%!PS-Adobe-2.0 %%Creator: dvips 5.516 Copyright 1986, 1993 Radical Eye Software %%Title: mpi-report.dvi %%CreationDate: Thu Apr 21 10:34:05 1994 %%Pages: 236 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips -o mpi-report.ps mpi-report %DVIPSSource: TeX output 1994.04.21:1032 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /@rigin{isls{[ 0 -1 1 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{Resolution hsize -72 div mul 0 TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{ /nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{ /sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0] N df-tail}B /E{pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get}B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{ 128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]} if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{ cc 1 add D}B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet %%BeginProcSet: special.pro TeXDict begin /SDict 200 dict N SDict begin /@SpecialDefaults{/hs 612 N /vs 792 N /ho 0 N /vo 0 N /hsc 1 N /vsc 1 N /ang 0 N /CLIP 0 N /rwiSeen false N /rhiSeen false N /letter{}N /note{}N /a4{}N /legal{}N}B /@scaleunit 100 N /@hscale{@scaleunit div /hsc X}B /@vscale{@scaleunit div /vsc X}B /@hsize{/hs X /CLIP 1 N}B /@vsize{/vs X /CLIP 1 N}B /@clip{ /CLIP 2 N}B /@hoffset{/ho X}B /@voffset{/vo X}B /@angle{/ang X}B /@rwi{ 10 div /rwi X /rwiSeen true N}B /@rhi{10 div /rhi X /rhiSeen true N}B /@llx{/llx X}B /@lly{/lly X}B /@urx{/urx X}B /@ury{/ury X}B /magscale true def end /@MacSetUp{userdict /md known{userdict /md get type /dicttype eq{userdict begin md length 10 add md maxlength ge{/md md dup length 20 add dict copy def}if end md begin /letter{}N /note{}N /legal{} N /od{txpose 1 0 mtx defaultmatrix dtransform S atan/pa X newpath clippath mark{transform{itransform moveto}}{transform{itransform lineto} }{6 -2 roll transform 6 -2 roll transform 6 -2 roll transform{ itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll curveto}}{{ closepath}}pathforall newpath counttomark array astore /gc xdf pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack}if}N /txpose{pxs pys scale ppr aload pop por{noflips{pop S neg S TR pop 1 -1 scale}if xflip yflip and{pop S neg S TR 180 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{pop S neg S TR pop 180 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{ppr 1 get neg ppr 0 get neg TR}if}{noflips{TR pop pop 270 rotate 1 -1 scale}if xflip yflip and{TR pop pop 90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{TR pop pop 90 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{TR pop pop 270 rotate ppr 2 get ppr 0 get neg sub neg 0 S TR}if}ifelse scaleby96{ppr aload pop 4 -1 roll add 2 div 3 1 roll add 2 div 2 copy TR .96 dup scale neg S neg S TR}if}N /cp {pop pop showpage pm restore}N end}if}if}N /normalscale{Resolution 72 div VResolution 72 div neg scale magscale{DVImag dup scale}if 0 setgray} N /psfts{S 65781.76 div N}N /startTexFig{/psf$SavedState save N userdict maxlength dict begin /magscale false def normalscale currentpoint TR /psf$ury psfts /psf$urx psfts /psf$lly psfts /psf$llx psfts /psf$y psfts /psf$x psfts currentpoint /psf$cy X /psf$cx X /psf$sx psf$x psf$urx psf$llx sub div N /psf$sy psf$y psf$ury psf$lly sub div N psf$sx psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub TR /showpage{}N /erasepage{}N /copypage{}N /p 3 def @MacSetUp}N /doclip{ psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll S lineto S lineto S lineto closepath clip newpath moveto}N /endTexFig{end psf$SavedState restore}N /@beginspecial{SDict begin /SpecialSave save N gsave normalscale currentpoint TR @SpecialDefaults count /ocount X /dcount countdictstack N}N /@setspecial {CLIP 1 eq{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto closepath clip}if ho vo TR hsc vsc scale ang rotate rwiSeen{rwi urx llx sub div rhiSeen{rhi ury lly sub div}{dup}ifelse scale llx neg lly neg TR }{rhiSeen{rhi ury lly sub div dup scale llx neg lly neg TR}if}ifelse CLIP 2 eq{newpath llx lly moveto urx lly lineto urx ury lineto llx ury lineto closepath clip}if /showpage{}N /erasepage{}N /copypage{}N newpath }N /@endspecial{count ocount sub{pop}repeat countdictstack dcount sub{ end}repeat grestore SpecialSave restore end}N /@defspecial{SDict begin} N /@fedspecial{end}B /li{lineto}B /rl{rlineto}B /rc{rcurveto}B /np{ /SaveX currentpoint /SaveY X N 1 setlinecap newpath}N /st{stroke SaveX SaveY moveto}N /fil{fill SaveX SaveY moveto}N /ellipse{/endangle X /startangle X /yrad X /xrad X /savematrix matrix currentmatrix N TR xrad yrad scale 0 0 1 startangle endangle arc savematrix setmatrix}N end %%EndProcSet TeXDict begin 40258431 52099146 1000 300 300 (/ogi/staff/otto/papers/Editor/MPI/draftFinal/mpi-report.dvi) @start /Fa 2 63 df<0000038000000F0000003C000000F0000003C000000F0000003C 000000F0000003C000000F0000003C000000F0000000F00000003C0000000F00000003C0 000000F00000003C0000000F00000003C0000000F00000003C0000000F00000003801918 7D9520>60 D 62 D E /Fb 17 118 df<387C7E7E3E0E1E1C78F060070B798416>44 D<1FE0003FF0007FF800783C00300E00000E00000E0003FE001FFE003E0E00700E00E00E 00E00E00E00E00783E007FFFE03FE7E00F83E013127E9116>97 D<03F80FFC1FFE3C1E78 0C7000E000E000E000E000E000F000700778073E0E1FFC0FF803F010127D9116>99 D<003F00007F00003F0000070000070000070000070003C7000FF7001FFF003C1F00780F 00700700E00700E00700E00700E00700E00700E00700700F00700F003C1F001FFFE00FE7 F007C7E014197F9816>I<03E00FF81FFC3C1E780E7007E007FFFFFFFFFFFFE000E00070 0778073C0F1FFE0FFC03F010127D9116>I<001F00007F8000FF8001E78001C30001C000 01C0007FFF00FFFF00FFFF0001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C0003FFE007FFF003FFE0011197F9816>I<018003C003C00180 00000000000000007FC07FC07FC001C001C001C001C001C001C001C001C001C001C001C0 01C07FFFFFFF7FFF101A7D9916>105 D<003000780078003000000000000000001FF81F F81FF8003800380038003800380038003800380038003800380038003800380038003800 38003800386070F0F0FFE07FC03F800D237E9916>I108 DI<7E3C00FEFE007FFF000F 87800F03800E03800E03800E03800E03800E03800E03800E03800E03800E03800E03807F C7F0FFE7F87FC7F01512809116>I<03E0000FF8001FFC003C1E00780F00700700E00380 E00380E00380E00380E00380F00780700700780F003C1E001FFC000FF80003E00011127E 9116>I<7E3E00FEFF007FFF800F83C00F00E00E00E00E00700E00700E00700E00700E00 700E00700E00E00F01E00F83C00FFF800EFF000E3C000E00000E00000E00000E00000E00 000E00007FC000FFE0007FC000141B809116>I114 D<0FEC3FFC7FFCF03CE01CE01C70007F801FF007F8003C600EE0 0EF00EF81EFFFCFFF8C7E00F127D9116>I<0300000700000700000700000700007FFF00 FFFF00FFFF00070000070000070000070000070000070000070000070100070380070380 07038007870003FE0001FC0000F80011177F9616>I<7E1F80FE3F807E1F800E03800E03 800E03800E03800E03800E03800E03800E03800E03800E03800E03800E0F800FFFF007FB F803E3F01512809116>I E /Fc 1 1 df0 D E /Fd 1 106 df<040C0000000000705898983030606464683006127E910B>105 D E /Fe 11 90 df22 D<0038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 38003800380038003800380038FFF8FFF8FFF80D63808118>I<00001800003000006000 00E00000C0000180000380000700000600000E00000C00001C0000380000380000700000 700000E00000E00001E00001C00001C0000380000380000380000780000700000700000F 00000E00000E00001E00001E00001E00001C00001C00003C00003C00003C00003C000038 0000780000780000780000780000780000780000780000780000700000F00000F00000F0 0000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0 0000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0000070 00007800007800007800007800007800007800007800007800003800003C00003C00003C 00003C00001C00001C00001E00001E00001E00000E00000E00000F000007000007000007 800003800003800003800001C00001C00001E00000E00000E00000700000700000380000 3800001C00000C00000E000006000007000003800001800000C00000E000006000003000 0018157C768121>32 DI<00001C00003C00 00F80001E00003C0000780000F00000E00001E00003C00003C00003C0000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 F00000F00000F00001E00001E00003C0000380000700000E00001C0000780000E00000E0 00007800001C00000E000007000003800003C00001E00001E00000F00000F00000F00000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 7800007800003C00003C00003C00001E00000E00000F000007800003C00001E00000F800 003C00001C167C7B8121>40 D<0018007800F001E003C007800F001F001E003E003C007C 007C007800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F8 00F800F800F800F800F800F800F8000D25707E25>56 D58 D<007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C00F800F800F800F001F001E003E003C0078007000E001C 003800F000C000F00038001C000E000700078003C003E001E001F000F000F800F800F800 7C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C0E4D798025>60 D62 D80 D89 D E /Ff 7 116 df<60F0F070101020204040040A7D830A>59 D<003E000C000C000C000C0018001800180018073018F0307060706060C060C060C06080 C080C480C4C1C446C838700F177E9612>100 D<03000380030000000000000000000000 00001C002400460046008C000C0018001800180031003100320032001C0009177F960C> 105 D<00180038001000000000000000000000000001C002200430043008600060006000 6000C000C000C000C001800180018001806300E300C60078000D1D80960E>I<383C1E00 44C6630047028100460301008E0703000C0603000C0603000C060600180C0600180C0620 180C0C20180C0C4030180440301807801B0E7F8D1F>109 D<383C0044C6004702004602 008E06000C06000C06000C0C00180C00180C40181840181880300880300F00120E7F8D15 >I<07C00C201870187038001E000FC003E000606060E060C0C0C1803F000C0E7E8D10> 115 D E /Fg 12 94 df<07C018303018701C600C600CE00EE00EE00EE00EE00EE00EE0 0EE00EE00E600C600C701C30181C7007C00F157F9412>48 D<03000700FF000700070007 00070007000700070007000700070007000700070007000700070007007FF00C157E9412 >I<0F8030E040708030C038E0384038003800700070006000C00180030006000C080808 10183FF07FF0FFF00D157E9412>I<0FE030306018701C701C001C00180038006007E000 300018000C000E000EE00EE00EC00C401830300FE00F157F9412>I<00300030007000F0 01F001700270047008701870107020704070C070FFFE0070007000700070007003FE0F15 7F9412>I<20303FE03FC0240020002000200020002F8030E020700030003800384038E0 38E0388030406020C01F000D157E9412>I<01F00608080C181C301C70006000E000E3E0 EC30F018F00CE00EE00EE00E600E600E300C3018183007C00F157F9412>I<40007FFE7F FC7FF8C008801080200040008000800100010003000200060006000E000E000E000E000E 0004000F167E9512>I<07E018302018600C600C700C78183E101F600FC00FF018F8607C 601EC00EC006C006C004600C38300FE00F157F9412>I61 D91 D93 D E /Fh 5 82 df0 D<020002000200C218F2 783AE00F800F803AE0F278C2180200020002000D0E7E8E12>3 D<000300030006000600 0C000C00180018003000300060006000C000C00180018003000300060006000C000C0018 0018003000300060006000C0004000101E7B9600>54 D<01FFE00FFFF818E0FC20E03E60 E01EC0E00E00E00E00E00C00C00C00C01801C01001C02001804001818003BE0003780003 00000700000600000600000E00000C00000C000018000017187F9619>80 D<001FC0007FE00181F00200F804007808003C18003C30001C30001C60001C60001CE000 18E00018E00030E00030F00060F808407C30803FE1001F02000004000038003FE0067FE0 0C0FFC0800FFF0001FC0171B7C961D>I E /Fi 61 122 df<01F1C003F1C007F1C00701 C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFF1C0FFF1C00E01C00E01C00E01 C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01 C00E01C0121D809C16>13 D<00F0000001F8000003FC0000039C0000070E0000070E0000 070E0000070E0000070E0000071E0000073C0000073800000778000003F0000003E00300 07C003000F8006001FC006003DC0060078E00C0070F00C00F0701800E0783800E03C3000 E01C6000E00EC000700F800070078040381FE1C01FF8FFC007E03F001A1F7E9D1F>38 D<00E001C0038007000E000E001C001C003800380038007000700070007000E000E000E0 00E000E000E000E000E000E000E000E000E00070007000700070003800380038001C001C 000E000E000700038001C000E00B2A7E9E10>40 DI<018001C0018001806186F99F7DBE1FF807E007E01FF87DBEF99F618601800180 01C0018010127E9E15>I<00060000000600000006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000FFFFFFE0FFFFFFE000 060000000600000006000000060000000600000006000000060000000600000006000000 0600000006000000060000000600001B1C7E9720>II< FFE0FFE00B0280890E>I<03C00FF01FF83C3C381C700E700E700EE007E007E007E007E0 07E007E007E007E007E007E007E007E007700E700E700E381C3C3C1FF80FF007E0101D7E 9B15>48 D<010007003F00FF00C700070007000700070007000700070007000700070007 000700070007000700070007000700070007000700FFF8FFF80D1C7C9B15>I<07C01FF0 3878701C601EE00EC00F400F400700070007000F000E001E001C003C007800F001E001C0 038007000E001C0038007000FFFFFFFF101C7E9B15>I58 D<7FFFFFC0FFFFFFE0000000000000000000000000000000 0000000000000000000000000000000000FFFFFFE07FFFFFC01B0C7E8F20>61 D<001C0000003E0000003E0000002E0000006700000067000000E7800000C7800000C380 0001C3C0000183C0000181C0000381E0000381E0000700F0000700F0000600F0000E0078 000FFFF8000FFFF8001C003C001C003C0018003C0038001E0038001E0070001F0070000F 0070000F00E0000780191D7F9C1C>65 DI<003FC000FFF003C0F00780300F00001E00003C00003C0000780000780000780000 F00000F00000F00000F00000F00000F00000F00000F00000F00000780000780000780000 3C00003C00001E00000F000807801803C07800FFF0003F80151F7D9D1B>IIII<003F8001FFF003C0F80780 380F00181E00003C00003C0000780000780000780000F00000F00000F00000F00000F000 00F00000F007F8F007F8F000387800387800387800383C00383C00381E00380F00380780 3803C0F801FFF0003F80151F7D9D1C>III75 DIII<003F000001FFE00003FFF00007C0F800 0F807C001E001E003E001F003C000F00780007807800078078000780F00003C0F00003C0 F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F80007C07800078078000780 7C000F803C000F003E001F001F003E000F807C0007C0F80003FFF00001FFE000003F0000 1A1F7E9D1F>II<003F000001FFE0 0003FFF00007C0F8000F807C001F003E003E001F003C000F007800078078000780780007 80F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003 C07800078078000780780E07803C0F0F003E079F001E03DE000F83FC0007C1F80003FFF0 0001FFF800003F780000003C0000003E0000001F0000000F801A237E9D1F>II<03F8000FFE001C0F00380700700300600000 E00000E00000E00000E00000F000007800007F00003FE0001FFC0007FE0001FF00001F80 0007800003C00003C00001C00001C00001C00001C0C00180E00380F007007C0E001FFC00 07F000121F7E9D17>IIII I<78000E007C001E003C003C001E0038000F0070000F00F0000781E00003C1C00001C3C0 0001E7800000F70000007E0000003E0000003C0000003C0000007E00000077000000E780 0001E3800003C1C0000381E0000700F0000F00F8000E0078001C003C003C003E0078001F 0070000F00F0000F80191D7F9C1C>II<7FFFF07FFF F00001E00003E00003C00007C0000780000F00001F00001E00003E00003C0000780000F8 0000F00001F00001E00003C00007C0000780000F80000F00001E00003E00003C00007C00 00780000FFFFF0FFFFF0141D7E9C19>I<0FC03FF07FF87038401C001C001C00FC0FFC3F FC781CE01CE01CE01CF07C7FFC7FDC3F1C0E127E9114>97 DI<07E00FF81FFC3C1C70047000E000E000E000 E000E000E000700070043C1C1FFC0FF807E00E127E9112>I<000E000E000E000E000E00 0E000E000E000E000E000E0F8E1FEE3FFE7C3E700E700EE00EE00EE00EE00EE00EE00EF0 0E701E7C3E3FFE1FEE0F8E0F1D7E9C15>I<07C01FE03FF078787018601CFFFCFFFCFFFC E000E000E000700070043C1C3FFC1FF807E00E127E9112>I<00FC01FC03FC07000E000E 000E000E000E000E000E00FFE0FFE00E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E1D809C0D>I<03C3C00FFFC01FFFC01C3800381C00381C00 381C00381C00381C001C38001FF8001FF0003BC0003800003800001FFC001FFF003FFF80 700780E001C0E001C0E001C0F003C07C0F803FFF001FFE0007F800121B7F9115>III107 DIII< 03F0000FFC001FFE003C0F00780780700380E001C0E001C0E001C0E001C0E001C0F003C0 7003807807803C0F001FFE000FFC0003F00012127F9115>II114 D<1FC03FF07FF0F030E000E000F0007F 003FC01FE000F0003800388038F078FFF07FE01FC00D127F9110>I<1C001C001C001C00 1C001C00FFE0FFE01C001C001C001C001C001C001C001C001C001C001C001C001C201FF0 0FF007C00C187F970F>IIII<70038078 07003C0E001C1C000E1C0007380003F00001E00001C00001E00003F0000738000E18000E 1C001C0E00380700700380F003C01212809113>II E /Fj 64 122 df<003F83F800FF8FF801FF9FF803E03E0003803800 0780780007807800078078000780780007807800078078000780780007807800FFFC7FC0 FFFC7FC0FFFC7FC007807800078078000780780007807800078078000780780007807800 078078000780780007807800078078000780780007807800078078000780780007807800 0780780007807800078078001D2380A21C>11 D<000078007C7800FC7801FC7803C00003 8000078000078000078000078000078000078000078000078000FFFC78FFFC78FFFC7807 807807807807807807807807807807807807807807807807807807807807807807807807 8078078078078078078078078078078078078078152480A31A>I<0000C018000000C018 000000C01800000180300000018030000001803000000180300000030060000003006000 000300600000030060000003006000000600C000000600C000000600C000000600C00000 0C018000FFFFFFFFC0FFFFFFFFC000180300000018030000001803000000180300000030 060000003006000000300600000030060000FFFFFFFFC0FFFFFFFFC000600C000000C018 000000C018000000C018000000C018000001803000000180300000018030000001803000 0003006000000300600000030060000003006000000600C000000600C000000600C00000 222D7DA229>35 D<787878781830306060E0050A7CA20E>39 D<007000E001C003C00780 07000F000E001E001C003C003C00380078007800780078007000F000F000F000F000F000 F000F000F000F000F000F000F000F000F0007000780078007800780038003C003C001C00 1E000E000F000700078003C001C000E000700C327DA413>II<787878781830306060E0050A 7C830E>44 DII<00C001C007C0FF C0FFC0FBC003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003 C003C003C003C003C003C003C003C003C003C0FFFFFFFFFFFF10227CA118>49 D<03F0000FFC001FFE003C1F003007807007C06003C0E003E0C001E04001E04001E00001 E00001E00001E00003C00003C0000780000780000F00001E00003C0000780000F00001E0 0001C0000380000700000E00001C0000380000700000FFFFE0FFFFE0FFFFE013227EA118 >I<01F00007FC001FFF003E0F003807807003C02003C02003C00003C00003C00003C000 0780000780000F00001E0003FC0003F80003FE00000F000007800003C00003C00001E000 01E00001E00001E00001E08001E0C003C0E003C07007803C0F801FFF000FFC0003F00013 237EA118>I<001F00001F00002F00002F00006F0000EF0000CF0001CF0001CF00038F00 038F00078F00070F000F0F000E0F001E0F003C0F003C0F00780F00780F00F00F00FFFFF8 FFFFF8FFFFF8000F00000F00000F00000F00000F00000F00000F00000F00000F0015217F A018>I<3FFF803FFF803FFF803C00003C00003C00003C00003C00003C00003C00003C00 003C00003CF8003FFE003FFF003F0F803E07803C03C03803C00001E00001E00001E00001 E00001E00001E00001E04003C04003C0E003C07007807C1F003FFE000FFC0003F0001322 7EA018>I<007E0001FF0003FF0007C1000F00001E00001E00003C00003C000078000078 000078F800F1FE00F7FF00FF0F80FC0780F803C0F803C0F801C0F001E0F001E0F001E0F0 01E0F001E07001E07001E07801E07803C03803C03C03801C07801F0F000FFE0007FC0001 F00013237EA118>II<01F00007FC000FFE001E0F003C07803C07807803C07803C07803C07803 C07803C03803803C07801E0F000F1E0007FC0003F8000FFE001E0F003C07807803C07803 C0F001E0F001E0F001E0F001E0F001E0F001E07803C07803C03C07803E0F801FFF0007FC 0001F00013237EA118>I<01F00007FC000FFE001E0F003C07003807807803807803C0F0 03C0F001C0F001E0F001E0F001E0F001E0F001E0F001E07003E07803E07803E03C07E03E 1FE01FFDE00FF1E003E3C00003C00003C0000380000780000700000F00101E00183C003F F8001FF00007C00013237EA118>II<001F0000001F0000003F8000003F8000003B8000007BC0000073C000 0071C00000F1E00000F1E00000E0E00001E0F00001E0F00001C0F00003C0780003C07800 0380780007803C0007803C0007003C000F001E000F001E000FFFFE001FFFFF001FFFFF00 1C000F003C0007803C00078038000780780003C0780003C0700003C0F00001E0F00001E0 E00001E01B237EA220>65 DI<000FF000007FFC0000FFFF0001F01F0003C00700 078000000F0000001E0000003E0000003C0000003C000000780000007800000078000000 F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000 7800000078000000780000003C0000003C0000003E0000001E0000000F00000007800080 03C0038001F00F8000FFFF00007FFC00000FF00019257DA31F>IIII<000FF000003FFE0000FFFF8001F80F8003E00380078000000F0000001E0000001E00 00003C0000003C000000780000007800000078000000F0000000F0000000F0000000F000 0000F0000000F0000000F000FFC0F000FFC0F000FFC0780003C0780003C0780003C03C00 03C03C0003C01E0003C01E0003C00F0003C0078003C003E003C001F807C000FFFFC0003F FF00000FF8001A257DA321>III76 DII<001FC000007FF00001FFFC0003F07E0007C01F000F800F801F0007C0 1E0003C03C0001E03C0001E0780000F0780000F0780000F070000070F0000078F0000078 F0000078F0000078F0000078F0000078F0000078F0000078F0000078780000F0780000F0 780000F07C0001F03C0001E03E0003E01E0003C01F0007C00F800F8007C01F0003F07E00 01FFFC00007FF000001FC0001D257DA324>II82 D<00FE0003FFC007FFE00F81E01E00603C00003C00007800007800007800007800007800 007C00003C00003F00001FC0000FFC0007FF0001FF80003FC00007E00001F00000F00000 F8000078000078000078000078000078000078C000F0E000F0F801E07E07C03FFF800FFF 0001FC0015257EA31B>IIII<780003E07C0003C03E0007C0 1E000F800F000F000F801F0007C03E0003C03C0003E07C0001F0780000F0F0000079F000 007DE000003FC000001FC000001F8000000F0000001F8000001FC000003FC000007DE000 0079F00000F0F00001F0780001E03C0003C03E0007C01E0007800F000F000F801F000780 1E0003C03C0003E07C0001F0780000F0F80000F81D237FA220>88 D<07E01FF83FFC381E201E000F000F000F000F00FF07FF1FFF3E0F780FF00FF00FF00FF0 0FF83F7FFF3FEF1F8F10167E9517>97 DI<01FC0007FF000FFF801F03803C0180 780000780000700000F00000F00000F00000F00000F00000F00000780000780000780000 3C00401F03C00FFFC007FF8001FC0012167E9516>I<0003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C003E3C00FFBC01FFFC03F0F C03C07C07803C07803C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07803 C07803C03C07C03E0FC01FFFC00FFBC003E3C012237EA219>I<03F00007FC001FFE003E 0F003C0780780380780380F001C0FFFFC0FFFFC0FFFFC0F00000F00000F0000070000078 00007800003C00801F07800FFF8007FF0001F80012167E9516>I<003F00FF01FF03C003 8007800780078007800780078007800780FFF8FFF8FFF807800780078007800780078007 80078007800780078007800780078007800780078007800780102380A20F>I<01F07807 FFF80FFFF81F1F001E0F003C07803C07803C07803C07803C07801E0F001F1F000FFE001F FC0019F0003800003800003C00001FFE001FFFC01FFFE03FFFF07801F07800F8F00078F0 0078F00078F000787800F03E03E01FFFC00FFF8001FC0015217F9518>III<00F000F000F000F000000000000000000000000000000000000000F000F000 F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000 F000F000F000F000F000F000F080F0E3E0FFE07FC01F000C2D83A20D>IIIII<01FC0007FF000FFF801F07C03C01E07800F07800F0700070F00078F00078 F00078F00078F00078F000787800F07800F07C01F03E03E01F07C00FFF8007FF0001FC00 15167F9518>II<03E3C007FBC00FFFC01F0FC03C07C07C03C07803C0F803C0F003C0F003C0F003C0F0 03C0F003C0F003C0F803C07803C07803C03C07C03F0FC01FFFC00FFBC003E3C00003C000 03C00003C00003C00003C00003C00003C00003C00003C00003C012207E9519>II<07F01FFC3FFE3C0E7806780078007C003F003FF01FF80FFC 01FE001F000F000F000FC00FF81EFFFE3FFC0FF010167F9513>I<0F000F000F000F000F 000F00FFF8FFF8FFF80F000F000F000F000F000F000F000F000F000F000F000F000F000F 000F080F1C07FC07F803E00E1C7F9B12>IIII<7801F07C01E03E03C01E07C00F0780 078F0007DE0003FC0001FC0000F80000700000F80001FC0003DC00039E00078F000F0780 1E07801E03C03C01E07800F0F800F81516809516>II E /Fk 60 124 df<007E1F0001C1B1800303E3 C00703C3C00E03C1800E01C0000E01C0000E01C0000E01C0000E01C0000E01C000FFFFFC 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0007F87FC001A1D80 9C18>11 D<007E0001C1800301800703C00E03C00E01800E00000E00000E00000E00000E 0000FFFFC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C07F87F8151D809C17>I<6060F0F0F8F8686808 0808080808101010102020404080800D0C7F9C15>34 D<004000800100020006000C000C 0018001800300030007000600060006000E000E000E000E000E000E000E000E000E000E0 00E000E000600060006000700030003000180018000C000C00060002000100008000400A 2A7D9E10>40 D<800040002000100018000C000C00060006000300030003800180018001 8001C001C001C001C001C001C001C001C001C001C001C001C00180018001800380030003 00060006000C000C00180010002000400080000A2A7E9E10>I<60F0F070101010102020 4080040C7C830C>44 DI<60F0F06004047C830C>I<0001000300 0600060006000C000C000C0018001800180030003000300060006000C000C000C0018001 800180030003000300060006000C000C000C001800180018003000300030006000600060 00C000C00010297E9E15>I<03C00C301818300C300C700E60066006E007E007E007E007 E007E007E007E007E007E007E007E007E00760066006700E300C300C18180C3007E0101D 7E9B15>I<030007003F00C7000700070007000700070007000700070007000700070007 0007000700070007000700070007000700070007000F80FFF80D1C7C9B15>I<07C01830 201C400C400EF00FF80FF807F8077007000F000E000E001C001C00380070006000C00180 030006010C01180110023FFE7FFEFFFE101C7E9B15>I<60F0F060000000000000000000 0060F0F06004127C910C>58 D<60F0F0600000000000000000000060F0F0701010101020 204080041A7C910C>I<7FFFFFC0FFFFFFE0000000000000000000000000000000000000 0000000000000000000000000000FFFFFFE07FFFFFC01B0C7E8F20>61 D<000600000006000000060000000F0000000F0000000F00000017800000178000001780 000023C0000023C0000023C0000041E0000041E0000041E0000080F0000080F0000180F8 000100780001FFF80003007C0002003C0002003C0006003E0004001E0004001E000C001F 001E001F00FF80FFF01C1D7F9C1F>65 D<001F808000E0618001801980070007800E0003 801C0003801C00018038000180780000807800008070000080F0000000F0000000F00000 00F0000000F0000000F0000000F0000000F0000000700000807800008078000080380000 801C0001001C0001000E000200070004000180080000E03000001FC000191E7E9C1E>67 DIII<001F808000E0618001801980070007800E0003801C00 03801C00018038000180780000807800008070000080F0000000F0000000F0000000F000 0000F0000000F0000000F000FFF0F0000F80700007807800078078000780380007801C00 07801C0007800E00078007000B800180118000E06080001F80001C1E7E9C21>I73 D76 DII<003F800000E0E0000380380007 001C000E000E001C0007003C00078038000380780003C0780003C0700001C0F00001E0F0 0001E0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0700001C0780003C078 0003C0380003803C0007801C0007000E000E0007001C000380380000E0E000003F80001B 1E7E9C20>II82 D<07E0801C1980300580700380600180E00180E00080E00080E00080F00000F8 00007C00007FC0003FF8001FFE0007FF0000FF80000F800007C00003C00001C08001C080 01C08001C0C00180C00180E00300D00200CC0C0083F800121E7E9C17>I<7FFFFFC0700F 01C0600F00C0400F0040400F0040C00F0020800F0020800F0020800F0020000F0000000F 0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F 0000000F0000000F0000000F0000000F0000000F0000000F0000001F800003FFFC001B1C 7F9B1E>II89 D<08081010202040404040808080808080B0B0F8F8787830300D0C7A9C15>92 D<1FC000307000783800781C00301C00001C00001C0001FC000F1C00381C00701C00601C 00E01C40E01C40E01C40603C40304E801F870012127E9115>97 DI<07E00C301878307870306000E000E000E000E000E000E0 0060007004300418080C3007C00E127E9112>I<003F0000070000070000070000070000 070000070000070000070000070000070003E7000C1700180F00300700700700600700E0 0700E00700E00700E00700E00700E00700600700700700300700180F000C370007C7E013 1D7E9C17>I<03E00C301818300C700E6006E006FFFEE000E000E000E000600070023002 18040C1803E00F127F9112>I<00F8018C071E061E0E0C0E000E000E000E000E000E00FF E00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E007FE00F 1D809C0D>I<00038003C4C00C38C01C3880181800381C00381C00381C00381C00181800 1C38000C300013C0001000003000001800001FF8001FFF001FFF803003806001C0C000C0 C000C0C000C06001803003001C0E0007F800121C7F9215>II<18003C003C0018000000000000000000000000000000FC001C 001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00FF80091D7F 9C0C>I<00C001E001E000C000000000000000000000000000000FE000E000E000E000E0 00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E060E0F0C0 F1C061803E000B25839C0D>IIIII<03F0000E1C00180600300300700380600180E001C0E001C0E001C0E001C0 E001C0E001C06001807003803003001806000E1C0003F00012127F9115>II<03C1000C3300180B00300F00700700700700E00700E00700E00700E0 0700E00700E00700600700700700300F00180F000C370007C70000070000070000070000 0700000700000700000700003FE0131A7E9116>II<1F9030704030C010 C010E010F8007F803FE00FF000F880388018C018C018E010D0608FC00D127F9110>I<04 000400040004000C000C001C003C00FFE01C001C001C001C001C001C001C001C001C001C 101C101C101C101C100C100E2003C00C1A7F9910>III< FF3FCFE03C0F03801C0701801C0701001C0B01000E0B82000E0B82000E1182000711C400 0711C4000720C40003A0E80003A0E80003C0680001C0700001C070000180300000802000 1B127F911E>I<7F8FF00F03800F030007020003840001C80001D80000F0000070000078 0000F800009C00010E00020E000607000403801E07C0FF0FF81512809116>II<7FFC70386038407040F040E041C003C0038007000F040E041C043C 0C380870087038FFF80E127F9112>II E /Fl 58 124 df<00003FE00000E0100001803800038078000300780007003000070000000700 0000070000000E0000000E0000000E000000FFFFE0000E00E0001C01C0001C01C0001C01 C0001C01C0001C0380003803800038038000380380003807000038070000700700007007 1000700E2000700E2000700E2000E00E2000E0064000E0038000E0000000C0000001C000 0001C000003180000079800000F3000000620000003C0000001D29829F1A>12 D<0E1F3F3F1D0102020404081020C0080E779F0E>39 D<00010002000400080010002000 6000C0018001800300070006000E000C001C0018003800380030007000700060006000E0 00E000C000C000C000C000C000C000C000C000C000C000C000C000C00040006000600020 00100010000800102E79A113>I<00100000080000040000060000020000030000030000 030000010000018000018000018000018000018000018000018000038000038000038000 0300000300000300000700000700000600000600000E00000C00000C00001C0000180000 380000300000700000600000E00000C0000180000100000300000600000C000018000030 0000600000800000112E80A113>I<1C3C3C3C3C040408081020204080060E7D840E>44 D<7FF0FFE07FE00C037D8A10>I<70F8F8F0E005057B840E>I<000F800030E000E07001C0 700380300380380700380F00780F00780E00781E00781E00703C00F03C00F03C00F03C00 F07801E07801E07801E07801C07003C0F003C0F00380F00780F00700700700700E00701C 003038001870000FC000151F7C9D17>48 D<000200020006000E003C00DC031C001C0038 003800380038007000700070007000E000E000E000E001C001C001C001C0038003800380 03800780FFF80F1E7B9D17>I<001F000061800080E00100E00200700220700420700410 700820F00820F00820F00840E00881E00703C0000380000700000C000018000060000080 000300000400000800401000401000802001807E030047FF0041FE0080FC00807800141F 7C9D17>I<0000600000E00000E00000E00001C00001C00001C000038000038000030000 0700000700000600000E00000C0000180000180000300000300000630000C70000870001 0700030700060E00040E00080E003F8E00607C00801FC0001C00001C0000380000380000 380000380000700000700000600013277E9D17>52 D<001F0000718000C0C00180C00380 E00700E00F00E00F01E01E01E01E01E01E01E01E01C01C03C01C03C01C03C01C07C01C0F 800C0F8006378003C700000F00000E00000E00001C00601C00F03800F07000E0600080C0 004380003E0000131F7B9D17>57 D<00000200000006000000060000000E0000001E0000 001E0000003F0000002F0000004F0000004F0000008F0000010F0000010F0000020F0000 020F0000040F00000C0F0000080F0000100F0000100F0000200F80003FFF800040078000 C007800080078001000780010007800200078002000780060007801E000F80FF807FF81D 207E9F22>65 D<01FFFFC0001E00F0001E0078001E0038001E003C003C003C003C003C00 3C003C003C003C0078007800780078007800F0007801E000F0078000FFFE0000F00F8000 F003C001E001C001E001E001E001E001E001E003C001E003C001E003C001E003C001C007 8003C00780078007800F0007801E000F007800FFFFE0001E1F7D9E20>I<0000FE020007 8186001C004C0038003C0060003C00C0001C01C0001803800018070000180F0000181E00 00101E0000103C0000003C00000078000000780000007800000078000000F0000000F000 0000F0000000F0000000F000008070000080700000807000010038000100380002001800 04000C001800060020000381C00000FE00001F217A9F21>I<01FFFF80001E00E0001E00 70001E0038001E001C003C001C003C000E003C000E003C000E0078000E0078000E007800 0E0078000E00F0001E00F0001E00F0001E00F0001E01E0003C01E0003C01E0003C01E000 7803C0007003C0007003C000E003C001C0078001C00780038007800E0007801C000F0070 00FFFFC0001F1F7D9E22>I<01FFFFFE001E001C001E000C001E0004001E0004003C0004 003C0004003C0004003C00040078080800780800007808000078180000F0300000FFF000 00F0300000F0300001E0200001E0200001E0200001E0001003C0002003C0002003C00040 03C00040078000800780018007800100078007000F001F00FFFFFE001F1F7D9E1F>I<01 FFFFFC001E0038001E0018001E0008001E0008003C0008003C0008003C0008003C000800 78001000780800007808000078080000F0100000F0300000FFF00000F0300001E0200001 E0200001E0200001E0200003C0000003C0000003C0000003C00000078000000780000007 800000078000000F800000FFF800001E1F7D9E1E>I<0000FC040007030C001C00980030 007800E0007801C000380380003003800030070000300E0000301E0000201E0000203C00 00003C00000078000000780000007800000078000000F0000000F000FFF0F0000780F000 0780F0000F0070000F0070000F0070000F0070001E0038001E0018003E001C002E000E00 CC000383040000FC00001E217A9F23>I<01FFF3FFE0001F003E00001E003C00001E003C 00001E003C00003C007800003C007800003C007800003C007800007800F000007800F000 007800F000007800F00000F001E00000FFFFE00000F001E00000F001E00001E003C00001 E003C00001E003C00001E003C00003C007800003C007800003C007800003C00780000780 0F000007800F000007800F000007800F00000F801F0000FFF1FFE000231F7D9E22>I<01 FFF0001F00001E00001E00001E00003C00003C00003C00003C0000780000780000780000 780000F00000F00000F00000F00001E00001E00001E00001E00003C00003C00003C00003 C0000780000780000780000780000F8000FFF800141F7D9E12>I<001FFF0000F80000F0 0000F00000F00001E00001E00001E00001E00003C00003C00003C00003C0000780000780 000780000780000F00000F00000F00000F00001E00001E00301E00781E00F83C00F83C00 F0780080700040E00021C0001F000018207D9E18>I<01FFF03FE0001F000F80001E000E 00001E000800001E001000003C002000003C004000003C010000003C0200000078040000 00780800000078100000007830000000F0F0000000F1F8000000F278000000F478000001 E83C000001F03C000001E03C000001E01E000003C01E000003C01E000003C00F000003C0 0F000007800F00000780078000078007800007800780000F8007C000FFF03FF800231F7D 9E23>I<01FFF800001F0000001E0000001E0000001E0000003C0000003C0000003C0000 003C00000078000000780000007800000078000000F0000000F0000000F0000000F00000 01E0000001E0000001E0000001E0008003C0010003C0010003C0030003C0020007800600 0780060007800C0007801C000F007800FFFFF800191F7D9E1D>I<01FE00007FC0001E00 00FC00001E0000F80000170001780000170001780000270002F00000270004F000002700 04F00000270008F00000470009E00000470011E00000470021E00000470021E000008700 43C00000838043C00000838083C00000838083C000010381078000010382078000010382 0780000103840780000203840F00000203880F00000203900F00000203900F00000401E0 1E00000401E01E00000401C01E00000C01801E00001C01803E0000FF8103FFC0002A1F7D 9E29>I<01FF007FE0001F000F00001F0004000017800400001780040000278008000023 C008000023C008000023C008000041E010000041E010000041F010000040F010000080F0 200000807820000080782000008078200001003C400001003C400001003C400001001E40 0002001E800002001E800002000F800002000F800004000F000004000700000400070000 0C000700001C00020000FF80020000231F7D9E22>I<0001FC0000070700001C01C00030 00E000E0006001C000700380007007800038070000380E0000381E0000381C0000383C00 00383C00003878000078780000787800007878000078F00000F0F00000F0F00000E0F000 01E0F00001C0F00003C0700003807000070078000F0038001E0038003C001C0070000E00 E0000783800001FC00001D217A9F23>I<01FFFF80001E00E0001E0070001E0038001E00 3C003C003C003C003C003C003C003C003C0078007800780078007800F0007800E000F003 C000F00F0000FFFC0000F0000001E0000001E0000001E0000001E0000003C0000003C000 0003C0000003C00000078000000780000007800000078000000F800000FFF000001E1F7D 9E1F>I<01FFFF00001E03C0001E00E0001E0070001E0078003C0078003C0078003C0078 003C0078007800F0007800F0007801E0007801C000F0070000F01E0000FFF00000F03800 01E01C0001E01E0001E00E0001E00F0003C01E0003C01E0003C01E0003C01E0007803C00 07803C0807803C0807803C100F801C10FFF00C20000007C01D207D9E21>82 D<0007E040001C18C0003005800060038000C0038001C001800180010003800100038001 00038001000380000003C0000003C0000003F8000001FF800001FFE000007FF000001FF0 000001F80000007800000078000000380000003800200038002000380020003000600070 00600060006000E0007000C000E8038000C606000081F800001A217D9F1A>I<0FFFFFF0 1E0780E0180780201007802020078020200F0020600F0020400F0020400F0020801E0040 001E0000001E0000001E0000003C0000003C0000003C0000003C00000078000000780000 007800000078000000F0000000F0000000F0000000F0000001E0000001E0000001E00000 01E0000003E00000FFFF00001C1F789E21>I<7FFC1FF807C003C0078001000780010007 8001000F0002000F0002000F0002000F0002001E0004001E0004001E0004001E0004003C 0008003C0008003C0008003C00080078001000780010007800100078001000F0002000F0 002000F0002000F0004000F0004000700080007001000030020000380400000C18000007 E000001D20779E22>I I<00F1800389C00707800E03801C03803C0380380700780700780700780700F00E00F00E 00F00E00F00E20F01C40F01C40703C40705C40308C800F070013147C9317>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E380F38 0F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207B9F15>I< 007E0001C1000300800E07801E07801C07003C0200780000780000780000F00000F00000 F00000F00000F0000070010070020030040018380007C00011147C9315>I<0000780003 F80000700000700000700000700000E00000E00000E00000E00001C00001C000F1C00389 C00707800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E 20F01C40F01C40703C40705C40308C800F070015207C9F17>I<007C01C207010E011C01 3C013802780C7BF07C00F000F000F000F0007000700170023804183807C010147C9315> I<00007800019C00033C00033C000718000700000700000E00000E00000E00000E00000E 0001FFE0001C00001C00001C00001C000038000038000038000038000038000070000070 0000700000700000700000700000E00000E00000E00000E00000C00001C00001C0000180 003180007B0000F300006600003C00001629829F0E>I<003C6000E27001C1E00380E007 00E00F00E00E01C01E01C01E01C01E01C03C03803C03803C03803C03803C07003C07001C 0F001C17000C2E0003CE00000E00000E00001C00001C00301C00783800F0700060E0003F 8000141D7E9315>I<01E0000FE00001C00001C00001C00001C000038000038000038000 038000070000070000071E000763000E81800F01C00E01C00E01C01C03801C03801C0380 1C0380380700380700380700380E10700E20700C20701C20700C40E00CC060070014207D 9F17>I<00C001E001E001C000000000000000000000000000000E003300230043804300 470087000E000E000E001C001C001C003840388030807080310033001C000B1F7C9E0E> I<01E0000FE00001C00001C00001C00001C0000380000380000380000380000700000700 000703C00704200E08E00E11E00E21E00E40C01C80001D00001E00001FC00038E0003870 00387000383840707080707080707080703100E03100601E0013207D9F15>107 D<03C01FC0038003800380038007000700070007000E000E000E000E001C001C001C001C 0038003800380038007000700070007100E200E200E200E200640038000A207C9F0C>I< 1C0F80F0002630C318004740640C004780680E004700700E004700700E008E00E01C000E 00E01C000E00E01C000E00E01C001C01C038001C01C038001C01C038001C01C070803803 8071003803806100380380E10038038062007007006600300300380021147C9325>I<1C 0F802630C04740604780604700704700708E00E00E00E00E00E00E00E01C01C01C01C01C 01C01C03843803883803083807083803107003303001C016147C931A>I<007C0001C300 0301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F00780 F00700700F00700E0030180018700007C00013147C9317>I<01C1E002621804741C0478 1C04701E04701E08E01E00E01E00E01E00E01E01C03C01C03C01C03C01C0380380780380 700380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00001C00 00FFC000171D809317>I<00F0400388C00705800E03801C03803C038038070078070078 0700780700F00E00F00E00F00E00F00E00F01C00F01C00703C00705C0030B8000F380000 380000380000700000700000700000700000E00000E0000FFE00121D7C9315>I<1C1E00 2661004783804787804707804703008E00000E00000E00000E00001C00001C00001C0000 1C000038000038000038000038000070000030000011147C9313>I<00FC030206010C03 0C070C060C000F800FF007F803FC003E000E700EF00CF00CE008401020601F8010147D93 13>I<018001C0038003800380038007000700FFF007000E000E000E000E001C001C001C 001C003800380038003820704070407080708031001E000C1C7C9B0F>I<0E00C03300E0 2301C04381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C0710 1C0E20180E20180E201C1E200C264007C38014147C9318>I<0E03803307802307C04383 C04301C04700C08700800E00800E00800E00801C01001C01001C01001C02001C02001C04 001C04001C08000E300003C00012147C9315>I<0E00C1C03300E3C02301C3E04381C1E0 4301C0E04701C060870380400E0380400E0380400E0380401C0700801C0700801C070080 1C0701001C0701001C0602001C0F02000C0F04000E13080003E1F0001B147C931E>I<03 83800CC4401068E01071E02071E02070C040E00000E00000E00000E00001C00001C00001 C00001C040638080F38080F38100E5810084C60078780013147D9315>I<0E00C03300E0 2301C04381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C0700 1C0E00180E00180E001C1E000C3C0007DC00001C00001C00003800F03800F07000E06000 C0C0004380003E0000131D7C9316>I123 D E /Fm 49 121 df<1C3E7E7E3A0202040408081020C0070E7D840D>44 DI<3078F8787005057C840D>I<00000400000C00000C00001800 00180000300000300000600000600000C00000C000018000018000030000030000060000 0600000C00000C0000180000180000300000300000600000600000C00000C00001800001 80000300000300000600000600000600000C00000C000018000018000030000030000060 0000600000C00000C00000800000162D7EA117>I<007E0001C3000301800701C00E00C0 0E00E01C00E01C00E03C01E03801E07801E07801E07801E07801E07801E0F003C0F003C0 F003C0F003C0F003C0F00380F00780E00780E00700E00700E00E00600E00701C00303800 3870000FC000131F7C9D17>I<000C001C00FC0F38003800380038003800380070007000 7000700070007000E000E000E000E000E000E001C001C001C001C001C001C0038003C0FF FE0F1E7C9D17>I<003F8000C1E00100F00200780400780400780F007C0F807C0F807C0F 00780600780000F80000F00001E00001C0000380000700000E00001C0000380000600000 C0000180000300200600200800401000403FFFC07FFF80FFFF80161E7E9D17>I<007F00 0183C00201E00400F00700F00F00F00F01F00F01F00001E00001E00003C0000380000700 000E0000F800000E000007000007800007C00003C00007C03007C07807C0F807C0F807C0 F00780800F00400E00201C0018780007E000141F7D9D17>I<0000600000600000E00001 C00003C00005C0000DC00009C00011C000238000438000C3800083800103800203800407 00080700180700100700200700400700FFFFF0000E00000E00000E00000E00000E00001C 00001E0001FFE0141E7E9D17>I<01803001FFE003FFC003FF0003FC0002000002000002 0000040000040000040000047C000587000603800C01800801C00001C00001E00001E000 01E00001E07003C0F803C0F003C0E00380800780400700400E00201C0018700007C00014 1F7D9D17>I<000F8000704000C0200180E00301E00701E00E00C01E00001C00003C0000 38000078F800790E007A07007C0300F80380F80380F003C0F003C0F003C0F003C0F00780 E00780E00780E00700E00F00600E00701C0030180018700007C000131F7C9D17>I<2000 003FFFE07FFFC07FFF80400100C002008002008004000008000010000020000040000040 000080000180000300000300000700000600000E00000E00001E00001C00001C00003C00 003C00003C0000780000780000780000300000131F799D17>I<003F0000C1C001006002 00600400300C00300C00300C00300C00600E00600F80C00FC18007F60003FC0001FC0001 FF00063F800C0F801007C03003C06001C06000C0C000C0C000C0C000C0C00080C0010060 030030040018180007E000141F7D9D17>I<007E0001C3000301800601C00E01C01C00C0 3C00E03C00E03C01E07801E07801E07801E07801E07803E07803E03803C03807C01C0BC0 0C13C003E380000780000780000700000E00600E00F01C00F01800E0300080600041C000 3F0000131F7C9D17>I<07E01838201C401C701CF03CF03C603C0038007000E001C00180 0300020006000400040008000800080008000000000000000000000030007800F8007800 70000E20799F15>63 D<0000100000001800000038000000380000007800000078000000 FC000001BC0000013C0000033C0000023C0000063C0000043E0000081E0000081E000010 1E0000101E0000201E0000200F0000400F0000400F0000FFFF0000800F0001000F800100 0780020007800200078004000780040007800C0007C03E0007C0FF807FFC1E207E9F22> 65 D<07FFFF00007C01C0003C01E0003C00F0007800F8007800F8007800F8007800F800 7800F8007800F000F001F000F001E000F003C000F00F8000FFFE0000F00F0001E007C001 E003C001E003E001E001E001E001E001E001E003C001E003C003E003C003E003C003C003 C007C003C00F8007800F0007803E00FFFFF0001D1F7E9E20>I<0001F808000E06180038 0138007000F801E0007803C0007007800030078000300F0000301F0000301E0000303E00 00203C0000007C0000007C0000007C0000007C000000F8000000F8000000F8000000F800 0000F80000007800004078000080780000803C0000803C0001001C0002000E0002000600 0C000300100001C0E000003F00001D217B9F21>I<07FFFF00007C01E0003C00F0003C00 780078003C0078003C0078001E0078001E0078001E0078001F00F0001F00F0001F00F000 1F00F0001F00F0001F00F0001F01E0001E01E0003E01E0003E01E0003E01E0003C01E000 7C03C0007803C000F003C000F003C001E003C003C003C0078007800F0007803C00FFFFE0 00201F7E9E23>I<07FFFFF8007C0078003C0038003C0018007800180078000800780008 00780008007800080078080800F0100000F0100000F0100000F0300000FFF00000F07000 01E0200001E0200001E0200001E0200001E0000801E0001003C0001003C0001003C00020 03C0002003C0006003C000C0078001C0078007C0FFFFFF801D1F7E9E1F>I<07FFFFF800 7C0078003C0038003C001800780018007800080078000800780008007800080078000800 F0100000F0100000F0100000F0300000F0700000FFF00001E0600001E0200001E0200001 E0200001E0200001E0000003C0000003C0000003C0000003C0000003C0000003C0000007 80000007C00000FFFE00001D1F7E9E1E>I<0001FC04000F030C003C009C0070007C00E0 003C01C0003803800018078000180F0000181F0000181E0000183E0000103C0000007C00 00007C0000007C0000007C000000F8000000F8000000F8007FFCF80003E0780001E07800 01E0780003C0780003C03C0003C03C0003C01C0003C00E0007C007000B800380118001E0 6080003F80001E217B9F24>I<07FFC7FFC0007C00F800003C007800003C007800007800 F000007800F000007800F000007800F000007800F000007800F00000F001E00000F001E0 0000F001E00000F001E00000FFFFE00000F001E00001E003C00001E003C00001E003C000 01E003C00001E003C00001E003C00003C007800003C007800003C007800003C007800003 C007800003C007800007800F000007C00F8000FFF8FFF800221F7E9E22>I<07FFE0007C 00003C00003C0000780000780000780000780000780000780000F00000F00000F00000F0 0000F00000F00001E00001E00001E00001E00001E00001E00003C00003C00003C00003C0 0003C00003C00007800007C000FFFC00131F7F9E10>I<07FFE07FE0007C001F00003C00 0C00003C0018000078001000007800400000780080000078010000007802000000780400 0000F008000000F010000000F060000000F0F0000000F1F0000000F278000001E4780000 01E878000001F03C000001E03C000001E01E000001E01E000003C00F000003C00F000003 C00F000003C007800003C007800003C003C000078003C00007C007E000FFFC3FFC00231F 7E9E23>75 D<07FFF000007E0000003C0000003C00000078000000780000007800000078 0000007800000078000000F0000000F0000000F0000000F0000000F0000000F0000001E0 000001E0000001E0000001E0000001E0008001E0010003C0010003C0010003C0030003C0 020003C0060003C0060007801E0007807C00FFFFFC00191F7E9E1C>I<07FC0000FFC000 7C0000F800003C00017800003C00017800004E0002F000004E0002F000004E0004F00000 4E0004F000004E0008F000004E0008F00000870011E00000870011E00000870021E00000 870021E00000870041E00000838041E00001038083C00001038083C00001038103C00001 038203C0000101C203C0000101C403C0000201C40780000201C80780000201C807800002 01D00780000200F00780000600E00780000600E00F00000F00C00F8000FFE0C1FFF8002A 1F7E9E2A>I<07FC01FFC0003E003E00003E001800003E001800004F001000004F001000 004780100000478010000043C010000043C010000083C020000081E020000081E0200000 80F020000080F020000080782000010078400001007C400001003C400001003C40000100 1E400001001E400002000F800002000F800002000F800002000780000200078000060003 800006000300000F00010000FFE0010000221F7E9E22>I<0003F800001E0E0000380700 00E0038001C001C003C001E0078000E00F0000F00F0000F01E0000F01E0000F83E0000F8 3C0000F87C0000F87C0000F87C0000F87C0000F8F80001F0F80001F0F80001F0F80001F0 F80003E0780003E0780003C0780007C07C0007803C000F003C001E001E001C000E003800 0700F00003C3C00000FE00001D217B9F23>I<07FFFF00007C03C0003C01E0003C00F000 7800F0007800F8007800F8007800F8007800F8007800F000F001F000F001E000F003C000 F0078000F00F0000FFF80001E0000001E0000001E0000001E0000001E0000001E0000003 C0000003C0000003C0000003C0000003C0000003C000000780000007C00000FFFC00001D 1F7E9E1F>I<0003F800001E0E000038070000F0038001E001C003C001E0078001E00F00 00F00F0000F01F0000F01E0000F83E0000F83C0000F87C0000F87C0000F87C0000F87C00 00F8F80001F0F80001F0F80001F0F80001F0F80003E0780003E0780003C0780007C0781E 07803C210F003C409E001E409C000E80B8000740F00003C1C04000FEC0400000C0400000 C0800000E1800000FF800000FF000000FF0000007E0000003C001D297B9F23>I<07FFFC 00007C0700003C03C0003C01E0007801E0007801F0007801F0007801F0007801F0007801 E000F003E000F003C000F0078000F00F0000F03C0000FFF00001E0300001E0380001E01C 0001E01C0001E01C0001E01E0003C03E0003C03E0003C03E0003C03E0003C03E0003C03E 0207803E0407C01F04FFFC0F18000003E01F207E9E21>I<003F040060CC01803C03801C 03001C0700180600080E00080E00080E00080E00000F00000F80000FE00007FE0003FF80 01FFC0007FE00007E00001E00000E00000F00000F04000E04000E04000E04000E06000C0 600180E00380F80300C60C0081F80016217D9F19>I<3FFFFFF03C0780F0300780306007 8030400F0010400F0010C00F0010800F0010800F0010800F0010001E0000001E0000001E 0000001E0000001E0000001E0000003C0000003C0000003C0000003C0000003C0000003C 000000780000007800000078000000780000007800000078000000F0000001F800007FFF E0001C1F7A9E21>II< FFF003FE1F8000F80F0000600F0000400F0000400F800080078001800780010007800200 0780020007C0040003C0040003C0080003C0080003C0100003E0100001E0200001E02000 01E0400001E0400001F0800000F1000000F1000000F2000000F2000000FC0000007C0000 00780000007800000070000000700000002000001F207A9E22>II<03FFC0FFC0007F007E00003E003800001E003000001E00200000 0F004000000F008000000F81000000078200000007C600000003C400000003E800000001 F000000001F000000000F000000000F800000000F8000000017C000000023C000000043C 0000000C1E000000081E000000101F000000200F000000400F800000C0078000008007C0 00010003C000070003E0001F8007E000FFE01FFE00221F7F9E22>II<03FFFFC003E007800380078007000F0006001E0004 003C000C003C00080078000800F0000801F0000001E0000003C000000780000007800000 0F0000001E0000003C0000003C00000078000000F0010001E0010001E0020003C0020007 8002000F8006000F0004001E000C003C001C007C003C007800F800FFFFF8001A1F7D9E1C >I<01FC07060E0F1C0F380E78007000F000F000F000F000E000E000E000E000F0027004 300818300FC010147C9314>99 D<0000700003F00000F00000700000700000E00000E000 00E00000E00000E00000E00001C000F9C00305C00E03C01C03C03801C0780380700380F0 0380F00380F00380F00380E00700E00700E00700E00700E00700700F00301E00186F000F 8FE014207C9F19>I<00F800070E000E07001C0700380380780380700380F00380F00380 FFFF80F00000E00000E00000E00000E00000F001007002003004001C180007E00011147D 9314>I<01C003E003E003E001C000000000000000000000000003801F80078003800380 0700070007000700070007000E000E000E000E000E000E001C001E00FF800B1F7F9E0C> 105 D<038F801F90E007A0E003C0600380600780E00700E00700E00700E00700E00700E0 0E01C00E01C00E01C00E01C00E01C00E01C01C03801E03C0FFCFF815147E9319>110 D<00FC000387000E01801C00C03800E03800E07000F0F000F0F000F0F000F0F000F0E001 E0E001E0E001C0E003C0F00380700700380E001C1C0007E00014147D9317>I<00800100 01000100030007000F001E00FFF80E000E000E000E001C001C001C001C001C001C003800 38203820382038203840384018800F000D1C7C9B12>116 D<1C0380FC1F803C07801C03 801C0380380700380700380700380700380700380700700E00700E00700E00700E00701E 00701E00703C00305E001F9FC012147B9319>I<1FF0FF03C07801C06001C04000E08000 E180007300007600003C00003C00001C00002E00004E000087000107000203800603800C 01C03E03E0FF07FC18147F9318>120 D E /Fn 62 123 df<000FF01FE0007FF07FE000 FFF0FFE001FFF1FFE003F013F02003E003E00007C007C00007C007C00007C007C00007C0 07C00007C007C00007C007C00007C007C00007C007C000FFFF07FF00FFFF07FF00FFFF07 FF00FFFF07FF0007C007C00007C007C00007C007C00007C007C00007C007C00007C007C0 0007C007C00007C007C00007C007C00007C007C00007C007C00007C007C00007C007C000 07C007C00007C007C00007C007C00007C007C00007C007C00007C007C00007C007C00007 C007C00007C007C00007C007C00023297FA823>11 D<00001F80003F9F80007F9F8000FF 9F8001FF9F8003E09F8003C0000007C0000007C0000007C0000007C0000007C0000007C0 000007C0000007C00000FFFF8F80FFFF8F80FFFF8F80FFFF8F8007C00F8007C00F8007C0 0F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C0 0F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C0 0F8007C00F8007C00F80192A7FA920>I<7E7E7E7E7E7E1E3C3C387870F0070D7C8511> 44 DII<00000600000E 00000E00001C00001C00001C0000380000380000380000700000700000E00000E00000E0 0001C00001C00001C0000380000380000380000700000700000700000E00000E00000E00 001C00001C00001C0000380000380000700000700000700000E00000E00000E00001C000 01C00001C0000380000380000380000700000700000700000E00000E00000E00001C0000 1C0000380000380000380000700000700000700000E00000E00000C00000173C7DAC1E> I<007F000001FFC00007FFF0000FFFF8000FC1F8001F007C003F007E003E003E003C001E 007C001F007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F80F8000F 80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F 80F8000F8078000F007C001F007C001F007C001F003E003E003E003E003F007E001F80FC 000FC1F8000FFFF80007FFF00001FFC000007F000019297EA71E>I<00180000380000F8 0007F800FFF800FFF800FFF800F8F80000F80000F80000F80000F80000F80000F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F8007FFFF07FFFF07FFF F07FFFF014287CA71E>I<00FE0003FFC00FFFE01FFFF03F03F83C00FC78007E78007EF0 003EF0003F70003F60001F20001F00001F00003F00003F00003E00003E00007C00007C00 00F80001F00003E00007E00007C0000F80001E00003C0000780000F00001E00003C00007 80000F00001E00003C00007FFFFF7FFFFF7FFFFF7FFFFF18287EA71E>I<007F000003FF C00007FFF0000FFFF8001F81F8003E00FC007C007E0038007E0030003E0010003E000000 7E0000007E0000007C0000007C000000FC000001F8000007F00000FFE00000FFC00000FF E00000FFF0000001F80000007C0000007E0000003F0000003F0000001F8000001F800000 1F8000001F8000001F8040001F8060003F00F0003F00F8007F007C007E003F81FC001FFF F8000FFFF00003FFC000007F000019297EA71E>I<0003F0000007F0000007F000000DF0 00000DF000001DF000003DF0000039F0000079F0000079F00000F1F00000F1F00001F1F0 0003E1F00003E1F00007C1F00007C1F0000F81F0000F81F0001F01F0001F01F0003E01F0 007C01F0007C01F000F801F000FFFFFF80FFFFFF80FFFFFF80FFFFFF800001F0000001F0 000001F0000001F0000001F0000001F0000001F0000001F0000001F0000001F00019277E A61E>I<3FFFFC3FFFFC3FFFFC3FFFFC3E00003E00003E00003E00003E00003E00003E00 003E00003E00003E00003E3F003EFFC03FFFE03FFFF03FC1F83F80FC3F007C3E007E0000 3E00003F00003F00003F00003F00003F00003F00003F20003F60007E70007EF800FC7C01 FC3F03F83FFFF00FFFE007FF8000FE0018287EA61E>I<000FF000003FFC0000FFFC0001 FFFC0003F80C0007E000000FC000000F8000001F0000001E0000003E0000003C0000007C 0000007C0000007C3FE000F8FFF000F9FFF800FBFFFC00FF807E00FF003E00FE003F00FC 001F00FC001F00FC000F80F8000F80F8000F80F8000F80F8000F8078000F807C000F807C 000F807C000F003E001F003E001F001F003E001F807C000FC1FC0007FFF80003FFF00001 FFC000007F000019297EA71E>II<007F00 0001FFC00007FFF0000FFFF8001FC1FC003F007E003E003E007E003F007C001F007C001F 007C001F007C001F007C001F003E003E003E003E001F007C000FC1F80007FFF00003FFE0 0003FFE0000FFFF8001FC1FC003F007E003E003E007C001F007C001F00F8000F80F8000F 80F8000F80F8000F80F8000F80F8000F807C001F007C001F007E003F003F007E001FC1FC 000FFFF80007FFF00003FFE000007F000019297EA71E>I<007F000001FFC00003FFE000 0FFFF0000FC1F8001F007C003E007C007C003E007C001E007C001F00F8001F00F8001F00 F8000F00F8000F80F8000F80F8000F80F8000F80F8001F807C001F807C001F807E003F80 3E007F803F00FF801FFFEF800FFFCF8007FF8F8003FE1F0000001F0000001F0000001E00 00003E0000003E0000007C0000007C000000F8001801F0001E07E0003FFFC0001FFF8000 0FFE000003F8000019297EA71E>I<03FC001FFF807FFFC0FFFFE0FE03F07801F06000F8 0000F80000F80000F80000F80001F00003F00007E0000FC0001F80003F00003E00007C00 007C0000F80000F80000F80000F80000F80001F80001F80001F800000000000000000000 00000000000000000000000001F80001F80001F80001F80001F80001F80015297DA81C> 63 D<0001F000000003F800000003F800000007FC00000007FC000000077C0000000F3E 0000000F3E0000000F3E0000001E3F0000001E1F0000003E1F8000003C1F8000003C1F80 00007C0FC00000780FC00000780FC00000F807E00000F007E00000F007E00001F003F000 01E003F00003E003F80003E001F80003C001F80007C000FC0007FFFFFC0007FFFFFC000F FFFFFE000FFFFFFE000F00007E001F00003F001F00003F003E00003F803E00001F803C00 001F807C00001FC07C00000FC07800000FC0F8000007E0F8000007E023297EA828>65 DI<0001FF00000FFFE0 003FFFF8007FFFF801FF03F803F8007007F0003007E000100FC000001F8000001F800000 3F0000003F0000007E0000007E0000007E0000007E000000FC000000FC000000FC000000 FC000000FC000000FC000000FC000000FC000000FC0000007E0000007E0000007E000000 7E0000003F0000003F0000001F8000001F8000000FC0000007E0000407F0000C03F8003C 01FF00FC007FFFFC003FFFF8000FFFE00001FF001E2B7CA926>IIII<0001FF00000FFFE0003FFFFC007FFFFE01FF01FE03FC003E07F0001C07E000 0C0FC000041F8000001F8000003F0000003F0000007E0000007E0000007E0000007E0000 00FC000000FC000000FC000000FC000000FC000000FC000000FC001FFEFC001FFEFC001F FE7E001FFE7E00003E7E00003E7E00003E3F00003E3F00003E1F80003E1F80003E0FC000 3E07E0003E07F0003E03FC003E01FF01FE007FFFFE003FFFFC000FFFE00001FF001F2B7C A928>III76 DII<0001FC0000000F FF8000003FFFE00000FFFFF80001FE03FC0003F800FE0007F0007F000FE0003F800FC000 1F801F80000FC01F000007C03F000007E03E000003E07E000003F07E000003F07E000003 F07C000001F0FC000001F8FC000001F8FC000001F8FC000001F8FC000001F8FC000001F8 FC000001F8FC000001F8FC000001F8FE000003F87E000003F07E000003F07E000003F03F 000007E03F000007E01F80000FC01F80000FC00FC0001F800FE0003F8007F0007F0003F8 00FE0001FF07FC0000FFFFF800003FFFE000000FFF80000001FC0000252B7DA92C>II82 D<007FC00001FFF80007FFFE000FFFFF001FC0 7F003F001F007E000E007C00060078000200F8000000F8000000F8000000F8000000FC00 0000FC0000007E0000007F8000003FF800001FFF80000FFFE00007FFF80003FFFC0000FF FE00000FFF000000FF0000003F8000001F8000000FC000000FC0000007C0000007C00000 07C0000007C0400007C060000F8070000F80F8001F00FE003F00FF80FE007FFFFC001FFF F80007FFE00000FF80001A2B7DA921>IIIII<00FE0007FF801FFFC03FFFE03E03F03801F03001F82000F80000 F80000F80000F80000F8007FF807FFF81FFFF83FE0F87F00F8FC00F8F800F8F800F8F800 F8FC01F87E07F87FFFF83FFFF81FFCF80FE0F8151B7E9A1D>97 DI<007F8001FFE007FFF80FFFFC1FC07C1F001C3E00087C00 007C00007C0000F80000F80000F80000F80000F80000F80000F800007C00007C00007E00 003E00041F001C1FC07C0FFFFC07FFFC01FFF0007F80161B7E9A1B>I<00003E00003E00 003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00 FC3E03FF3E07FFFE0FFFFE1FC1FE3F007E3E003E7C003E7C003EFC003EF8003EF8003EF8 003EF8003EF8003EF8003EF8003EFC003E7C003E7C003E3E007E3F00FE1FC1FE0FFFFE07 FFBE03FF3E00FC3E17297EA81F>I<007E0003FF8007FFC00FFFE01F83F03F00F03E0078 7C00787C003878003CFFFFFCFFFFFCFFFFFCFFFFFCF80000F80000F800007800007C0000 7C00003E00003F000C1FC07C0FFFFC07FFFC01FFF0007F80161B7E9A1B>I<001FC0007F C000FFC001FFC003E04003C00007C00007C00007C00007C00007C00007C00007C00007C0 00FFFE00FFFE00FFFE00FFFE0007C00007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00012297FA812>I<00F8078003FE7FC00FFFFFC01FFFFFC01F07C0 003E03E0003E03E0007C01F0007C01F0007C01F0007C01F0007C01F0007C01F0003E03E0 003E03E0001F07C0001FFFC0003FFF80003BFE000038F8000078000000780000003C0000 003FFFC0003FFFF8001FFFFC001FFFFE003FFFFF007C007F00F8001F80F8000F80F8000F 80F8000F80FC001F807E003F003F80FE003FFFFE000FFFF80007FFF00000FF80001A287E 9A1E>III107 DI II<007F000001FFC00007FFF0000FFFF8001FC1FC 003F007E003E003E007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F 80F8000F80F8000F80F8000F807C001F007C001F007E003F003E003E003F007E001FC1FC 000FFFF80007FFF00001FFC000007F0000191B7E9A1E>II<00FC3E03FF3E07FFBE0FFFFE1FC1FE3F80FE3E007E7E003E7C003E7C003EF800 3EF8003EF8003EF8003EF8003EF8003EF8003EFC003E7C003E7E003E3E007E3F007E1FC1 FE1FFFFE0FFFBE03FF3E01F83E00003E00003E00003E00003E00003E00003E00003E0000 3E00003E00003E00003E00003E17277E9A1F>II<03FC001FFF803FFFC07FFFC07C07C0F80180F80000F80000F80000 FC00007F80007FF8003FFE001FFF0007FF8000FFC0000FE00007E00003E04003E06003E0 F007E0FC0FC0FFFFC07FFF801FFE0003F800131B7E9A17>I<07C00007C00007C00007C0 0007C00007C00007C000FFFFC0FFFFC0FFFFC0FFFFC007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 0007C04007E1C003FFE003FFE001FF8000FC0013227FA116>IIII<7C000FC03E001F803F001F001F803E000F807C0007C0FC0003 E0F80001F1F00001FBE00000FFC000007FC000003F8000001F0000001F0000003F800000 7FC00000FBC00000F3E00001F1F00003E0F80007C07C000F807C000F803E001F001F003E 000F807E000FC0FC0007E01B1B809A1C>I II E /Fo 18 121 df<00001E000000003E000000 00FE00000003FE0000003FFE0000FFFFFE0000FFFFFE0000FFFFFE0000FFCFFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE 0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE000000 0FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00007FFFFF FFC07FFFFFFFC07FFFFFFFC07FFFFFFFC0223879B731>49 D<0007FE0000007FFFE00001 FFFFF80003FFFFFE0007F01FFF000F8007FF801F0001FFC03E0000FFE07F8000FFF07FC0 007FF0FFE0007FF8FFF0003FF8FFF0003FFCFFF0003FFCFFF0003FFCFFF0003FFC7FE000 3FFC3FC0003FFC1F80003FFC0000003FFC0000003FF80000007FF80000007FF00000007F F0000000FFE0000000FFC0000001FF80000001FF00000003FE00000007FC00000007F800 00000FF00000001FC00000003F800000007F00000000FC00000001F800000001F0003C00 03E0003C0007C0003C000F000078001E000078003C00007800780000F800F00000F801FF FFFFF803FFFFFFF007FFFFFFF00FFFFFFFF01FFFFFFFF03FFFFFFFF07FFFFFFFF0FFFFFF FFF0FFFFFFFFE0FFFFFFFFE0FFFFFFFFE026387BB731>I<0003FF8000001FFFF000007F FFFE0000FE03FF0001F000FF8003C000FFC00780007FE00FF0007FF00FF8007FF01FFC00 7FF81FFE007FF81FFE007FF81FFE007FF81FFE007FF81FFE007FF80FFC007FF007F8007F F003F0007FF0000000FFE0000000FFC0000001FF80000001FF00000003FE00000007FC00 00001FF000000FFFC000000FFF8000000FFFF800000003FE00000000FF800000007FE000 00003FF00000003FF80000003FFC0000001FFC0000001FFE0000001FFE0200001FFF1FC0 001FFF3FE0001FFF7FF0001FFF7FF0001FFFFFF8001FFFFFF8001FFFFFF8001FFEFFF800 1FFEFFF0001FFE7FF0003FFC7FE0003FFC3FC0003FF81F80007FF01FE000FFE007FC03FF C003FFFFFF0001FFFFFE00003FFFF0000007FF800028397CB731>I<00000007C0000000 000FC0000000000FC0000000001FC0000000003FC0000000007FC000000000FFC0000000 00FFC000000001FFC000000003FFC000000007FFC00000000FFFC00000000FFFC0000000 1EFFC00000003CFFC00000007CFFC0000000F8FFC0000000F0FFC0000001E0FFC0000003 C0FFC0000007C0FFC000000F80FFC000000F00FFC000001E00FFC000003C00FFC000007C 00FFC00000F800FFC00000F000FFC00001E000FFC00003C000FFC00007C000FFC0000F80 00FFC0000F0000FFC0001E0000FFC0003C0000FFC0007C0000FFC000F80000FFC000FFFF FFFFFFC0FFFFFFFFFFC0FFFFFFFFFFC0FFFFFFFFFFC0000001FFC000000001FFC0000000 01FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000000 01FFC000000001FFC0000007FFFFFFC00007FFFFFFC00007FFFFFFC00007FFFFFFC02A37 7DB631>I<04000000C00F800007C00FF8007FC00FFFFFFF800FFFFFFF000FFFFFFE000F FFFFFC000FFFFFF8000FFFFFF0000FFFFFE0000FFFFF80000FFFFE00000FFFF800000F80 0000000F800000000F800000000F800000000F800000000F800000000F800000000F8000 00000F81FF00000F8FFFE0000FBFFFF8000FFE03FE000FF000FF000FC000FF800F80007F C00F00007FE00700007FF00000003FF00000003FF80000003FF80000003FF80000003FFC 0000003FFC0600003FFC1F80003FFC3FC0003FFC7FE0003FFCFFE0003FFCFFF0003FFCFF F0003FFCFFF0003FF8FFE0003FF8FFE0003FF87FC0007FF07F00007FF03C00007FE03E00 00FFC01F0000FF800FC003FF0007F00FFE0003FFFFFC0001FFFFF000007FFFC000000FFC 000026397BB731>I<00000FF80000007FFF000003FFFF80000FFC07C0001FE001E0003F C001F0007F0007F000FF000FF001FE001FF803FC003FF807FC003FF80FFC003FF80FF800 3FF81FF8001FF01FF8000FE03FF80007C03FF00000003FF00000007FF00000007FF00000 007FF00000007FF07FF000FFF0FFFE00FFF1F7FF00FFF3807F80FFF6003FE0FFFE001FF0 FFFC001FF0FFFC000FF8FFF8000FFCFFF8000FFCFFF8000FFEFFF8000FFEFFF0000FFEFF F0000FFFFFF0000FFFFFF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0 000FFF3FF0000FFF3FF0000FFE3FF0000FFE1FF0000FFE1FF8000FFC0FF8000FFC0FF800 1FF807FC001FF003FC001FF001FE003FE000FF80FFC0007FFFFF00001FFFFE000007FFF8 000000FFC00028397CB731>I<1E00000000001F00000000001FF0000000001FFFFFFFFF C01FFFFFFFFFC01FFFFFFFFFC03FFFFFFFFFC03FFFFFFFFF803FFFFFFFFF003FFFFFFFFE 003FFFFFFFFC003FFFFFFFF8003FFFFFFFF0007FFFFFFFF0007C000007E0007C00000FC0 007800001F80007800001F00007800003E0000F000007E0000F00000FC0000F00001F800 00000003F00000000003E00000000007E0000000000FC0000000000F80000000001F8000 0000003F80000000003F00000000007F00000000007F0000000000FF0000000000FE0000 000001FE0000000001FE0000000003FE0000000003FE0000000003FE0000000007FC0000 000007FC0000000007FC000000000FFC000000000FFC000000000FFC000000000FFC0000 00000FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC0000 00001FFC000000001FFC000000001FFC000000001FFC000000000FF80000000007F00000 000003E00000002A3B7BB931>I<0001FF8000000FFFF800003FFFFE00007F00FF0000F8 003F8001F0000FC003E0000FE007C00007F007C00007F00FC00003F80FC00003F81FC000 03F81FC00003F81FE00003F81FF00003F81FF80003F81FFC0007F01FFF0007F01FFFC007 E00FFFF00FE00FFFFC1FC007FFFE3F8007FFFFFF0003FFFFFC0001FFFFF80000FFFFFE00 007FFFFF00003FFFFF80003FFFFFE000FFFFFFF001FDFFFFF007F07FFFF80FE01FFFFC1F C007FFFC3F8001FFFE3F8000FFFE7F00003FFF7F00000FFFFE000003FFFE000001FFFE00 0000FFFE000000FFFE0000007FFE0000007FFE0000007EFE0000007E7F0000007E7F0000 00FC3F800000FC3FC00001F81FE00003F00FF00007E007FE007FC003FFFFFF8000FFFFFE 00003FFFF8000003FF800028397CB731>I<00000001F80000000000000001F800000000 00000003FC0000000000000003FC0000000000000007FE0000000000000007FE00000000 00000007FE000000000000000FFF000000000000000FFF000000000000001FFF80000000 0000001FFF800000000000001FFF800000000000003FFFC00000000000003FFFC0000000 0000007FFFE00000000000007DFFE00000000000007DFFE0000000000000FDFFF0000000 000000F8FFF0000000000001F8FFF8000000000001F07FF8000000000001F07FF8000000 000003F07FFC000000000003E03FFC000000000007E03FFE000000000007C01FFE000000 00000FC01FFF00000000000F801FFF00000000000F800FFF00000000001F800FFF800000 00001F0007FF80000000003F0007FFC0000000003E0003FFC0000000003E0003FFC00000 00007E0003FFE0000000007C0001FFE000000000FC0001FFF000000000F80000FFF00000 0000F80000FFF000000001FFFFFFFFF800000001FFFFFFFFF800000003FFFFFFFFFC0000 0003FFFFFFFFFC00000003E000003FFC00000007E000003FFE00000007C000001FFE0000 000FC000001FFF0000000F8000000FFF0000001F8000000FFF8000001F0000000FFF8000 001F00000007FF8000003F00000007FFC000003E00000003FFC000007E00000003FFE000 00FF00000001FFE000FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0 FFFFFC0001FFFFFFF0443B7DBA4B>65 D<0000001FFF000030000001FFFFE000F000000F FFFFFC01F000007FFFFFFE03F00001FFFE007F87F00003FFE0000FCFF0000FFF000003FF F0001FFC000001FFF0003FF80000007FF0007FF00000003FF000FFC00000003FF001FFC0 0000001FF003FF800000000FF007FF000000000FF00FFF0000000007F00FFE0000000007 F01FFE0000000003F01FFE0000000003F03FFC0000000003F03FFC0000000001F03FFC00 00000001F07FFC0000000001F07FF80000000001F07FF80000000000007FF80000000000 00FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF800 0000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF80000000000 00FFF8000000000000FFF80000000000007FF80000000000007FF80000000000007FF800 00000000007FFC0000000000F03FFC0000000000F03FFC0000000000F03FFC0000000000 F01FFE0000000000F01FFE0000000001E00FFE0000000001E00FFF0000000001E007FF00 00000003C003FF8000000003C001FFC0000000078000FFE00000000F00007FF00000001F 00003FF80000003E00001FFC0000007C00000FFF000001F8000003FFE00007F0000001FF FE003FC00000007FFFFFFF000000000FFFFFFC0000000001FFFFF000000000001FFF0000 003C3D7BBB47>67 D<001FFF00000001FFFFF0000003FFFFFC000007F007FE00000FF801 FF00001FFC00FF80001FFC007FC0001FFC007FE0001FFC003FE0000FF8003FF0000FF800 3FF00007F0003FF00001C0003FF0000000003FF0000000003FF0000000003FF0000000FF FFF000000FFFFFF000007FF83FF00001FF803FF00007FE003FF0000FF8003FF0001FF000 3FF0003FE0003FF0007FE0003FF0007FE0003FF000FFC0003FF000FFC0003FF000FFC000 3FF000FFC0003FF000FFC0007FF0007FE0007FF0007FE000DFF0003FF0039FF8001FFC0F 0FFFF007FFFE0FFFF001FFFC07FFF0003FE000FFF02C267DA530>97 D<0001FFC000000FFFF800003FFFFE0000FF80FF0001FE003F8007FC001FC00FF8000FE0 0FF8000FF01FF00007F03FF00007F83FF00007F87FE00007F87FE00003FC7FE00003FC7F E00003FCFFE00003FCFFFFFFFFFCFFFFFFFFFCFFFFFFFFFCFFE0000000FFE0000000FFE0 000000FFE00000007FE00000007FE00000007FE00000003FE00000003FF000003C1FF000 003C1FF000003C0FF800007807FC0000F803FE0001F001FF0007E000FFC03FC0003FFFFF 000007FFFC000000FFE00026267DA52D>101 D<00FF00000000FFFF00000000FFFF0000 0000FFFF00000000FFFF0000000007FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000 000003FF0000000003FF007FC00003FF01FFF80003FF07FFFC0003FF0F03FE0003FF1C01 FF0003FF3001FF8003FF6000FF8003FFE000FFC003FFC000FFC003FF8000FFC003FF8000 FFC003FF8000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000 FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3F FFFFFFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFF303C7CBB37>104 D<00FE007FC000FFFE01FFF800FFFE07FFFC00FFFE0F03FE00FFFE1C01FF0007FE3001FF 8003FE6000FF8003FEE000FFC003FEC000FFC003FF8000FFC003FF8000FFC003FF8000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC0FFFFFC3FFFFFFFFFFC3FFF FFFFFFFC3FFFFFFFFFFC3FFFFF30267CA537>110 D<00FF01FF8000FFFF0FFFF000FFFF 3FFFFC00FFFFFE03FF00FFFFF000FF8003FFC0007FC003FF80003FE003FF00003FF003FF 00001FF803FF00001FFC03FF00000FFC03FF00000FFE03FF00000FFE03FF000007FE03FF 000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF 000007FF03FF000007FF03FF000007FF03FF000007FE03FF000007FE03FF00000FFE03FF 00000FFC03FF00000FFC03FF00001FF803FF00001FF803FF00003FF003FF80003FE003FF C0007FC003FFF001FF8003FFFC07FF0003FF3FFFFC0003FF0FFFF00003FF01FF000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 00000000FFFFFC000000FFFFFC000000FFFFFC000000FFFFFC00000030377DA537>112 D<00FE03F000FFFE0FFE00FFFE1FFF00FFFE3C3F80FFFE707FC007FE60FFE003FEE0FFE0 03FEC0FFE003FFC0FFE003FF807FC003FF807FC003FF803F8003FF800E0003FF00000003 FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF 00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00 000003FF00000003FF00000003FF00000003FF00000003FF000000FFFFFE0000FFFFFE00 00FFFFFE0000FFFFFE000023267DA529>114 D<00078000000780000007800000078000 00078000000F8000000F8000000F8000000F8000001F8000001F8000003F8000003F8000 007F800000FF800001FF800007FF80001FFFFFF0FFFFFFF0FFFFFFF0FFFFFFF001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C00FF8078 00FFC078007FC070003FE0E0001FFFC00007FF800001FF001E377EB626>116 D120 D E /Fp 31 123 df<007E01C007000E001C003C003800780078007FF0F000F000F0007000700070003000 18000C1807E00F147E9312>15 D<70F8F8F87005057C840D>58 D<70F8FCFC7404040408 0810102040060E7C840D>I<000001C00000078000001E00000078000001E00000078000 000E00000038000000F0000003C000000F0000003C000000F0000000F00000003C000000 0F00000003C0000000F0000000380000000E0000000780000001E0000000780000001E00 00000780000001C01A1A7C9723>I62 D<000002000000060000000E0000000E0000 001E0000001F0000002F0000002F0000004F0000008F0000008F0000010F0000010F0000 020F0000040F0000040F0000080F8000080780001007800020078000200780007FFF8000 40078000800780018007800100078002000780020007C0040003C00C0003C01E0007C0FF 807FFC1E207E9F22>65 D<0FFFFFFC1E03C0381803C0181003C0082003C0082007800860 0780084007800840078008800F0010000F0000000F0000000F0000001E0000001E000000 1E0000001E0000003C0000003C0000003C0000003C000000780000007800000078000000 78000000F0000000F0000000F0000000F0000001F000007FFFC0001E1F7F9E1B>84 D<00F1800389C00707800E03801C03803C0380380700780700780700780700F00E00F00E 00F00E00F00E10F01C20F01C20703C20705C40308C400F078014147E9318>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E380F38 0F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207E9F14>I< 007C01C207010E0F1E0F1C0E3C04780078007800F000F000F000F000F000700170023004 18380FC010147E9314>I<0000780003F80000700000700000700000700000E00000E000 00E00000E00001C00001C000F1C00389C00707800E03801C03803C038038070078070078 0700780700F00E00F00E00F00E00F00E10F01C20F01C20703C20705C40308C400F078015 207E9F18>I<007C01C207010E011C013C013802780C7BF07C00F000F000F000F0007000 700170023004183807C010147E9315>I<00007C0000CE00019E00039E00030C00070000 0700000700000700000E00000E00000E0000FFF0000E00000E00001C00001C00001C0000 1C00001C0000380000380000380000380000380000700000700000700000700000700000 E00000E00000E00000E00000C00001C000318000798000F300006200003C000017297E9F 16>I<001E3000713800E0F001C0700380700780700700E00F00E00F00E00F00E01E01C0 1E01C01E01C01E01C01E03801E03800E07800E0B8006170001E700000700000700000E00 000E00300E00781C00F038006070003FC000151D809316>I<00E001E001E000C0000000 00000000000000000000000E00130023804380438043808700070007000E000E001C001C 001C20384038403840388019000E000B1F7E9E10>105 D<0000C00001E00001E00001C0 000000000000000000000000000000000000000000001E00006300004380008380010380 010380020700000700000700000700000E00000E00000E00000E00001C00001C00001C00 001C0000380000380000380000380000700000700030700078E000F1C0006380003E0000 1328819E13>I<01E0000FE00001C00001C00001C00001C0000380000380000380000380 000700000700000701E00706100E08700E10F00E20F00E40601C80001D00001E00001FC0 00387000383800383800381C20703840703840703840701880E01880600F0014207E9F18 >I<03C01FC0038003800380038007000700070007000E000E000E000E001C001C001C00 1C0038003800380038007000700070007100E200E200E200E200640038000A207E9F0E> I<1E07C07C00231861860023A032030043C0340300438038038043803803808700700700 0700700700070070070007007007000E00E00E000E00E00E000E00E00E000E00E01C101C 01C01C201C01C038201C01C038401C01C0184038038018801801800F0024147E9328>I< 1E07802318C023A06043C0704380704380708700E00700E00700E00700E00E01C00E01C0 0E01C00E03821C03841C07041C07081C03083803101801E017147E931B>I<007C0001C3 000301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F007 80F00700700F00700E0030180018700007C00013147E9316>I<03C1E004621804741C08 781C08701E08701E10E01E00E01E00E01E00E01E01C03C01C03C01C03C01C03803807803 80700380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00001C 0000FFC000171D819317>I<1E1E0023210023C38043C780438780438300870000070000 0700000700000E00000E00000E00000E00001C00001C00001C00001C0000380000180000 11147E9315>114 D<007C018203010603060706060E00078007F803FC01FE001F000770 07F006F006E004400820301FC010147E9315>I<00C000E001C001C001C001C003800380 FFF8038007000700070007000E000E000E000E001C001C001C001C103820382038203840 18800F000D1C7F9B10>I<0F00601180702180E021C0E041C0E04380E08381C00701C007 01C00701C00E03800E03800E03800E03840E07080C07080C07080E0F1006131003E1E016 147E931A>I<0F01801183C02183E021C1E041C0E0438060838040070040070040070040 0E00800E00800E00800E01000E01000C02000E04000E040006180001E00013147E9316> I<0F006060118070F02180E0F821C0E07841C0E0384380E0188381C0100701C0100701C0 100701C0100E0380200E0380200E0380200E0380400E0380400E0380800E078080060781 000709860001F078001D147E9321>I<03C1C00C62201034701038F02038F02038604070 0000700000700000700000E00000E00000E00000E02061C040F1C040F1C080E2C0804463 00383C0014147E931A>I<0F00601180702180E021C0E041C0E04380E08381C00701C007 01C00701C00E03800E03800E03800E03800E07000C07000C07000E0F00061E0003EE0000 0E00000E00001C0078180078380070700060600021C0001F0000141D7E9316>I<01E020 03F04007F8C00C1F8008010000020000040000080000100000600000C000010000020000 0400800801001003003F060061FC0040F80080700013147E9315>I E /Fq 56 122 df<000FF07F00007FF9FF8000F83FC7C001E07F8FC003E07F0FC007C07F 0FC007C03F078007C01F000007C01F000007C01F000007C01F000007C01F0000FFFFFFF8 00FFFFFFF80007C01F000007C01F000007C01F000007C01F000007C01F000007C01F0000 07C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007 C01F000007C01F000007C01F00003FF8FFF0003FF8FFF0002220809F1F>11 D<000FE000007FF80000F81C0001E07C0003E07C0007C07C0007C07C0007C0380007C000 0007C0000007C0000007C1FE00FFFFFE00FFFFFE0007C03E0007C03E0007C03E0007C03E 0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E 0007C03E0007C03E0007C03E003FF9FFC03FF9FFC01A20809F1D>I<000FF03FFC00007F FDFFFC0000F83FE0FC0001E03F81FC0003E07F81FC0007C07F00FC0007C03F00FC0007C0 1F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C00FFFFFFFFFC00FFFF FFFFFC0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C0 1F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C0 1F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C003FF8FFE3FF803FF8 FFE3FF802920809F2C>15 D<387CFEFFFF7F3B030306060E0C18702008107C860F>44 DI<387CFEFEFE7C3807077C860F>I<01FC0007FF001F 07C01E03C03E03E07C01F07C01F07C01F0FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC 01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F87C01F07C01F07C01F03E03E01E03C01F 8FC007FF0001FC00151D7E9C1A>48 D<00E00001E0000FE000FFE000F3E00003E00003E0 0003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E0 0003E00003E00003E00003E00003E00003E00003E00003E000FFFF80FFFF80111D7C9C1A >I<07F0001FFE00383F007C1F80FE0FC0FE0FC0FE0FE0FE07E07C07E03807E0000FE000 0FC0000FC0001F80001F00003E0000780000F00000E00001C0000380600700600E00601C 00E01FFFC03FFFC07FFFC0FFFFC0FFFFC0131D7D9C1A>I<01FC0007FF000E0F801E0FC0 3F07E03F07E03F07E03F07E01E0FC0000FC0000F80001F0001FC0001FC00000F800007C0 0003E00003F00003F83803F87C03F8FE03F8FE03F8FE03F0FC03F07807E03C0FC01FFF80 03FC00151D7E9C1A>I<0001C00003C00007C00007C0000FC0001FC0003BC00073C00063 C000C3C00183C00383C00703C00E03C00C03C01803C03803C07003C0E003C0FFFFFEFFFF FE0007C00007C00007C00007C00007C00007C000FFFE00FFFE171D7F9C1A>I<3803803F FF803FFF003FFE003FFC003FF0003F800030000030000030000030000033F80037FE003C 1F00380F801007C00007C00007E00007E07807E0FC07E0FC07E0FC07E0FC07C0780FC060 0F80381F001FFC0007F000131D7D9C1A>I<003F0001FFC007E0E00F81E01F03F01E03F0 3E03F07C03F07C01E07C0000FC1000FCFF00FDFFC0FD03E0FE01F0FE01F0FC01F8FC01F8 FC01F8FC01F87C01F87C01F87C01F83C01F03E01F01E03E00F07C007FF8001FE00151D7E 9C1A>I<6000007FFFF87FFFF87FFFF07FFFE07FFFC0E00180C00300C00300C00600000C 0000180000380000380000780000700000F00000F00001F00001F00001F00001F00003F0 0003F00003F00003F00003F00003F00001E00000C000151E7D9D1A>I<01FC0007FF000F 07801E03C01C01E03C01E03C01E03E01E03F01E03FC3C01FE3801FFF000FFE0007FF8007 FFC01FFFE03C3FF0780FF07803F8F001F8F000F8F00078F00078F000707800707C00E03E 03C00FFF8003FC00151D7E9C1A>I<01FC000FFF001F07803E03C07C03E07C01E0FC01F0 FC01F0FC01F0FC01F8FC01F8FC01F8FC01F87C03F87C03F83E05F81FFDF807F9F80041F8 0001F03C01F07E01F07E03E07E03E07E07C03C0780381F001FFC0007F000151D7E9C1A> I<7FFFFFFF80FFFFFFFFC0FFFFFFFFC00000000000000000000000000000000000000000 00000000000000000000000000000000000000000000000000FFFFFFFFC0FFFFFFFFC07F FFFFFF80220F7D9229>61 D<0000E000000000E000000001F000000001F000000001F000 000003F800000003F800000006FC00000006FC0000000EFE0000000C7E0000000C7E0000 00183F000000183F000000303F800000301F800000701FC00000600FC00000600FC00000 C007E00000FFFFE00001FFFFF000018003F000018003F000030001F800030001F8000600 01FC00060000FC000E0000FE00FFE00FFFE0FFE00FFFE0231F7E9E28>65 DI<0007FC02003FFF0E00FE03DE 03F000FE07E0003E0FC0001E1F80001E3F00000E3F00000E7F0000067E0000067E000006 FE000000FE000000FE000000FE000000FE000000FE000000FE0000007E0000007E000006 7F0000063F0000063F00000C1F80000C0FC0001807E0003803F0007000FE01C0003FFF80 0007FC001F1F7D9E26>I69 DI<0007FC0200003FFF0E0000FE 03DE0003F000FE0007E0003E000FC0001E001F80001E003F00000E003F00000E007F0000 06007E000006007E00000600FE00000000FE00000000FE00000000FE00000000FE000000 00FE003FFFE0FE003FFFE07E00007E007E00007E007F00007E003F00007E003F00007E00 1F80007E000FC0007E0007E0007E0003F000FE0000FE01FE00003FFF8E000007FC060023 1F7D9E29>I73 D76 DI 80 D<001FF80000FFFF0001F81F8007E007E00FC003F01F8001F81F8001F83F0000FC7F 0000FE7F0000FE7E00007EFE00007FFE00007FFE00007FFE00007FFE00007FFE00007FFE 00007FFE00007FFE00007F7E00007E7E00007E7F0000FE3F0000FC3F87C1FC1F8FE1F80F D833F007F81FE001F81F8000FFFF00001FFE0300000E0300000F0700000FFF000007FF00 0007FE000007FE000003FC000001F8000000F020287D9E27>I<03FC080FFF381E03F838 00F8700078700038F00038F00018F00018F80000FC00007FC0007FFE003FFF801FFFE00F FFF007FFF000FFF80007F80000FC00007C00003CC0003CC0003CC0003CE00038E00078F8 0070FE01E0E7FFC081FF00161F7D9E1D>83 D<7FFFFFFC7FFFFFFC7C07E07C7007E01C60 07E00C6007E00CE007E00EC007E006C007E006C007E006C007E0060007E0000007E00000 07E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E00000 07E0000007E0000007E0000007E0000007E0000007E00003FFFFC003FFFFC01F1E7E9D24 >I86 D<07FC001FFF003F0F803F07C03F03E0 3F03E00C03E00003E0007FE007FBE01F03E03C03E07C03E0F803E0F803E0F803E0FC05E0 7E0DE03FF8FE0FE07E17147F9319>97 DI<01FE0007FF801F0FC03E0FC03E0FC07C0FC07C0300FC0000 FC0000FC0000FC0000FC0000FC00007C00007E00003E00603F00C01F81C007FF0001FC00 13147E9317>I<0007F80007F80000F80000F80000F80000F80000F80000F80000F80000 F80000F80000F801F8F80FFEF81F83F83E01F87E00F87C00F87C00F8FC00F8FC00F8FC00 F8FC00F8FC00F8FC00F87C00F87C00F87E00F83E01F81F07F80FFEFF03F8FF18207E9F1D >I<01FE0007FF800F83C01E01E03E00F07C00F07C00F8FC00F8FFFFF8FFFFF8FC0000FC 0000FC00007C00007C00003E00181E00180F807007FFE000FF8015147F9318>I<001F80 00FFC001F3E003E7E003C7E007C7E007C3C007C00007C00007C00007C00007C000FFFC00 FFFC0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C000 07C00007C00007C00007C00007C0003FFC003FFC0013207F9F10>I<01FC3C07FFFE0F07 9E1E03DE3E03E03E03E03E03E03E03E03E03E01E03C00F07800FFF0009FC001800001800 001C00001FFF800FFFF007FFF81FFFFC3C007C70003EF0001EF0001EF0001E78003C7800 3C3F01F80FFFE001FF00171E7F931A>II<1C003E003F007F003F003E001C0000000000000000000000 0000FF00FF001F001F001F001F001F001F001F001F001F001F001F001F001F001F001F00 1F00FFE0FFE00B217EA00E>I<0038007C00FE00FE00FE007C0038000000000000000000 00000001FE01FE003E003E003E003E003E003E003E003E003E003E003E003E003E003E00 3E003E003E003E003E003E303E783EFC3CFC7C78783FF01FC00F2A83A010>IIIII<01FF0007FFC01F83F03E00F83E00F87C 007C7C007CFC007EFC007EFC007EFC007EFC007EFC007E7C007C7C007C3E00F83E00F81F 83F007FFC001FF0017147F931A>I I<01F81807FE381F87783F01F83E01F87E00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00 F8FC00F87C00F87E00F87E00F83F01F81F87F80FFEF803F8F80000F80000F80000F80000 F80000F80000F80000F80007FF0007FF181D7E931C>II<0FE63FFE701E600EE006E006F800FFC07FF8 3FFC1FFE03FE001FC007C007E007F006F81EFFFCC7F010147E9315>I<01800180018003 800380038007800F803F80FFFCFFFC0F800F800F800F800F800F800F800F800F800F800F 860F860F860F860F8607CC03F801F00F1D7F9C14>IIIIII E /Fr 37 122 df<00000007FF800000000001FFFFF0000000000FFFFFFC000000003FFFFFFE00 000000FFFC00FF00000001FFC0003F80000007FF00007FC000000FFE0001FFC000001FFC 0001FFE000001FF80003FFE000003FF00003FFE000003FF00003FFE000007FE00003FFE0 00007FE00003FFE000007FE00003FFE000007FE00003FFE000007FE00001FFC000007FE0 0000FF8000007FE000003E0000007FE00000000000007FE00000000000007FE000000000 00007FE00000000000007FE00000000000007FE00000000000007FE0003FFFF000FFFFFF FFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF0 00007FF00000FFF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 003FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFF E03FFFFFE03B487EC742>12 D<07C0001FF0003FFC007FFC007FFE00FFFE00FFFF00FFFF 00FFFF80FFFF80FFFF807FFF807FFF803FFF801FFF8007E780000780000780000F00000F 00000F00000F00001E00001E00003C00003C0000780000F80000F00001F00003E00007C0 000F80001F00001E00000C00001124788F21>44 DI<00000000 03E000000000000000000007F000000000000000000007F00000000000000000000FF800 00000000000000000FF80000000000000000000FF80000000000000000001FFC00000000 00000000001FFC0000000000000000003FFE0000000000000000003FFE00000000000000 00007FFF0000000000000000007FFF0000000000000000007FFF000000000000000000FF FF800000000000000000FFFF800000000000000001FFFFC00000000000000001FFFFC000 00000000000001FFFFC00000000000000003FFFFE00000000000000003EFFFE000000000 00000007EFFFF00000000000000007CFFFF00000000000000007C7FFF000000000000000 0FC7FFF8000000000000000F83FFF8000000000000001F83FFFC000000000000001F03FF FC000000000000001F01FFFC000000000000003F01FFFE000000000000003E00FFFE0000 00000000007E00FFFF000000000000007C007FFF00000000000000FC007FFF8000000000 0000F8007FFF80000000000000F8003FFF80000000000001F8003FFFC0000000000001F0 001FFFC0000000000003F0001FFFE0000000000003E0001FFFE0000000000003E0000FFF E0000000000007E0000FFFF0000000000007C00007FFF000000000000FC00007FFF80000 0000000F800007FFF800000000000F800003FFF800000000001F800003FFFC0000000000 1F000001FFFC00000000003FFFFFFFFFFE00000000003FFFFFFFFFFE00000000007FFFFF FFFFFF00000000007FFFFFFFFFFF00000000007FFFFFFFFFFF0000000000FC0000007FFF 8000000000F80000003FFF8000000001F80000003FFFC000000001F00000003FFFC00000 0001F00000001FFFC000000003F00000001FFFE000000003E00000000FFFE000000007E0 0000000FFFF000000007C00000000FFFF000000007C000000007FFF00000000FC0000000 07FFF80000000F8000000003FFF80000001F8000000003FFFC0000001F0000000003FFFC 0000007FC000000001FFFC0000FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FF FFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF8051487CC7 5A>65 DI<000000003FFE00000E0000000FFFFFC0001E0000007FFFFFF8003E000003FFFFFFFE 00FE00000FFFFFFFFF81FE00003FFFF800FFC3FE0000FFFF80000FF7FE0001FFFC000003 FFFE0007FFF0000001FFFE000FFFC00000007FFE001FFF800000003FFE003FFF00000000 1FFE007FFE000000000FFE00FFFC0000000007FE01FFF80000000007FE03FFF000000000 03FE03FFF00000000001FE07FFE00000000001FE07FFE00000000000FE0FFFC000000000 00FE0FFFC000000000007E1FFFC000000000007E1FFF8000000000007E3FFF8000000000 007E3FFF8000000000003E3FFF8000000000003E7FFF8000000000003E7FFF0000000000 003E7FFF000000000000007FFF00000000000000FFFF00000000000000FFFF0000000000 0000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF0000000000 0000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF0000000000 0000FFFF00000000000000FFFF00000000000000FFFF000000000000007FFF0000000000 00007FFF000000000000007FFF000000000000007FFF8000000000003E3FFF8000000000 003E3FFF8000000000003E3FFF8000000000003E1FFF8000000000003E1FFFC000000000 003E0FFFC000000000007C0FFFC000000000007C07FFE000000000007C07FFE000000000 00F803FFF00000000000F803FFF00000000001F801FFF80000000001F000FFFC00000000 03E0007FFE0000000007E0003FFF000000000FC0001FFF800000001F80000FFFC0000000 3F000007FFF0000000FE000001FFFC000001FC000000FFFF80000FF80000003FFFF8007F F00000000FFFFFFFFFC000000003FFFFFFFF00000000007FFFFFFC00000000000FFFFFE0 0000000000003FFE000000474979C756>I69 DI<000000 003FFE00000E000000000FFFFFC0001E000000007FFFFFF8003E00000003FFFFFFFE00FE 0000000FFFFFFFFF81FE0000003FFFF800FFC3FE000000FFFF80000FF7FE000001FFFC00 0003FFFE000007FFF0000001FFFE00000FFFC00000007FFE00001FFF800000003FFE0000 3FFF000000001FFE00007FFE000000000FFE0000FFFC0000000007FE0001FFF800000000 07FE0003FFF00000000003FE0003FFF00000000001FE0007FFE00000000001FE0007FFE0 0000000000FE000FFFC00000000000FE000FFFC000000000007E001FFFC000000000007E 001FFF8000000000007E003FFF8000000000007E003FFF8000000000003E003FFF800000 0000003E007FFF8000000000003E007FFF0000000000003E007FFF00000000000000007F FF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000 000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF00 00000000000000FFFF0000000000000000FFFF0000000000000000FFFF00000000000000 00FFFF0000000000000000FFFF0000000000000000FFFF0000007FFFFFFFFE7FFF000000 7FFFFFFFFE7FFF0000007FFFFFFFFE7FFF8000007FFFFFFFFE7FFF8000007FFFFFFFFE3F FF8000000000FFFE003FFF8000000000FFFE003FFF8000000000FFFE001FFF8000000000 FFFE001FFFC000000000FFFE000FFFC000000000FFFE000FFFC000000000FFFE0007FFE0 00000000FFFE0007FFE000000000FFFE0003FFF000000000FFFE0003FFF800000000FFFE 0001FFF800000000FFFE0000FFFC00000000FFFE00007FFE00000000FFFE00003FFF0000 0000FFFE00001FFF80000000FFFE00000FFFE0000001FFFE000007FFF0000003FFFE0000 01FFFC000007FFFE000000FFFF80001FFFFE0000003FFFFC00FFC7FE0000000FFFFFFFFF 83FE00000003FFFFFFFE00FE000000007FFFFFF8003E000000000FFFFFE0000E00000000 003FFE000000004F4979C75D>I73 D76 DI80 D<3FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFF FFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFF0003FFF8003FFF007FF8 0003FFF80007FF807FE00003FFF80001FF807FC00003FFF80000FF807F800003FFF80000 7F807F000003FFF800003F807F000003FFF800003F807E000003FFF800001F807E000003 FFF800001F807E000003FFF800000F807C000003FFF800000F807C000003FFF800000F80 7C000003FFF800000F807C000003FFF800000F80FC000003FFF800000FC0F8000003FFF8 000007C0F8000003FFF8000007C0F8000003FFF8000007C0F8000003FFF8000007C0F800 0003FFF8000007C000000003FFF80000000000000003FFF80000000000000003FFF80000 000000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003 FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800000000 00000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF80000000000000003FFF8000000000000 0003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF80000 000000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003 FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800000000 00000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8 0000000000000003FFF80000000000000003FFF80000000000000003FFF8000000000000 0003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF80000 000000000003FFF8000000000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFF FFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF800004A467CC553>84 D<0007FFFC000000007FFFFFC0000001FFFFFFF8000003FFFFFFFE000007FE001FFF0000 07FF0003FFC0000FFF8001FFE0000FFF8000FFF0000FFF80007FF0000FFF80007FF8000F FF80007FF80007FF00003FFC0007FF00003FFC0003FE00003FFC0000F800003FFC000000 00003FFC00000000003FFC00000000003FFC00000000003FFC00000007FFFFFC000000FF FFFFFC000007FFFFFFFC00003FFFE03FFC0000FFFE003FFC0003FFF0003FFC0007FFC000 3FFC000FFF00003FFC001FFE00003FFC003FFC00003FFC007FF800003FFC007FF800003F FC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC 00FFF000007FFC007FF80000FFFC007FF80001EFFC003FFC0003EFFC003FFF0007CFFF00 0FFFC03F8FFFF807FFFFFF07FFFC01FFFFFC03FFFC007FFFF001FFFC0003FF80007FF836 2E7DAD3A>97 D<007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000 000000FFFFC00000000000FFFFC0000000000003FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC00FFC00000001FFC07FFFC0000001FFC3FFFFF0000001FFCFFFFFFC000001FFDFF00F FF000001FFFF8003FF800001FFFE0001FFC00001FFF800007FE00001FFF000007FF00001 FFE000003FF80001FFE000001FFC0001FFE000001FFC0001FFE000001FFE0001FFE00000 1FFE0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001 FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE00000 0FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001 FFE000000FFF8001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001FFE00000 1FFE0001FFE000001FFE0001FFE000001FFC0001FFE000001FFC0001FFE000003FF80001 FFF000003FF80001FFF800007FF00001FFFC0000FFE00001FFFE0001FFC00001FFBF0007 FF800001FF1FE01FFE000001FE0FFFFFFC000001FC03FFFFF0000001F800FFFF80000001 F0001FF800000039487CC742>I<00001FFFC0000000FFFFF8000007FFFFFE00001FFFFF FF80007FFC00FFC000FFE001FFC001FFC003FFE003FF8003FFE007FF0003FFE00FFE0003 FFE00FFE0003FFE01FFC0001FFC01FFC0001FFC03FFC0000FF803FFC00003E007FF80000 00007FF8000000007FF800000000FFF800000000FFF800000000FFF800000000FFF80000 0000FFF800000000FFF800000000FFF800000000FFF800000000FFF800000000FFF80000 00007FF8000000007FF8000000007FFC000000003FFC000000003FFC000000001FFC0000 00F81FFE000000F80FFE000000F80FFF000001F007FF800003F003FFC00007E001FFE000 0FC000FFF0001F80007FFE00FF00001FFFFFFE000007FFFFF8000000FFFFE00000001FFE 00002D2E7CAD35>I<00000000007FC00000000000FFFFC00000000000FFFFC000000000 00FFFFC00000000000FFFFC00000000000FFFFC0000000000003FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 0000000FFC01FFC0000000FFFF81FFC0000007FFFFE1FFC000001FFFFFF9FFC000007FFC 03FFFFC00000FFF0007FFFC00001FFC0001FFFC00003FF80000FFFC00007FF000007FFC0 000FFE000003FFC0000FFE000003FFC0001FFC000003FFC0001FFC000003FFC0003FFC00 0003FFC0003FFC000003FFC0007FF8000003FFC0007FF8000003FFC0007FF8000003FFC0 00FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF800 0003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC0 00FFF8000003FFC0007FF8000003FFC0007FF8000003FFC0007FF8000003FFC0003FF800 0003FFC0003FFC000003FFC0003FFC000003FFC0001FFC000003FFC0001FFC000003FFC0 000FFE000007FFC00007FF00000FFFC00003FF00001FFFC00001FFC0003FFFC00000FFE0 00FFFFE000007FF807FBFFFF80001FFFFFF3FFFF800007FFFFC3FFFF800001FFFF03FFFF 8000001FF803FFFF8039487CC742>I<00001FFE00000001FFFFE0000007FFFFF800001F FFFFFE00007FFC07FF0000FFE001FF8001FFC0007FC003FF80003FE007FF00003FF00FFE 00001FF01FFE00000FF81FFC00000FF83FFC00000FFC3FFC000007FC7FFC000007FC7FF8 000007FC7FF8000007FE7FF8000007FEFFF8000007FEFFF8000007FEFFFFFFFFFFFEFFFF FFFFFFFEFFFFFFFFFFFEFFFFFFFFFFFCFFF800000000FFF800000000FFF800000000FFF8 000000007FF8000000007FF8000000007FFC000000003FFC000000003FFC000000003FFC 0000001C1FFE0000003E0FFE0000003E07FF0000007E07FF000000FC03FF800001F801FF C00003F0007FF0001FE0003FFE00FFC0001FFFFFFF800007FFFFFE000000FFFFF8000000 0FFF80002F2E7DAD36>I<000000FFC000000007FFF80000003FFFFC000000FFFFFF0000 01FFC1FF000007FF03FF80000FFC03FF80000FF807FFC0001FF807FFC0003FF007FFC000 3FF007FFC0003FE003FF80007FE003FF80007FE001FF00007FE000FE00007FE000380000 7FE0000000007FE0000000007FE0000000007FE0000000007FE0000000007FE000000000 7FE0000000007FE0000000007FE0000000007FE0000000FFFFFFFE0000FFFFFFFE0000FF FFFFFE0000FFFFFFFE0000FFFFFFFE0000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF00000003FFFFFF800003FFFFFF800003FFFFFF800003F FFFFF800003FFFFFF800002A487DC724>I<00000000001F8000007FF000FFE00007FFFF 03FFF0001FFFFFC7FFF0007FFFFFFFC7F800FFE03FFE0FF801FF800FFC0FF803FF0007FE 0FF807FE0003FF07F007FE0003FF07F00FFC0001FF81C00FFC0001FF80000FFC0001FF80 001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC000 1FFC0001FFC0000FFC0001FF80000FFC0001FF80000FFC0001FF800007FE0003FF000007 FE0003FF000003FF0007FE000001FF800FFC000000FFE03FF8000001FFFFFFF0000001DF FFFFC0000003C7FFFF00000003C07FF000000007C0000000000007C0000000000007C000 0000000007C0000000000007E0000000000007F0000000000007F8000000000007FFFFFF F0000007FFFFFFFF000003FFFFFFFFE00003FFFFFFFFF80001FFFFFFFFFE0001FFFFFFFF FF0000FFFFFFFFFF80007FFFFFFFFF8003FFFFFFFFFFC00FFFFFFFFFFFC01FF800001FFF E03FE0000001FFE07FC00000007FF07FC00000003FF0FF800000001FF0FF800000001FF0 FF800000001FF0FF800000001FF0FF800000001FF07FC00000003FE07FC00000003FE03F E00000007FC03FF0000000FFC01FFC000003FF800FFF00000FFF0003FFF000FFFC0000FF FFFFFFF000003FFFFFFFC0000007FFFFFE000000003FFFC0000035447DAE3B>I<007FC0 0000000000FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC000000000 00FFFFC0000000000003FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC001FFC0000001FFC0 0FFFF8000001FFC03FFFFE000001FFC0FFFFFF000001FFC1FC07FF800001FFC3E003FFC0 0001FFC7C001FFC00001FFCF0001FFE00001FFDE0000FFE00001FFDC0000FFE00001FFFC 0000FFF00001FFF80000FFF00001FFF00000FFF00001FFF00000FFF00001FFF00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF000FFFFFFC07FFFFF E0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B487C C742>I<00FC0001FF0003FF8007FFC00FFFC01FFFE01FFFE01FFFE01FFFE01FFFE01FFF E00FFFC007FFC003FF8001FF0000FC000000000000000000000000000000000000000000 00000000000000000000000000007FC0FFFFC0FFFFC0FFFFC0FFFFC0FFFFC003FFC001FF C001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FF C001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FF C001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0FFFFFFFFFFFFFFFF FFFFFFFFFFFFFF18497CC820>I<007FC000000000FFFFC000000000FFFFC000000000FF FFC000000000FFFFC000000000FFFFC00000000003FFC00000000001FFC00000000001FF C00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC0 0000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC000 00000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000 000001FFC00000000001FFC00000000001FFC0007FFFF801FFC0007FFFF801FFC0007FFF F801FFC0007FFFF801FFC0007FFFF801FFC0000FFE0001FFC00007F00001FFC0000FE000 01FFC0003FC00001FFC0007F800001FFC000FE000001FFC001FC000001FFC007F8000001 FFC00FF0000001FFC01FC0000001FFC03F80000001FFC0FF00000001FFC1FE00000001FF C3FF00000001FFCFFF80000001FFDFFFC0000001FFFFFFC0000001FFFFFFE0000001FFFF FFF0000001FFFCFFF8000001FFF87FFC000001FFE03FFC000001FFC01FFE000001FFC01F FF000001FFC00FFF800001FFC007FFC00001FFC003FFC00001FFC001FFE00001FFC001FF F00001FFC000FFF80001FFC0007FFC0001FFC0003FFC0001FFC0001FFE0001FFC0000FFF 0001FFC0000FFF8001FFC0000FFFC0FFFFFF807FFFFFFFFFFF807FFFFFFFFFFF807FFFFF FFFFFF807FFFFFFFFFFF807FFFFF38487CC73F>107 D<007FC000FFFFC000FFFFC000FF FFC000FFFFC000FFFFC00003FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001 FFC000FFFFFF80FFFFFF80FFFFFF80FFFFFF80FFFFFF8019487CC720>I<007FC001FFC0 0000FFE00000FFFFC00FFFF80007FFFC0000FFFFC03FFFFE001FFFFF0000FFFFC0FFFFFF 007FFFFF8000FFFFC1FC07FF80FE03FFC000FFFFC3E003FFC1F001FFE00003FFC7C001FF C3E000FFE00001FFCF0001FFE78000FFF00001FFDE0000FFEF00007FF00001FFDC0000FF EE00007FF00001FFFC0000FFFE00007FF80001FFF80000FFFC00007FF80001FFF00000FF F800007FF80001FFF00000FFF800007FF80001FFF00000FFF800007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FF F000007FF80001FFE00000FFF000007FF800FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFF FFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFF FFE03FFFFFF05C2E7CAD63>I<007FC001FFC00000FFFFC00FFFF80000FFFFC03FFFFE00 00FFFFC0FFFFFF0000FFFFC1FC07FF8000FFFFC3E003FFC00003FFC7C001FFC00001FFCF 0001FFE00001FFDE0000FFE00001FFDC0000FFE00001FFFC0000FFF00001FFF80000FFF0 0001FFF00000FFF00001FFF00000FFF00001FFF00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF000FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFF C07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B2E7CAD42>I<00000FFF00000000 00FFFFF000000007FFFFFE0000001FFFFFFF8000003FFC03FFC00000FFE0007FF00001FF 80001FF80003FF00000FFC0007FE000007FE000FFE000007FF000FFC000003FF001FFC00 0003FF803FFC000003FFC03FF8000001FFC03FF8000001FFC07FF8000001FFE07FF80000 01FFE07FF8000001FFE0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001 FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FF F0FFF8000001FFF07FF8000001FFE07FF8000001FFE07FF8000001FFE07FF8000001FFE0 3FFC000003FFC03FFC000003FFC01FFC000003FF801FFE000007FF800FFE000007FF0007 FF00000FFE0003FF80001FFC0001FFC0003FF80000FFE0007FF000007FFC03FFE000001F FFFFFF80000007FFFFFE00000000FFFFF0000000000FFF000000342E7DAD3B>I<007FC0 0FFC000000FFFFC07FFFC00000FFFFC3FFFFF00000FFFFCFFFFFFC0000FFFFDFF01FFF00 00FFFFFF8007FF800003FFFE0001FFC00001FFF80000FFE00001FFF00000FFF00001FFE0 00007FF80001FFE000003FFC0001FFE000003FFC0001FFE000003FFE0001FFE000001FFE 0001FFE000001FFF0001FFE000001FFF0001FFE000001FFF0001FFE000000FFF0001FFE0 00000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF 8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE0 00000FFF8001FFE000000FFF0001FFE000001FFF0001FFE000001FFF0001FFE000001FFE 0001FFE000001FFE0001FFE000003FFC0001FFE000003FFC0001FFE000007FF80001FFF0 00007FF80001FFF80000FFF00001FFFC0001FFE00001FFFE0003FFC00001FFFF0007FF80 0001FFFFE03FFE000001FFEFFFFFFC000001FFE3FFFFF0000001FFE0FFFF80000001FFE0 1FF800000001FFE0000000000001FFE0000000000001FFE0000000000001FFE000000000 0001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0 000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE000000000 0001FFE0000000000001FFE00000000000FFFFFFC000000000FFFFFFC000000000FFFFFF C000000000FFFFFFC000000000FFFFFFC00000000039427CAD42>I<00FF803F8000FFFF 80FFF000FFFF83FFFC00FFFF87FFFE00FFFF8FC3FF00FFFF8F07FF0003FF9E0FFF8001FF BC0FFF8001FFB80FFF8001FFF80FFF8001FFF00FFF8001FFF007FF0001FFF007FF0001FF E003FE0001FFE000F80001FFE000000001FFE000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC000000001FFC0000000FFFFFFE00000FFFFFFE00000FFFF FFE00000FFFFFFE00000FFFFFFE00000292E7CAD31>114 D<000FFF00E0007FFFF3E001 FFFFFFE007FFFFFFE00FF800FFE01FC0001FE03F80000FE03F000007E07F000003E07F00 0003E0FF000003E0FF000003E0FF800003E0FFC0000000FFF0000000FFFE000000FFFFF8 00007FFFFFC0007FFFFFF0003FFFFFFC001FFFFFFF000FFFFFFF8007FFFFFFC003FFFFFF E000FFFFFFF0003FFFFFF00003FFFFF800001FFFF8000000FFFC0000001FFC7800000FFC F8000007FCF8000003FCFC000003FCFC000003FCFE000003F8FE000003F8FF000003F8FF 800007F0FFC0000FF0FFF0001FE0FFFC00FFC0FFFFFFFF80FC7FFFFE00F81FFFF800E003 FF8000262E7CAD2F>I<0001F000000001F000000001F000000001F000000001F0000000 01F000000003F000000003F000000003F000000007F000000007F000000007F00000000F F00000000FF00000001FF00000003FF00000003FF00000007FF0000001FFF0000003FFF0 00000FFFFFFFC0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC000FFF0000000FFF000 0000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF00000 00FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000 FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FF F001F000FFF001F000FFF001F000FFF001F000FFF001F000FFF001F000FFF001F000FFF0 01F000FFF001F0007FF001E0007FF803E0003FF803E0003FFC07C0001FFE0F80000FFFFF 800007FFFE000001FFFC0000001FF00024427EC12E>I<007FE000003FF000FFFFE0007F FFF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF00003 FFE00001FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00001FFF00001FFE00001FFF00001FFE00001FFF00001FFE00003FFF00000FFE00007 FFF00000FFE0000F7FF000007FE0001F7FF000007FF0003E7FF800003FFC00FC7FFFE000 1FFFFFF87FFFE00007FFFFE07FFFE00001FFFF807FFFE000003FFE007FFFE03B2E7CAD42 >III<7FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFF FF007FFFFF801FFFFF00007FF80001FF0000007FFC0001FC0000003FFE0003F80000001F FF0007F00000000FFF0007E000000007FF800FC000000003FFC01F8000000003FFE03F80 00000001FFF07F0000000000FFF8FE00000000007FF9FC00000000003FFFF80000000000 3FFFF000000000001FFFE000000000000FFFC0000000000007FFC0000000000003FFC000 0000000001FFE0000000000001FFF0000000000001FFF8000000000003FFFC0000000000 03FFFE000000000007FFFE00000000000FEFFF00000000001FCFFF80000000003F87FFC0 000000007F03FFE000000000FE01FFE000000001FC00FFF000000001F8007FF800000003 F0007FFC00000007F0003FFE0000000FE0001FFF0000001FC0000FFF0000003F800007FF 800000FF800007FFC000FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFF F8003FFFFFC0FFFFF8003FFFFFC03A2E7EAD3F>I<7FFFFFC000FFFF807FFFFFC000FFFF 807FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000FFFF8000FFF000000FE00000FFF8 00000FC00000FFF800000FC000007FFC00000F8000007FFC00001F8000003FFC00001F00 00003FFE00003F0000001FFE00003E0000001FFF00007E0000000FFF00007C0000000FFF 8000FC00000007FF8000F800000007FFC001F800000003FFC001F000000003FFE003F000 000003FFE003F000000001FFF003E000000001FFF007E000000000FFF007C000000000FF F80FC0000000007FF80F80000000007FFC1F80000000003FFC1F00000000003FFE3F0000 0000001FFE3E00000000001FFF7E00000000000FFF7C00000000000FFFFC00000000000F FFFC000000000007FFF8000000000007FFF8000000000003FFF0000000000003FFF00000 00000001FFE0000000000001FFE0000000000000FFC0000000000000FFC0000000000000 7F800000000000007F800000000000003F000000000000003F000000000000003F000000 000000003E000000000000007E000000000000007C00000000000000FC000000001F8000 F8000000003FC001F8000000007FE001F000000000FFF003F000000000FFF003E0000000 00FFF007E000000000FFF00FC000000000FFF01F8000000000FFF03F80000000007FE07F 00000000007F43FE00000000003FFFF800000000001FFFF0000000000007FFC000000000 0001FE00000000000039427EAD3F>I E /Fs 10 58 df<1F00318060C04040C060C060C0 60C060C060C060C060C060404060C031801F000B107F8F0F>48 D<0C003C00CC000C000C 000C000C000C000C000C000C000C000C000C000C00FF8009107E8F0F>I<1F00618040C0 8060C0600060006000C00180030006000C00102020207FC0FFC00B107F8F0F>I<1F0021 8060C060C000C0008001800F00008000400060C060C060804060801F000B107F8F0F>I< 0300030007000F000B001300330023004300C300FFE003000300030003001FE00B107F8F 0F>I<20803F002C002000200020002F0030802040006000600060C06080C061801F000B 107F8F0F>I<0780184030C060C06000C000CF00F080E040C060C060C060406060C03080 1F000B107F8F0F>I<40007FE07FC08080808001000200040004000C0008000800180018 001800180018000B117E900F>I<1F00318060C060C060C071803F000F00338061C0C060 C060C060404060801F000B107F8F0F>I<1F00318060C0C040C060C060C06040E021E01E 600060004060C0608043003E000B107F8F0F>I E /Ft 16 104 df0 D<70F8F8F87005057C8D0D>I<400004C0000C6000183000301800600C00 C006018003030001860000CC0000780000300000300000780000CC000186000303000601 800C00C0180060300030600018C0000C40000416187A9623>I<01800180018001800180 C183F18F399C0FF003C003C00FF0399CF18FC1830180018001800180018010147D9417> I<0001FE00000007FF8000001E01E000007800780000E0001C0001800006000300000300 06000001800C000000C00C000000C0180000006030000000303000000030300000003060 0000001860000000186000000018C00000000CC00000000CC00000000CC00000000CC000 00000CC00000000CC00000000CC00000000CC00000000C60000000186000000018600000 001830000000303000000030300000003018000000600C000000C00C000000C006000001 800300000300018000060000E0001C000078007800001E01E0000007FF80000001FE0000 262B7DA02D>13 D<03C00FF01C38300C60066006C003C003C003C003C003C00360066006 300C1C380FF003C010127D9317>I<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFFFF FFFFFF7FFE7FFE3FFC1FF80FF003C010127D9317>I<000000C0000003C000000F000000 3C000000F0000003C00000070000001C00000078000001E00000078000001E0000007800 0000E0000000780000001E0000000780000001E0000000780000001C0000000700000003 C0000000F00000003C0000000F00000003C0000000C00000000000000000000000000000 00000000000000000000000000007FFFFF80FFFFFFC01A247C9C23>20 DI<003FF800FFF803C0000700000C000018000030000030000060000060 0000C00000C00000C00000FFFFF8FFFFF8C00000C00000C0000060000060000030000030 00001800000C000007000003C00000FFF8003FF8151C7C981E>50 D<00000C00000C0000180000180000300000300000600000600000C00000C00001800001 80000180000300000300000600000600000C00000C000018000018000030000030000060 0000600000C00000C0000180000180000300000300000600000600000600000C00000C00 00180000180000300000300000600000600000C00000400000162C7AA000>54 D<003FFFC001FFFFF0071E03FC081E00FE181E003E701E001F701E001FC01E000F001C00 0F001C000F003C000E003C000E003C001C0038001C003800380078003000780040007000 800070030000F01C0000F3F00000E7800000E0000001E0000001C0000001C0000003C000 00038000000380000007000000070000000E0000000C00000020217F9E20>80 D<0000FE000007FF80001C1FC0002007E000C003F0018001F0030000F0060000F80E0000 F81C0000781C00007838000078380000787000007870000078F0000070F0000070F00000 F0F00000E0F00000E0F80001C0F80001C07C0003807E0083003F0707001FFC0E000FE00C 0000001000000020000000C000000F00001FFC000C7FF8001807FF0030007FF060000FFF C00000FF001E257B9F25>I<400002C00006C00006C00006C00006C00006C00006C00006 C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006 C0000660000C60000C3000181C00700F01E003FF8000FE00171C7D9A1E>91 D<000F0038006000E001C001C001C001C001C001C001C001C001C001C001C001C001C001 C001C0038007001E00F8001E000700038001C001C001C001C001C001C001C001C001C001 C001C001C001C001C001C000E000600038000F102D7DA117>102 DI E /Fu 89 126 df<70F8F8F8F8F8F8F8F8F8F8F8F8F8F8F8F870000000000070F8F8F87005 1C779B18>33 D<4010E038F078E038E038E038E038E038E038E038E038E038E03860300D 0E7B9C18>I<030600078F00078F00078F00078F00078F00078F007FFFC0FFFFE0FFFFE0 7FFFC00F1E000F1E000F1E000F1E000F1E000F1E007FFFC0FFFFE0FFFFE07FFFC01E3C00 1E3C001E3C001E3C001E3C001E3C000C1800131C7E9B18>I<3803007C07807C0780EE0F 80EE0F00EE0F00EE1F00EE1E00EE1E00EE3E007C3C007C3C00387C0000780000780000F8 0000F00001F00001E00001E00003E00003C00003C00007C0000783800787C00F87C00F0E E00F0EE01F0EE01E0EE01E0EE03E0EE03C07C03C07C018038013247E9F18>37 D<01C00007E0000FF0000E70001C38001C38001C38001C38001C73F01C73F01CE3F00FE3 800FC7000F87000F07001F0E003F0E007B8E0073DC00E1DC00E0F800E0F800E07070E078 7070FC707FFFE03FCFE00F03C0141C7F9B18>I<387C7C7E3E0E0E0E1C1C38F8F0C0070E 789B18>I<007000F001E003C007800F001E001C00380038007000700070007000E000E0 00E000E000E000E000E000E0007000700070007000380038001C001E000F00078003C001 F000F000700C24799F18>I<6000F00078003C001E000F000780038001C001C000E000E0 00E000E00070007000700070007000700070007000E000E000E000E001C001C003800780 0F001E003C007800F00060000C247C9F18>I<01C00001C00001C00001C000C1C180F1C7 80F9CF807FFF001FFC0007F00007F0001FFC007FFF00F9CF80F1C780C1C18001C00001C0 0001C00001C00011147D9718>I<00600000F00000F00000F00000F00000F00000F00000 F0007FFFC0FFFFE0FFFFE07FFFC000F00000F00000F00000F00000F00000F00000F00000 600013147E9718>I<1C3E7E7F3F1F070E1E7CF860080C788518>I<7FFF00FFFF80FFFF80 7FFF0011047D8F18>I<3078FCFC78300606778518>I<000300000780000780000F80000F 00001F00001E00001E00003E00003C00007C0000780000780000F80000F00001F00001E0 0003E00003C00003C00007C0000780000F80000F00000F00001F00001E00003E00003C00 003C00007C0000780000F80000F00000F0000060000011247D9F18>I<01F00007FC000F FE001F1F001C07003803807803C07001C07001C0E000E0E000E0E000E0E000E0E000E0E0 00E0E000E0E000E0E000E0F001E07001C07001C07803C03803801C07001F1F000FFE0007 FC0001F000131C7E9B18>I<01800380038007800F803F80FF80FB804380038003800380 03800380038003800380038003800380038003800380038003807FFCFFFE7FFC0F1C7B9B 18>I<03F0000FFE003FFF007C0F807003C0E001C0F000E0F000E06000E00000E00000E0 0001C00001C00003C0000780000F00001E00003C0000780000F00001E00007C0000F8000 1E00E03C00E07FFFE0FFFFE07FFFE0131C7E9B18>I<07F8001FFE003FFF007807807803 C07801C03001C00001C00003C0000380000F0003FF0003FE0003FF000007800003C00001 C00000E00000E00000E0F000E0F000E0F001C0F003C07C07803FFF001FFE0003F800131C 7E9B18>I<001F00003F0000770000770000E70001E70001C7000387000787000707000E 07001E07003C0700380700780700F00700FFFFF8FFFFF8FFFFF800070000070000070000 0700000700000700007FF000FFF8007FF0151C7F9B18>I<1FFF803FFF803FFF80380000 3800003800003800003800003800003800003800003BF8003FFE003FFF003C07801803C0 0001C00000E00000E06000E0F000E0F000E0E001C07003C07C0F803FFF001FFC0003F000 131C7E9B18>I<007E0001FF0007FF800F83C01E03C01C03C03801803800007000007000 00E1F800E7FE00FFFF00FE0780F803C0F001C0F000E0E000E0F000E07000E07000E07000 E03801C03C03C01E07800FFF0007FE0001F800131C7E9B18>II<03F8000FFE001FFF003E0F803803807001C07001C07001C0 7001C03803803C07801FFF0007FC000FFE001F1F003C07807001C0F001E0E000E0E000E0 E000E0E000E07001C07803C03E0F801FFF000FFE0003F800131C7E9B18>I<03F0000FFC 001FFE003C0F00780780700380E001C0E001C0E001C0E001E0E001E07001E07803E03C0F E01FFFE00FFEE003F0E00000E00001C00001C00001C0300380780780780F00783E003FFC 001FF00007C000131C7E9B18>I<3078FCFC783000000000000000003078FCFC78300614 779318>I<183C7E7E3C180000000000000000183C7E7E3E1E0E1C3C78F060071A789318> I<000300000780001F80003F00007E0001FC0003F00007E0001FC0003F00007E0000FC00 00FC00007E00003F00001FC00007E00003F00001FC00007E00003F00001F800007800003 0011187D9918>I<7FFFC0FFFFE0FFFFE0FFFFE0000000000000000000000000FFFFE0FF FFE0FFFFE07FFFC0130C7E9318>I<600000F00000FC00007E00003F00001FC00007E000 03F00001FC00007E00003F00001F80001F80003F00007E0001FC0003F00007E0001FC000 3F00007E0000FC0000F0000060000011187D9918>I<007C0001FE0007FF000F87801E03 C03C1DC0387FC070FFE071E3E071C1E0E1C1E0E380E0E380E0E380E0E380E0E380E0E380 E0E1C1C071C1C071E3C070FF80387F003C1C001E00E00F83E007FFC001FF80007E00131C 7E9B18>64 D<00700000F80000F80000D80000D80001DC0001DC0001DC00018C00038E00 038E00038E00038E000306000707000707000707000707000FFF800FFF800FFF800E0380 0E03801C01C01C01C07F07F0FF8FF87F07F0151C7F9B18>II<00F8E003FEE007FFE00F07E01E03E03C01E03800E07000E07000E070 0000E00000E00000E00000E00000E00000E00000E00000E000007000007000E07000E038 00E03C00E01E01C00F07C007FF8003FE0000F800131C7E9B18>I<7FF800FFFE007FFF00 1C0F801C03C01C03C01C01E01C00E01C00E01C00F01C00701C00701C00701C00701C0070 1C00701C00701C00701C00F01C00E01C00E01C01E01C01C01C03C01C0F807FFF00FFFE00 7FF800141C7F9B18>III<01F1C003FDC00FFFC01F0FC01C03C03803C03801C07001C0 7001C0700000E00000E00000E00000E00000E00000E00FF0E01FF0E00FF07001C07001C0 7003C03803C03803C01C07C01F0FC00FFFC003FDC001F1C0141C7E9B18>I<7F07F0FF8F F87F07F01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01FFFC01FFF C01FFFC01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C01C07F07 F0FF8FF87F07F0151C7F9B18>I<7FFF00FFFF807FFF0001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C0007FFF00FFFF807FFF00111C7D9B18>I<7F07F0 FF87F87F07F01C03C01C07801C07001C0E001C1E001C3C001C38001C70001CF0001DF000 1DF0001FB8001FB8001F1C001E1C001C0E001C0E001C07001C07001C03801C03801C01C0 7F03F0FF87F87F03F0151C7F9B18>75 D<7FE000FFE0007FE0000E00000E00000E00000E 00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E 00000E00000E00700E00700E00700E00700E00707FFFF0FFFFF07FFFF0141C7F9B18>I< FC01F8FE03F8FE03F83B06E03B06E03B06E03B06E03B8EE03B8EE0398CE0398CE039DCE0 39DCE039DCE038D8E038D8E038F8E03870E03870E03800E03800E03800E03800E03800E0 3800E0FE03F8FE03F8FE03F8151C7F9B18>I<7E07F0FF0FF87F07F01D81C01D81C01D81 C01DC1C01CC1C01CC1C01CE1C01CE1C01CE1C01C61C01C71C01C71C01C31C01C39C01C39 C01C39C01C19C01C19C01C1DC01C0DC01C0DC01C0DC07F07C0FF87C07F03C0151C7F9B18 >I<0FF8003FFE007FFF00780F00700700F00780E00380E00380E00380E00380E00380E0 0380E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380F0078070 0700780F007FFF003FFE000FF800111C7D9B18>II<0FF8003FFE007FFF00780F00700700F00780E00380E00380E00380E00380E003 80E00380E00380E00380E00380E00380E00380E00380E00380E00380E1E380E1E380F0E7 8070F700787F007FFF003FFE000FFC00001C00001E00000E00000F000007000007001122 7D9B18>I<7FF800FFFE007FFF001C0F801C03801C03C01C01C01C01C01C01C01C03C01C 03801C0F801FFF001FFE001FFE001C0F001C07001C03801C03801C03801C03801C03801C 039C1C039C1C039C7F01F8FF81F87F00F0161C7F9B18>I<03F3801FFF803FFF807C0F80 700780E00380E00380E00380E000007000007800003F00001FF00007FE0000FF00000F80 0003C00001C00000E00000E06000E0E000E0E001E0F001C0F80780FFFF80FFFE00E7F800 131C7E9B18>I<7FFFF8FFFFF8FFFFF8E07038E07038E07038E070380070000070000070 000070000070000070000070000070000070000070000070000070000070000070000070 0000700000700000700007FF0007FF0007FF00151C7F9B18>IIII<7F8FE07F9FE07F8FE00E07000F0700070E00078E00039C0003 DC0001F80001F80000F00000F00000700000F00000F80001F80001DC00039E00038E0007 0F000707000E07800E03801E03C07F07F0FF8FF87F07F0151C7F9B18>II<3FFFE07FFFE07FFFE07001C07003C0700780700700000F 00001E00001C00003C0000780000700000F00001E00001C00003C0000780000700000F00 001E00E01C00E03C00E07800E07000E0FFFFE0FFFFE0FFFFE0131C7E9B18>II<600000F00000F00000F800007800007C00003C00003C00003E00001E00001F0000 0F00000F00000F800007800007C00003C00003C00003E00001E00001F00000F00000F800 007800007800007C00003C00003E00001E00001E00001F00000F00000F80000780000780 00030011247D9F18>II<7FFF00FFFF80FFFF807FFF0011047D7F18>95 D<061E3E387070E0E0E0F8FC7C7C38070E789E18>I<1FE0003FF8007FFC00781E00300E 0000070000070000FF0007FF001FFF007F0700780700E00700E00700E00700F00F00781F 003FFFF01FFBF007E1F014147D9318>I<7E0000FE00007E00000E00000E00000E00000E 00000E00000E3E000EFF800FFFC00FC1E00F80E00F00700E00700E00380E00380E00380E 00380E00380E00380F00700F00700F80E00FC1E00FFFC00EFF80063E00151C809B18>I< 01FE0007FF001FFF803E0780380300700000700000E00000E00000E00000E00000E00000 E000007000007001C03801C03E03C01FFF8007FF0001FC0012147D9318>I<001F80003F 80001F8000038000038000038000038000038003E3800FFB801FFF803C1F80380F807007 80700380E00380E00380E00380E00380E00380E00380700780700780380F803C1F801FFF F00FFBF803E3F0151C7E9B18>I<01F00007FC001FFE003E0F00380780700380700380E0 01C0E001C0FFFFC0FFFFC0FFFFC0E000007000007001C03801C03E03C01FFF8007FF0001 FC0012147D9318>I<001F80007FC000FFE000E1E001C0C001C00001C00001C0007FFFC0 FFFFC0FFFFC001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C0007FFF007FFF007FFF00131C7F9B18>I<01E1F007FFF80FFF F81E1E301C0E003807003807003807003807003807001C0E001E1E001FFC001FF80039E0 003800001C00001FFE001FFFC03FFFE07801F0700070E00038E00038E00038E000387800 F07E03F01FFFC00FFF8001FC00151F7F9318>I<7E0000FE00007E00000E00000E00000E 00000E00000E00000E3E000EFF800FFFC00FC1C00F80E00F00E00E00E00E00E00E00E00E 00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E07FC3FCFFE7FE7FC3FC171C809B 18>I<03800007C00007C00007C0000380000000000000000000000000007FC000FFC000 7FC00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C000FFFF00FFFF80FFFF00111D7C9C18>I<0038007C007C007C003800 000000000000000FFC1FFC0FFC001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C001C001C001C001C001C001C001C001C6038F078FFF07FE03F800E277E9C18 >II<7FE000FFE0007FE00000E00000E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00000E00000E00000E00000E00000E0007FFFC0FFFFE07FFFC0131C7E 9B18>I<7CE0E000FFFBF8007FFFF8001F1F1C001E1E1C001E1E1C001C1C1C001C1C1C00 1C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C00 7F1F1F00FFBFBF807F1F1F001914819318>I<7E3E00FEFF807FFFC00FC1C00F80E00F00 E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E07FC3 FCFFE7FE7FC3FC1714809318>I<01F0000FFE001FFF003E0F803803807001C07001C0E0 00E0E000E0E000E0E000E0E000E0F001E07001C07803C03C07803E0F801FFF000FFE0001 F00013147E9318>I<7E3E00FEFF807FFFC00FC1E00F80E00F00700E00700E00380E0038 0E00380E00380E00380E00380F00700F00700F80E00FC1E00FFFC00EFF800E3E000E0000 0E00000E00000E00000E00000E00000E00007FC000FFE0007FC000151E809318>I<01E3 8007FB801FFF803E1F80380F80700780700780E00380E00380E00380E00380E00380E003 80700780700780380F803C1F801FFF800FFB8003E3800003800003800003800003800003 80000380000380003FF8003FF8003FF8151E7E9318>I<7F87E0FF9FF07FBFF803F87803 F03003E00003C00003C00003800003800003800003800003800003800003800003800003 80007FFE00FFFF007FFE0015147F9318>I<07F7003FFF007FFF00780F00E00700E00700 E007007C00007FE0001FFC0003FE00001F00600780E00380E00380F00380F80F00FFFF00 FFFC00E7F00011147D9318>I<0180000380000380000380000380007FFFC0FFFFC0FFFF C00380000380000380000380000380000380000380000380000380000380400380E00380 E00380E001C1C001FFC000FF80003E0013197F9818>I<7E07E0FE0FE07E07E00E00E00E 00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E01E00F 03E007FFFC03FFFE01FCFC1714809318>I<7F8FF0FF8FF87F8FF01E03C00E03800E0380 0E0380070700070700070700038E00038E00038E00038E0001DC0001DC0001DC0000F800 00F80000700015147F9318>II<7F8FF07F9FF07F8FF0070700078E00039E0001DC0001F80000F80000 700000F00000F80001DC00039E00038E000707000F07807F8FF0FF8FF87F8FF015147F93 18>I<7F8FF0FF8FF87F8FF00E01C00E03800E0380070380070700070700038700038600 038E0001CE0001CE0000CC0000CC0000DC00007800007800007800007000007000007000 00F00000E00079E0007BC0007F80003F00001E0000151E7F9318>I<3FFFF07FFFF07FFF F07001E07003C0700780000F00001E00003C0000F80001F00003C0000780000F00701E00 703C0070780070FFFFF0FFFFF0FFFFF014147F9318>I<0007E0001FE0007FE000780000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00001E0007F C000FF8000FF80007FC00001E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E000007800007FE0001FE00007E013247E9F18>I<60F0F0F0F0F0 F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0600424769F18> I<7C0000FF0000FFC00003C00000E00000E00000E00000E00000E00000E00000E00000E0 0000E00000E00000E00000F000007FC0003FE0003FE0007FC000F00000E00000E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00003C000FFC000FF00007C00 0013247E9F18>I E /Fv 78 123 df<007F07F001FF1FF003FF3FF00780780007007000 0F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F000FFF8FF80FFF8FF80 FFF8FF800F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F000 0F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F000 1C20809F1B>11 D<0000F000F8F001F8F003F8F00780000700000F00000F00000F00000F 00000F00000F00000F0000FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F 00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F014 2180A018>I<00F8F001F8F003F8F00780F00700F00F00F00F00F00F00F00F00F00F00F0 0F00F00F00F0FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F00F00F00F0 0F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F01420809F18> I<783C783C783C783C180C3018301860306030E0700E0A7F9F17>34 D<007C000000FE000001FF000003C78000038780000383C0000783C0000703C0000703C0 000783C00007878000078F8000078F0000079E000003BC008003F800E003F001E003E001 C007E001C00FF003C01EF003803CF8078078780700787C0F00F03E0E00F01F1E00F01F3C 00F00FF800F807F0007803E0007C0FF8303FFFFFF01FFE3FF007F00FC01C227EA022>38 D<007000E001C00380078007000E001E001E003C003C003C0078007800780078007000F0 00F000F000F000F000F000F000F000F000F000F000F000700078007800780078003C003C 003C001E001E000E0007000780038001C000E000700C2E7EA112>40 DI<018001C001800180 C183E187F99F7DBE1FF807E007E01FF87DBEF99FE187C1830180018001C0018010147DA1 17>I<000600000006000000060000000600000006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000FFFFFFF0FFFFFFF000 060000000600000006000000060000000600000006000000060000000600000006000000 06000000060000000600000006000000060000000600001C207D9A23>I<787878781830 306060E0050A7D830D>III<03F0 0007F8001FFE001E1E003C0F00380700780780780780700380F003C0F003C0F003C0F003 C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07807807807 807807803807003C0F001E1E001FFE000FFC0003F00012207E9E17>48 D<00C001C00FC0FFC0FFC0F3C003C003C003C003C003C003C003C003C003C003C003C003 C003C003C003C003C003C003C003C003C003C003C0FFFEFFFEFFFE0F1F7C9E17>I<07F0 000FFC001FFE00383F00700F00600780E00780E003C04003C04003C00003C00003C00003 C0000780000780000F00000E00001C00003C0000780000E00001C0000380000700000E00 001C0000380000700000FFFFC0FFFFC0FFFFC0121F7E9E17>I<03F0000FFC001FFE003C 1F00780F00300780200780000780000780000780000F00000F00003E0003FC0003F80003 FC00001E00000F000007800003800003C00003C00003C00003C08003C0C003C0C0078070 0F807C1F003FFE000FFC0003F00012207E9E17>I<003E00003E00005E00005E0000DE00 01DE00019E00039E00039E00079E00071E000F1E000E1E001E1E003C1E003C1E00781E00 781E00F01E00FFFFF0FFFFF0FFFFF0001E00001E00001E00001E00001E00001E00001E00 001E00141E7F9D17>I<7FFF007FFF007FFF007800007800007800007800007800007800 0078000079F0007FFC007FFE007F1F007C07007C07807807800003C00003C00003C00003 C00003C00003C00003C0400780600780F00F007C1E003FFC001FF80007E000121F7E9D17 >I<007C0001FE0003FE000782000F00001E00003C00003C000078000078000078F800F3 FC00F7FE00FE1F00FC0F80F80780F80780F003C0F003C0F003C0F003C0F003C07003C078 03C07803C07807803807803C0F001E1E000FFE0007F80001F00012207E9E17>II<03F0000FFC001FFE003E1F003C0F 007807807807807807807807807807807807803C0F001E1E000FFC0007F8000FFC001F3E 003C0F00780780780780F003C0F003C0F003C0F003C0F003C0F003C07807807C0F803E1F 001FFE000FFC0003F00012207E9E17>I<03F00007F8000FFC001E1E003C0F0078070078 0780F00780F00380F003C0F003C0F003C0F003C0F003C0F003C07807C07807C07C0FC03E 1FC01FFBC00FF3C007C780000780000780000700000F00001E00201E00307C007FF8003F F0000FC00012207E9E17>II61 D<001F0000001F0000003F800000 3B8000003B8000007BC0000073C0000071C00000F1E00000E1E00000E0E00001E0F00001 E0F00001C0F00003C0780003C078000380780007803C0007803C0007003C000FFFFE000F FFFE000FFFFE001E000F001E000F003C000F803C0007803C000780780007C0780003C078 0003C0F00003E01B207F9F1E>65 DI<001FC000FFF801FFFC03E03C07800C0F00001E00003E00003C00 007C0000780000780000780000F00000F00000F00000F00000F00000F00000F00000F000 007800007800007800007C00003C00003E00001E00000F000207800E03E03E01FFFC00FF F0001FC017227DA01D>II II<001FE000 FFF801FFFE03E03E07800E0F00001E00003E00003C00007C0000780000780000780000F0 0000F00000F00000F00000F00000F00000F003FEF003FE7803FE78001E78001E7C001E3C 001E3E001E1E001E0F001E07801E03E03E01FFFE00FFF8001FC017227DA01E>III75 DIII<003F000000FFC00003FFF00007E1F8000F807C001F003E001E001E00 3C000F003C000F00780007807800078078000780F00003C0F00003C0F00003C0F00003C0 F00003C0F00003C0F00003C0F00003C0F00003C0F80007C0780007807800078078000780 3C000F003C000F001E001E001F003E000F807C0007E1F80003FFF00000FFC000003F0000 1A227DA021>II<003F000000FFC00003FFF00007E1F8000F807C001F003E001E001E003C000F003C00 0F00780007807800078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F000 03C0F00003C0F00003C0F00003C0F00003C07800078078000780781E07803C0F0F003C0F 8F001E079E001F03FE000F83FC0007E1F80003FFF00000FFF800003F780000007C000000 3E0000001E0000001F0000000F801A277DA021>II<01FC0007FF800FFFC01F03C03C00C03C00007800 007800007800007800007800007C00003C00003F00001FE0000FFC0007FE0001FF00003F 800007C00003C00003E00001E00001E00001E00001E00001E00001C0C003C0F007C0FC0F 807FFF001FFE0003F80013227EA019>IIIII<780007807C000F003E001F001E001E00 0F003C000F807C000780780003C0F00003E1F00001F1E00000F3C000007FC000007F8000 003F0000001F0000001E0000003F0000007F8000007FC00000F3C00001F1E00001E0F000 03C0F80007C0780007803C000F003E001F001E001E000F003C000F807C0007C0780003C0 F00003E01B207F9F1E>IIII<381C3018301860306030C060F078F078F078F0780E0A79 9F17>II<07E03FF87FFC701E401F000F000F000F 003F07FF1FFF7E0FF80FF00FF00FF00FF83F7FFF3FEF1F8F10147E9316>97 DI<03F00FFC 1FFE3E0E3C0278007800F000F000F000F000F000F000780078003C013E0F1FFF0FFE03F0 10147E9314>I<0007800007800007800007800007800007800007800007800007800007 8000078000078007C7800FF7801FFF803E1F807C0780780780F80780F00780F00780F007 80F00780F00780F00780F00780780780780F803E1F801FFF800FF78007C78011207E9F17 >I<03F0000FFC001FFE003E1F003C0700780700700380FFFF80FFFF80FFFF80F00000F0 0000F000007000007800003C01003E07001FFF0007FE0001F80011147F9314>I<007E01 FE03FE078007000F000F000F000F000F000F000F00FFF0FFF0FFF00F000F000F000F000F 000F000F000F000F000F000F000F000F000F000F000F000F000F20809F0E>I<03E0F00F FFF01FFFF03E3E003C1E00780F00780F00780F00780F00780F003C1E003E3E001FFC003F F80033E0003000003800003FFE003FFF801FFFC03FFFE07803F0F000F0F000F0F000F0F8 01F07E07E03FFFC00FFF0003FC00141E7F9317>III<01E001E001E001E00000000000 000000000000000000000001E001E001E001E001E001E001E001E001E001E001E001E001 E001E001E001E001E001E001E001E001E001E001E001E001E0C3C0FFC0FF803F000B2983 9F0C>IIIII<01F80007FE001FFF 803F0FC03C03C07801E07801E0F000F0F000F0F000F0F000F0F000F0F000F07801E07801 E03C03C03F0FC01FFF8007FE0001F80014147F9317>II<03C7800FF7801FFF803E1F807C0F80780780780780F00780F00780 F00780F00780F00780F00780F807807807807C0F803E1F801FFF800FF78007C780000780 000780000780000780000780000780000780000780000780111D7E9317>II<07F01FFC3FFC780C7800780078007C003FC01FF00FF803F8007C003C003C C03CF07CFFF87FF00FC00E147F9311>I<1E001E001E001E001E001E00FFF0FFF0FFF01E 001E001E001E001E001E001E001E001E001E001E001E001E001E201FF00FF007C00C1A7F 9910>IIII<7801E07C03C03E07801E0F 000F0F00079E0003FC0003F80001F80000F00001F00001F80003FC00079E000F0F000E0F 001E07803C03C07801E0F801F01414809315>II<7FFF7FFF7FFF003E003C007800F800F001E003E007C007800F001F001E00 3C007C00FFFFFFFFFFFF10147F9314>I E /Fw 1 98 df<001800001800001800003C00 003C00004E00004E00004E000087000087000187800103800103800201C00201C003FFC0 0400E00400E00800700800701800703C0078FE01FF18177F961C>97 D E /Fx 91 128 df<001F83E000F06E3001C078780380F8780300F03007007000070070 000700700007007000070070000700700007007000FFFFFF800700700007007000070070 000700700007007000070070000700700007007000070070000700700007007000070070 000700700007007000070070000700700007007000070070007FE3FF001D20809F1B>11 D<003F0000E0C001C0C00381E00701E00701E00700000700000700000700000700000700 00FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700 E00700E00700E00700E00700E00700E00700E00700E07FC3FE1720809F19>I<003FE000 E0E001C1E00381E00700E00700E00700E00700E00700E00700E00700E00700E0FFFFE007 00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E007 00E00700E00700E00700E00700E00700E07FE7FE1720809F19>I<001F81F80000F04F04 0001C07C06000380F80F000300F00F000700F00F00070070000007007000000700700000 070070000007007000000700700000FFFFFFFF0007007007000700700700070070070007 007007000700700700070070070007007007000700700700070070070007007007000700 700700070070070007007007000700700700070070070007007007000700700700070070 07007FE3FE3FF02420809F26>I<001F81FF0000F06F070001C07C0F000380F80F000300 F00700070070070007007007000700700700070070070007007007000700700700070070 0700FFFFFFFF000700700700070070070007007007000700700700070070070007007007 000700700700070070070007007007000700700700070070070007007007000700700700 070070070007007007000700700700070070070007007007007FE3FE3FF02420809F26> I22 D<70F8F8F8F8F8F8F87070707070707070707020202020200000 00000070F8F8F87005217CA00D>33 D<7038F87CFC7EFC7E743A04020402040208040804 10081008201040200F0E7E9F17>I<000300C0000300C0000300C0000300C00006018000 060180000601800006018000060180000C0300000C0300000C0300000C0300000C030000 1806007FFFFFFCFFFFFFFE00300C0000300C0000300C0000300C00006018000060180000 60180000601800FFFFFFFE7FFFFFFC00C030000180600001806000018060000180600001 8060000300C0000300C0000300C0000300C0000300C0000601800006018000060180001F 297D9F26>I<007800000084000001840000030200000702000007020000070200000702 0000070400000704000007080000070800000310000003A00FFC03C003E0038001C001C0 008001C0010003E0010004E0020008F00200187004003078080070380800701C1000F01E 1000F00E2000F0074000F003C0087003C0087801C010380670301C18386007E00F801E22 7EA023>38 D<70F8FCFC74040404080810102040060E7C9F0D>I<002000400080010002 0006000C000C00180018003000300030007000600060006000E000E000E000E000E000E0 00E000E000E000E000E000E0006000600060007000300030003000180018000C000C0006 00020001000080004000200B2E7DA112>I<800040002000100008000C00060006000300 030001800180018001C000C000C000C000E000E000E000E000E000E000E000E000E000E0 00E000E000C000C000C001C001800180018003000300060006000C000800100020004000 80000B2E7DA112>I<000600000006000000060000000600000006000000060000000600 000006000000060000000600000006000000060000000600000006000000060000FFFFFF F0FFFFFFF000060000000600000006000000060000000600000006000000060000000600 00000600000006000000060000000600000006000000060000000600001C207D9A23>43 D<70F8FCFC74040404080810102040060E7C840D>II<70F8F8F8 7005057C840D>I<000100030003000600060006000C000C000C00180018001800300030 003000600060006000C000C000C00180018001800300030003000600060006000C000C00 0C00180018001800300030003000600060006000C000C000C000102D7DA117>I<03F000 0E1C001C0E00180600380700700380700380700380700380F003C0F003C0F003C0F003C0 F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0700380700380700380 7807803807001806001C0E000E1C0003F000121F7E9D17>I<018003800F80F380038003 800380038003800380038003800380038003800380038003800380038003800380038003 80038003800380038007C0FFFE0F1E7C9D17>I<03F0000C1C00100E0020070040078080 0780F007C0F803C0F803C0F803C02007C00007C0000780000780000F00000E00001C0000 380000700000600000C0000180000300000600400C00401800401000803FFF807FFF80FF FF80121E7E9D17>I<03F0000C1C00100E00200F00780F80780780780780380F80000F80 000F00000F00000E00001C0000380003F000003C00000E00000F000007800007800007C0 2007C0F807C0F807C0F807C0F00780400780400F00200E001C3C0003F000121F7E9D17> I<000600000600000E00000E00001E00002E00002E00004E00008E00008E00010E00020E 00020E00040E00080E00080E00100E00200E00200E00400E00C00E00FFFFF0000E00000E 00000E00000E00000E00000E00000E0000FFE0141E7F9D17>I<1803001FFE001FFC001F F8001FE00010000010000010000010000010000010000011F000161C00180E0010070010 07800003800003800003C00003C00003C07003C0F003C0F003C0E0038040038040070020 0600100E000C380003E000121F7E9D17>I<007C000182000701000E03800C07801C0780 380300380000780000700000700000F1F000F21C00F40600F80700F80380F80380F003C0 F003C0F003C0F003C0F003C07003C07003C07003803803803807001807000C0E00061C00 01F000121F7E9D17>I<4000007FFFC07FFF807FFF804001008002008002008004000008 0000080000100000200000200000400000400000C00000C00001C0000180000380000380 00038000038000078000078000078000078000078000078000078000030000121F7D9D17 >I<03F0000C0C001006003003002001806001806001806001807001807803003E03003F 06001FC8000FF00003F80007FC000C7E00103F00300F806003804001C0C001C0C000C0C0 00C0C000C0C000806001802001001002000C0C0003F000121F7E9D17>I<03F0000E1800 1C0C00380600380700700700700380F00380F00380F003C0F003C0F003C0F003C0F003C0 7007C07007C03807C0180BC00E13C003E3C0000380000380000380000700300700780600 780E00700C002018001070000FC000121F7E9D17>I<70F8F8F870000000000000000000 0070F8F8F87005147C930D>I<70F8F8F8700000000000000000000070F0F8F878080808 101010202040051D7C930D>I<7FFFFFE0FFFFFFF0000000000000000000000000000000 0000000000000000000000000000000000FFFFFFF07FFFFFE01C0C7D9023>61 D<0FC0307040384038E03CF03CF03C603C0038007000E000C00180018001000300020002 0002000200020002000000000000000000000007000F800F800F8007000E207D9F15>63 D<000100000003800000038000000380000007C0000007C0000007C0000009E0000009E0 000009E0000010F0000010F0000010F00000207800002078000020780000403C0000403C 0000403C0000801E0000801E0000FFFE0001000F0001000F0001000F0002000780020007 8002000780040003C00E0003C01F0007E0FFC03FFE1F207F9F22>65 DI<000FC040007030C001C009C0 038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00004078000040 F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003800200 01C0040000703800000FC0001A217D9F21>IIII<000FE0200078186000E004E0038002E0070001E0 0F0000E01E0000601E0000603C0000603C0000207C00002078000020F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F8007FFCF80003E0780001E07C0001E0 3C0001E03C0001E01E0001E01E0001E00F0001E0070001E0038002E000E0046000781820 000FE0001E217D9F24>III<0FFFC0007C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00203C00F83C00F83C 00F83C00F0380040780040700030E0000F800012207E9E17>IIIII<001F800000F0F00001C0380007801E000F000F000E0007001E00 07803C0003C03C0003C07C0003E0780001E0780001E0F80001F0F80001F0F80001F0F800 01F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07C0003E07C0003E03C00 03C03C0003C01E0007800E0007000F000F0007801E0001C0380000F0F000001F80001C21 7D9F23>II<001F800000F0F000 01C0380007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C0003E0 780001E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0 F80001F0780001E0780001E07C0003E03C0003C03C0F03C01E1087800E2047000F204F00 07A03E0001E0380000F0F010001FB01000003010000038300000387000003FF000001FE0 00001FE000000FC0000007801C297D9F23>II<07E0800C1980100780300380600180600180E00180E00080E000 80E00080F00000F000007800007F00003FF0001FFC000FFE0003FF00001F800007800003 C00003C00001C08001C08001C08001C08001C0C00180C00380E00300F00600CE0C0081F8 0012217D9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F0030800F0010 800F0010800F0010800F0010000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000001F800007FFFE001C1F7E9E21>IIII<7FF83FF8 0FE00FC007C0070003C0020001E0040001F00C0000F0080000781000007C1000003C2000 003E4000001E4000000F8000000F8000000780000003C0000007E0000005E0000009F000 0018F8000010780000207C0000603C0000401E0000801F0001800F0001000780020007C0 070003C01F8007E0FFE01FFE1F1F7F9E22>II<7FFFF87C00F87000F06001E04001E0C003C0C003C0800780800F80800F00 001E00001E00003C00003C0000780000F80000F00001E00001E00003C00403C004078004 0F80040F000C1E000C1E00083C00183C0018780038F801F8FFFFF8161F7D9E1C>II<080410082010201040204020804080408040B85CFC7E FC7E7C3E381C0F0E7B9F17>II<08102020404080 8080B8FCFC7C38060E7D9F0D>96 D<1FE000303000781800781C00300E00000E00000E00 000E0000FE00078E001E0E00380E00780E00F00E10F00E10F00E10F01E10781E10386720 0F83C014147E9317>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E3E000EC3800F01C00F00E00E00E00E00700E00700E00780E00 780E00780E00780E00780E00780E00700E00700E00E00F00E00D01C00CC300083E001520 7F9F19>I<03F80E0C1C1E381E380C70007000F000F000F000F000F000F0007000700038 0138011C020E0C03F010147E9314>I<000380003F800003800003800003800003800003 8000038000038000038000038000038003E380061B801C07803803803803807003807003 80F00380F00380F00380F00380F00380F003807003807003803803803807801C07800E1B 8003E3F815207E9F19>I<03F0000E1C001C0E00380700380700700700700380F00380F0 0380FFFF80F00000F00000F000007000007000003800801800800C010007060001F80011 147F9314>I<007C00C6018F038F07060700070007000700070007000700FFF007000700 07000700070007000700070007000700070007000700070007000700070007007FF01020 809F0E>I<0000E003E3300E3C301C1C30380E00780F00780F00780F00780F00780F0038 0E001C1C001E380033E0002000002000003000003000003FFE001FFF800FFFC03001E060 0070C00030C00030C00030C000306000603000C01C038003FC00141F7F9417>I<0E0000 FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E3E00 0E43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16207F9F19>I<1C001E003E001E00 1C000000000000000000000000000E007E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E00FFC00A1F809E0C>I<00E001F001F001F000E000 0000000000000000000000007007F000F000700070007000700070007000700070007000 70007000700070007000700070007000700070007000706070F060F0C061803F000C2882 9E0E>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E0FF00E03C00E03000E02000E04000E08000E10000E30000E70000EF8000F38 000E1C000E1E000E0E000E07000E07800E03800E03C00E03E0FFCFF815207F9F18>I<0E 00FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B20809F0C>I<0E1F 01F000FE618618000E81C81C000F00F00E000F00F00E000E00E00E000E00E00E000E00E0 0E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E 000E00E00E000E00E00E000E00E00E000E00E00E00FFE7FE7FE023147F9326>I<0E3E00 FE43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16147F9319>I<01F800070E001C03 803801C03801C07000E07000E0F000F0F000F0F000F0F000F0F000F0F000F07000E07000 E03801C03801C01C0380070E0001F80014147F9317>I<0E3E00FEC3800F01C00F00E00E 00E00E00F00E00700E00780E00780E00780E00780E00780E00780E00700E00F00E00E00F 01E00F01C00EC3000E3E000E00000E00000E00000E00000E00000E00000E00000E0000FF E000151D7F9319>I<03E0800619801C05803C0780380380780380700380F00380F00380 F00380F00380F00380F003807003807803803803803807801C0B800E138003E380000380 000380000380000380000380000380000380000380003FF8151D7E9318>I<0E78FE8C0F 1E0F1E0F0C0E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00F 147F9312>I<1F9030704030C010C010C010E00078007F803FE00FF00070803880188018 C018C018E030D0608F800D147E9312>I<020002000200060006000E000E003E00FFF80E 000E000E000E000E000E000E000E000E000E000E000E080E080E080E080E080610031001 E00D1C7F9B12>I<0E01C0FE1FC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E03C00603C0030DC001F1FC16147F93 19>III<7FC3FC0F01E00701C007018003810001C20000E40000EC 00007800003800003C00007C00004E000087000107000303800201C00601E01E01E0FF07 FE1714809318>II<3FFF380E200E 201C40384078407000E001E001C00380078007010E011E011C0338027006700EFFFE1014 7F9314>III<30307878F87C787830 300E057C9E17>127 D E /Fy 48 122 df<00200040008001000300060004000C000C00 180018003000300030007000600060006000E000E000E000E000E000E000E000E000E000 E000E000E000E000E0006000600060007000300030003000180018000C000C0004000600 030001000080004000200B327CA413>40 D<800040002000100018000C00040006000600 0300030001800180018001C000C000C000C000E000E000E000E000E000E000E000E000E0 00E000E000E000E000E000C000C000C001C0018001800180030003000600060004000C00 180010002000400080000B327DA413>I<70F8FCFC7404040404080810102040060F7C84 0E>44 DI<70F8F8F87005057C840E>I<01F000071C000C060018 03003803803803807001C07001C07001C07001C0F001E0F001E0F001E0F001E0F001E0F0 01E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E07001C07001C07001C078 03C03803803803801C07000C0600071C0001F00013227EA018>48 D<008003800F80F380038003800380038003800380038003800380038003800380038003 80038003800380038003800380038003800380038003800380038007C0FFFE0F217CA018 >I<03F0000C1C001007002007804003C04003C08003E0F003E0F801E0F801E0F801E020 03E00003E00003C00003C0000780000700000E00001C0000180000300000600000C00001 80000100000200200400200800201800603000403FFFC07FFFC0FFFFC013217EA018>I< 03F8000C1E001007002007804007C07807C07803C07807C03807C0000780000780000700 000F00000E0000380003F000001C00000F000007800007800003C00003C00003E02003E0 7003E0F803E0F803E0F003C04003C0400780200780100F000C1C0003F00013227EA018> I<000200000600000E00000E00001E00001E00002E00004E00004E00008E00008E00010E 00020E00020E00040E00040E00080E00100E00100E00200E00200E00400E00800E00FFFF F8000E00000E00000E00000E00000E00000E00000E00001F0001FFF015217FA018>I<10 00801E07001FFF001FFE001FF80013E00010000010000010000010000010000010000010 F800130E001407001803801003800001C00001C00001E00001E00001E00001E07001E0F0 01E0F001E0E001C08001C04003C04003802007001006000C1C0003F00013227EA018>I< 007E0001C1000300800601C00E03C01C03C0180180380000380000780000700000700000 F0F800F30C00F40600F40300F80380F801C0F001C0F001E0F001E0F001E0F001E0F001E0 7001E07001E07001E03801C03801C01803801C03000C0600070C0001F00013227EA018> I<01F800060E000803001001802001802000C06000C06000C06000C07000C07801803E01 003F02001FC4000FF80003F80003FC00067F00083F80100F803007C06001C06000E0C000 E0C00060C00060C00060C000606000406000C03000801803000E0E0003F00013227EA018 >56 D<01F000060C000C0600180700380380700380700380F001C0F001C0F001C0F001E0 F001E0F001E0F001E0F001E07001E07003E03803E01805E00C05E00619E003E1E00001C0 0001C00001C0000380000380300300780700780600700C002018001030000FC00013227E A018>I<0001800000018000000180000003C0000003C0000003C0000005E0000005E000 000DF0000008F0000008F0000010F800001078000010780000203C0000203C0000203C00 00401E0000401E0000401E0000800F0000800F0000FFFF000100078001000780030007C0 020003C0020003C0040003E0040001E0040001E00C0000F00C0000F03E0001F8FF800FFF 20237EA225>65 D<0007E0100038183000E0063001C00170038000F0070000F00E000070 1E0000701C0000303C0000303C0000307C0000107800001078000010F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F800000078000000780000107C000010 3C0000103C0000101C0000201E0000200E000040070000400380008001C0010000E00200 00381C000007E0001C247DA223>67 D69 DI73 D77 DI80 D82 D<03F0200C0C601802603001E07000E06000 60E00060E00060E00020E00020E00020F00000F000007800007F00003FF0001FFE000FFF 0003FF80003FC00007E00001E00000F00000F0000070800070800070800070800070C000 60C00060E000C0F000C0C80180C6070081FC0014247DA21B>I<7FFFFFF8780780786007 8018400780084007800840078008C007800C800780048007800480078004800780040007 800000078000000780000007800000078000000780000007800000078000000780000007 800000078000000780000007800000078000000780000007800000078000000780000007 80000007800000078000000FC00003FFFF001E227EA123>I<0FE0001838003C0C003C0E 0018070000070000070000070000FF0007C7001E07003C0700780700700700F00708F007 08F00708F00F087817083C23900FC1E015157E9418>97 D<0E0000FE00001E00000E0000 0E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E1F000E61C0 0E80600F00300E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E001E0E001E 0E001C0E003C0E00380F00700C80600C41C0083F0017237FA21B>I<01FE000703000C07 801C0780380300780000700000F00000F00000F00000F00000F00000F00000F000007000 007800403800401C00800C010007060001F80012157E9416>I<0000E0000FE00001E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E001F8E007 04E00C02E01C01E03800E07800E07000E0F000E0F000E0F000E0F000E0F000E0F000E0F0 00E07000E07800E03800E01801E00C02E0070CF001F0FE17237EA21B>I<01FC00070700 0C03801C01C03801C07801E07000E0F000E0FFFFE0F00000F00000F00000F00000F00000 7000007800203800201C00400E008007030000FC0013157F9416>I<003C00C6018F038F 030F070007000700070007000700070007000700FFF80700070007000700070007000700 0700070007000700070007000700070007000700070007807FF8102380A20F>I<000070 01F198071E180E0E181C07001C07003C07803C07803C07803C07801C07001C07000E0E00 0F1C0019F0001000001000001800001800001FFE000FFFC00FFFE03800F0600030400018 C00018C00018C000186000306000303800E00E038003FE0015217F9518>I<0E0000FE00 001E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E1F800E60C00E80E00F00700F00700E00700E00700E00700E00700E00700E00700E00 700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18237FA21B>I<1C 001E003E001E001C00000000000000000000000000000000000E00FE001E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFC00A227FA10E >I<01C003E003E003E001C00000000000000000000000000000000001E00FE001E000E0 00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E0 00E000E000E000E060E0F0C0F18061803E000B2C82A10F>I<0E0000FE00001E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E03FC0E01 F00E01C00E01800E02000E04000E08000E10000E38000EF8000F1C000E1E000E0E000E07 000E07800E03C00E01C00E01E00E00F00E00F8FFE3FE17237FA21A>I<0E00FE001E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B237FA20E>I<0E1F C07F00FE60E183801E807201C00F003C00E00F003C00E00E003800E00E003800E00E0038 00E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800 E00E003800E00E003800E00E003800E00E003800E00E003800E0FFE3FF8FFE27157F942A >I<0E1F80FE60C01E80E00F00700F00700E00700E00700E00700E00700E00700E00700E 00700E00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18157F941B>I< 01FC000707000C01801800C03800E0700070700070F00078F00078F00078F00078F00078 F00078F000787000707800F03800E01C01C00E038007070001FC0015157F9418>I<0E1F 00FE61C00E80600F00700E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E00 1E0E001E0E003C0E003C0E00380F00700E80E00E41C00E3F000E00000E00000E00000E00 000E00000E00000E00000E00000E0000FFE000171F7F941B>I<0E3CFE461E8F0F0F0F06 0F000E000E000E000E000E000E000E000E000E000E000E000E000E000F00FFF010157F94 13>114 D<0F8830786018C018C008C008E008F0007F803FE00FF001F8003C801C800C80 0CC00CC008E018D0308FC00E157E9413>I<02000200020002000600060006000E001E00 3E00FFF80E000E000E000E000E000E000E000E000E000E000E000E040E040E040E040E04 0E040708030801F00E1F7F9E13>I<0E0070FE07F01E00F00E00700E00700E00700E0070 0E00700E00700E00700E00700E00700E00700E00700E00700E00700E00F00E00F0060170 03827800FC7F18157F941B>III121 D E /Fz 18 117 df45 D<78FCFCFCFC780000000000000000 000000000000000000000078FCFCFCFC78061F7A9E12>58 D<0000030000000000030000 0000000300000000000780000000000780000000000FC0000000000FC0000000000FC000 00000017E00000000013E00000000013E00000000023F00000000021F00000000021F000 00000040F80000000040F80000000040F800000000807C00000000807C00000001807E00 000001003E00000001003E00000002003F00000002001F00000002001F00000004000F80 000004000F80000004000F800000080007C00000080007C00000180007E000001FFFFFE0 00001FFFFFE00000200003F00000200001F00000200001F00000400001F80000400000F8 0000400000F800008000007C00008000007C00008000007C00010000003E00010000003E 00030000003F00030000001F00070000001F001F8000003F80FFE00003FFFCFFE00003FF FC2E327EB132>65 D73 D77 D80 D<007F802001FFE02007C078600F001C601E0006E03C0003E0 380001E0780000E0700000E070000060F0000060F0000060F0000020F0000020F0000020 F8000020F80000007C0000007E0000003F0000003FC000001FF800000FFF800007FFF800 03FFFC0000FFFF00000FFF800000FFC000001FE0000007E0000003F0000001F0000000F0 000000F8000000F88000007880000078800000788000007880000078C0000078C0000070 E00000F0E00000E0F00000E0F80001C0EC000380C7000700C1F01E00807FFC00800FF000 1D337CB125>83 D<00FE00000303C0000C00E00010007000100038003C003C003E001C00 3E001E003E001E0008001E0000001E0000001E0000001E00000FFE0000FC1E0003E01E00 0F801E001F001E003E001E003C001E007C001E00F8001E04F8001E04F8001E04F8003E04 F8003E0478003E047C005E043E008F080F0307F003FC03E01E1F7D9E21>97 D<003F8000E0600380180700040F00041E001E1C003E3C003E7C003E7C0008780000F800 00F80000F80000F80000F80000F80000F80000F80000F800007800007C00007C00003C00 011E00011E00020F000207000403801800E060003F80181F7D9E1D>99 D<000001E000003FE000003FE0000003E0000001E0000001E0000001E0000001E0000001 E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001 E0000001E0001F81E000F061E001C019E0078005E00F0003E00E0003E01E0001E03C0001 E03C0001E07C0001E0780001E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001 E0F80001E0F80001E0F80001E0780001E0780001E03C0001E03C0001E01C0001E01E0003 E00E0005E0070009E0038011F000E061FF003F81FF20327DB125>I<003F800000E0E000 0380380007003C000E001E001E001E001C000F003C000F007C000F0078000F8078000780 F8000780F8000780FFFFFF80F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000003C0000003C0000801E0000800E0001000F0002000780020001C00C00 00F03000001FC000191F7E9E1D>I<0007E0001C1000383800707C00E07C01E07C01C038 03C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C000 FFFFC0FFFFC003C00003C00003C00003C00003C00003C00003C00003C00003C00003C000 03C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C000 03C00003C00003C00003C00007E0007FFF007FFF0016327FB114>I<000000F0007F0308 01C1C41C0380E81C070070080F0078001E003C001E003C003E003E003E003E003E003E00 3E003E003E003E003E003E001E003C001E003C000F007800070070000780E00009C1C000 087F000018000000180000001800000018000000180000001C0000000E0000000FFFF800 07FFFF0003FFFF800E000FC0180001E0300000F070000070E0000038E0000038E0000038 E0000038E00000387000007070000070380000E01C0001C00700070001C01C00003FE000 1E2F7E9F21>I<07000F801F801F800F8007000000000000000000000000000000000000 00000000000780FF80FF800F800780078007800780078007800780078007800780078007 800780078007800780078007800780078007800780078007800FC0FFF8FFF80D307EAF12 >105 D<0780FE0000FF83078000FF8C03C0000F9001E00007A001E00007A000F00007C0 00F00007C000F000078000F000078000F000078000F000078000F000078000F000078000 F000078000F000078000F000078000F000078000F000078000F000078000F000078000F0 00078000F000078000F000078000F000078000F000078000F000078000F000078000F000 0FC001F800FFFC1FFF80FFFC1FFF80211F7E9E25>110 D<0783E0FF8C18FF907C0F907C 07A07C07C03807C00007C00007C000078000078000078000078000078000078000078000 078000078000078000078000078000078000078000078000078000078000078000078000 0FC000FFFE00FFFE00161F7E9E19>114 D<01FC100E03301800F0300070600030E00030 E00010E00010E00010F00010F800007E00003FF0001FFF000FFFC003FFE0003FF00001F8 0000F880003C80003C80001CC0001CC0001CE0001CE00018F00038F00030CC0060C301C0 80FE00161F7E9E1A>I<00400000400000400000400000400000C00000C00000C00001C0 0001C00003C00007C0000FC0001FFFE0FFFFE003C00003C00003C00003C00003C00003C0 0003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C01003C0 1003C01003C01003C01003C01003C01003C01001C02001E02000E0400078C0001F00142C 7FAB19>I E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 0 1 0 0 bop 340 938 a Fz(MPI:)22 b(A)g(Message-P)n(assing)e(In)n(terface)j (Standard)621 1133 y Fy(Message)16 b(P)o(assing)h(In)o(terface)e(F)l (orum)824 1259 y(April)g(21,)i(1994)77 1317 y(This)g(w)o(ork)f(w)o(as)h (supp)q(orted)g(in)f(part)h(b)o(y)e(ARP)l(A)h(and)h(NSF)e(under)h(gran) o(t)h(ASC-9310330,)i(the)192 1375 y(National)d(Science)f(F)l(oundation) i(Science)e(and)i(T)l(ec)o(hnology)f(Cen)o(ter)f(Co)q(op)q(erativ)o(e) 76 1433 y(Agreemen)o(t)e(No.)22 b(CCR-8809615,)d(and)e(b)o(y)e(the)h (Commission)e(of)j(the)f(Europ)q(ean)i(Comm)o(unit)n(y)568 1492 y(through)g(Esprit)e(pro)s(ject)g(P6643\(PPPE\).)p eop %%Page: 1 2 1 1 bop 166 45 a Fx(The)21 b(Message)f(P)o(assing)h(In)o(terface)g(F)l (orum)g(\(MPIF\),)e(with)j(participation)g(from)e(o)o(v)o(er)g(40)g (or-)75 102 y(ganizations,)e(has)g(b)q(een)h(meeting)f(since)h(No)o(v)o (em)o(b)q(er)e(1992)g(to)g(discuss)i(and)f(de\014ne)h(a)e(set)h(of)f (library)75 158 y(in)o(terface)12 b(standards)f(for)f(message)h (passing.)19 b(MPIF)12 b(is)g(not)f(sanctioned)h(or)f(supp)q(orted)h(b) o(y)f(an)o(y)g(o\016cial)75 214 y(standards)k(organization.)166 271 y(The)i(goal)f(of)g(the)g(Message)g(P)o(assing)g(In)o(terface,)g (simply)i(stated,)e(is)g(to)g(dev)o(elop)h(a)g(widely)g(used)75 327 y(standard)h(for)g(writing)h(message-passing)g(programs.)29 b(As)19 b(suc)o(h)g(the)g(in)o(terface)g(should)g(establish)h(a)75 384 y(practical,)c(p)q(ortable,)f(e\016cien)o(t,)g(and)h(\015exible)h (standard)e(for)g(message)f(passing.)166 440 y(This)20 b(is)g(the)g(\014nal)h(rep)q(ort,)f(V)l(ersion)g(1.0,)g(of)f(the)h (Message)f(P)o(assing)h(In)o(terface)f(F)l(orum.)33 b(This)75 497 y(do)q(cumen)o(t)18 b(con)o(tains)g(all)h(the)f(tec)o(hnical)i (features)d(prop)q(osed)h(for)g(the)g(in)o(terface.)28 b(This)19 b(cop)o(y)e(of)h(the)75 553 y(draft)d(w)o(as)f(pro)q(cessed)i (b)o(y)f(L)554 547 y Fw(a)575 553 y Fx(T)600 567 y(E)626 553 y(X)g(on)g(April)h(21,)f(1994.)166 610 y(Please)i(send)g(commen)o (ts)f(on)g Fv(MPI)g Fx(to)g Fu(mpi-comments@cs.utk.edu)p Fx(.)21 b(Y)l(our)16 b(commen)o(t)g(will)i(b)q(e)75 666 y(forw)o(arded)c(to)h(MPIF)g(committee)g(mem)o(b)q(ers)g(who)g(will)i (attempt)d(to)h(resp)q(ond.)179 1922 y(c)166 1923 y Ft(\015)p Fx(1993,)f(1994)h(Univ)o(ersit)o(y)h(of)f(T)l(ennessee,)i(Kno)o (xville,)g(T)l(ennessee.)22 b(P)o(ermission)16 b(to)f(cop)o(y)g(with-) 75 1979 y(out)d(fee)g(all)h(or)f(part)g(of)f(this)i(material)g(is)f (gran)o(ted,)g(pro)o(vided)h(the)f(Univ)o(ersit)o(y)h(of)f(T)l (ennessee)h(cop)o(yrigh)o(t)75 2035 y(notice)i(and)f(the)g(title)h(of)e (this)i(do)q(cumen)o(t)f(app)q(ear,)g(and)h(notice)f(is)h(giv)o(en)g (that)e(cop)o(ying)h(is)h(b)o(y)f(p)q(ermis-)75 2092 y(sion)i(of)e(the)i(Univ)o(ersit)o(y)g(of)e(T)l(ennessee.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 3 2 2 bop 75 377 a Fr(Con)m(ten)m(ts)75 645 y Fq(Ac)o(kno)o(wledgmen)o (ts)1346 b(vi)75 747 y(1)42 b(In)o(tro)q(duction)19 b(to)f(MPI)1230 b(1)143 804 y Fx(1.1)46 b(Ov)o(erview)16 b(and)f(Goals)38 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(1)143 860 y(1.2)46 b(Who)15 b(Should)h(Use)f(This)h(Standard?)32 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(2)143 916 y(1.3)46 b(What)14 b(Platforms)h(Are)g(T)l(argets)f(F)l(or)h(Implemen)o(tation?) 38 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(3)143 973 y(1.4)46 b(What)14 b(Is)i(Included)h(In)f(The)g (Standard?)42 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(3)143 1029 y(1.5)46 b(What)14 b(Is)i(Not)e(Included)k(In)e(The)f(Standard?)23 b Fp(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)91 b Fx(3)143 1086 y(1.6)46 b(Organization)16 b(of)e(this)i(Do)q(cumen)o(t)34 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(4)75 1188 y Fq(2)42 b(MPI)17 b(T)l(erms)f(and)i(Con)o(v)o(en)o (tions)1042 b(6)143 1244 y Fx(2.1)46 b(Do)q(cumen)o(t)15 b(Notation)44 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)91 b Fx(6)143 1301 y(2.2)46 b(Pro)q(cedure)16 b(Sp)q(eci\014cation) 44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(6)143 1357 y(2.3)46 b(Seman)o(tic)15 b(T)l(erms)g Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 91 b Fx(7)143 1414 y(2.4)46 b(Data)14 b(T)o(yp)q(es)30 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)91 b Fx(8)248 1470 y(2.4.1)50 b(Opaque)16 b(ob)s(jects)21 b Fp(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(8)248 1526 y(2.4.2)50 b(Arra)o(y)14 b(argumen)o(ts)28 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(9)248 1583 y(2.4.3)50 b(State)14 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(9)248 1639 y(2.4.4)50 b(Named)15 b(constan)o(ts)23 b Fp(:)f(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(9)248 1696 y(2.4.5)50 b(Choice)20 b Fp(:)i(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fx(9)248 1752 y(2.4.6)50 b(Addresses)28 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(10)143 1809 y(2.5)46 b(Language)15 b(Binding)45 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b Fx(10)248 1865 y(2.5.1)50 b(F)l(ortran)14 b(77)h(Binding)i(Issues)30 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(10)248 1922 y(2.5.2)50 b(C)15 b(Binding)i(Issues)31 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(11)143 1978 y(2.6)46 b(Pro)q(cesses)40 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b Fx(12)143 2035 y(2.7)46 b(Error)14 b(Handling)32 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fx(12)143 2091 y(2.8)46 b(Implemen)o(tation)16 b(issues)36 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(13)248 2147 y(2.8.1)50 b(Indep)q(endence)19 b(of)14 b(Basic)i(Run)o(time)g(Routines)42 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(13)248 2204 y(2.8.2)50 b(In)o(teraction)16 b(with)f(signals)h(in)g(POSIX)42 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)69 b Fx(14)75 2306 y Fq(3)42 b(P)o(oin)o(t-to-P)o(oin)o(t)19 b(Comm)o(unication)975 b(15)143 2362 y Fx(3.1)46 b(In)o(tro)q(duction) 15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)69 b Fx(15)143 2419 y(3.2)46 b(Blo)q(c)o(king)16 b(Send)g(and)g(Receiv)o(e)g(Op)q(erations)24 b Fp(:)e(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(16)248 2475 y(3.2.1)50 b(Blo)q(c)o(king)17 b(send)h Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(16)248 2532 y(3.2.2)50 b(Message)15 b(data)26 b Fp(:)d(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(16)248 2588 y(3.2.3)50 b(Message)15 b(en)o(v)o(elop)q(e)j Fp(:)k(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(18)248 2645 y(3.2.4)50 b(Blo)q(c)o(king)17 b(receiv)o(e)44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(19)248 2701 y(3.2.5)50 b(Return)16 b(status)j Fp(:)k(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(20)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 4 3 3 bop 143 45 a Fx(3.3)46 b(Data)14 b(t)o(yp)q(e)h(matc)o(hing)g(and)h (data)e(con)o(v)o(ersion)j Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(21)248 102 y(3.3.1)50 b(T)o(yp)q(e)16 b(matc)o(hing)f(rules)27 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(21)248 158 y(3.3.2)50 b(Data)14 b(con)o(v)o(ersion)46 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(24)143 214 y(3.4)46 b(Comm)o(unication)15 b(Mo)q(des)i Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(25)143 271 y(3.5)46 b(Seman)o(tics)15 b(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h (comm)o(unication)40 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(29)143 327 y(3.6)46 b(Bu\013er)15 b(allo)q(cation)h(and)g(usage)38 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(32)248 384 y(3.6.1)50 b(Mo)q(del)16 b(implemen)o(tation)h(of)d(bu\013ered)i(mo)q (de)27 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)69 b Fx(34)143 440 y(3.7)46 b(Non)o(blo)q(c)o(king)16 b(comm)o(unication)j Fp(:)j(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(34)248 497 y(3.7.1)50 b(Comm)o(unication)16 b(Ob)s(jects)23 b Fp(:)g(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(36)248 553 y(3.7.2)50 b(Comm)o(unication)16 b(initiation)32 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fx(36)248 610 y(3.7.3)50 b(Comm)o(unication)16 b(Completion)i Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(39)248 666 y(3.7.4)50 b(Seman)o(tics)16 b(of)f(Non)o(blo)q(c)o(king)h(Comm)o(unications)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(42)248 723 y(3.7.5)50 b(Multiple)17 b(Completions)41 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(43)143 779 y(3.8)46 b(Prob)q(e)15 b(and)g(Cancel)g Fp(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(48)143 835 y(3.9)46 b(P)o(ersisten)o(t)15 b(comm)o(unication)h(requests)33 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(52)143 892 y(3.10)23 b(Send-receiv)o(e)d Fp(:)i(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(56)143 948 y(3.11)23 b(Null)16 b(pro)q(cesses)i Fp(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(58)143 1005 y(3.12)23 b(Deriv)o(ed)15 b(datat)o(yp)q(es)42 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(59)248 1061 y(3.12.1)27 b(Datat)o(yp)q(e)14 b(constructors)27 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b Fx(60)248 1118 y(3.12.2)27 b(Address)16 b(and)f(exten)o(t)g(functions)32 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(67)248 1174 y(3.12.3)27 b(Lo)o(w)o(er-b)q(ound)16 b(and)f(upp)q(er-b)q(ound)j(mark)o(ers)33 b Fp(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(69)248 1231 y(3.12.4)27 b(Commit)15 b(and)g(free)28 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(70)248 1287 y(3.12.5)27 b(Use)16 b(of)e(general)i(datat)o(yp)q(es)f(in)h(comm) o(unication)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)69 b Fx(72)248 1344 y(3.12.6)27 b(Correct)15 b(use)g(of)g (addresses)26 b Fp(:)d(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(74)248 1400 y(3.12.7)27 b(Examples)33 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(75)143 1456 y(3.13)23 b(P)o(ac)o(k)14 b(and)i(unpac)o(k)29 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(83)75 1558 y Fq(4)42 b(Collectiv)o(e)19 b(Comm)o(unication)1077 b(90)143 1615 y Fx(4.1)46 b(In)o(tro)q(duction)16 b(and)f(Ov)o(erview) 44 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(90)143 1671 y(4.2)46 b(Comm)o(unicator)14 b(argumen)o(t)e Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(93)143 1728 y(4.3)46 b(Barrier)15 b(sync)o(hronization)44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(93)143 1784 y(4.4)46 b(Broadcast)27 b Fp(:)c(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(93)248 1841 y(4.4.1)50 b(Example)16 b(using)g Fv(MPI)p 786 1841 14 2 v 16 w(BCAST)26 b Fp(:)d(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(94)143 1897 y(4.5)46 b(Gather)17 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fx(94)248 1954 y(4.5.1)50 b(Examples)16 b(using)g Fv(MPI)p 804 1954 V 16 w(GA)l(THER)p Fx(,)g Fv(MPI)p 1106 1954 V 16 w(GA)l(THERV)35 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b Fx(96)143 2010 y(4.6)46 b(Scatter)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b Fx(103)248 2066 y(4.6.1)k(Examples)16 b(using)g Fv(MPI)p 804 2066 V 16 w(SCA)l(TTER)p Fx(,)g Fv(MPI)p 1129 2066 V 15 w(SCA)l(TTERV)25 b Fp(:)e(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b Fx(105)143 2123 y(4.7)g(Gather-to-all)40 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b Fx(107)248 2179 y(4.7.1)k(Examples)16 b(using)g Fv(MPI)p 804 2179 V 16 w(ALLGA)l(THER)p Fx(,)f Fv(MPI)p 1185 2179 V 16 w(ALLGA)l(THERV)i Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(109)143 2236 y(4.8)g(All-to-All)17 b(Scatter/Gather)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(109)143 2292 y(4.9)g(Global)16 b(Reduction)g(Op)q(erations)34 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(111)248 2349 y(4.9.1)k(Reduce)45 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(111)248 2405 y(4.9.2)k(Prede\014ned) 17 b(reduce)f(op)q(erations)37 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(112)248 2462 y(4.9.3)k(MINLOC)16 b(and)f(MAXLOC)44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(114)248 2518 y(4.9.4)k (User-De\014ned)16 b(Op)q(erations)h Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fx(118)248 2575 y(4.9.5)k(All-Reduce)43 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(122)143 2631 y(4.10)23 b(Reduce-Scatter)33 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(123)143 2687 y(4.11)23 b(Scan)28 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(124)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 5 4 4 bop 248 45 a Fx(4.11.1)27 b(Example)16 b(using)g Fv(MPI)p 786 45 14 2 v 16 w(SCAN)k Fp(:)i(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(124)143 102 y(4.12)23 b(Correctness)32 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fx(126)75 203 y Fq(5)c(Groups,)17 b(Con)o(texts,)f(and)i(Comm)o (unicators)757 b(130)143 260 y Fx(5.1)46 b(In)o(tro)q(duction)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b Fx(130)248 316 y(5.1.1)k(F)l(eatures)15 b(Needed)h(to)f(Supp)q(ort)h(Libraries)40 b Fp(:)23 b(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(130)248 373 y(5.1.2)k Fv(MPI)p Fx('s)14 b(Supp)q(ort)i(for)f(Libraries)30 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(131)143 429 y(5.2)g(Basic)16 b(Concepts)30 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b Fx(133)248 486 y(5.2.1)k(Groups)43 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fx(133)248 542 y(5.2.2)k(Con)o(texts)c Fp(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(133)248 599 y(5.2.3)k(In)o(tra-Comm)o(unicators)44 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(134)248 655 y(5.2.4)k (Prede\014ned)17 b(In)o(tra-Comm)o(unicators)30 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b Fx(134)143 712 y(5.3)g(Group)15 b(Managemen)o(t)41 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(135)248 768 y(5.3.1)k(Group)15 b(Accessors)34 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(135)248 824 y(5.3.2)k(Group)15 b(Constructors)37 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(136)248 881 y(5.3.3)k(Group)15 b(Destructors)27 b Fp(:)c(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b Fx(140)143 937 y(5.4)g(Comm)o(unicator)14 b(Managemen)o(t)i Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(141)248 994 y(5.4.1)k(Comm)o(unicator)15 b(Accessors)43 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(141)248 1050 y(5.4.2)k(Comm)o (unicator)15 b(Constructors)10 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(142)248 1107 y(5.4.3)k(Comm)o(unicator)15 b(Destructors)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(145)143 1163 y(5.5)g(Motiv)m(ating) 15 b(Examples)h Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b Fx(146)248 1220 y(5.5.1)k(Curren)o(t)15 b(Practice)g(#1)27 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(146)248 1276 y(5.5.2)k(Curren)o(t)15 b(Practice)g(#2)27 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(147)248 1333 y(5.5.3)k(\(Appro)o(ximate\))15 b(Curren)o(t)f(Practice)i(#3)35 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b Fx(147)248 1389 y(5.5.4)k(Example)16 b(#4)44 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(148)248 1445 y(5.5.5)k(Library)16 b(Example)g(#1)22 b Fp(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(149)248 1502 y(5.5.6)k(Library)16 b(Example)g(#2)22 b Fp(:)g(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b Fx(151)143 1558 y(5.6)g(In)o(ter-Comm)o(unication) 14 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(153)248 1615 y(5.6.1)k(In)o(ter-comm)o(unicator)15 b(Accessors)h Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(154)248 1671 y(5.6.2)k(In)o(ter-comm)o(unicator)15 b(Op)q(erations)22 b Fp(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b Fx(156)248 1728 y(5.6.3)k(In)o(ter-Comm)o (unication)16 b(Examples)g Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(158)143 1784 y(5.7)g(Cac)o(hing)32 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(165)248 1841 y(5.7.1)k(F)l(unctionalit)o(y)32 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(165)248 1897 y(5.7.2)k(A)o (ttributes)15 b(Example)43 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fx(169)143 1954 y(5.8)g(F)l(ormalizing)16 b(the)f(Lo)q(osely)h (Sync)o(hronous)g(Mo)q(del)j Fp(:)j(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(171)248 2010 y(5.8.1)k(Basic)16 b(Statemen)o(ts)22 b Fp(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(171)248 2066 y(5.8.2)k(Mo)q(dels)16 b(of)f(Execution)29 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(172)75 2168 y Fq(6)c(Pro)q(cess)17 b(T)l(op)q(ologies)1224 b(174)143 2225 y Fx(6.1)46 b(In)o(tro)q(duction)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(174)143 2281 y(6.2)g(Virtual)16 b(T)l(op)q(ologies)37 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(175)143 2338 y(6.3)g(Em)o(b)q(edding)16 b(in)g Fv(MPI)29 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(175)143 2394 y(6.4)g(Ov)o(erview)16 b(of)e(the)i(F)l(unctions)22 b Fp(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(176)143 2451 y(6.5)g(T)l(op)q(ology)15 b(Constructors)20 b Fp(:)i(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(177)248 2507 y(6.5.1)k(Cartesian)15 b(Constructor)27 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b Fx(177)248 2564 y(6.5.2)k(Cartesian)15 b(Con)o(v)o (enience)h(F)l(unction:)21 b Fv(MPI)p 1152 2564 V 16 w(DIMS)p 1279 2564 V 16 w(CREA)l(TE)39 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b Fx(177)248 2620 y(6.5.3)k(General)16 b(\(Graph\))e (Constructor)27 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(178)248 2676 y(6.5.4)k(T)l(op)q(ology)15 b(inquiry)i(functions)42 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(180)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 6 5 5 bop 248 45 a Fx(6.5.5)50 b(Cartesian)15 b(Shift)h(Co)q(ordinates)23 b Fp(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(184)248 102 y(6.5.6)k(P)o(artitioning) 16 b(of)e(Cartesian)h(structures)45 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(185)248 158 y(6.5.7)k(Lo)o(w-lev)o(el)17 b(top)q(ology)d(functions)45 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b Fx(185)143 214 y(6.6)g(An)15 b(Application)i(Example)g Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b Fx(187)75 316 y Fq(7)c(MPI)17 b(En)o(vironmen)o(tal)g (Managemen)o(t)883 b(189)143 373 y Fx(7.1)46 b(Implemen)o(tation)16 b(information)22 b Fp(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(189)248 429 y(7.1.1)k(En)o(vironmen)o(tal)16 b(Inquiries)28 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(189)143 486 y(7.2)g(Error)14 b(handling)41 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(191)143 542 y(7.3)g(Error)14 b(co)q(des)i(and)f(classes)e Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b Fx(194)143 599 y(7.4)g(Timers)18 b Fp(:)k(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(195)143 655 y(7.5)g(Startup)40 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(196)75 757 y Fq(8)c(Pro\014ling)18 b(In)o(terface)1238 b(198)143 813 y Fx(8.1)46 b(Requiremen)o(ts)28 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(198)143 870 y(8.2)g(Discussion)21 b Fp(:)i(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(198)143 926 y(8.3)g(Logic)16 b(of)e(the)i(design)29 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(199)248 983 y(8.3.1)k(Miscellaneous)17 b(con)o(trol)e(of)g (pro\014ling)28 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(199)143 1039 y(8.4)g(Examples)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b Fx(200)248 1096 y(8.4.1)k(Pro\014ler)16 b(implemen)o(tation)35 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(200)248 1152 y(8.4.2)k(MPI)15 b(library)h(implemen)o(tation)h Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(201)248 1209 y(8.4.3)k (Complications)e Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fx(202)143 1265 y(8.5)g(Multiple)17 b(lev)o(els)f(of)f(in)o (terception)25 b Fp(:)d(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(203)75 1367 y Fq(Bibliograph)o(y)1430 b(204)75 1469 y(A)28 b(Language)19 b(Binding)1242 b(207)143 1525 y Fx(A.1)35 b(In)o(tro)q(duction)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(207)143 1582 y(A.2)35 b(De\014ned)16 b(Constan)o(ts)e(for)g(C)h(and)h(F)l (ortran)21 b Fp(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(207)143 1638 y(A.3)35 b(C)15 b(bindings)i(for)d(P)o(oin)o(t-to-P)o(oin)o(t)h(Comm)o (unication)21 b Fp(:)i(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b Fx(211)143 1695 y(A.4)35 b(C)15 b(Bindings)i(for)d (Collectiv)o(e)j(Comm)o(unication)34 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(214)143 1751 y(A.5)35 b(C)15 b(Bindings)i(for)d(Groups,)h(Con)o (texts,)f(and)h(Comm)o(unicators)26 b Fp(:)c(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b Fx(215)143 1808 y(A.6)35 b(C)15 b(Bindings)i(for)d(Pro)q(cess)h(T)l(op)q(ologies)43 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(216)143 1864 y(A.7)35 b(C)15 b(bindings)i(for)d(En)o(vironmen)o(tal)i(Inquiry)42 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b Fx(217)143 1921 y(A.8)35 b(C)15 b(Bindings)i(for)d(Pro\014ling)35 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b Fx(217)143 1977 y(A.9)35 b(F)l(ortran)14 b(Bindings)j(for)d(P)o(oin)o(t-to-P)o(oin)o(t)h(Comm)o(unication)39 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(217)143 2033 y(A.10)12 b(F)l(ortran)i(Bindings)j(for)d(Collectiv)o (e)j(Comm)o(unication)23 b Fp(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b Fx(221)143 2090 y(A.11)12 b(F)l(ortran)i(Bindings)j(for)d(Groups,)h(Con)o(texts,)f(etc.)27 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b Fx(223)143 2146 y(A.12)12 b(F)l(ortran)i(Bindings)j(for)d (Pro)q(cess)h(T)l(op)q(ologies)32 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(224)143 2203 y(A.13)12 b(F)l(ortran)i(Bindings)j(for)d(En)o (vironmen)o(tal)i(Inquiry)24 b Fp(:)e(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(225)143 2259 y(A.14)12 b(F)l(ortran)i(Bindings)j(for)d(Pro\014ling)24 b Fp(:)e(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fx(226)75 2361 y Fq(MPI)17 b(F)l(unction)h(Index)1252 b(227)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 7 6 6 bop 75 75 a Fr(Ac)m(kno)m(wledgmen)m(ts)166 282 y Fx(The)19 b(tec)o(hnical)i(dev)o(elopmen)o(t)f(w)o(as)e(carried)i(out)f (b)o(y)g(subgroups,)h(whose)f(w)o(ork)g(w)o(as)f(review)o(ed)75 338 y(b)o(y)d(the)h(full)h(committee.)k(During)15 b(the)h(p)q(erio)q(d) h(of)e(dev)o(elopmen)o(t)h(of)f(the)h(Message)f(P)o(assing)g(In)o (terface)75 395 y(\()p Fv(MPI)p Fx(\),)f(man)o(y)g(p)q(eople)j(serv)o (ed)e(in)h(p)q(ositions)g(of)f(resp)q(onsibilit)o(y)j(and)d(are)g (listed)i(b)q(elo)o(w.)143 497 y Ft(\017)23 b Fx(Jac)o(k)15 b(Dongarra,)e(Da)o(vid)i(W)l(alk)o(er,)g(Con)o(v)o(eners)g(and)g (Meeting)h(Chairs)143 589 y Ft(\017)23 b Fx(Ewing)15 b(Lusk,)h(Bob)f(Knigh)o(ten,)h(Min)o(utes)143 682 y Ft(\017)23 b Fx(Marc)14 b(Snir,)i(William)h(Gropp,)d(Ewing)i(Lusk,)f(P)o(oin)o (t-to-P)o(oin)o(t)f(Comm)o(unications)143 774 y Ft(\017)23 b Fx(Al)16 b(Geist,)e(Marc)h(Snir,)h(Stev)o(e)f(Otto,)f(Collectiv)o(e)j (Comm)o(unications)143 867 y Ft(\017)23 b Fx(Stev)o(e)15 b(Otto,)f(Editor)143 959 y Ft(\017)23 b Fx(Rolf)15 b(Hemp)q(el,)i(Pro)q (cess)e(T)l(op)q(ologies)143 1052 y Ft(\017)23 b Fx(Ewing)15 b(Lusk,)h(Language)f(Binding)143 1144 y Ft(\017)23 b Fx(William)17 b(Gropp,)d(En)o(vironmen)o(tal)i(Managemen)o(t)143 1237 y Ft(\017)23 b Fx(James)15 b(Co)o(wnie,)g(Pro\014ling)143 1330 y Ft(\017)23 b Fx(T)l(on)o(y)16 b(Skjellum,)j(Lyndon)f(Clark)o(e,) f(Marc)g(Snir,)h(Ric)o(hard)g(Little\014eld,)h(Mark)e(Sears,)g(Groups,) 189 1386 y(Con)o(texts,)c(and)j(Comm)o(unicators)143 1479 y Ft(\017)23 b Fx(Stev)o(en)15 b(Huss-Lederman,)h(Initial)h (Implemen)o(tation)f(Subset)166 1580 y(The)23 b(follo)o(wing)g(list)g (includes)i(some)d(of)g(the)g(activ)o(e)h(participan)o(ts)g(in)g(the)g Fv(MPI)f Fx(pro)q(cess)g(not)75 1637 y(men)o(tioned)16 b(ab)q(o)o(v)o(e.)100 1771 y(Ed)f(Anderson)114 b(Rob)q(ert)15 b(Babb)135 b(Jo)q(e)16 b(Baron)186 b(Eric)16 b(Barszcz)100 1827 y(Scott)f(Berryman)56 b(Rob)15 b(Bjornson)119 b(Nathan)15 b(Doss)133 b(Anne)16 b(Elster)100 1883 y(Jim)g(F)l(eeney)149 b(Vince)16 b(F)l(ernando)81 b(Sam)15 b(Fineb)q(erg)115 b(Jon)16 b(Flo)o(w)o(er)100 1940 y(Daniel)g(F)l(ry)o(e)143 b(Ian)15 b(Glendinning)74 b(Adam)15 b(Green)o(b)q(erg)50 b(Rob)q(ert)16 b(Harrison)100 1996 y(Leslie)h(Hart)153 b(T)l(om)15 b(Haupt)163 b(Don)15 b(Heller)176 b(T)l(om)15 b(Henderson)100 2053 y(Alex)h(Ho)210 b(C.T.)14 b(Ho)o(w)o(ard)g(Ho)62 b(Gary)14 b(Ho)o(w)o(ell)143 b(John)16 b(Kap)q(enga)100 2109 y(James)f(Kohl)141 b(Susan)15 b(Krauss)125 b(Bob)15 b(Leary)182 b(Arth)o(ur)15 b(Maccab)q(e)100 2166 y(P)o(eter)g(Madams)83 b(Alan)16 b(Main)o(w)o(aring)49 b(Oliv)o(er)17 b(McBry)o(an)70 b(Phil)17 b(McKinley)100 2222 y(Charles)e(Mosher)63 b(Dan)15 b(Nessett)149 b(P)o(eter)15 b(P)o(ac)o(heco)109 b(Ho)o(w)o(ard)14 b(P)o(almer)100 2279 y(P)o(aul)h(Pierce)147 b(Sanja)o(y)14 b(Rank)m(a)121 b(P)o(eter)15 b(Rigsb)q(ee)117 b(Arc)o(h)15 b(Robison)100 2335 y(Eric)o(h)h(Sc)o(hikuta)81 b(Am)o(buj)15 b(Singh)133 b(Alan)16 b(Sussman)108 b(Rob)q(ert)16 b(T)l(omlinson)100 2392 y(Rob)q(ert)f(G.)g(V)l(oigt)50 b(Dennis)16 b(W)l(eeks)117 b(Stephen)16 b(Wheat)83 b(Stev)o(en)16 b(Zenith)166 2534 y(The)d(Univ)o(ersit)o(y)h(of)f(T)l(ennessee)i(and)e(Oak)g(Ridge)i (National)e(Lab)q(oratory)g(made)g(the)g(draft)g(a)o(v)m(ail-)75 2591 y(able)j(b)o(y)f(anon)o(ymous)f(FTP)h(mail)g(serv)o(ers)g(and)g(w) o(ere)g(instrumen)o(tal)g(in)h(distributing)h(the)e(do)q(cumen)o(t.)166 2647 y Fv(MPI)c Fx(op)q(erated)h(on)f(a)h(v)o(ery)f(tigh)o(t)h(budget)g (\(in)g(realit)o(y)l(,)g(it)g(had)g(no)g(budget)g(when)g(the)g(\014rst) f(meeting)75 2704 y(w)o(as)i(announced\).)20 b(ARP)l(A)14 b(and)f(NSF)h(ha)o(v)o(e)e(supp)q(orted)i(researc)o(h)g(at)e(v)m (arious)i(institutions)h(that)d(ha)o(v)o(e)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 8 7 7 bop 75 45 a Fx(made)17 b(a)f(con)o(tribution)h(to)o(w)o(ards)e(tra) o(v)o(el)h(for)g(the)g(U.S.)g(academics.)25 b(Supp)q(ort)17 b(for)f(sev)o(eral)h(Europ)q(ean)75 102 y(participan)o(ts)f(w)o(as)e (pro)o(vided)i(b)o(y)f(ESPRIT.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 1 9 1 8 bop 75 356 a Fo(Chapter)34 b(1)75 564 y Fr(In)m(tro)s(duction)41 b(to)g(MPI)75 805 y Fn(1.1)59 b(Overview)19 b(and)g(Goals)75 906 y Fx(Message)11 b(passing)i(is)f(a)g(paradigm)g(used)g(widely)i(on) d(certain)i(classes)f(of)g(parallel)h(mac)o(hines,)g(esp)q(ecially)75 963 y(those)j(with)i(distributed)g(memory)l(.)24 b(Although)17 b(there)g(are)f(man)o(y)h(v)m(ariations,)g(the)g(basic)g(concept)g(of) 75 1019 y(pro)q(cesses)j(comm)o(unicating)h(through)e(messages)h(is)g (w)o(ell)h(understo)q(o)q(d.)34 b(Ov)o(er)20 b(the)g(last)g(ten)g(y)o (ears,)75 1076 y(substan)o(tial)11 b(progress)f(has)g(b)q(een)h(made)g (in)g(casting)f(signi\014can)o(t)i(applications)g(in)f(this)g (paradigm.)18 b(Eac)o(h)75 1132 y(v)o(endor)13 b(has)g(implemen)o(ted)i (its)e(o)o(wn)g(v)m(arian)o(t.)19 b(More)13 b(recen)o(tly)l(,)h(sev)o (eral)f(systems)g(ha)o(v)o(e)f(demonstrated)75 1188 y(that)k(a)h (message)f(passing)h(system)g(can)g(b)q(e)g(e\016cien)o(tly)h(and)f(p)q (ortably)h(implemen)o(ted.)26 b(It)17 b(is)g(th)o(us)g(an)75 1245 y(appropriate)c(time)g(to)f(try)h(to)f(de\014ne)i(b)q(oth)f(the)g (syn)o(tax)f(and)h(seman)o(tics)g(of)g(a)g(core)f(of)h(library)h (routines)75 1301 y(that)i(will)i(b)q(e)f(useful)h(to)e(a)g(wide)h (range)f(of)h(users)f(and)h(e\016cien)o(tly)h(implemen)o(table)g(on)f (a)f(wide)h(range)75 1358 y(of)e(computers.)166 1414 y(In)c(designing)h Fv(MPI)e Fx(w)o(e)h(ha)o(v)o(e)f(sough)o(t)g(to)g (mak)o(e)g(use)h(of)f(the)h(most)e(attractiv)o(e)h(features)g(of)g(a)h (n)o(um)o(b)q(er)75 1471 y(of)16 b(existing)i(message)e(passing)h (systems,)g(rather)f(than)g(selecting)j(one)d(of)h(them)f(and)h (adopting)g(it)g(as)75 1527 y(the)g(standard.)26 b(Th)o(us,)18 b Fv(MPI)e Fx(has)h(b)q(een)i(strongly)e(in\015uenced)j(b)o(y)d(w)o (ork)f(at)h(the)g(IBM)h(T.)f(J.)g(W)l(atson)75 1584 y(Researc)o(h)e (Cen)o(ter)g([1)o(,)g(2)o(],)f(In)o(tel's)i(NX/2)e([23)o(],)g(Express)h ([22)o(],)f(nCUBE's)h(V)l(ertex)g([21)o(],)f(p4)h([7)o(,)g(6],)f(and)75 1640 y(P)l(ARMA)o(CS)h([5,)f(8].)19 b(Other)c(imp)q(ortan)o(t)g(con)o (tributions)h(ha)o(v)o(e)e(come)h(from)f(Zip)q(co)q(de)j([24)o(,)d (25],)g(Chimp)75 1697 y([14)o(,)h(15)o(],)g(PVM)g([4)o(,)g(11)o(],)g (Chameleon)g([19)o(],)g(and)g(PICL)h([18)o(].)166 1753 y(The)e Fv(MPI)f Fx(standardization)h(e\013ort)e(in)o(v)o(olv)o(ed)j (ab)q(out)e(60)g(p)q(eople)i(from)e(40)g(organizations)h(mainly)75 1810 y(from)j(the)g(United)i(States)e(and)h(Europ)q(e.)27 b(Most)16 b(of)h(the)h(ma)s(jor)e(v)o(endors)h(of)g(concurren)o(t)h (computers)75 1866 y(w)o(ere)11 b(in)o(v)o(olv)o(ed)h(in)h Fv(MPI)p Fx(,)d(along)h(with)h(researc)o(hers)f(from)g(univ)o (ersities,)j(go)o(v)o(ernmen)o(t)c(lab)q(oratories,)i(and)75 1922 y(industry)l(.)21 b(The)14 b(standardization)h(pro)q(cess)f(b)q (egan)h(with)f(the)h(W)l(orkshop)f(on)g(Standards)g(for)g(Message)75 1979 y(P)o(assing)j(in)i(a)e(Distributed)h(Memory)f(En)o(vironmen)o(t,) h(sp)q(onsored)g(b)o(y)f(the)h(Cen)o(ter)f(for)g(Researc)o(h)h(on)75 2035 y(P)o(arallel)11 b(Computing,)g(held)h(April)f(29-30,)f(1992,)g (in)h(Williamsburg,)h(Virginia)g([29)o(].)18 b(A)o(t)9 b(this)i(w)o(orkshop)75 2092 y(the)17 b(basic)h(features)f(essen)o (tial)h(to)f(a)g(standard)g(message)g(passing)g(in)o(terface)h(w)o(ere) f(discussed,)i(and)e(a)75 2148 y(w)o(orking)e(group)g(established)i(to) d(con)o(tin)o(ue)i(the)f(standardization)h(pro)q(cess.)166 2205 y(A)e(preliminary)i(draft)e(prop)q(osal,)g(kno)o(wn)g(as)g(MPI1,)g (w)o(as)f(put)h(forw)o(ard)f(b)o(y)h(Dongarra,)f(Hemp)q(el,)75 2261 y(Hey)l(,)24 b(and)f(W)l(alk)o(er)g(in)g(No)o(v)o(em)o(b)q(er)f (1992,)h(and)f(a)h(revised)g(v)o(ersion)g(w)o(as)e(completed)j(in)f(F)l (ebruary)75 2318 y(1993)e([12)o(].)39 b(MPI1)22 b(em)o(b)q(o)q(died)h (the)f(main)g(features)g(that)f(w)o(ere)g(iden)o(ti\014ed)j(at)d(the)h (Williamsburg)75 2374 y(w)o(orkshop)d(as)h(b)q(eing)h(necessary)g(in)g (a)f(message)f(passing)i(standard.)34 b(Since)22 b(MPI1)d(w)o(as)h (primarily)75 2431 y(in)o(tended)12 b(to)d(promote)h(discussion)i(and)e (\\get)g(the)h(ball)g(rolling,")h(it)f(fo)q(cused)g(mainly)g(on)f(p)q (oin)o(t-to-p)q(oin)o(t)75 2487 y(comm)o(unications.)33 b(MPI1)20 b(brough)o(t)f(to)g(the)g(forefron)o(t)f(a)h(n)o(um)o(b)q(er) h(of)f(imp)q(ortan)o(t)h(standardization)75 2543 y(issues,)c(but)f(did) h(not)f(include)i(an)o(y)e(collectiv)o(e)i(comm)o(unication)f(routines) f(and)h(w)o(as)e(not)h(thread-safe.)166 2600 y(In)j(No)o(v)o(em)o(b)q (er)f(1992,)g(a)g(meeting)h(of)f(the)h Fv(MPI)f Fx(w)o(orking)g(group)h (w)o(as)e(held)j(in)g(Minneap)q(olis,)h(at)75 2656 y(whic)o(h)15 b(it)g(w)o(as)e(decided)k(to)c(place)j(the)e(standardization)h(pro)q (cess)g(on)f(a)g(more)g(formal)g(fo)q(oting,)g(and)h(to)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 10 2 9 bop 75 -100 a Fx(2)903 b Fm(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION)17 b(TO)e(MPI)75 45 y Fx(generally)i(adopt)f(the)g(pro)q(cedures)i(and)e (organization)g(of)g(the)g(High)h(P)o(erformance)f(F)l(ortran)f(F)l (orum.)75 102 y(Sub)q(committees)i(w)o(ere)e(formed)g(for)g(the)h(ma)s (jor)e(comp)q(onen)o(t)i(areas)f(of)g(the)h(standard,)f(and)g(an)h (email)75 158 y(discussion)23 b(service)g(established)g(for)e(eac)o(h.) 39 b(In)22 b(addition,)i(the)d(goal)h(of)f(pro)q(ducing)i(a)e(draft)g Fv(MPI)75 214 y Fx(standard)13 b(b)o(y)g(the)g(F)l(all)h(of)e(1993)g(w) o(as)g(set.)19 b(T)l(o)13 b(ac)o(hiev)o(e)h(this)f(goal)g(the)g Fv(MPI)g Fx(w)o(orking)g(group)f(met)h(ev)o(ery)75 271 y(6)k(w)o(eeks)g(for)f(t)o(w)o(o)g(da)o(ys)h(throughout)f(the)h (\014rst)g(9)g(mon)o(ths)f(of)h(1993,)f(and)h(presen)o(ted)h(the)f (draft)g Fv(MPI)75 327 y Fx(standard)d(at)g(the)g(Sup)q(ercomputing)i (93)e(conference)h(in)g(No)o(v)o(em)o(b)q(er)f(1993.)19 b(These)14 b(meetings)h(and)g(the)75 384 y(email)h(discussion)g (together)f(constituted)g(the)g Fv(MPI)f Fx(F)l(orum,)g(mem)o(b)q (ership)i(of)f(whic)o(h)h(has)e(b)q(een)i(op)q(en)75 440 y(to)f(all)h(mem)o(b)q(ers)f(of)g(the)g(high)h(p)q(erformance)f (computing)h(comm)o(unit)o(y)l(.)166 497 y(The)j(main)f(adv)m(an)o (tages)g(of)g(establishing)j(a)d(message-passing)g(standard)g(are)g(p)q (ortabilit)o(y)i(and)75 553 y(ease-of-use.)g(In)14 b(a)g(distributed)h (memory)e(comm)o(unication)i(en)o(vironmen)o(t)f(in)g(whic)o(h)h(the)f (higher)h(lev)o(el)75 610 y(routines)c(and/or)g(abstractions)f(are)h (build)i(up)q(on)e(lo)o(w)o(er)g(lev)o(el)h(message)e(passing)i (routines)f(the)g(b)q(ene\014ts)75 666 y(of)22 b(standardization)h(are) f(particularly)h(apparen)o(t.)41 b(F)l(urthermore,)24 b(the)e(de\014nition)i(of)e(a)g(message)75 723 y(passing)16 b(standard,)e(suc)o(h)i(as)e(that)h(prop)q(osed)g(here,)h(pro)o(vides)f (v)o(endors)h(with)f(a)g(clearly)h(de\014ned)h(base)75 779 y(set)10 b(of)g(routines)g(that)g(they)g(can)h(implemen)o(t)g (e\016cien)o(tly)l(,)i(or)c(in)i(some)f(cases)h(pro)o(vide)f(hardw)o (are)g(supp)q(ort)75 835 y(for,)k(thereb)o(y)h(enhancing)i(scalabilit)o (y)l(.)166 892 y(The)h(goal)g(of)g(the)g(Message)g(P)o(assing)g(In)o (terface)g(simply)i(stated)d(is)i(to)f(dev)o(elop)h(a)f(widely)h(used) 75 948 y(standard)f(for)g(writing)h(message-passing)g(programs.)29 b(As)19 b(suc)o(h)g(the)g(in)o(terface)g(should)g(establish)h(a)75 1005 y(practical,)c(p)q(ortable,)f(e\016cien)o(t,)g(and)h(\015exible)h (standard)e(for)g(message)f(passing.)166 1061 y(A)h(complete)h(list)g (of)f(goals)g(follo)o(ws.)143 1144 y Ft(\017)23 b Fx(Design)11 b(an)f(application)i(programming)e(in)o(terface)h(\(not)f(necessarily)i (for)d(compilers)j(or)e(a)g(system)189 1201 y(implemen)o(tation)16 b(library\).)143 1288 y Ft(\017)23 b Fx(Allo)o(w)15 b(e\016cien)o(t)h (comm)o(unication:)21 b(Av)o(oid)16 b(memory-to-memory)e(cop)o(ying)h (and)h(allo)o(w)f(o)o(v)o(erlap)189 1344 y(of)c(computation)h(and)g (comm)o(unication)h(and)f(o\017oad)f(to)g(comm)o(unication)i(co-pro)q (cessor,)f(where)189 1401 y(a)o(v)m(ailable.)143 1488 y Ft(\017)23 b Fx(Allo)o(w)15 b(for)g(implemen)o(tations)h(that)f(can)g (b)q(e)h(used)g(in)g(a)f(heterogeneous)g(en)o(vironmen)o(t.)143 1575 y Ft(\017)23 b Fx(Allo)o(w)15 b(con)o(v)o(enien)o(t)h(C)f(and)h(F) l(ortran)e(77)g(bindings)j(for)e(the)g(in)o(terface.)143 1662 y Ft(\017)23 b Fx(Assume)14 b(a)h(reliable)h(comm)o(unication)g (in)o(terface:)j(the)c(user)g(need)g(not)f(cop)q(e)h(with)g(comm)o (unica-)189 1718 y(tion)c(failures.)19 b(Suc)o(h)12 b(failures)f(are)g (dealt)g(with)g(b)o(y)g(the)g(underlying)i(comm)o(unication)e (subsystem.)143 1806 y Ft(\017)23 b Fx(De\014ne)15 b(an)f(in)o(terface) h(that)f(is)h(not)f(to)q(o)g(di\013eren)o(t)g(from)g(curren)o(t)h (practice,)f(suc)o(h)h(as)f(PVM,)g(NX,)189 1862 y(Express,)g(p4,)h (etc.,)g(and)g(pro)o(vides)h(extensions)g(that)e(allo)o(w)i(greater)e (\015exibilit)o(y)l(.)143 1949 y Ft(\017)23 b Fx(De\014ne)d(an)f(in)o (terface)h(that)e(can)i(b)q(e)g(implemen)o(ted)h(on)e(man)o(y)g(v)o (endor's)g(platforms,)g(with)h(no)189 2006 y(signi\014can)o(t)c(c)o (hanges)f(in)h(the)f(underlying)i(comm)o(unication)f(and)g(system)e (soft)o(w)o(are.)143 2093 y Ft(\017)23 b Fx(Seman)o(tics)15 b(of)g(the)g(in)o(terface)h(should)g(b)q(e)g(language)f(indep)q(enden)o (t.)143 2180 y Ft(\017)23 b Fx(The)15 b(in)o(terface)h(should)g(b)q(e)g (designed)g(to)f(allo)o(w)g(for)g(thread-safet)o(y)l(.)75 2320 y Fn(1.2)59 b(Who)20 b(Should)g(Use)g(This)f(Standa)n(rd?)75 2421 y Fx(This)d(standard)f(is)h(in)o(tended)h(for)e(use)g(b)o(y)h(all) g(those)f(who)g(w)o(an)o(t)g(to)f(write)i(p)q(ortable)g (message-passing)75 2478 y(programs)9 b(in)j(F)l(ortran)d(77)i(and)f (C.)h(This)g(includes)i(individual)h(application)e(programmers,)e(dev)o (elop)q(ers)75 2534 y(of)17 b(soft)o(w)o(are)e(designed)j(to)f(run)g (on)g(parallel)i(mac)o(hines,)f(and)f(creators)f(of)h(en)o(vironmen)o (ts)g(and)g(to)q(ols.)75 2591 y(In)g(order)f(to)f(b)q(e)i(attractiv)o (e)e(to)h(this)g(wide)i(audience,)f(the)g(standard)e(m)o(ust)h(pro)o (vide)h(a)e(simple,)j(easy-)75 2647 y(to-use)f(in)o(terface)g(for)f (the)g(basic)i(user)f(while)h(not)e(seman)o(tically)i(precluding)h(the) e(high-p)q(erformance)75 2704 y(message-passing)e(op)q(erations)h(a)o (v)m(ailable)h(on)e(adv)m(anced)h(mac)o(hines.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 11 3 10 bop 75 -100 a Fm(1.3.)34 b(WHA)l(T)15 b(PLA)l(TF)o(ORMS)h(ARE)g(T) l(AR)o(GETS)f(F)o(OR)g(IMPLEMENT)l(A)l(TION?)296 b Fx(3)75 45 y Fn(1.3)59 b(What)20 b(Platfo)n(rms)g(Are)g(T)-5 b(a)n(rgets)19 b(F)n(o)n(r)i(Implementat)o(ion?)75 147 y Fx(The)c(attractiv)o(eness)f(of)g(the)h(message-passing)g(paradigm)f (at)g(least)h(partially)h(stems)e(from)g(its)h(wide)75 203 y(p)q(ortabilit)o(y)l(.)k(Programs)13 b(expressed)j(this)f(w)o(a)o (y)f(ma)o(y)g(run)h(on)g(distributed-memory)h(m)o(ultipro)q(cessors,)75 259 y(net)o(w)o(orks)i(of)i(w)o(orkstations,)e(and)i(com)o(binations)g (of)g(all)g(of)f(these.)34 b(In)20 b(addition,)h(shared-memory)75 316 y(implemen)o(tations)g(are)e(p)q(ossible.)34 b(The)20 b(paradigm)g(will)h(not)e(b)q(e)h(made)f(obsolete)h(b)o(y)g(arc)o (hitectures)75 372 y(com)o(bining)15 b(the)e(shared-)h(and)g (distributed-memory)h(views,)f(or)f(b)o(y)g(increases)i(in)f(net)o(w)o (ork)f(sp)q(eeds.)20 b(It)75 429 y(th)o(us)d(should)i(b)q(e)f(b)q(oth)g (p)q(ossible)h(and)f(useful)g(to)f(implemen)o(t)i(this)f(standard)f(on) h(a)f(great)g(v)m(ariet)o(y)g(of)75 485 y(mac)o(hines,)i(including)h (those)e(\\mac)o(hines")g(consisting)g(of)g(collections)h(of)e(other)h (mac)o(hines,)g(parallel)75 542 y(or)d(not,)f(connected)i(b)o(y)f(a)g (comm)o(unication)h(net)o(w)o(ork.)166 598 y(The)21 b(in)o(terface)f (is)h(suitable)h(for)e(use)h(b)o(y)f(fully)i(general)f(MIMD)f (programs,)g(as)g(w)o(ell)h(as)f(those)75 655 y(written)d(in)g(the)g (more)f(restricted)h(st)o(yle)g(of)f(SPMD.)g(Although)h(no)f(explicit)j (supp)q(ort)e(for)f(threads)g(is)75 711 y(pro)o(vided,)e(the)f(in)o (terface)h(has)f(b)q(een)h(designed)g(so)f(as)g(not)f(to)h(prejudice)i (their)e(use.)20 b(With)13 b(this)h(v)o(ersion)75 768 y(of)h Fv(MPI)f Fx(no)h(supp)q(ort)h(is)g(pro)o(vided)g(for)e(dynamic)i (spa)o(wning)g(of)e(tasks.)166 824 y Fv(MPI)e Fx(pro)o(vides)h(man)o(y) g(features)f(in)o(tended)i(to)e(impro)o(v)o(e)h(p)q(erformance)g(on)g (scalable)h(parallel)g(com-)75 880 y(puters)23 b(with)g(sp)q(ecialized) j(in)o(terpro)q(cessor)c(comm)o(unication)i(hardw)o(are.)42 b(Th)o(us,)24 b(w)o(e)e(exp)q(ect)i(that)75 937 y(nativ)o(e,)18 b(high-p)q(erformance)h(implemen)o(tations)g(of)f Fv(MPI)f Fx(will)i(b)q(e)g(pro)o(vided)f(on)g(suc)o(h)g(mac)o(hines.)28 b(A)o(t)75 993 y(the)17 b(same)f(time,)h(implemen)o(tations)h(of)e Fv(MPI)g Fx(on)g(top)h(of)f(standard)g(Unix)i(in)o(terpro)q(cessor)e (comm)o(uni-)75 1050 y(cation)e(proto)q(cols)g(will)i(pro)o(vide)e(p)q (ortabilit)o(y)h(to)f(w)o(orkstation)e(clusters)j(and)f(heterogenous)g (net)o(w)o(orks)75 1106 y(of)h(w)o(orkstations.)j(Sev)o(eral)e (proprietary)l(,)e(nativ)o(e)i(implemen)o(tations)g(of)f Fv(MPI)p Fx(,)f(and)h(a)g(public)i(domain,)75 1163 y(p)q(ortable)f (implemen)o(tation)g(of)f Fv(MPI)g Fx(are)f(in)i(progress)f(at)g(the)g (time)g(of)g(this)h(writing)f([17,)f(13].)75 1306 y Fn(1.4)59 b(What)20 b(Is)f(Included)g(In)g(The)g(Standa)n(rd?)75 1407 y Fx(The)c(standard)g(includes:)143 1514 y Ft(\017)23 b Fx(P)o(oin)o(t-to-p)q(oin)o(t)15 b(comm)o(unication)143 1608 y Ft(\017)23 b Fx(Collectiv)o(e)16 b(op)q(erations)143 1701 y Ft(\017)23 b Fx(Pro)q(cess)15 b(groups)143 1795 y Ft(\017)23 b Fx(Comm)o(unication)15 b(con)o(texts)143 1889 y Ft(\017)23 b Fx(Pro)q(cess)15 b(top)q(ologies)143 1983 y Ft(\017)23 b Fx(Bindings)17 b(for)d(F)l(ortran)g(77)h(and)g(C) 143 2077 y Ft(\017)23 b Fx(En)o(vironmen)o(tal)15 b(Managemen)o(t)f (and)i(inquiry)143 2170 y Ft(\017)23 b Fx(Pro\014ling)16 b(in)o(terface)75 2314 y Fn(1.5)59 b(What)20 b(Is)f(Not)h(Included)e (In)i(The)f(Standa)n(rd?)75 2415 y Fx(The)c(standard)g(do)q(es)h(not)f (sp)q(ecify:)143 2521 y Ft(\017)23 b Fx(Explicit)17 b(shared-memory)e (op)q(erations)143 2615 y Ft(\017)23 b Fx(Op)q(erations)c(that)e (require)i(more)f(op)q(erating)h(system)f(supp)q(ort)g(than)g(is)h (curren)o(tly)g(standard;)189 2672 y(for)14 b(example,)i(in)o (terrupt-driv)o(en)g(receiv)o(es,)g(remote)f(execution,)h(or)e(activ)o (e)i(messages)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 12 4 11 bop 75 -100 a Fx(4)903 b Fm(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION) 17 b(TO)e(MPI)143 45 y Ft(\017)23 b Fx(Program)13 b(construction)j(to)q (ols)143 139 y Ft(\017)23 b Fx(Debugging)15 b(facilities)143 233 y Ft(\017)23 b Fx(Explicit)17 b(supp)q(ort)e(for)g(threads)143 327 y Ft(\017)23 b Fx(Supp)q(ort)15 b(for)g(task)f(managemen)o(t)143 420 y Ft(\017)23 b Fx(I/O)15 b(functions)166 527 y(There)g(are)f(man)o (y)h(features)f(that)g(ha)o(v)o(e)g(b)q(een)i(considered)g(and)f(not)g (included)i(in)f(this)f(standard.)75 583 y(This)i(happ)q(ened)h(for)e (a)g(n)o(um)o(b)q(er)h(of)f(reasons,)g(one)h(of)f(whic)o(h)h(is)g(the)g (time)g(constrain)o(t)f(that)g(w)o(as)f(self-)75 640 y(imp)q(osed)h(in)h(\014nishing)g(the)e(standard.)20 b(F)l(eatures)15 b(that)f(are)h(not)g(included)j(can)e(alw)o(a)o(ys)e (b)q(e)i(o\013ered)f(as)75 696 y(extensions)j(b)o(y)f(sp)q(eci\014c)h (implemen)o(tations.)27 b(P)o(erhaps)17 b(future)g(v)o(ersions)g(of)f Fv(MPI)h Fx(will)h(address)f(some)75 752 y(of)e(these)g(issues.)75 896 y Fn(1.6)59 b(Organization)20 b(of)g(this)f(Do)r(cument)75 997 y Fx(The)k(follo)o(wing)h(is)f(a)f(list)i(of)e(the)h(remaining)h(c) o(hapters)e(in)i(this)f(do)q(cumen)o(t,)i(along)e(with)g(a)f(brief)75 1054 y(description)17 b(of)d(eac)o(h.)143 1160 y Ft(\017)23 b Fx(Chapter)c(2,)i Fv(MPI)e(T)l(erms)f(and)i(Conventions)p Fx(,)j(explains)e(notational)f(terms)g(and)g(con)o(v)o(en)o(tions)189 1216 y(used)15 b(throughout)g(the)g Fv(MPI)g Fx(do)q(cumen)o(t.)143 1310 y Ft(\017)23 b Fx(Chapter)16 b(3,)g Fv(P)o(oint)h(to)f(P)o(oint)h (Comm)m(unication)p Fx(,)e(de\014nes)i(the)g(basic,)g(pairwise)g(comm)o (unication)189 1367 y(subset)d(of)g Fv(MPI)p Fx(.)f Fl(send)h Fx(and)g Fl(r)n(e)n(c)n(eive)f Fx(are)h(found)h(here,)f(along)g(with)h (man)o(y)f(asso)q(ciated)g(functions)189 1423 y(designed)i(to)f(mak)o (e)f(basic)i(comm)o(unication)g(p)q(o)o(w)o(erful)g(and)f(e\016cien)o (t.)143 1517 y Ft(\017)23 b Fx(Chapter)12 b(4,)h Fv(Collective)g(Com)o (m)n(unications)p Fx(,)e(de\014nes)j(pro)q(cess-group)f(collectiv)o(e)h (comm)o(unication)189 1573 y(op)q(erations.)28 b(W)l(ell)19 b(kno)o(wn)f(examples)h(of)e(this)i(are)e(barrier)i(and)f(broadcast)f (o)o(v)o(er)g(a)h(group)f(of)189 1630 y(pro)q(cesses)e(\(not)g (necessarily)h(all)g(the)g(pro)q(cesses\).)143 1724 y Ft(\017)23 b Fx(Chapter)16 b(5,)g Fv(Groups,)h(Contexts,)i(and)e(Comm)m (unicato)o(rs)p Fx(,)d(sho)o(ws)i(ho)o(w)g(groups)g(of)g(pro)q(cesses)h (are)189 1780 y(formed)c(and)h(manipulated,)i(ho)o(w)d(unique)i(comm)o (unication)g(con)o(texts)e(are)h(obtained,)g(and)g(ho)o(w)189 1836 y(the)h(t)o(w)o(o)f(are)h(b)q(ound)h(together)e(in)o(to)h(a)g Fl(c)n(ommunic)n(ator)p Fx(.)143 1930 y Ft(\017)23 b Fx(Chapter)18 b(6,)h Fv(Pro)q(cess)h(T)l(op)q(ologies)p Fx(,)f(explains)h(a)f(set)f(of)h(utilit)o(y)h(functions)f(mean)o(t)f (to)h(assist)f(in)189 1987 y(the)13 b(mapping)i(of)e(pro)q(cess)g (groups)h(\(a)f(linearly)i(ordered)f(set\))f(to)g(ric)o(her)h(top)q (ological)g(structures)189 2043 y(suc)o(h)h(as)g(m)o(ulti-dimensional)j (grids.)143 2137 y Ft(\017)23 b Fx(Chapter)10 b(7,)h Fv(MPI)g(Environmental)e(Management)p Fx(,)g(explains)j(ho)o(w)f(the)g (programmer)e(can)i(manage)189 2193 y(and)g(mak)o(e)f(inquiries)j(of)d (the)h(curren)o(t)f Fv(MPI)g Fx(en)o(vironmen)o(t.)19 b(These)11 b(functions)g(are)g(needed)h(for)e(the)189 2250 y(writing)k(of)f(correct,)g(robust)g(programs,)f(and)i(are)f(esp)q (ecially)j(imp)q(ortan)o(t)d(for)g(the)h(construction)189 2306 y(of)g(highly-p)q(ortable)k(message-passing)d(programs.)143 2400 y Ft(\017)23 b Fx(Chapter)d(8,)h Fv(Pro\014ling)f(Interface)p Fx(,)i(explains)g(a)e(simple)i(name-shifting)g(con)o(v)o(en)o(tion)e (that)g(an)o(y)189 2457 y Fv(MPI)h Fx(implemen)o(tation)i(m)o(ust)f (supp)q(ort.)40 b(One)23 b(motiv)m(ation)f(for)f(this)i(is)f(the)g (abilit)o(y)h(to)f(put)189 2513 y(p)q(erformance)c(pro\014ling)i(calls) g(in)o(to)f Fv(MPI)f Fx(without)g(the)h(need)g(for)f(access)h(to)f(the) h Fv(MPI)f Fx(source)189 2570 y(co)q(de.)31 b(The)19 b(name)g(shift)g(is)h(merely)f(an)g(in)o(terface,)h(it)f(sa)o(ys)f (nothing)i(ab)q(out)e(ho)o(w)h(the)g(actual)189 2626 y(pro\014ling)d(should)g(b)q(e)f(done)h(and)f(in)h(fact,)e(the)h(name)g (shift)g(can)g(b)q(e)h(useful)g(for)e(other)h(purp)q(oses.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 13 5 12 bop 75 -100 a Fm(1.6.)34 b(OR)o(GANIZA)l(TION)17 b(OF)e(THIS)h(DOCUMENT)810 b Fx(5)143 45 y Ft(\017)23 b Fx(Annex)18 b(A,)g Fv(Language)g(Bindings)p Fx(,)i(giv)o(es)d(sp)q (eci\014c)j(syn)o(tax)d(in)i(F)l(ortran)d(77)i(and)g(C,)f(for)g(all)i Fv(MPI)189 102 y Fx(functions,)c(constan)o(ts,)f(and)h(t)o(yp)q(es.)143 195 y Ft(\017)23 b Fx(The)12 b Fv(MPI)g(F)o(unction)i(Index)f Fx(is)g(a)f(simple)i(index)g(sho)o(wing)e(the)h(lo)q(cation)g(of)f(the) h(precise)g(de\014nition)189 252 y(of)h(eac)o(h)i Fv(MPI)e Fx(function,)i(together)e(with)i(b)q(oth)f(C)g(and)h(F)l(ortran)e (bindings.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 14 6 13 bop 75 356 a Fo(Chapter)34 b(2)75 564 y Fr(MPI)40 b(T)-10 b(erms)39 b(and)g(Con)m(v)m(en)m(tions)75 786 y Fx(This)12 b(c)o(hapter)f(explains)i(notational)e(terms)g(and)g(con)o (v)o(en)o(tions)g(used)h(throughout)f(the)g Fv(MPI)g Fx(do)q(cumen)o(t,)75 843 y(some)k(of)g(the)g(c)o(hoices)h(that)e(ha)o (v)o(e)h(b)q(een)i(made,)d(and)i(the)f(rationale)h(b)q(ehind)h(those)e (c)o(hoices.)75 986 y Fn(2.1)59 b(Do)r(cument)18 b(Notation)189 1088 y Fl(R)n(ationale.)45 b Fx(Throughout)15 b(this)i(do)q(cumen)o(t,) f(the)g(rationale)h(for)e(the)h(design)h(c)o(hoices)g(made)f(in)189 1144 y(the)j(in)o(terface)g(sp)q(eci\014cation)i(is)e(set)g(o\013)f(in) i(this)f(format.)30 b(Some)19 b(readers)g(ma)o(y)f(wish)i(to)e(skip)189 1201 y(these)i(sections,)h(while)h(readers)e(in)o(terested)g(in)h(in)o (terface)f(design)h(ma)o(y)f(w)o(an)o(t)f(to)g(read)h(them)189 1257 y(carefully)l(.)h(\()p Fl(End)16 b(of)g(r)n(ationale.)p Fx(\))189 1363 y Fl(A)n(dvic)n(e)21 b(to)h(users.)78 b Fx(Throughout)21 b(this)h(do)q(cumen)o(t,)h(material)f(that)f(sp)q (eaks)g(to)g(users)h(and)189 1420 y(illustrates)12 b(usage)g(is)g(set)f (o\013)g(in)i(this)f(format.)17 b(Some)12 b(readers)f(ma)o(y)g(wish)i (to)e(skip)h(these)g(sections,)189 1476 y(while)g(readers)e(in)o (terested)h(in)g(programming)f(in)h Fv(MPI)f Fx(ma)o(y)f(w)o(an)o(t)h (to)f(read)i(them)f(carefully)l(.)20 b(\()p Fl(End)189 1533 y(of)c(advic)n(e)g(to)h(users.)p Fx(\))189 1639 y Fl(A)n(dvic)n(e)k(to)i(implementors.)80 b Fx(Throughout)22 b(this)g(do)q(cumen)o(t,)i(material)e(that)f(is)i(primarily)189 1695 y(commen)o(tary)13 b(to)g(implemen)o(tors)i(is)f(set)g(o\013)f(in) i(this)g(format.)j(Some)c(readers)g(ma)o(y)f(wish)i(to)e(skip)189 1752 y(these)20 b(sections,)i(while)g(readers)f(in)o(terested)g(in)g Fv(MPI)f Fx(implemen)o(tations)i(ma)o(y)e(w)o(an)o(t)f(to)h(read)189 1808 y(them)15 b(carefully)l(.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)g (to)h(implementors.)p Fx(\))75 1951 y Fn(2.2)59 b(Pro)r(cedure)19 b(Sp)r(eci\014cation)75 2053 y Fv(MPI)g Fx(pro)q(cedures)i(are)e(sp)q (eci\014ed)j(using)f(a)f(language)g(indep)q(enden)o(t)i(notation.)33 b(The)20 b(argumen)o(ts)f(of)75 2109 y(pro)q(cedure)d(calls)g(are)f (mark)o(ed)g(as)g Fk(IN)p Fx(,)g Fk(OUT)g Fx(or)g Fk(INOUT)p Fx(.)g(The)h(meanings)f(of)g(these)h(are:)143 2203 y Ft(\017)23 b Fx(the)15 b(call)h(uses)g(but)f(do)q(es)h(not)e(up)q(date) i(an)f(argumen)o(t)g(mark)o(ed)f Fk(IN)p Fx(,)143 2297 y Ft(\017)23 b Fx(the)15 b(call)h(ma)o(y)f(up)q(date)h(an)f(argumen)o (t)f(mark)o(ed)h Fk(OUT)p Fx(,)143 2391 y Ft(\017)23 b Fx(the)15 b(call)h(b)q(oth)g(uses)f(and)g(up)q(dates)h(an)f(argumen)o (t)g(mark)o(ed)f Fk(INOUT)p Fx(.)166 2485 y(There)20 b(is)g(one)g(sp)q(ecial)i(case)e(|)g(if)h(an)e(argumen)o(t)h(is)g(a)g (handle)h(to)e(an)h(opaque)g(ob)s(ject)f(\(these)75 2541 y(terms)d(are)h(de\014ned)h(in)g(Section)f(2.4.1\),)e(and)i(the)g(ob)s (ject)f(is)i(up)q(dated)f(b)o(y)g(the)g(pro)q(cedure)h(call,)g(then)75 2598 y(the)g(argumen)o(t)g(is)g(mark)o(ed)g Fk(OUT)p Fx(.)g(It)h(is)f(mark)o(ed)g(this)h(w)o(a)o(y)e(ev)o(en)i(though)f(the) g(handle)h(itself)h(is)e(not)75 2654 y(mo)q(di\014ed)c(|)f(w)o(e)f(use) h(the)g Fk(OUT)g Fx(attribute)f(to)g(denote)h(that)f(what)g(the)g (handle)i Fl(r)n(efer)n(enc)n(es)c Fx(is)j(up)q(dated.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 15 7 14 bop 75 -100 a Fm(2.3.)34 b(SEMANTIC)15 b(TERMS)1244 b Fx(7)166 45 y(The)17 b(de\014nition)i(of)d Fv(MPI)g Fx(tries)h(to)f(a)o(v)o(oid,)h(to)f(the)h(largest)f(p)q(ossible)j (exten)o(t,)d(the)h(use)g(of)f Fk(INOUT)75 102 y Fx(argumen)o(ts,)e(b)q (ecause)i(suc)o(h)g(use)f(is)h(error-prone,)e(esp)q(ecially)k(for)d (scalar)g(argumen)o(ts.)166 158 y(A)k(common)f(o)q(ccurrence)h(for)f Fv(MPI)g Fx(functions)i(is)f(an)f(argumen)o(t)g(that)g(is)h(used)g(as)f Fk(IN)h Fx(b)o(y)f(some)75 214 y(pro)q(cesses)12 b(and)g Fk(OUT)f Fx(b)o(y)h(other)f(pro)q(cesses.)19 b(Suc)o(h)12 b(argumen)o(t)f(is,)h(syn)o(tactically)l(,)h(an)f Fk(INOUT)g Fx(argumen)o(t)75 271 y(and)j(is)h(mark)o(ed)e(as)g(suc)o(h,)h (although,)g(seman)o(tically)l(,)h(it)f(is)h(not)e(used)i(in)f(one)g (call)i(b)q(oth)e(for)f(input)i(and)75 327 y(for)f(output.)166 384 y(Another)h(frequen)o(t)g(situation)h(arises)g(when)f(an)g(argumen) o(t)g(v)m(alue)h(is)g(needed)h(only)f(b)o(y)f(a)g(subset)75 440 y(of)f(the)g(pro)q(cesses.)20 b(When)c(an)f(argumen)o(t)f(is)i(not) f(signi\014can)o(t)h(at)e(a)h(pro)q(cess)h(then)f(an)g(arbitrary)g(v)m (alue)75 497 y(can)g(b)q(e)h(passed)g(as)e(argumen)o(t.)166 553 y(Unless)h(sp)q(eci\014ed)g(otherwise,)f(an)g(argumen)o(t)f(of)g(t) o(yp)q(e)h Fk(OUT)g Fx(or)f(t)o(yp)q(e)h Fk(INOUT)g Fx(cannot)g(b)q(e)g (aliased)75 610 y(with)h(an)o(y)f(other)g(argumen)o(t)g(passed)g(to)g (an)g Fv(MPI)g Fx(pro)q(cedure.)21 b(An)15 b(example)g(of)f(argumen)o (t)f(aliasing)j(in)75 666 y(C)f(app)q(ears)g(b)q(elo)o(w.)21 b(If)15 b(w)o(e)g(de\014ne)h(a)f(C)g(pro)q(cedure)h(lik)o(e)h(this,)75 760 y Fu(void)23 b(copyIntBuffer\()f(int)i(*pin,)f(int)g(*pout,)g(int)h (len)f(\))75 816 y({)71 b(int)24 b(i;)170 873 y(for)g(\(i=0;)f(i)i Fx(to)e(represen)o(t)g(a)g(c)o(hoice)h(v)m(ariable,)g(for)f(C,)f (w)o(e)h(use)h Fv(\(void)e(*\))p Fx(.)75 223 y Fj(2.4.6)49 b(Addresses)75 309 y Fx(Some)22 b Fv(MPI)f Fx(pro)q(cedures)h(use)g Fl(addr)n(ess)g Fx(argumen)o(ts)e(that)h(represen)o(t)h(an)g(absolute)g (address)f(in)i(the)75 366 y(calling)16 b(program.)i(The)d(datat)o(yp)q (e)e(of)h(suc)o(h)g(an)g(argumen)o(t)g(is)g(an)g(in)o(teger)g(of)g(the) g(size)h(needed)h(to)d(hold)75 422 y(an)o(y)i(v)m(alid)i(address)e(in)h (the)f(execution)i(en)o(vironmen)o(t.)75 565 y Fn(2.5)59 b(Language)19 b(Binding)75 667 y Fx(This)e(section)g(de\014nes)h(the)f (rules)g(for)f Fv(MPI)g Fx(language)h(binding)i(in)e(general)g(and)g (for)f(F)l(ortran)g(77)g(and)75 723 y(ANSI)d(C)e(in)i(particular.)19 b(De\014ned)13 b(here)f(are)g(v)m(arious)g(ob)s(ject)g(represen)o (tations,)g(as)f(w)o(ell)i(as)f(the)g(naming)75 780 y(con)o(v)o(en)o (tions)21 b(used)g(for)e(expressing)j(this)f(standard.)35 b(The)21 b(actual)g(calling)h(sequences)f(are)g(de\014ned)75 836 y(elsewhere.)166 893 y(It)i(is)g(exp)q(ected)h(that)e(an)o(y)g(F)l (ortran)g(90)g(and)h(C++)g(implemen)o(tations)h(use)f(the)g(F)l(ortran) f(77)75 949 y(and)f(ANSI)h(C)e(bindings,)k(resp)q(ectiv)o(ely)l(.)39 b(Although)22 b(w)o(e)e(consider)i(it)f(premature)g(to)f(de\014ne)i (other)75 1005 y(bindings)e(to)d(F)l(ortran)g(90)g(and)h(C++,)h(the)f (curren)o(t)f(bindings)j(are)e(designed)h(to)e(encourage,)i(rather)75 1062 y(than)c(discourage,)g(exp)q(erimen)o(tation)i(with)e(b)q(etter)g (bindings)i(that)e(migh)o(t)g(b)q(e)h(adopted)f(later.)166 1118 y(Since)i(the)e(w)o(ord)g(P)l(ARAMETER)h(is)g(a)f(k)o(eyw)o(ord)g (in)h(the)g(F)l(ortran)e(language,)h(w)o(e)g(use)h(the)g(w)o(ord)75 1175 y(\\argumen)o(t")21 b(to)i(denote)g(the)g(argumen)o(ts)e(to)i(a)f (subroutine.)44 b(These)23 b(are)f(normally)i(referred)f(to)75 1231 y(as)e(parameters)f(in)h(C,)g(ho)o(w)o(ev)o(er,)g(w)o(e)f(exp)q (ect)i(that)e(C)h(programmers)f(will)i(understand)g(the)f(w)o(ord)75 1288 y(\\argumen)o(t")c(\(whic)o(h)i(has)f(no)g(sp)q(eci\014c)i (meaning)f(in)g(C\),)e(th)o(us)h(allo)o(wing)h(us)g(to)e(a)o(v)o(oid)h (unnecessary)75 1344 y(confusion)e(for)f(F)l(ortran)f(programmers.)166 1401 y(There)k(are)g(sev)o(eral)g(imp)q(ortan)o(t)f(language)h(binding) i(issues)f(not)e(addressed)i(b)o(y)e(this)i(standard.)75 1457 y(This)11 b(standard)g(do)q(es)g(not)g(discuss)g(the)g(in)o(terop) q(erabilit)o(y)i(of)e(message)f(passing)h(b)q(et)o(w)o(een)g (languages.)19 b(It)75 1514 y(is)14 b(fully)g(exp)q(ected)h(that)d(man) o(y)h(implemen)o(tations)i(will)f(ha)o(v)o(e)f(suc)o(h)h(features,)f (and)g(that)g(suc)o(h)g(features)75 1570 y(are)i(a)g(sign)g(of)g(the)h (qualit)o(y)f(of)g(the)g(implemen)o(tation.)75 1692 y Fj(2.5.1)49 b(F)o(o)o(rtran)15 b(77)i(Binding)h(Issues)75 1777 y Fx(All)j Fv(MPI)d Fx(names)h(ha)o(v)o(e)g(an)g Fu(MPI)p 648 1777 15 2 v 36 w Fx(pre\014x,)h(and)g(all)g(c)o(haracters) e(are)h(capitals.)33 b(Programs)18 b(m)o(ust)g(not)75 1834 y(declare)13 b(v)m(ariables)h(or)e(functions)h(with)f(names)g(b)q (eginning)j(with)d(the)h(pre\014x,)g Fu(MPI)p 1489 1834 V 17 w Fx(.)18 b(This)13 b(is)g(mandated)75 1890 y(to)i(a)o(v)o(oid)g (p)q(ossible)h(name)g(collisions.)166 1947 y(All)h Fv(MPI)f Fx(F)l(ortran)e(subroutines)j(ha)o(v)o(e)f(a)g(return)g(co)q(de)g(in)h (the)f(last)g(argumen)o(t.)22 b(A)16 b(few)f Fv(MPI)h Fx(op-)75 2003 y(erations)e(are)f(functions,)i(whic)o(h)f(do)g(not)f (ha)o(v)o(e)h(the)g(return)g(co)q(de)g(argumen)o(t.)19 b(The)14 b(return)f(co)q(de)i(v)m(alue)75 2060 y(for)d(successful)h (completion)g(is)g Fi(MPI)p 694 2060 13 2 v 14 w(SUCCESS)p Fx(.)e(Other)h(error)g(co)q(des)g(are)g(implemen)o(tation)i(dep)q (enden)o(t;)75 2116 y(see)h(Chapter)g(7.)166 2173 y(Handles)k(are)e (represen)o(ted)h(in)g(F)l(ortran)f(as)g Fu(INTEGER)p Fx(s.)26 b(Binary-v)m(alued)20 b(v)m(ariables)e(are)g(of)f(t)o(yp)q(e) 75 2229 y Fu(LOGICAL)p Fx(.)166 2286 y(Arra)o(y)d(argumen)o(ts)h(are)f (indexed)j(from)e(one.)166 2342 y(Unless)e(explicitly)i(stated,)d(the)g Fv(MPI)g Fx(F77)f(binding)k(is)d(consisten)o(t)h(with)f(ANSI)h (standard)f(F)l(ortran)75 2398 y(77.)21 b(There)16 b(are)f(sev)o(eral)h (p)q(oin)o(ts)g(where)g(this)h(standard)e(div)o(erges)h(from)f(the)h (ANSI)h(F)l(ortran)d(77)h(stan-)75 2455 y(dard.)20 b(These)14 b(exceptions)i(are)e(consisten)o(t)g(with)h(common)f(practice)h(in)g (the)f(F)l(ortran)g(comm)o(unit)o(y)l(.)19 b(In)75 2511 y(particular:)143 2605 y Ft(\017)k Fv(MPI)14 b Fx(iden)o(ti\014ers)j (are)e(limited)i(to)d(thirt)o(y)l(,)h(not)g(six,)g(signi\014can)o(t)h (c)o(haracters.)143 2699 y Ft(\017)23 b Fv(MPI)14 b Fx(iden)o (ti\014ers)j(ma)o(y)d(con)o(tain)i(underscores)f(after)g(the)g(\014rst) g(c)o(haracter.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 11 19 11 18 bop 75 -100 a Fm(2.5.)34 b(LANGUA)o(GE)15 b(BINDING)1165 b Fx(11)290 81 y Fu(double)23 b(precision)f(a)290 138 y(integer)h(b)290 194 y(...)290 251 y(call)g(MPI_send\(a,...\))290 307 y(call)g(MPI_send\(b,...\))75 455 y Fx(Figure)14 b(2.1:)k(An)c(example)h(of)e(calling)i(a)f(routine)g(with)g(mismatc)o (hed)g(formal)f(and)h(actual)g(argumen)o(ts.)143 588 y Ft(\017)23 b Fx(An)c Fv(MPI)g Fx(subroutine)h(with)g(a)f(c)o(hoice)h (argumen)o(t)f(ma)o(y)g(b)q(e)h(called)g(with)g(di\013eren)o(t)g (argumen)o(t)189 645 y(t)o(yp)q(es.)32 b(An)20 b(example)g(is)g(sho)o (wn)f(in)h(Figure)g(2.1.)32 b(This)20 b(violates)g(the)f(letter)h(of)f (the)g(F)l(ortran)189 701 y(standard,)c(but)h(suc)o(h)h(a)f(violation)h (is)f(common)g(practice.)23 b(An)16 b(alternativ)o(e)h(w)o(ould)f(b)q (e)h(to)e(ha)o(v)o(e)189 758 y(a)g(separate)f(v)o(ersion)i(of)e Fv(MPI)p 697 758 14 2 v 16 w(SEND)i Fx(for)e(eac)o(h)i(data)e(t)o(yp)q (e.)143 851 y Ft(\017)23 b Fx(Although)10 b(not)g(required,)i(it)e(is)h (strongly)f(suggested)g(that)f(named)i Fv(MPI)e Fx(constan)o(ts)g(\()p Fu(PARAMETER)p Fx(s\))189 908 y(b)q(e)14 b(pro)o(vided)g(in)g(an)g (include)i(\014le,)e(called)h Fu(mpif.h)p Fx(.)k(On)14 b(systems)f(that)f(do)i(not)f(supp)q(ort)h(include)189 964 y(\014les,)h(the)h(implemen)o(tation)g(should)g(sp)q(ecify)h(the)e (v)m(alues)i(of)d(named)i(constan)o(ts.)143 1058 y Ft(\017)23 b Fx(V)l(endors)c(are)g(encouraged)h(to)f(pro)o(vide)h(t)o(yp)q(e)f (declarations)h(in)g(the)g Fu(mpif.h)e Fx(\014le)j(on)e(F)l(ortran)189 1115 y(systems)i(that)f(supp)q(ort)i(user-de\014ned)h(t)o(yp)q(es.)39 b(One)22 b(should)h(de\014ne,)h(if)e(p)q(ossible,)i(the)e(t)o(yp)q(e) 189 1171 y Fv(MPI)p 274 1171 V 15 w(ADDRESS)p Fx(,)e(whic)o(h)f(is)g (an)g Fv(INTEGER)g Fx(of)f(the)h(size)h(needed)g(to)e(hold)h(an)g (address)f(in)i(the)189 1227 y(execution)14 b(en)o(vironmen)o(t.)19 b(On)14 b(systems)e(where)i(t)o(yp)q(e)f(de\014nition)h(is)g(not)f (supp)q(orted,)g(it)h(is)f(up)h(to)189 1284 y(the)h(user)g(to)f(use)h (an)g Fv(INTEGER)h Fx(of)f(the)g(righ)o(t)f(kind)i(to)f(represen)o(t)g (addresses)g(\(i.e.,)f Fu(INTEGER*4)189 1340 y Fx(on)h(a)g(32)f(bit)i (mac)o(hine,)g Fu(INTEGER*8)e Fx(on)h(a)g(64)f(bit)i(mac)o(hine,)g (etc.\).)75 1499 y Fj(2.5.2)49 b(C)16 b(Binding)i(Issues)75 1585 y Fx(W)l(e)f(use)g(the)g(ANSI)h(C)f(declaration)g(format.)24 b(All)18 b Fv(MPI)e Fx(names)h(ha)o(v)o(e)g(an)f Fu(MPI)p 1460 1585 15 2 v 34 w Fx(pre\014x,)h(de\014ned)i(con-)75 1642 y(stan)o(ts)12 b(are)h(in)i(all)f(capital)g(letters,)g(and)f (de\014ned)i(t)o(yp)q(es)f(and)f(functions)h(ha)o(v)o(e)f(one)h (capital)g(letter)g(after)75 1698 y(the)k(pre\014x.)30 b(Programs)17 b(m)o(ust)h(not)g(declare)h(v)m(ariables)g(or)f (functions)h(with)g(names)f(b)q(eginning)j(with)75 1755 y(the)15 b(pre\014x,)h Fu(MPI)p 369 1755 V 16 w Fx(.)k(This)c(is)g (mandated)f(to)f(a)o(v)o(oid)h(p)q(ossible)i(name)e(collisions.)166 1811 y(The)i(de\014nition)h(of)e(named)h(constan)o(ts,)e(function)i (protot)o(yp)q(es,)f(and)h(t)o(yp)q(e)f(de\014nitions)i(m)o(ust)e(b)q (e)75 1867 y(supplied)i(in)e(an)f(include)i(\014le)g Fv(mpi.h)p Fx(.)166 1924 y(Almost)24 b(all)h(C)e(functions)i(return)f (an)g(error)f(co)q(de.)47 b(The)24 b(successful)h(return)f(co)q(de)h (will)g(b)q(e)75 1980 y Fu(MPI)p 150 1980 V 17 w(SUCCESS)p Fx(,)16 b(but)i(failure)h(return)f(co)q(des)g(are)f(implemen)o(tation)i (dep)q(enden)o(t.)29 b(A)18 b(few)g(C)f(functions)75 2037 y(do)e(not)g(return)g(v)m(alues,)h(so)f(that)f(they)i(can)f(b)q(e) h(implemen)o(ted)h(as)e(macros.)166 2093 y(T)o(yp)q(e)g(declarations)h (are)e(pro)o(vided)i(for)e(handles)i(to)e(eac)o(h)h(category)f(of)g (opaque)h(ob)s(jects.)20 b(Either)75 2150 y(a)15 b(p)q(oin)o(ter)h(or)e (an)h(in)o(teger)h(t)o(yp)q(e)f(is)h(used.)166 2206 y(Arra)o(y)e (argumen)o(ts)h(are)f(indexed)j(from)e(zero.)166 2263 y(Logical)j(\015ags)e(are)g(in)o(tegers)h(with)g(v)m(alue)h(0)f (meaning)g(\\false")g(and)g(a)f(non-zero)h(v)m(alue)h(meaning)75 2319 y(\\true.")166 2376 y(Choice)e(argumen)o(ts)e(are)h(p)q(oin)o (ters)h(of)f(t)o(yp)q(e)g Fu(void*)p Fx(.)166 2432 y(Address)f(argumen) o(ts)e(are)h(of)g Fv(MPI)g Fx(de\014ned)i(t)o(yp)q(e)e Fi(MPI)p 1108 2432 13 2 v 14 w(Aint)p Fx(.)20 b(This)14 b(is)f(de\014ned)i(to)e(b)q(e)h(an)f Fi(int)h Fx(of)f(the)75 2489 y(size)j(needed)h(to)d(hold)i(an)o(y)f(v)m(alid)i(address)e(on)g (the)h(target)e(arc)o(hitecture.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 12 20 12 19 bop 75 -100 a Fx(12)692 b Fm(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y Fn(2.6)59 b(Pro)r(cesses)75 147 y Fx(An)15 b Fv(MPI)e Fx(program)g(consists)i(of) f(autonomous)f(pro)q(cesses,)i(executing)g(their)g(o)o(wn)f(co)q(de,)g (in)h(an)g(MIMD)75 203 y(st)o(yle.)29 b(The)18 b(co)q(des)h(executed)g (b)o(y)f(eac)o(h)g(pro)q(cess)h(need)g(not)f(b)q(e)h(iden)o(tical.)30 b(The)19 b(pro)q(cesses)f(comm)o(u-)75 259 y(nicate)h(via)f(calls)h(to) e Fv(MPI)g Fx(comm)o(unication)i(primitiv)o(es.)29 b(T)o(ypically)l(,) 20 b(eac)o(h)e(pro)q(cess)g(executes)h(in)g(its)75 316 y(o)o(wn)g(address)h(space,)h(although)f(shared-memory)f(implemen)o (tations)i(of)f Fv(MPI)f Fx(are)g(p)q(ossible.)35 b(This)75 372 y(do)q(cumen)o(t)20 b(sp)q(eci\014es)i(the)e(b)q(eha)o(vior)h(of)e (a)h(parallel)i(program)d(assuming)h(that)f(only)i Fv(MPI)e Fx(calls)i(are)75 429 y(used)16 b(for)f(comm)o(unication.)22 b(The)16 b(in)o(teraction)g(of)g(an)f Fv(MPI)g Fx(program)g(with)h (other)f(p)q(ossible)j(means)d(of)75 485 y(comm)o(unication)h(\(e.g.,)e (shared)h(memory\))f(is)i(not)f(sp)q(eci\014ed.)166 542 y Fv(MPI)c Fx(do)q(es)h(not)f(sp)q(ecify)i(the)e(execution)i(mo)q(del)f (for)f(eac)o(h)h(pro)q(cess.)19 b(A)11 b(pro)q(cess)h(can)g(b)q(e)g (sequen)o(tial,)75 598 y(or)e(can)h(b)q(e)h(m)o(ulti-threaded,)g(with)f (threads)g(p)q(ossibly)h(executing)g(concurren)o(tly)l(.)20 b(Care)10 b(has)g(b)q(een)i(tak)o(en)75 655 y(to)h(mak)o(e)g Fv(MPI)f Fx(\\thread-safe,")h(b)o(y)g(a)o(v)o(oiding)h(the)f(use)h(of)f (implicit)j(state.)i(The)c(desired)h(in)o(teraction)e(of)75 711 y Fv(MPI)j Fx(with)g(threads)g(is)h(that)e(concurren)o(t)i(threads) f(b)q(e)h(all)g(allo)o(w)o(ed)f(to)g(execute)h Fv(MPI)e Fx(calls,)i(and)g(calls)75 768 y(b)q(e)g(reen)o(tran)o(t;)e(a)h(blo)q (c)o(king)h Fv(MPI)f Fx(call)h(blo)q(c)o(ks)g(only)g(the)f(in)o(v)o (oking)h(thread,)f(allo)o(wing)h(the)f(sc)o(heduling)75 824 y(of)f(another)g(thread.)166 880 y Fv(MPI)j Fx(do)q(es)g(not)g(pro) o(vide)h(mec)o(hanisms)g(to)e(sp)q(ecify)j(the)e(initial)j(allo)q (cation)e(of)f(pro)q(cesses)g(to)g(an)75 937 y Fv(MPI)c Fx(computation)g(and)h(their)g(binding)i(to)d(ph)o(ysical)h(pro)q (cessors.)20 b(It)14 b(is)h(exp)q(ected)h(that)e(v)o(endors)g(will)75 993 y(pro)o(vide)20 b(mec)o(hanisms)h(to)e(do)g(so)h(either)g(at)f (load)h(time)g(or)g(at)f(run)h(time.)34 b(Suc)o(h)20 b(mec)o(hanisms)g(will)75 1050 y(allo)o(w)c(the)f(sp)q(eci\014cation)j (of)d(the)h(initial)i(n)o(um)o(b)q(er)d(of)h(required)g(pro)q(cesses,)g (the)g(co)q(de)g(to)f(b)q(e)h(executed)75 1106 y(b)o(y)k(eac)o(h)h (initial)i(pro)q(cess,)e(and)g(the)g(allo)q(cation)g(of)f(pro)q(cesses) h(to)f(pro)q(cessors.)36 b(Also,)21 b(the)g(curren)o(t)75 1163 y(prop)q(osal)f(do)q(es)h(not)e(pro)o(vide)i(for)e(dynamic)i (creation)f(or)g(deletion)h(of)f(pro)q(cesses)g(during)h(program)75 1219 y(execution)c(\(the)f(total)f(n)o(um)o(b)q(er)i(of)f(pro)q(cesses) g(is)h(\014xed\),)f(although)g(it)h(is)f(in)o(tended)i(to)d(b)q(e)i (consisten)o(t)75 1276 y(with)e(suc)o(h)h(extensions.)k(Finally)l(,)d (w)o(e)e(alw)o(a)o(ys)f(iden)o(tify)i(pro)q(cesses)g(according)f(to)g (their)h(relativ)o(e)f(rank)75 1332 y(in)h(a)f(group,)f(that)h(is,)g (consecutiv)o(e)h(in)o(tegers)g(in)g(the)f(range)g Fu(0..groupsize-1)p Fx(.)75 1473 y Fn(2.7)59 b(Erro)n(r)21 b(Handling)75 1574 y Fv(MPI)11 b Fx(pro)o(vides)i(the)f(user)g(with)h(reliable)h (message)d(transmission.)19 b(A)13 b(message)e(sen)o(t)h(is)g(alw)o(a)o (ys)g(receiv)o(ed)75 1631 y(correctly)l(,)17 b(and)g(the)g(user)g(do)q (es)g(not)f(need)i(to)e(c)o(hec)o(k)h(for)f(transmission)h(errors,)f (time-outs,)h(or)f(other)75 1687 y(error)c(conditions.)20 b(In)13 b(other)g(w)o(ords,)f Fv(MPI)g Fx(do)q(es)h(not)f(pro)o(vide)h (mec)o(hanisms)g(for)f(dealing)i(with)f(failures)75 1744 y(in)i(the)f(comm)o(unication)g(system.)19 b(If)14 b(the)g Fv(MPI)f Fx(implemen)o(tation)i(is)g(built)g(on)e(an)h(unreliable)i (underly-)75 1800 y(ing)g(mec)o(hanism,)f(then)h(it)f(is)h(the)f(job)g (of)g(the)g(implemen)o(tor)h(of)f(the)g Fv(MPI)g Fx(subsystem)g(to)f (insulate)j(the)75 1857 y(user)12 b(from)g(this)g(unreliabili)q(t)o(y)l (,)j(or)c(to)h(re\015ect)g(unreco)o(v)o(erable)h(errors)f(as)f (failures.)20 b(Whenev)o(er)13 b(p)q(ossible,)75 1913 y(suc)o(h)k(failures)g(will)h(b)q(e)g(re\015ected)f(as)f(errors)g(in)h (the)g(relev)m(an)o(t)g(comm)o(unication)g(call.)25 b(Similarly)l(,)19 b Fv(MPI)75 1970 y Fx(itself)e(pro)o(vides)f(no)g(mec)o(hanisms)g(for)g (handling)h(pro)q(cessor)f(failures.)23 b(The)16 b(error)f(handling)j (facilities)75 2026 y(describ)q(ed)c(in)f(section)g(7.2)e(can)i(b)q(e)f (used)h(to)f(restrict)g(the)g(scop)q(e)h(of)f(an)g(unreco)o(v)o(erable) h(error,)e(or)h(design)75 2083 y(error)i(reco)o(v)o(ery)h(at)g(the)g (application)i(lev)o(el.)166 2139 y(Of)g(course,)g Fv(MPI)f Fx(programs)g(ma)o(y)g(still)i(b)q(e)g(erroneous.)24 b(A)17 b Fq(program)i(error)c Fx(can)i(o)q(ccur)g(when)75 2195 y(an)f Fv(MPI)g Fx(call)i(is)e(called)i(with)f(an)f(incorrect)h (argumen)o(t)f(\(non-existing)h(destination)g(in)h(a)e(send)h(op)q(er-) 75 2252 y(ation,)g(bu\013er)g(to)q(o)f(small)h(in)h(a)e(receiv)o(e)i (op)q(eration,)f(etc.\))25 b(This)17 b(t)o(yp)q(e)g(of)f(error)g(w)o (ould)h(o)q(ccur)h(in)f(an)o(y)75 2308 y(implemen)o(tation.)31 b(In)19 b(addition,)h(a)f Fq(resource)h(error)d Fx(ma)o(y)h(o)q(ccur)h (when)g(a)f(program)f(exceeds)j(the)75 2365 y(amoun)o(t)d(of)f(a)o(v)m (ailable)j(system)e(resources)h(\(n)o(um)o(b)q(er)f(of)g(p)q(ending)i (messages,)e(system)g(bu\013ers,)g(etc.\).)75 2421 y(The)h(o)q (ccurrence)h(of)f(this)g(t)o(yp)q(e)g(of)g(error)f(dep)q(ends)j(on)e (the)g(amoun)o(t)f(of)g(a)o(v)m(ailable)j(resources)e(in)h(the)75 2478 y(system)13 b(and)h(the)g(resource)f(allo)q(cation)i(mec)o(hanism) f(used;)h(this)f(ma)o(y)f(di\013er)h(from)f(system)g(to)g(system.)75 2534 y(A)k(high-qualit)o(y)h(implemen)o(tation)h(will)f(pro)o(vide)g (generous)f(limits)h(on)f(the)g(imp)q(ortan)o(t)f(resources)h(so)75 2591 y(as)e(to)f(alleviate)j(the)e(p)q(ortabilit)o(y)h(problem)g(this)g (represen)o(ts.)166 2647 y(Almost)11 b(all)h Fv(MPI)f Fx(calls)i(return)e(a)g(co)q(de)h(that)f(indicates)h(successful)h (completion)g(of)e(the)g(op)q(eration.)75 2704 y(Whenev)o(er)20 b(p)q(ossible,)j Fv(MPI)c Fx(calls)i(return)f(an)g(error)f(co)q(de,)i (if)g(an)f(error)f(o)q(ccurred)h(during)h(the)f(call.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 13 21 13 20 bop 75 -100 a Fm(2.8.)34 b(IMPLEMENT)l(A)l(TION)16 b(ISSUES)1045 b Fx(13)75 45 y(By)18 b(default,)h(an)g(error)e(detected) i(during)g(the)f(execution)h(of)f(the)g Fv(MPI)g Fx(library)h(causes)f (the)g(parallel)75 102 y(computation)d(to)g(ab)q(ort.)k(Ho)o(w)o(ev)o (er,)14 b Fv(MPI)g Fx(pro)o(vides)i(mec)o(hanisms)f(for)g(users)g(to)g (c)o(hange)g(this)g(default)75 158 y(and)i(to)g(handle)h(reco)o(v)o (erable)f(errors.)25 b(The)17 b(user)g(ma)o(y)f(sp)q(ecify)j(that)d(no) h(error)f(is)i(fatal,)e(and)i(handle)75 214 y(error)h(co)q(des)h (returned)h(b)o(y)e Fv(MPI)h Fx(calls)g(b)o(y)g(himself)h(or)f (herself.)34 b(Also,)21 b(the)f(user)g(ma)o(y)f(pro)o(vide)h(his)75 271 y(or)f(her)h(o)o(wn)f(error-handling)i(routines,)f(whic)o(h)h(will) g(b)q(e)f(in)o(v)o(ok)o(ed)g(whenev)o(er)g(an)f Fv(MPI)g Fx(call)i(returns)75 327 y(abnormally)l(.)g(The)15 b Fv(MPI)g Fx(error)f(handling)j(facilities)g(are)e(describ)q(ed)i(in)f (section)g(7.2.)166 384 y(Sev)o(eral)21 b(factors)f(limit)j(the)e (abilit)o(y)h(of)f Fv(MPI)f Fx(calls)i(to)e(return)h(with)h(meaningful) g(error)e(co)q(des)75 440 y(when)15 b(an)g(error)f(o)q(ccurs.)20 b Fv(MPI)14 b Fx(ma)o(y)h(not)f(b)q(e)h(able)h(to)e(detect)h(some)g (errors;)f(other)g(errors)g(ma)o(y)g(b)q(e)i(to)q(o)75 497 y(exp)q(ensiv)o(e)j(to)e(detect)g(in)i(normal)e(execution)i(mo)q (de;)f(\014nally)h(some)e(errors)g(ma)o(y)g(b)q(e)h(\\catastrophic")75 553 y(and)d(ma)o(y)g(prev)o(en)o(t)g Fv(MPI)f Fx(from)h(returning)h (con)o(trol)f(to)f(the)h(caller)i(in)f(a)f(consisten)o(t)g(state.)166 610 y(Another)c(subtle)g(issue)h(arises)f(b)q(ecause)h(of)f(the)f (nature)h(of)g(async)o(hronous)f(comm)o(unications:)19 b Fv(MPI)75 666 y Fx(calls)13 b(ma)o(y)e(initiate)i(op)q(erations)f (that)f(con)o(tin)o(ue)i(async)o(hronously)f(after)f(the)h(call)h (returned.)19 b(Th)o(us,)12 b(the)75 723 y(op)q(eration)i(ma)o(y)g (return)g(with)g(a)g(co)q(de)h(indicating)h(successful)f(completion,)g (y)o(et)f(later)g(cause)h(an)f(error)75 779 y(exception)h(to)e(b)q(e)h (raised.)20 b(If)14 b(there)f(is)i(a)e(subsequen)o(t)h(call)h(that)e (relates)h(to)f(the)g(same)h(op)q(eration)g(\(e.g.,)75 835 y(a)g(call)h(that)e(v)o(eri\014es)i(that)e(an)h(async)o(hronous)g (op)q(eration)g(has)g(completed\))h(then)f(the)g(error)g(argumen)o(t)75 892 y(asso)q(ciated)19 b(with)f(this)h(call)g(will)h(b)q(e)f(used)g(to) f(indicate)i(the)e(nature)g(of)g(the)h(error.)28 b(In)19 b(a)f(few)g(cases,)75 948 y(the)e(error)g(ma)o(y)g(o)q(ccur)g(after)g (all)h(calls)h(that)d(relate)i(to)e(the)i(op)q(eration)f(ha)o(v)o(e)g (completed,)h(so)f(that)g(no)75 1005 y(error)d(v)m(alue)j(can)e(b)q(e)h (used)g(to)e(indicate)j(the)f(nature)f(of)f(the)i(error)e(\(e.g.,)g(an) h(error)g(in)h(a)f(send)g(with)h(the)75 1061 y(ready)f(mo)q(de\).)20 b(Suc)o(h)15 b(an)f(error)g(m)o(ust)g(b)q(e)h(treated)f(as)g(fatal,)g (since)i(information)e(cannot)g(b)q(e)i(returned)75 1118 y(for)f(the)g(user)g(to)g(reco)o(v)o(er)f(from)h(it.)166 1174 y(This)f(do)q(cumen)o(t)g(do)q(es)g(not)f(sp)q(ecify)i(the)f (state)e(of)h(a)h(computation)f(after)g(an)g(erroneous)h Fv(MPI)f Fx(call)75 1231 y(has)h(o)q(ccurred.)20 b(The)14 b(desired)h(b)q(eha)o(vior)f(is)h(that)e(a)g(relev)m(an)o(t)i(error)e (co)q(de)i(b)q(e)f(returned,)g(and)g(the)g(e\013ect)75 1287 y(of)g(the)g(error)g(b)q(e)h(lo)q(calized)h(to)e(the)g(greatest)f (p)q(ossible)j(exten)o(t.)k(E.g.,)13 b(it)h(is)h(highly)g(desireable)h (that)e(an)75 1344 y(erroneous)j(receiv)o(e)i(call)f(will)h(not)e (cause)h(an)o(y)f(part)f(of)h(the)h(receiv)o(er's)g(memory)e(to)h(b)q (e)h(o)o(v)o(erwritten,)75 1400 y(b)q(ey)o(ond)e(the)f(area)g(sp)q (eci\014ed)i(for)e(receiving)h(the)g(message.)166 1456 y(Implemen)o(tations)f(ma)o(y)f(go)g(b)q(ey)o(ond)h(this)g(do)q(cumen)o (t)g(in)g(supp)q(orting)h(in)f(a)f(meaningful)i(manner)75 1513 y Fv(MPI)k Fx(calls)h(that)e(are)h(de\014ned)i(here)e(to)g(b)q(e)h (erroneous.)35 b(F)l(or)19 b(example,)j Fv(MPI)e Fx(sp)q(eci\014es)i (strict)e(t)o(yp)q(e)75 1569 y(matc)o(hing)g(rules)g(b)q(et)o(w)o(een)g (matc)o(hing)f(send)h(and)g(receiv)o(e)g(op)q(erations:)29 b(it)20 b(is)g(erroneous)f(to)g(send)h(a)75 1626 y(\015oating)c(p)q (oin)o(t)g(v)m(ariable)i(and)e(receiv)o(e)h(an)f(in)o(teger.)22 b(Implemen)o(tations)17 b(ma)o(y)f(go)f(b)q(ey)o(ond)i(these)f(t)o(yp)q (e)75 1682 y(matc)o(hing)d(rules,)i(and)e(pro)o(vide)h(automatic)f(t)o (yp)q(e)h(con)o(v)o(ersion)f(in)i(suc)o(h)e(situations.)20 b(It)14 b(will)h(b)q(e)f(helpful)75 1739 y(to)h(generate)g(w)o(arnings) g(for)f(suc)o(h)i(nonconforming)f(b)q(eha)o(vior.)75 1882 y Fn(2.8)59 b(Implementation)17 b(issues)75 1983 y Fx(There)c(are)g(a)f(n)o(um)o(b)q(er)i(of)e(areas)h(where)g(an)g Fv(MPI)f Fx(implemen)o(tation)i(ma)o(y)f(in)o(teract)f(with)i(the)f(op) q(erating)75 2040 y(en)o(vironmen)o(t)k(and)g(system.)24 b(While)19 b Fv(MPI)d Fx(do)q(es)h(not)f(mandate)h(that)f(an)o(y)g (services)i(\(suc)o(h)f(as)f(I/O)h(or)75 2096 y(signal)g(handling\))g (b)q(e)f(pro)o(vided,)g(it)g(do)q(es)g(strongly)g(suggest)f(the)h(b)q (eha)o(vior)g(to)f(b)q(e)h(pro)o(vided)h(if)f(those)75 2153 y(services)g(are)g(a)o(v)m(ailable.)23 b(This)16 b(is)g(an)f(imp)q(ortan)o(t)h(p)q(oin)o(t)g(in)g(ac)o(hieving)h(p)q (ortabilit)o(y)g(across)d(platforms)75 2209 y(that)g(pro)o(vide)i(the)g (same)e(set)h(of)g(services.)75 2331 y Fj(2.8.1)49 b(Indep)q(endence)15 b(of)i(Basic)g(Runtime)d(Routines)75 2417 y Fv(MPI)f Fx(programs)f(require)i(that)f(library)h(routines)g(that)f(are)g(part)g (of)f(the)i(basic)g(language)g(en)o(vironmen)o(t)75 2473 y(\(suc)o(h)h(as)g Fu(date)f Fx(and)i Fu(write)e Fx(in)i(F)l(ortran)e (and)h Fu(printf)f Fx(and)h Fu(malloc)g Fx(in)h(ANSI)f(C\))g(and)g(are) g(executed)75 2530 y(after)g Fu(MPI)p 258 2530 15 2 v 17 w(INIT)g Fx(and)i(b)q(efore)f Fu(MPI)p 685 2530 V 17 w(FINALIZE)f Fx(op)q(erate)g(indep)q(enden)o(tly)k(and)e(that)e (their)h Fl(c)n(ompletion)75 2586 y Fx(is)g(indep)q(enden)o(t)h(of)e (the)g(action)h(of)e(other)h(pro)q(cesses)h(in)g(an)f Fv(MPI)g Fx(program.)166 2643 y(Note)f(that)g(this)h(in)g(no)f(w)o(a)o (y)g(prev)o(en)o(ts)g(the)g(creation)h(of)f(library)h(routines)g(that)e (pro)o(vide)i(parallel)75 2699 y(services)h(whose)f(op)q(eration)h(is)g (collectiv)o(e.)22 b(Ho)o(w)o(ev)o(er,)14 b(the)h(follo)o(wing)h (program)e(is)i(exp)q(ected)g(to)f(com-)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 14 22 14 21 bop 75 -100 a Fx(14)692 b Fm(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y Fx(plete)j(in)g(an)f(ANSI) h(C)f(en)o(vironmen)o(t)h(regardless)f(of)g(the)g(size)h(of)f Fu(MPI)p 1312 45 15 2 v 16 w(COMM)p 1424 45 V 17 w(WORLD)g Fx(\(assuming)g(that)75 102 y(I/O)f(is)f(a)o(v)m(ailable)i(at)e(the)g (executing)h(no)q(des\).)75 195 y Fu(int)23 b(rank;)75 252 y(MPI_Init\()g(argc,)g(argv)g(\);)75 308 y(MPI_Comm_rank\()f (MPI_COMM_WORLD,)g(&rank)h(\);)75 365 y(if)h(\(rank)f(==)g(0\))h (printf\()f("Starting)f(program\\n")h(\);)75 421 y(MPI_Finalize\(\);)75 515 y Fx(The)15 b(corresp)q(onding)i(F)l(ortran)d(77)g(program)g(is)i (also)f(exp)q(ected)i(to)d(complete.)166 571 y(An)k(example)h(of)f (what)f(is)i Fl(not)j Fx(required)d(is)g(an)o(y)f(particular)h (ordering)f(of)g(the)g(action)g(of)g(these)75 628 y(routines)g(when)g (called)i(b)o(y)d(sev)o(eral)h(tasks.)27 b(F)l(or)17 b(example,)i Fv(MPI)e Fx(mak)o(es)g(neither)i(requiremen)o(ts)f(nor)75 684 y(recommendations)g(for)f(the)g(output)g(from)g(the)h(follo)o(wing) g(program)e(\(again)h(assuming)h(that)f(I/O)g(is)75 741 y(a)o(v)m(ailable)g(at)d(the)i(executing)g(no)q(des\).)75 835 y Fu(MPI_Comm_rank\()22 b(MPI_COMM_WORLD,)g(&rank)h(\);)75 891 y(printf\()g("Output)g(from)g(task)g(rank)h(\045d\\n",)f(rank)g (\);)166 985 y Fx(In)18 b(addition,)h(calls)f(that)f(fail)h(b)q(ecause) g(of)f(resource)h(exhaustion)g(or)f(other)g(error)g(are)g(not)g(con-)75 1041 y(sidered)h(a)f(violation)h(of)e(the)h(requiremen)o(ts)h(here)f (\(ho)o(w)o(ev)o(er,)f(they)h(are)g(required)h(to)e(complete,)i(just)75 1098 y(not)d(to)f(complete)i(successfully\).)75 1220 y Fj(2.8.2)49 b(Interaction)16 b(with)g(signals)i(in)f(POSIX)75 1305 y Fv(MPI)10 b Fx(do)q(es)h(not)g(sp)q(ecify)h(either)f(the)g(in)o (teraction)g(of)f(pro)q(cesses)i(with)f(signals,)h(in)f(a)g(UNIX)g(en)o (vironmen)o(t,)75 1362 y(or)19 b(with)g(other)g(ev)o(en)o(ts)g(that)f (do)h(not)g(relate)g(to)g Fv(MPI)f Fx(comm)o(unication.)33 b(That)18 b(is,)j(signals)f(are)e(not)75 1418 y(signi\014can)o(t)j (from)f(the)g(view)h(p)q(oin)o(t)g(of)f Fv(MPI)p Fx(,)f(and)h(implemen) o(tors)h(should)h(attempt)d(to)h(implemen)o(t)75 1475 y Fv(MPI)15 b Fx(so)f(that)h(signals)h(are)f(transparen)o(t:)k(an)c Fv(MPI)g Fx(call)h(susp)q(ended)h(b)o(y)e(a)g(signal)h(should)g(resume) g(and)75 1531 y(complete)i(after)e(the)h(signal)h(is)f(handled.)27 b(Generally)l(,)19 b(the)e(state)f(of)g(a)h(computation)g(that)f(is)i (visible)75 1588 y(or)d(signi\014can)o(t)h(from)e(the)i(view-p)q(oin)o (t)g(of)f Fv(MPI)f Fx(should)j(only)e(b)q(e)h(a\013ected)f(b)o(y)g Fv(MPI)g Fx(calls.)166 1644 y(The)20 b(in)o(ten)o(t)f(of)g Fv(MPI)g Fx(to)f(b)q(e)i(thread)g(and)f(signal)h(safe)f(has)h(a)f(n)o (um)o(b)q(er)g(of)g(subtle)i(e\013ects.)32 b(F)l(or)75 1701 y(example,)20 b(on)e(Unix)h(systems,)g(a)f(catc)o(hable)h(signal)g (suc)o(h)g(as)f(SIGALRM)h(\(an)f(alarm)h(signal\))g(m)o(ust)75 1757 y(not)f(cause)h(an)g Fv(MPI)f Fx(routine)h(to)f(b)q(eha)o(v)o(e)h (di\013eren)o(tly)h(than)e(it)h(w)o(ould)h(ha)o(v)o(e)e(in)h(the)g (absence)h(of)e(the)75 1813 y(signal.)39 b(Of)21 b(course,)h(if)g(the)f (signal)h(handler)h(issues)f Fv(MPI)e Fx(calls)j(or)d(c)o(hanges)h(the) h(en)o(vironmen)o(t)f(in)75 1870 y(whic)o(h)d(the)f Fv(MPI)g Fx(routine)g(is)h(op)q(erating)g(\(for)e(example,)i(consuming)g(all)g (a)o(v)m(ailable)h(memory)d(space\),)75 1926 y(the)e Fv(MPI)g Fx(routine)g(should)i(b)q(eha)o(v)o(e)e(as)g(appropriate)g (for)f(that)h(situation)g(\(in)h(particular,)g(in)g(this)f(case,)75 1983 y(the)h(b)q(eha)o(vior)h(should)g(b)q(e)g(the)f(same)g(as)g(for)g (a)g(m)o(ultithreaded)h Fv(MPI)e Fx(implemen)o(tation\).)166 2039 y(A)20 b(second)h(e\013ect)g(is)g(that)e(a)h(signal)i(handler)f (that)f(p)q(erforms)g Fv(MPI)g Fx(calls)i(m)o(ust)e(not)g(in)o(terfere) 75 2096 y(with)d(the)g(op)q(eration)h(of)e Fv(MPI)p Fx(.)g(F)l(or)h (example,)h(an)f Fv(MPI)f Fx(receiv)o(e)i(of)f(an)o(y)f(t)o(yp)q(e)h (that)g(o)q(ccurs)g(within)h(a)75 2152 y(signal)13 b(handler)h(m)o(ust) d(not)h(cause)h(erroneous)f(b)q(eha)o(vior)h(b)o(y)f(the)h Fv(MPI)e Fx(implemen)o(tation.)20 b(Note)12 b(that)g(an)75 2209 y(implemen)o(tation)17 b(is)f(p)q(ermitted)g(to)f(prohibit)i(the)f (use)f(of)h Fv(MPI)e Fx(calls)j(from)e(within)i(a)e(signal)h(handler,) 75 2265 y(and)f(is)h(not)f(required)h(to)f(detect)g(suc)o(h)h(use.)166 2322 y(It)e(is)g(highly)h(desirable)g(that)e Fv(MPI)g Fx(not)g(use)h Fu(SIGALRM)p Fx(,)f Fu(SIGFPE)p Fx(,)f(or)h Fu(SIGIO)p Fx(.)g(An)h(implemen)o(tation)75 2378 y(is)h Fl(r)n(e)n(quir)n(e)n(d)j Fx(to)c(clearly)i(do)q(cumen)o(t)f(all)g(of)f (the)h(signals)g(that)f(the)h Fv(MPI)e Fx(implemen)o(tation)j(uses;)f (a)f(go)q(o)q(d)75 2434 y(place)i(for)f(this)g(information)h(is)f(a)g (Unix)h(`)p Fu(man)p Fx(')e(page)h(on)h Fu(MPI)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 15 23 15 22 bop 75 356 a Fo(Chapter)34 b(3)75 564 y Fr(P)m(oin)m(t-to-P)m (oin)m(t)42 b(Comm)m(unication)75 805 y Fn(3.1)59 b(Intro)r(duction)75 906 y Fx(Sending)14 b(and)e(receiving)h(of)f(messages)g(b)o(y)g(pro)q (cesses)g(is)h(the)f(basic)g Fv(MPI)g Fx(comm)o(unication)h(mec)o (hanism.)75 963 y(The)20 b(basic)g(p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o (unication)h(op)q(erations)e(are)g Fq(send)h Fx(and)f Fq(receiv)o(e)p Fx(.)32 b(Their)21 b(use)e(is)75 1019 y(illustrated)e(in)f(the)f(example)h(b)q(elo)o(w.)75 1125 y Fu(#include)23 b("mpi.h")75 1182 y(main\()g(argc,)g(argv)h(\))75 1238 y(int)f(argc;)75 1295 y(char)g(**argv;)75 1351 y({)170 1408 y(char)h(message[20];)170 1464 y(int)g(myrank;)170 1521 y(MPI_Status)f(status;)170 1577 y(MPI_Init\()g(&argc,)g(&argv)g (\);)170 1633 y(MPI_Comm_rank\()f(MPI_COMM_WORLD,)g(&myrank)h(\);)170 1690 y(if)h(\(myrank)f(==)h(0\))95 b(/*)23 b(code)h(for)f(process)g (zero)g(*/)170 1746 y({)266 1803 y(strcpy\(message,"Hello,)e(there"\);) 266 1859 y(MPI_Send\(message,)g(strlen\(message\),)h(MPI_CHAR,)h(1,)g (99,)h(MPI_COMM_WORLD\);)170 1916 y(})170 1972 y(else)382 b(/*)23 b(code)h(for)f(process)g(one)g(*/)170 2029 y({)266 2085 y(MPI_Recv\(message,)e(20,)j(MPI_CHAR,)f(0,)g(99,)h (MPI_COMM_WORLD,)d(&status\);)266 2142 y(printf\("received)h (:\045s:\\n",)g(message\);)170 2198 y(})170 2254 y(MPI_Finalize\(\);)75 2311 y(})166 2417 y Fx(In)17 b(this)h(example,)f(pro)q(cess)h(zero)e (\()p Fv(myrank)e(=)j(0)p Fx(\))f(sends)i(a)e(message)h(to)f(pro)q (cess)h(one)g(using)h(the)75 2474 y Fq(send)f Fx(op)q(eration)g Fv(MPI)p 485 2474 14 2 v 16 w(SEND)p Fx(.)g(The)g(op)q(eration)g(sp)q (eci\014es)i(a)e Fq(send)i(bu\013er)d Fx(in)i(the)f(sender)h(memory)75 2530 y(from)f(whic)o(h)i(the)f(message)g(data)f(is)h(tak)o(en.)28 b(In)19 b(the)f(example)h(ab)q(o)o(v)o(e,)f(the)g(send)g(bu\013er)g (consists)g(of)75 2587 y(the)f(storage)f(con)o(taining)i(the)f(v)m (ariable)i Fv(message)c Fx(in)j(the)f(memory)g(of)f(pro)q(cess)i(zero.) 25 b(The)18 b(lo)q(cation,)75 2643 y(size)k(and)f(t)o(yp)q(e)g(of)g (the)g(send)h(bu\013er)f(are)g(sp)q(eci\014ed)i(b)o(y)e(the)g(\014rst)g (three)g(parameters)g(of)f(the)h(send)75 2699 y(op)q(eration.)29 b(The)19 b(message)f(sen)o(t)g(will)h(con)o(tain)g(the)f(13)g(c)o (haracters)f(of)h(this)h(v)m(ariable.)30 b(In)19 b(addition,)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 16 24 16 23 bop 75 -100 a Fx(16)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fx(the)g(send)h(op)q (eration)g(asso)q(ciates)f(an)g Fq(en)o(v)o(elop)q(e)g Fx(with)h(the)f(message.)26 b(This)17 b(en)o(v)o(elop)q(e)i(sp)q (eci\014es)g(the)75 102 y(message)10 b(destination)h(and)f(con)o(tains) h(distinguishing)h(information)f(that)e(can)i(b)q(e)f(used)h(b)o(y)f (the)h Fq(receiv)o(e)75 158 y Fx(op)q(eration)17 b(to)f(select)i(a)e (particular)i(message.)24 b(The)17 b(last)f(three)h(parameters)f(of)h (the)f(send)i(op)q(eration)75 214 y(sp)q(ecify)f(the)e(en)o(v)o(elop)q (e)h(for)f(the)g(message)g(sen)o(t.)166 271 y(Pro)q(cess)d(one)h(\()p Fv(myr)o(ank)d(=)j(1)p Fx(\))f(receiv)o(es)i(this)f(message)f(with)h (the)f Fq(receiv)o(e)g Fx(op)q(eration)h Fv(MPI)p 1734 271 14 2 v 16 w(RECV)p Fx(.)75 327 y(The)e(message)f(to)h(b)q(e)g (receiv)o(ed)h(is)f(selected)h(according)g(to)e(the)h(v)m(alue)h(of)e (its)h(en)o(v)o(elop)q(e,)i(and)e(the)f(message)75 384 y(data)17 b(is)h(stored)f(in)o(to)g(the)h Fq(receiv)o(e)h(bu\013er)p Fx(.)26 b(In)18 b(the)g(example)g(ab)q(o)o(v)o(e,)f(the)h(receiv)o(e)g (bu\013er)g(consists)75 440 y(of)d(the)h(storage)e(con)o(taining)j(the) e(string)h Fv(message)d Fx(in)k(the)e(memory)h(of)f(pro)q(cess)h(one.) 21 b(The)16 b(\014rst)f(three)75 497 y(parameters)e(of)h(the)g(receiv)o (e)h(op)q(eration)f(sp)q(ecify)i(the)e(lo)q(cation,)g(size)h(and)g(t)o (yp)q(e)f(of)f(the)h(receiv)o(e)h(bu\013er.)75 553 y(The)e(next)g (three)g(parameters)f(are)h(used)g(for)g(selecting)h(the)f(incoming)h (message.)19 b(The)13 b(last)g(parameter)75 610 y(is)j(used)f(to)g (return)g(information)h(on)f(the)g(message)g(just)g(receiv)o(ed.)166 666 y(The)g(next)g(sections)g(describ)q(e)i(the)e(blo)q(c)o(king)h (send)g(and)f(receiv)o(e)h(op)q(erations.)k(W)l(e)15 b(discuss)h(send,)75 723 y(receiv)o(e,)e(blo)q(c)o(king)f(comm)o (unication)g(seman)o(tics,)g(t)o(yp)q(e)g(matc)o(hing)f(requiremen)o (ts,)h(t)o(yp)q(e)g(con)o(v)o(ersion)f(in)75 779 y(heterogeneous)18 b(en)o(vironmen)o(ts,)g(and)f(more)g(general)h(comm)o(unication)g(mo)q (des.)27 b(Non)o(blo)q(c)o(king)19 b(com-)75 835 y(m)o(unication)i(is)f (addressed)g(next,)h(follo)o(w)o(ed)f(b)o(y)g(c)o(hannel-lik)o(e)i (constructs)d(and)h(send-receiv)o(e)i(op)q(er-)75 892 y(ations.)32 b(W)l(e)19 b(then)h(consider)g(general)g(datat)o(yp)q(es)f (that)f(allo)o(w)i(one)f(to)g(transfer)f(e\016cien)o(tly)j(hetero-)75 948 y(geneous)g(and)f(noncon)o(tiguous)h(data.)35 b(W)l(e)21 b(conclude)h(with)f(the)f(description)i(of)e(calls)i(for)e(explicit)75 1005 y(pac)o(king)c(and)f(unpac)o(king)h(of)f(messages.)75 1145 y Fn(3.2)59 b(Blo)r(cking)20 b(Send)f(and)g(Receive)f(Op)r (erations)75 1248 y Fj(3.2.1)49 b(Blo)q(cking)18 b(send)75 1334 y Fx(The)d(syn)o(tax)g(of)g(the)g(blo)q(c)o(king)h(send)g(op)q (eration)g(is)f(giv)o(en)h(b)q(elo)o(w.)75 1485 y Fv(MPI)p 160 1485 V 16 w(SEND\(buf,)g(count,)g(datat)o(yp)q(e,)g(dest,)g(tag,)f (comm)m(\))117 1562 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1633 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts)g(in)g(send)h(bu\013er)h(\(nonnegativ)o(e)e(in)o (te-)905 1690 y(ger\))117 1761 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e (\(handle\))117 1833 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1905 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1976 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 2101 y Fu(int)23 b(MPI)p 245 2101 15 2 v 17 w(Send\(void*)g(buf,)g(int) g(count,)h(MPI)p 979 2101 V 16 w(Datatype)f(datatype,)g(int)g(dest,)393 2157 y(int)h(tag,)f(MPI)p 683 2157 V 17 w(Comm)g(comm\))75 2244 y(MPI)p 150 2244 V 17 w(SEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)h(COMM,)f(IERROR\))170 2300 y()g(BUF\(*\))170 2356 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 2443 y Fx(The)15 b(blo)q(c)o(king)i(seman)o(tics)e(of)g(this)h(call)g (are)f(describ)q(ed)i(in)f(Sec.)k(3.4.)75 2561 y Fj(3.2.2)49 b(Message)16 b(data)75 2647 y Fx(The)e(send)g(bu\013er)g(sp)q (eci\014ed)i(b)o(y)e(the)f Fv(MPI)p 803 2647 14 2 v 16 w(SEND)h Fx(op)q(eration)g(consists)g(of)f Fv(count)i Fx(successiv)o(e)g(en)o(tries)f(of)75 2704 y(the)f(t)o(yp)q(e)h (indicated)h(b)o(y)e Fv(datat)o(yp)q(e)p Fx(,)i(starting)e(with)h(the)f (en)o(try)g(at)g(address)h Fv(buf)p Fx(.)20 b(Note)13 b(that)g(w)o(e)g(sp)q(ecify)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 17 25 17 24 bop 75 -100 a Fm(3.2.)34 b(BLOCKING)17 b(SEND)e(AND)g(RECEIVE)g (OPERA)l(TIONS)598 b Fx(17)75 45 y(the)19 b(message)f(length)i(in)g (terms)e(of)g(n)o(um)o(b)q(er)h(of)g Fl(elements)p Fx(,)f(not)g(n)o(um) o(b)q(er)i(of)e Fl(bytes)p Fx(.)30 b(The)19 b(former)f(is)75 102 y(mac)o(hine)e(indep)q(enden)o(t)i(and)d(closer)h(to)e(the)h (application)i(lev)o(el.)166 158 y(The)g(data)f(part)g(of)g(the)g (message)h(consists)f(of)h(a)f(sequence)i(of)e Fv(count)i Fx(v)m(alues,)f(eac)o(h)g(of)f(the)h(t)o(yp)q(e)75 214 y(indicated)j(b)o(y)e Fv(datat)o(yp)q(e)p Fx(.)31 b Fv(count)20 b Fx(ma)o(y)d(b)q(e)i(zero,)g(in)g(whic)o(h)g(case)g(the)f(data)g(part) f(of)h(the)h(message)e(is)75 271 y(empt)o(y)l(.)i(The)13 b(basic)h(datat)o(yp)q(es)e(that)h(can)g(b)q(e)h(sp)q(eci\014ed)h(for)d (message)h(data)f(v)m(alues)i(corresp)q(ond)g(to)e(the)75 327 y(basic)k(datat)o(yp)q(es)f(of)g(the)g(host)g(language.)20 b(P)o(ossible)d(v)m(alues)f(of)f(this)h(argumen)o(t)e(for)h(F)l(ortran) f(and)i(the)75 384 y(corresp)q(onding)g(F)l(ortran)e(t)o(yp)q(es)h(are) g(listed)i(b)q(elo)o(w.)p 472 532 1006 2 v 471 589 2 57 v 497 572 a(MPI)e(datat)o(yp)q(e)p 1046 589 V 294 w(F)l(ortran)f(datat)o(yp)q(e)p 1477 589 V 472 591 1006 2 v 471 647 2 57 v 497 630 a Fv(MPI)p 582 630 14 2 v 16 w(INTEGER)p 1046 647 2 57 v 287 w Fu(INTEGER)p 1477 647 V 471 703 V 497 687 a Fv(MPI)p 582 687 14 2 v 16 w(REAL)p 1046 703 2 57 v 365 w Fu(REAL)p 1477 703 V 471 760 V 497 743 a Fv(MPI)p 582 743 14 2 v 16 w(DOUBLE)p 777 743 V 17 w(PRECISION)p 1046 760 2 57 v 50 w Fu(DOUBLE)23 b(PRECISION)p 1477 760 V 471 816 V 497 799 a Fv(MPI)p 582 799 14 2 v 16 w(COMPLEX)p 1046 816 2 57 v 263 w Fu(COMPLEX)p 1477 816 V 471 873 V 497 856 a Fv(MPI)p 582 856 14 2 v 16 w(LOGICAL)p 1046 873 2 57 v 291 w Fu(LOGICAL)p 1477 873 V 471 929 V 497 912 a Fv(MPI)p 582 912 14 2 v 16 w(CHARA)o(CTER)p 1046 929 2 57 v 211 w Fu(CHARACTER\(1\))p 1477 929 V 471 986 V 497 969 a Fv(MPI)p 582 969 14 2 v 16 w(BYTE)p 1046 986 2 57 v 1477 986 V 471 1042 V 497 1025 a(MPI)p 582 1025 14 2 v 16 w(P)l(A)o(CKED)p 1046 1042 2 57 v 1477 1042 V 472 1044 1006 2 v 75 1214 a Fx(P)o(ossible)16 b(v)m(alues)g(for)f(this)h(argumen)o(t)e(for)h(C)g(and)g(the)g(corresp) q(onding)h(C)f(t)o(yp)q(es)h(are)f(listed)h(b)q(elo)o(w.)p 466 1354 1019 2 v 465 1410 2 57 v 491 1393 a(MPI)f(datat)o(yp)q(e)p 1004 1410 V 259 w(C)g(datat)o(yp)q(e)p 1483 1410 V 466 1412 1019 2 v 465 1468 2 57 v 491 1451 a Fv(MPI)p 576 1451 14 2 v 16 w(CHAR)p 1004 1468 2 57 v 321 w Fu(signed)23 b(char)p 1483 1468 V 465 1525 V 491 1508 a Fv(MPI)p 576 1508 14 2 v 16 w(SHORT)p 1004 1525 2 57 v 291 w Fu(signed)g(short)p 1483 1525 V 465 1581 V 491 1564 a Fv(MPI)p 576 1564 14 2 v 16 w(INT)p 1004 1581 2 57 v 365 w Fu(signed)g(int)p 1483 1581 V 465 1638 V 491 1621 a Fv(MPI)p 576 1621 14 2 v 16 w(LONG)p 1004 1638 2 57 v 321 w Fu(signed)g(long)g(int)p 1483 1638 V 465 1694 V 491 1677 a Fv(MPI)p 576 1677 14 2 v 16 w(UNSIGNED)p 815 1677 V 17 w(CHAR)p 1004 1694 2 57 v 81 w Fu(unsigned)f(char)p 1483 1694 V 465 1750 V 491 1733 a Fv(MPI)p 576 1733 14 2 v 16 w(UNSIGNED)p 815 1733 V 17 w(SHORT)p 1004 1750 2 57 v 51 w Fu(unsigned)g(short)i (int)p 1483 1750 V 465 1807 V 491 1790 a Fv(MPI)p 576 1790 14 2 v 16 w(UNSIGNED)p 1004 1807 2 57 v 218 w Fu(unsigned)e(int)p 1483 1807 V 465 1863 V 491 1846 a Fv(MPI)p 576 1846 14 2 v 16 w(UNSIGNED)p 815 1846 V 17 w(LONG)p 1004 1863 2 57 v 81 w Fu(unsigned)g(long)i(int)p 1483 1863 V 465 1920 V 491 1903 a Fv(MPI)p 576 1903 14 2 v 16 w(FLO)o(A)l(T)p 1004 1920 2 57 v 301 w Fu(float)p 1483 1920 V 465 1976 V 491 1959 a Fv(MPI)p 576 1959 14 2 v 16 w(DOUBLE)p 1004 1976 2 57 v 262 w Fu(double)p 1483 1976 V 465 2033 V 491 2016 a Fv(MPI)p 576 2016 14 2 v 16 w(LONG)p 712 2016 V 16 w(DOUBLE)p 1004 2033 2 57 v 126 w Fu(long)f(double)p 1483 2033 V 465 2089 V 491 2072 a Fv(MPI)p 576 2072 14 2 v 16 w(BYTE)p 1004 2089 2 57 v 1483 2089 V 465 2146 V 491 2129 a(MPI)p 576 2129 14 2 v 16 w(P)l(A)o(CKED)p 1004 2146 2 57 v 1483 2146 V 466 2147 1019 2 v 166 2308 a Fx(The)f(datat)o(yp)q(es)f Fv(MPI)p 566 2308 14 2 v 16 w(BYTE)h Fx(and)g Fv(MPI)p 899 2308 V 15 w(P)l(A)o(CKED)g Fx(do)f(not)g(corresp)q(ond)h(to)f(a)h(F)l(ortran)e(or)h(C)75 2365 y(datat)o(yp)q(e.)42 b(A)23 b(v)m(alue)h(of)e(t)o(yp)q(e)h Fv(MPI)p 742 2365 V 15 w(BYTE)h Fx(consists)f(of)f(a)g(b)o(yte)h(\(8)f (binary)h(digits\).)43 b(A)23 b(b)o(yte)f(is)75 2421 y(unin)o(terpreted)h(and)f(is)h(di\013eren)o(t)f(from)f(a)h(c)o (haracter.)39 b(Di\013eren)o(t)22 b(mac)o(hines)h(ma)o(y)e(ha)o(v)o(e)h (di\013eren)o(t)75 2478 y(represen)o(tations)15 b(for)g(c)o(haracters,) f(or)h(ma)o(y)g(use)h(more)f(than)g(one)g(b)o(yte)h(to)e(represen)o(t)i (c)o(haracters.)j(On)75 2534 y(the)g(other)g(hand,)h(a)f(b)o(yte)g(has) g(the)g(same)g(binary)g(v)m(alue)i(on)e(all)h(mac)o(hines.)32 b(The)20 b(use)f(of)g(the)g(t)o(yp)q(e)75 2591 y Fv(MPI)p 160 2591 V 16 w(P)l(A)o(CKED)c Fx(is)h(explained)h(in)f(Section)g (3.13.)166 2647 y Fv(MPI)c Fx(requires)h(supp)q(ort)g(of)f(the)h(datat) o(yp)q(es)f(listed)i(ab)q(o)o(v)o(e,)e(whic)o(h)h(matc)o(h)f(the)h (basic)g(datat)o(yp)q(es)f(of)75 2704 y(F)l(ortran)h(77)g(and)h(ANSI)h (C.)f(Additional)h Fv(MPI)f Fx(datat)o(yp)q(es)f(should)i(b)q(e)g(pro)o (vided)g(if)f(the)g(host)g(language)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 18 26 18 25 bop 75 -100 a Fx(18)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fx(has)e(additional)i (data)e(t)o(yp)q(es:)20 b Fv(MPI)p 698 45 14 2 v 16 w(LONG)p 834 45 V 16 w(LONG)p 970 45 V 17 w(INT)p Fx(,)15 b(for)g(\(64)f(bit\))i (C)f(in)o(tegers)g(declared)i(to)d(b)q(e)i(of)75 102 y(t)o(yp)q(e)11 b Fv(longlong)f(int)p Fx(;)i Fv(MPI)p 503 102 V 16 w(DOUBLE)p 698 102 V 17 w(COMPLEX)f Fx(for)f(double)i (precision)g(complex)g(in)f(F)l(ortran)f(declared)75 158 y(to)16 b(b)q(e)h(of)f(t)o(yp)q(e)g Fv(DOUBLE)h(PRECISION)p Fx(;)g Fv(MPI)p 892 158 V 16 w(REAL2)p Fx(,)f Fv(MPI)p 1153 158 V 16 w(REAL4)g Fx(and)h Fv(MPI)p 1491 158 V 16 w(REAL8)f Fx(for)g(F)l(ortran)75 214 y(reals,)i(declared)h(to)e(b)q (e)h(of)f(t)o(yp)q(e)h Fv(REAL*2)p Fx(,)g Fv(REAL*4)f Fx(and)h Fv(REAL*8)p Fx(,)g(resp)q(ectiv)o(ely;)i Fv(MPI)p 1650 214 V 16 w(INTEGER1)75 271 y(MPI)p 160 271 V 16 w(INTEGER2)10 b Fx(and)h Fv(MPI)p 564 271 V 16 w(INTEGER4)f Fx(for)g(F)l(ortran)f(in)o(tegers,)i(declared)h(to)d(b)q(e)i(of)f(t)o (yp)q(e)g Fv(INTEGER*1)p Fx(,)75 327 y Fv(INTEGER*2)15 b Fx(and)h Fv(INTEGER*4)p Fx(,)f(resp)q(ectiv)o(ely;)h(etc.)189 407 y Fl(R)n(ationale.)80 b Fx(One)22 b(goal)g(of)f(the)h(design)h(is)g (to)e(allo)o(w)h(for)f Fv(MPI)g Fx(to)h(b)q(e)g(implemen)o(ted)i(as)d (a)189 464 y(library)l(,)d(with)f(no)g(need)h(for)e(additional)j (prepro)q(cessing)f(or)e(compilation.)27 b(Th)o(us,)17 b(one)g(cannot)189 520 y(assume)c(that)g(a)g(comm)o(unication)h(call)g (has)g(information)f(on)h(the)f(datat)o(yp)q(e)g(of)g(v)m(ariables)i (in)f(the)189 576 y(comm)o(unication)20 b(bu\013er;)i(this)f (information)f(m)o(ust)f(b)q(e)i(supplied)h(b)o(y)e(an)g(explicit)i (argumen)o(t.)189 633 y(The)17 b(need)h(for)f(suc)o(h)h(datat)o(yp)q(e) e(information)i(will)h(b)q(ecome)f(clear)g(in)g(Section)g(3.3.2.)24 b(\()p Fl(End)18 b(of)189 689 y(r)n(ationale.)p Fx(\))75 808 y Fj(3.2.3)49 b(Message)16 b(envelop)q(e)75 893 y Fx(In)j(addition)g(to)e(the)h(data)g(part,)f(messages)h(carry)f (information)i(that)e(can)h(b)q(e)h(used)f(to)g(distinguish)75 950 y(messages)12 b(and)h(selectiv)o(ely)i(receiv)o(e)f(them.)19 b(This)14 b(information)f(consists)g(of)f(a)h(\014xed)g(n)o(um)o(b)q (er)h(of)e(\014elds,)75 1006 y(whic)o(h)k(w)o(e)f(collectiv)o(ely)j (call)e(the)f Fq(message)i(en)o(v)o(elop)q(e)p Fx(.)i(These)d(\014elds) g(are)913 1081 y(source)865 1138 y(destination)943 1194 y(tag)834 1251 y(comm)o(unicator)166 1325 y(The)c(message)g(source)g (is)h(implicitly)i(determined)f(b)o(y)e(the)g(iden)o(tit)o(y)h(of)f (the)g(message)g(sender.)20 b(The)75 1382 y(other)15 b(\014elds)h(are)f(sp)q(eci\014ed)i(b)o(y)f(argumen)o(ts)e(in)i(the)f (send)h(op)q(eration.)166 1438 y(The)f(message)g(destination)h(is)g(sp) q(eci\014ed)h(b)o(y)e(the)h Fv(dest)g Fx(argumen)o(t.)166 1495 y(The)h(in)o(teger-v)m(alued)h(message)e(tag)f(is)i(sp)q (eci\014ed)i(b)o(y)d(the)g Fv(tag)h Fx(argumen)o(t.)22 b(This)17 b(in)o(teger)g(can)f(b)q(e)75 1551 y(used)21 b(b)o(y)f(the)h(program)e(to)h(distinguish)i(di\013eren)o(t)f(t)o(yp)q (es)f(of)g(messages.)35 b(The)21 b(range)f(of)g(v)m(alid)i(tag)75 1608 y(v)m(alues)17 b(is)f Fv(0,...,UB)p Fx(,)e(where)i(the)g(v)m(alue) h(of)f Fv(UB)g Fx(is)g(implemen)o(tation)h(dep)q(enden)o(t.)24 b(It)15 b(can)h(b)q(e)h(found)f(b)o(y)75 1664 y(querying)i(the)g(v)m (alue)g(of)f(the)h(attribute)f Fi(MPI)p 870 1664 13 2 v 14 w(T)m(A)o(G)p 964 1664 V 14 w(UB)p Fx(,)g(as)g(describ)q(ed)i(in)f (Chapter)f(7.)26 b Fv(MPI)17 b Fx(requires)75 1721 y(that)d Fv(UB)i Fx(b)q(e)g(no)f(less)h(than)f(32767.)166 1777 y(The)f Fv(comm)9 b Fx(argumen)o(t)k(sp)q(eci\014es)j(the)e Fq(comm)o(unicator)h Fx(that)e(is)i(used)g(for)e(the)h(send)h(op)q (eration.)75 1833 y(Comm)o(unicators)f(are)h(explained)j(in)e(Chapter)e (5;)h(b)q(elo)o(w)h(is)f(a)g(brief)h(summary)f(of)g(their)g(usage.)166 1890 y(A)h(comm)o(unicator)f(sp)q(eci\014es)j(the)d(comm)o(unication)i (con)o(text)e(for)g(a)h(comm)o(unication)g(op)q(eration.)75 1946 y(Eac)o(h)f(comm)o(unication)h(con)o(text)f(pro)o(vides)g(a)g (separate)g(\\comm)o(unication)h(univ)o(erse:")21 b(messages)14 b(are)75 2003 y(alw)o(a)o(ys)h(receiv)o(ed)j(within)f(the)f(con)o(text) f(they)h(w)o(ere)g(sen)o(t,)g(and)g(messages)g(sen)o(t)f(in)i (di\013eren)o(t)g(con)o(texts)75 2059 y(do)e(not)g(in)o(terfere.)166 2116 y(The)20 b(comm)o(unicator)f(also)g(sp)q(eci\014es)i(the)f(set)f (of)g(pro)q(cesses)g(that)g(share)g(this)h(comm)o(unication)75 2172 y(con)o(text.)i(This)17 b Fq(pro)q(cess)h(group)e Fx(is)g(ordered)h(and)f(pro)q(cesses)g(are)g(iden)o(ti\014ed)i(b)o(y)e (their)h(rank)f(within)75 2229 y(this)g(group.)21 b(Th)o(us,)15 b(the)g(range)h(of)f(v)m(alid)i(v)m(alues)f(for)f Fv(dest)i Fx(is)f Fv(0,)f(...)20 b(,)c(n-1)p Fx(,)f(where)h Fv(n)g Fx(is)g(the)g(n)o(um)o(b)q(er)f(of)75 2285 y(pro)q(cesses)i(in)h(the)e (group.)25 b(\(If)16 b(the)h(comm)o(unicator)f(is)i(an)e(in)o(ter-comm) o(unicator,)h(then)g(destinations)75 2342 y(are)e(iden)o(ti\014ed)i(b)o (y)e(their)h(rank)f(in)h(the)f(remote)g(group.)k(See)d(Chapter)f(5.\)) 166 2398 y(A)g(prede\014ned)i(comm)o(unicator)d Fi(MPI)p 808 2398 V 15 w(COMM)p 953 2398 V 14 w(W)o(ORLD)h Fx(is)h(pro)o(vided)g (b)o(y)f Fv(MPI)p Fx(.)f(It)h(allo)o(ws)g(comm)o(u-)75 2454 y(nication)22 b(with)f(all)h(pro)q(cesses)f(that)f(are)h (accessible)i(after)d Fv(MPI)g Fx(initialization)k(and)d(pro)q(cesses)g (are)75 2511 y(iden)o(ti\014ed)c(b)o(y)e(their)h(rank)f(in)h(the)f (group)g(of)g Fi(MPI)p 939 2511 V 14 w(COMM)p 1083 2511 V 15 w(W)o(ORLD)p Fx(.)189 2591 y Fl(A)n(dvic)n(e)j(to)i(users.)60 b Fx(Users)18 b(that)g(are)g(comfortable)g(with)h(the)g(notion)g(of)f (a)g(\015at)g(name)h(space)189 2647 y(for)d(pro)q(cesses,)h(and)g(a)g (single)h(comm)o(unication)g(con)o(text,)e(as)h(o\013ered)f(b)o(y)h (most)f(existing)i(com-)189 2704 y(m)o(unication)d(libraries,)h(need)f (only)h(use)e(the)h(prede\014ned)h(v)m(ariable)g Fi(MPI)p 1437 2704 V 15 w(COMM)p 1582 2704 V 14 w(W)o(ORLD)f Fx(as)f(the)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 19 27 19 26 bop 75 -100 a Fm(3.2.)34 b(BLOCKING)17 b(SEND)e(AND)g(RECEIVE)g (OPERA)l(TIONS)598 b Fx(19)189 45 y Fv(comm)12 b Fx(argumen)o(t.)29 b(This)19 b(will)h(allo)o(w)f(comm)o(unication)g(with)g(all)g(the)g (pro)q(cesses)f(a)o(v)m(ailable)j(at)189 102 y(initialization)d(time.) 189 177 y(Users)h(ma)o(y)g(de\014ne)i(new)f(comm)o(unicators,)g(as)g (explained)h(in)g(Chapter)f(5.)33 b(Comm)o(unicators)189 233 y(pro)o(vide)13 b(an)g(imp)q(ortan)o(t)g(encapsulation)h(mec)o (hanism)f(for)g(libraries)h(and)f(mo)q(dules.)21 b(They)13 b(allo)o(w)189 290 y(mo)q(dules)21 b(to)e(ha)o(v)o(e)h(their)g(o)o(wn)g (disjoin)o(t)h(comm)o(unication)f(univ)o(erse)h(and)g(their)f(o)o(wn)g (pro)q(cess)189 346 y(n)o(um)o(b)q(ering)c(sc)o(heme.)k(\()p Fl(End)c(of)g(advic)n(e)g(to)h(users.)p Fx(\))189 452 y Fl(A)n(dvic)n(e)i(to)h(implementors.)65 b Fx(The)20 b(message)f(en)o(v)o(elop)q(e)h(w)o(ould)g(normally)g(b)q(e)g(enco)q (ded)h(b)o(y)e(a)189 509 y(\014xed-length)d(message)e(header.)20 b(Ho)o(w)o(ev)o(er,)13 b(the)i(actual)g(enco)q(ding)h(is)f(implemen)o (tation)g(dep)q(en-)189 565 y(den)o(t.)k(Some)c(of)e(the)i(information) f(\(e.g.,)f(source)h(or)g(destination\))h(ma)o(y)e(b)q(e)i(implicit,)i (and)d(need)189 622 y(not)j(b)q(e)i(explicitly)i(carried)e(b)o(y)f (messages.)28 b(Also,)19 b(pro)q(cesses)f(ma)o(y)f(b)q(e)i(iden)o (ti\014ed)h(b)o(y)e(relativ)o(e)189 678 y(ranks,)c(or)h(absolute)g (ids,)h(etc.)k(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 800 y Fj(3.2.4)49 b(Blo)q(cking)18 b(receive)75 886 y Fx(The)d(syn)o(tax)g(of)g(the)g(blo)q(c)o(king)h(receiv)o(e)h(op) q(eration)e(is)h(giv)o(en)f(b)q(elo)o(w.)75 1037 y Fv(MPI)p 160 1037 14 2 v 16 w(RECV)h(\(buf,)f(count,)h(datat)o(yp)q(e,)h (source,)e(tag,)g(comm)m(,)d(status\))117 1114 y Fk(OUT)108 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g (\(c)o(hoice\))117 1189 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1264 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1339 y(IN)155 b Fv(source)468 b Fk(rank)14 b(of)f(source)j(\(in)o (teger\))117 1414 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1490 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1565 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1689 y Fu(int)23 b(MPI)p 245 1689 15 2 v 17 w(Recv\(void*)g(buf,)g(int) g(count,)h(MPI)p 979 1689 V 16 w(Datatype)f(datatype,)g(int)g(source,) 393 1746 y(int)h(tag,)f(MPI)p 683 1746 V 17 w(Comm)g(comm,)g(MPI)p 1034 1746 V 17 w(Status)g(*status\))75 1832 y(MPI)p 150 1832 V 17 w(RECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS,)g(IERROR\))170 1888 y()g(BUF\(*\))170 1945 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS\(MPI)p 1463 1945 V 16 w(STATUS)p 1623 1945 V 17 w(SIZE\),)170 2001 y(IERROR)166 2088 y Fx(The)15 b(blo)q(c)o(king)i (seman)o(tics)e(of)g(this)h(call)g(are)f(describ)q(ed)i(in)f(Sec.)k (3.4.)166 2144 y(The)15 b(receiv)o(e)h(bu\013er)f(consists)h(of)e(the)h (storage)f(con)o(taining)i Fv(count)g Fx(consecutiv)o(e)g(elemen)o(ts)g (of)f(the)75 2201 y(t)o(yp)q(e)e(sp)q(eci\014ed)i(b)o(y)e Fv(datat)o(yp)q(e)p Fx(,)h(starting)e(at)h(address)f Fv(buf)p Fx(.)20 b(The)13 b(length)h(of)e(the)h(receiv)o(ed)h(message)f (m)o(ust)75 2257 y(b)q(e)20 b(less)g(than)f(or)g(equal)h(to)f(the)h (length)g(of)f(the)g(receiv)o(e)h(bu\013er.)33 b(An)20 b(o)o(v)o(er\015o)o(w)e(error)g(o)q(ccurs)i(if)g(all)75 2314 y(incoming)c(data)f(do)q(es)h(not)e(\014t,)h(without)g (truncation,)g(in)o(to)g(the)h(receiv)o(e)g(bu\013er.)166 2370 y(If)i(a)f(message)g(that)f(is)i(shorter)f(than)g(the)h(receiv)o (e)g(bu\013er)f(arriv)o(es,)g(then)h(only)g(those)f(lo)q(cations)75 2427 y(corresp)q(onding)f(to)f(the)g(\(shorter\))f(message)h(are)g(mo)q (di\014ed.)189 2533 y Fl(A)n(dvic)n(e)h(to)i(users.)47 b Fx(The)16 b Fv(MPI)p 733 2533 14 2 v 16 w(PROBE)i Fx(function)f (describ)q(ed)h(in)f(Section)h(3.8)d(can)i(b)q(e)g(used)g(to)189 2589 y(receiv)o(e)f(messages)e(of)h(unkno)o(wn)h(length.)k(\()p Fl(End)c(of)g(advic)n(e)g(to)h(users.)p Fx(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 20 28 20 27 bop 75 -100 a Fx(20)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Fl(A)n(dvic)n(e)f(to)i (implementors.)46 b Fx(Ev)o(en)16 b(though)g(no)h(sp)q(eci\014c)h(b)q (eha)o(vior)f(is)f(mandated)h(b)o(y)f Fv(MPI)g Fx(for)189 102 y(erroneous)f(programs,)f(the)h(recommended)h(handling)h(of)e(o)o (v)o(er\015o)o(w)f(situations)h(is)h(to)f(return)g(in)189 158 y Fv(status)21 b Fx(information)e(ab)q(out)g(the)g(source)g(and)g (tag)f(of)h(the)g(incoming)h(message.)31 b(The)19 b(receiv)o(e)189 214 y(op)q(eration)e(will)j(return)d(an)g(error)g(co)q(de.)28 b(A)17 b(qualit)o(y)h(implemen)o(tation)h(will)g(also)f(ensure)g(that) 189 271 y(no)d(memory)f(that)h(is)h(outside)f(the)h(receiv)o(e)g (bu\013er)f(will)i(ev)o(er)e(b)q(e)h(o)o(v)o(erwritten.)189 344 y(In)g(the)f(case)g(of)g(a)g(message)g(shorter)g(than)g(the)g (receiv)o(e)h(bu\013er,)f Fv(MPI)g Fx(is)h(quite)g(strict)f(in)h(that)f (it)189 400 y(allo)o(ws)h(no)g(mo)q(di\014cation)h(of)f(the)g(other)g (lo)q(cations.)23 b(A)16 b(more)g(lenien)o(t)h(statemen)o(t)e(w)o(ould) i(allo)o(w)189 457 y(for)c(some)h(optimizations)g(but)g(this)h(is)f (not)g(allo)o(w)o(ed.)19 b(The)c(implemen)o(tation)g(m)o(ust)e(b)q(e)i (ready)f(to)189 513 y(end)i(a)g(cop)o(y)g(in)o(to)g(the)h(receiv)o(er)f (memory)g(exactly)h(at)e(the)h(end)h(of)f(the)g(receiv)o(e)h(bu\013er,) f(ev)o(en)g(if)189 569 y(it)f(is)h(an)f(o)q(dd)h(address.)k(\()p Fl(End)15 b(of)h(advic)n(e)h(to)f(implementors.)p Fx(\))166 659 y(The)22 b(selection)i(of)d(a)h(message)g(b)o(y)g(a)f(receiv)o(e)i (op)q(eration)g(is)f(go)o(v)o(erned)g(b)o(y)g(the)g(v)m(alue)h(of)f (the)75 715 y(message)13 b(en)o(v)o(elop)q(e.)20 b(A)14 b(message)f(can)g(b)q(e)h(receiv)o(ed)h(b)o(y)e(a)g(receiv)o(e)i(op)q (eration)e(if)h(its)g(en)o(v)o(elop)q(e)g(matc)o(hes)75 772 y(the)e Fv(source)p Fx(,)g Fv(tag)g Fx(and)g Fv(comm)6 b Fx(v)m(alues)13 b(sp)q(eci\014ed)h(b)o(y)e(the)f(receiv)o(e)i(op)q (eration.)19 b(The)12 b(receiv)o(er)h(ma)o(y)e(sp)q(ecify)75 828 y(a)18 b(wildcard)h Fi(MPI)p 380 828 13 2 v 15 w(ANY)p 480 828 V 14 w(SOURCE)e Fx(v)m(alue)i(for)f Fv(source)p Fx(,)h(and/or)e(a)h(wildcard)36 b Fi(MPI)p 1491 828 V 14 w(ANY)p 1590 828 V 15 w(T)m(A)o(G)17 b Fx(v)m(alue)j(for)75 885 y Fv(tag)p Fx(,)12 b(indicating)i(that)d(an)o(y)h(source)g(and/or)f (tag)g(are)g(acceptable.)20 b(It)12 b(cannot)g(sp)q(ecify)h(a)f (wildcard)h(v)m(alue)75 941 y(for)h Fv(comm)m Fx(.)j(Th)o(us,)d(a)g (message)g(can)h(b)q(e)g(receiv)o(ed)g(b)o(y)g(a)f(receiv)o(e)h(op)q (eration)g(only)g(if)g(it)g(is)g(addressed)g(to)75 998 y(the)i(receiving)i(pro)q(cess,)e(has)g(a)g(matc)o(hing)g(comm)o (unicator,)f(has)h(matc)o(hing)g(source)g(unless)i(source=)75 1054 y Fi(MPI)p 152 1054 V 14 w(ANY)p 251 1054 V 15 w(SOURCE)13 b Fx(in)j(the)g(pattern,)e(and)i(has)f(a)f(matc)o(hing)i(tag)e(unless)i (tag=)e Fi(MPI)p 1553 1054 V 14 w(ANY)p 1652 1054 V 15 w(T)m(A)o(G)g Fx(in)i(the)75 1110 y(pattern.)166 1167 y(The)24 b(message)g(tag)f(is)i(sp)q(eci\014ed)h(b)o(y)e(the)g Fv(tag)g Fx(argumen)o(t)g(of)g(the)g(receiv)o(e)h(op)q(eration.)47 b(The)75 1223 y(argumen)o(t)24 b Fv(source)p Fx(,)i(if)f(di\013eren)o (t)g(from)e Fi(MPI)p 879 1223 V 14 w(ANY)p 978 1223 V 15 w(SOURCE)p Fx(,)f(is)j(sp)q(eci\014ed)h(as)e(a)g(rank)g(within)i (the)75 1280 y(pro)q(cess)e(group)f(asso)q(ciated)g(with)h(that)e(same) i(comm)o(unicator)e(\(remote)h(pro)q(cess)h(group,)g(for)f(in-)75 1336 y(tercomm)o(unicators\).)37 b(Th)o(us,)23 b(the)e(range)g(of)g(v)m (alid)j(v)m(alues)e(for)f(the)g Fv(source)h Fx(argumen)o(t)f(is)h Ft(f)p Fv(0,...,n-)75 1393 y(1)p Ft(g[f)p Fv(MPI)p 259 1393 14 2 v 15 w(ANY)p 366 1393 V 17 w(SOURCE)p Ft(g)p Fx(,)16 b(where)g Fv(n)g Fx(is)f(the)h(n)o(um)o(b)q(er)f(of)g(pro)q (cesses)g(in)h(this)g(group.)166 1449 y(Note)i(the)i(asymmetry)d(b)q (et)o(w)o(een)j(send)f(and)g(receiv)o(e)h(op)q(erations:)28 b(A)19 b(receiv)o(e)h(op)q(eration)f(ma)o(y)75 1506 y(accept)13 b(messages)f(from)g(an)h(arbitrary)f(sender,)i(on)f(the)g(other)f (hand,)h(a)g(send)g(op)q(eration)g(m)o(ust)g(sp)q(ecify)75 1562 y(a)i(unique)j(receiv)o(er.)k(This)16 b(matc)o(hes)f(a)h(\\push")f (comm)o(unication)i(mec)o(hanism,)f(where)g(data)f(transfer)75 1619 y(is)i(e\013ected)h(b)o(y)f(the)g(sender)g(\(rather)f(than)h(a)g (\\pull")h(mec)o(hanism,)g(where)f(data)f(transfer)g(is)i(e\013ected)75 1675 y(b)o(y)d(the)g(receiv)o(er\).)166 1731 y(Source)g(=)h (destination)g(is)g(allo)o(w)o(ed,)f(that)f(is,)i(a)e(pro)q(cess)i(can) f(send)h(a)f(message)g(to)f(itself.)21 b(\(Ho)o(w-)75 1788 y(ev)o(er,)c(it)h(is)g(unsafe)f(to)g(do)g(so)g(with)h(the)g(blo)q (c)o(king)g(send)g(and)g(receiv)o(e)g(op)q(erations)f(describ)q(ed)j (ab)q(o)o(v)o(e,)75 1844 y(since)c(this)g(ma)o(y)e(lead)i(to)f(deadlo)q (c)o(k.)21 b(See)16 b(Sec.)k(3.5.\))189 1934 y Fl(A)n(dvic)n(e)15 b(to)i(implementors.)43 b Fx(Message)15 b(con)o(text)g(and)h(other)g (comm)o(unicator)f(information)h(can)189 1990 y(b)q(e)k(implemen)o(ted) h(as)e(an)h(additional)h(tag)d(\014eld.)34 b(It)20 b(di\013ers)g(from)f (the)g(regular)h(message)f(tag)189 2047 y(in)i(that)e(wild)j(card)e (matc)o(hing)h(is)f(not)g(allo)o(w)o(ed)h(on)f(this)h(\014eld,)i(and)d (that)g(v)m(alue)h(setting)g(for)189 2103 y(this)e(\014eld)h(is)g(con)o (trolled)g(b)o(y)f(comm)o(unicator)f(manipulation)j(functions.)32 b(\()p Fl(End)19 b(of)h(advic)n(e)g(to)189 2160 y(implementors.)p Fx(\))75 2279 y Fj(3.2.5)49 b(Return)16 b(status)75 2365 y Fx(The)g(source)f(or)g(tag)f(of)h(a)g(receiv)o(ed)h(message)f(ma)o(y) g(not)g(b)q(e)g(kno)o(wn)h(if)f(wildcard)i(v)m(alues)f(w)o(ere)f(used)h (in)75 2421 y(the)h(receiv)o(e)i(op)q(eration.)26 b(The)18 b(information)f(is)h(returned)g(b)o(y)f(the)h Fv(status)h Fx(argumen)o(t)e(of)g Fv(MPI)p 1734 2421 V 16 w(RECV)p Fx(.)75 2478 y(The)h(t)o(yp)q(e)f(of)h Fv(status)h Fx(is)f Fv(MPI)p Fx(-de\014ned.)28 b(Status)17 b(v)m(ariables)i(need)f(to)f(b)q (e)i(explicitly)h(allo)q(cated)e(b)o(y)g(the)75 2534 y(user,)d(that)f(is,)i(they)f(are)g(not)g(system)g(ob)s(jects.)166 2591 y(In)c(C,)f Fv(status)j Fx(is)e(a)g(structure)f(that)g(con)o (tains)h(t)o(w)o(o)e(\014elds)j(named)f Fi(MPI)p 1343 2591 13 2 v 14 w(SOURCE)e Fx(and)i Fi(MPI)p 1687 2591 V 15 w(T)m(A)o(G)p Fx(,)e(and)75 2647 y(the)k(structure)f(ma)o(y)g(con) o(tain)h(additional)h(\014elds.)20 b(Th)o(us,)13 b Fv(status.MPI)p 1277 2647 14 2 v 17 w(SOURCE)h Fx(and)f Fv(status.MPI)p 1774 2647 V 18 w(T)l(A)o(G)75 2704 y Fx(con)o(tain)i(the)h(source)f (and)g(tag,)f(resp)q(ectiv)o(ely)l(,)j(of)e(the)g(receiv)o(ed)h (message.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 21 29 21 28 bop 75 -100 a Fm(3.3.)34 b(D)o(A)l(T)l(A)15 b(TYPE)g(MA)l(TCHING) g(AND)g(D)o(A)l(T)l(A)g(CONVERSION)520 b Fx(21)166 45 y(In)15 b(F)l(ortran,)f Fv(status)i Fx(is)g(an)e(arra)o(y)g(of)g Fu(INTEGER)p Fx(s)g(of)g(size)i Fi(MPI)p 1219 45 13 2 v 14 w(ST)m(A)m(TUS)p 1386 45 V 14 w(SIZE)p Fx(.)e(The)h(t)o(w)o(o)f (constan)o(ts)75 102 y Fi(MPI)p 152 102 V 14 w(SOURCE)e Fx(and)i Fi(MPI)p 502 102 V 14 w(T)m(A)o(G)f Fx(are)g(the)h(indices)i (of)d(the)g(en)o(tries)h(that)f(store)g(the)h(source)g(and)f(tag)g (\014elds.)75 158 y(Th)o(us)k Fv(status\(MPI)p 406 158 14 2 v 17 w(SOURCE\))h Fx(and)e Fv(status\(MPI)p 933 158 V 18 w(T)l(A)o(G\))h Fx(con)o(tain,)f(resp)q(ectiv)o(ely)l(,)j(the) d(source)h(and)g(the)75 214 y(tag)d(of)h(the)g(receiv)o(ed)i(message.) 166 271 y(The)h(status)e(argumen)o(t)h(also)g(returns)g(information)h (on)f(the)h(length)g(of)f(the)g(message)g(receiv)o(ed.)75 327 y(Ho)o(w)o(ev)o(er,)12 b(this)i(information)f(is)h(not)f(directly)h (a)o(v)m(ailable)h(as)e(a)g(\014eld)h(of)f(the)h(status)e(v)m(ariable)j (and)e(a)g(call)75 384 y(to)i Fv(MPI)p 216 384 V 15 w(GET)p 319 384 V 17 w(COUNT)h Fx(is)g(required)g(to)f(\\deco)q(de")g(this)h (information.)75 535 y Fv(MPI)p 160 535 V 16 w(GET)p 264 535 V 17 w(COUNT\(status,)h(datat)o(yp)q(e,)f(count\))117 612 y Fk(IN)155 b Fv(status)476 b Fk(return)15 b(status)g(of)e(receiv)o (e)i(op)q(eration)f(\(Status\))117 687 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen) o(t)e(\(handle\))117 762 y(OUT)108 b Fv(count)482 b Fk(n)o(um)o(b)q(er) 13 b(of)h(receiv)o(ed)h(elemen)o(ts)f(\(in)o(teger\))75 887 y Fu(int)23 b(MPI)p 245 887 15 2 v 17 w(Get)p 334 887 V 17 w(count\(MPI)p 567 887 V 16 w(Status)g(status,)g(MPI)p 1013 887 V 17 w(Datatype)f(datatype,)h(int)h(*count\))75 973 y(MPI)p 150 973 V 17 w(GET)p 239 973 V 17 w(COUNT\(STATUS,)e (DATATYPE,)g(COUNT,)h(IERROR\))170 1030 y(INTEGER)g(STATUS\(MPI)p 604 1030 V 16 w(STATUS)p 764 1030 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)166 1116 y Fx(Returns)16 b(the)f(n)o(um)o(b)q(er)h(of)f(elemen) o(ts)h(receiv)o(ed.)21 b(\(Again,)15 b(w)o(e)g(coun)o(t)h Fl(elements)p Fx(,)d(not)i Fl(bytes)p Fx(.\))20 b(The)75 1173 y Fv(datat)o(yp)q(e)h Fx(argumen)o(t)e(should)i(matc)o(h)e(the)h (argumen)o(t)f(pro)o(vided)i(b)o(y)f(the)g(receiv)o(e)g(call)h(that)e (set)h(the)75 1229 y Fv(status)f Fx(v)m(ariable.)26 b(\(W)l(e)17 b(shall)h(later)f(see,)g(in)h(Section)g(3.12.5,)d(that)h Fv(MPI)p 1347 1229 14 2 v 16 w(GET)p 1451 1229 V 16 w(COUNT)i Fx(ma)o(y)e(return,)75 1285 y(in)g(certain)g(situations,)f(the)g(v)m (alue)i Fi(MPI)p 773 1285 13 2 v 14 w(UNDEFINED)p Fx(.\))189 1392 y Fl(R)n(ationale.)56 b Fx(Some)18 b(message)f(passing)h (libraries)i(use)e Fk(INOUT)h Fv(count)p Fx(,)g Fv(tag)f Fx(and)g Fv(source)g Fx(argu-)189 1448 y(men)o(ts,)f(th)o(us)h(using)g (them)g(b)q(oth)g(to)f(sp)q(ecify)i(the)f(selection)h(criteria)f(for)f (incoming)i(messages)189 1505 y(and)c(return)g(the)h(actual)f(en)o(v)o (elop)q(e)i(v)m(alues)f(of)f(the)g(receiv)o(ed)i(message.)i(The)d(use)g (of)f(a)f(separate)189 1561 y(status)g(argumen)o(t)h(prev)o(en)o(ts)g (errors)g(that)g(are)g(often)g(attac)o(hed)g(with)h Fk(INOUT)g Fx(argumen)o(t)f(\(e.g.,)189 1618 y(using)i(the)g Fi(MPI)p 466 1618 V 14 w(ANY)p 565 1618 V 15 w(T)m(A)o(G)e Fx(constan)o(t)h(as)g (the)g(tag)g(in)h(a)g(send\).)24 b(Some)16 b(libraries)i(use)f(calls)h (that)189 1674 y(refer)d(implicitly)j(to)d(the)g(\\last)g(message)f (receiv)o(ed.")21 b(This)16 b(is)g(not)e(thread)i(safe.)189 1749 y(The)e Fv(datat)o(yp)q(e)i Fx(argumen)o(t)e(is)h(passed)f(to)g Fv(MPI)p 991 1749 14 2 v 16 w(GET)p 1095 1749 V 17 w(COUNT)h Fx(so)f(as)g(to)g(impro)o(v)o(e)g(p)q(erformance.)189 1806 y(A)k(message)g(migh)o(t)g(b)q(e)h(receiv)o(ed)h(without)e(coun)o (ting)h(the)g(n)o(um)o(b)q(er)g(of)e(elemen)o(ts)j(it)e(con)o(tains,) 189 1862 y(and)g(the)h(coun)o(t)f(v)m(alue)h(is)g(often)f(not)g (needed.)31 b(Also,)19 b(this)g(allo)o(ws)g(the)f(same)g(function)i(to) d(b)q(e)189 1918 y(used)e(after)g(a)g(call)h(to)f Fv(MPI)p 662 1918 V 15 w(PROBE)p Fx(.)i(\()p Fl(End)e(of)i(r)n(ationale.)p Fx(\))166 2025 y(All)i(send)g(and)g(receiv)o(e)g(op)q(erations)f(use)h (the)f Fv(buf)p Fx(,)h Fv(count)p Fx(,)h Fv(datat)o(yp)q(e)p Fx(,)g Fv(source)p Fx(,)f Fv(dest)p Fx(,)h Fv(tag)p Fx(,)f Fv(comm)75 2081 y Fx(and)12 b Fv(status)i Fx(argumen)o(ts)d(in)h(the)g (same)g(w)o(a)o(y)f(as)g(the)h(blo)q(c)o(king)h Fv(MPI)p 1216 2081 V 16 w(SEND)f Fx(and)g Fv(MPI)p 1528 2081 V 16 w(RECV)h Fx(op)q(erations)75 2138 y(describ)q(ed)k(in)f(this)g (section.)75 2281 y Fn(3.3)59 b(Data)19 b(t)n(yp)r(e)g(matching)f(and)h (data)g(conversion)75 2384 y Fj(3.3.1)49 b(T)l(yp)q(e)17 b(matching)d(rules)75 2470 y Fx(One)i(can)f(think)h(of)f(message)g (transfer)f(as)h(consisting)h(of)f(the)g(follo)o(wing)h(three)f (phases.)131 2564 y(1.)22 b(Data)14 b(is)h(pulled)j(out)d(of)f(the)i (send)f(bu\013er)h(and)f(a)g(message)g(is)g(assem)o(bled.)131 2657 y(2.)22 b(A)15 b(message)g(is)g(transferred)g(from)f(sender)i(to)f (receiv)o(er.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 22 30 22 29 bop 75 -100 a Fx(22)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)131 45 y Fx(3.)22 b(Data)14 b(is)h(pulled)j(from)c(the)h(incoming)i(message)e(and)g (disassem)o(bled)i(in)o(to)e(the)g(receiv)o(e)h(bu\013er.)166 128 y(T)o(yp)q(e)k(matc)o(hing)f(has)h(to)e(b)q(e)j(observ)o(ed)e(at)g (eac)o(h)g(of)h(these)f(three)h(phases:)29 b(The)19 b(t)o(yp)q(e)h(of)f (eac)o(h)75 184 y(v)m(ariable)i(in)f(the)g(sender)g(bu\013er)g(has)g (to)f(matc)o(h)g(the)g(t)o(yp)q(e)h(sp)q(eci\014ed)i(for)d(that)g(en)o (try)g(b)o(y)h(the)f(send)75 241 y(op)q(eration;)d(the)f(t)o(yp)q(e)h (sp)q(eci\014ed)i(b)o(y)e(the)f(send)i(op)q(eration)e(has)h(to)f(matc)o (h)g(the)h(t)o(yp)q(e)f(sp)q(eci\014ed)j(b)o(y)e(the)75 297 y(receiv)o(e)g(op)q(eration;)e(and)h(the)g(t)o(yp)q(e)f(of)h(eac)o (h)f(v)m(ariable)i(in)g(the)e(receiv)o(e)i(bu\013er)f(has)f(to)g(matc)o (h)g(the)h(t)o(yp)q(e)75 354 y(sp)q(eci\014ed)f(for)e(that)g(en)o(try)g (b)o(y)g(the)g(receiv)o(e)h(op)q(eration.)20 b(A)12 b(program)f(that)h (fails)h(to)e(observ)o(e)h(these)h(three)75 410 y(rules)j(is)g (erroneous.)166 466 y(T)l(o)f(de\014ne)i(t)o(yp)q(e)f(matc)o(hing)g (more)f(precisely)l(,)j(w)o(e)d(need)i(to)e(deal)h(with)g(t)o(w)o(o)f (issues:)22 b(matc)o(hing)15 b(of)75 523 y(t)o(yp)q(es)e(of)g(the)h (host)f(language)g(with)h(t)o(yp)q(es)g(sp)q(eci\014ed)h(in)f(comm)o (unication)h(op)q(erations;)e(and)h(matc)o(hing)75 579 y(of)h(t)o(yp)q(es)g(at)g(sender)g(and)h(receiv)o(er.)166 636 y(The)k(t)o(yp)q(es)f(of)h(a)f(send)h(and)g(receiv)o(e)h(matc)o(h)e (\(phase)g(t)o(w)o(o\))f(if)j(b)q(oth)e(op)q(erations)h(use)g(iden)o (tical)75 692 y(names.)36 b(That)20 b(is,)i Fv(MPI)p 517 692 14 2 v 16 w(INTEGER)f Fx(matc)o(hes)f Fv(MPI)p 1006 692 V 16 w(INTEGER)p Fx(,)h Fv(MPI)p 1327 692 V 16 w(REAL)g Fx(matc)o(hes)f Fv(MPI)p 1738 692 V 16 w(REAL)p Fx(,)75 749 y(and)12 b(so)g(on.)18 b(There)12 b(is)h(one)f(exception)h (to)e(this)h(rule,)h(discussed)g(in)g(Sec.)20 b(3.13,)10 b(the)i(t)o(yp)q(e)g Fv(MPI)p 1687 749 V 16 w(P)l(A)o(CKED)75 805 y Fx(can)j(matc)o(h)g(an)o(y)g(other)g(t)o(yp)q(e.)166 862 y(The)k(t)o(yp)q(e)g(of)f(a)g(v)m(ariable)i(in)g(a)e(host)h (program)e(matc)o(hes)h(the)h(t)o(yp)q(e)g(sp)q(eci\014ed)i(in)e(the)g (comm)o(u-)75 918 y(nication)g(op)q(eration)g(if)f(the)h(datat)o(yp)q (e)f(name)g(used)h(b)o(y)f(that)f(op)q(eration)i(corresp)q(onds)f(to)g (the)g(basic)75 975 y(t)o(yp)q(e)f(of)g(the)g(host)g(program)f(v)m (ariable.)27 b(F)l(or)17 b(example,)h(an)f(en)o(try)g(with)g(t)o(yp)q (e)g(name)g Fv(MPI)p 1672 975 V 16 w(INTEGER)75 1031 y Fx(matc)o(hes)c(a)g(F)l(ortran)g(v)m(ariable)i(of)e(t)o(yp)q(e)g Fu(INTEGER)p Fx(.)f(A)i(table)g(giving)g(this)g(corresp)q(ondence)h (for)e(F)l(ortran)75 1087 y(and)20 b(C)f(app)q(ears)h(in)h(Sec.)33 b(3.2.2.)f(There)20 b(are)f(t)o(w)o(o)g(exceptions)h(to)f(this)i(last)e (rule:)30 b(an)19 b(en)o(try)h(with)75 1144 y(t)o(yp)q(e)f(name)g Fv(MPI)p 391 1144 V 16 w(BYTE)h Fx(or)f Fv(MPI)p 687 1144 V 16 w(P)l(A)o(CKED)g Fx(can)g(b)q(e)h(used)g(to)e(matc)o(h)h(an)o (y)g(b)o(yte)g(of)g(storage)e(\(on)i(a)75 1200 y(b)o(yte-addressable)g (mac)o(hine\),)g(irresp)q(ectiv)o(e)h(of)e(the)g(datat)o(yp)q(e)g(of)g (the)g(v)m(ariable)i(that)d(con)o(tains)i(this)75 1257 y(b)o(yte.)f(The)12 b(t)o(yp)q(e)g Fv(MPI)p 466 1257 V 16 w(P)l(A)o(CKED)f Fx(is)h(used)g(to)f(send)i(data)e(that)f(has)i(b) q(een)h(explicitly)h(pac)o(k)o(ed,)e(or)f(receiv)o(e)75 1313 y(data)k(that)g(will)i(b)q(e)g(explicitly)h(unpac)o(k)o(ed,)e(see) g(Section)g(3.13.)k(The)c(t)o(yp)q(e)g Fv(MPI)p 1468 1313 V 16 w(BYTE)g Fx(allo)o(ws)g(one)g(to)75 1370 y(transfer)e(the)i (binary)g(v)m(alue)g(of)f(a)g(b)o(yte)g(in)h(memory)e(unc)o(hanged.)166 1426 y(T)l(o)h(summarize,)g(the)g(t)o(yp)q(e)h(matc)o(hing)f(rules)h (fall)g(in)o(to)f(the)g(three)h(categories)f(b)q(elo)o(w.)143 1509 y Ft(\017)23 b Fx(Comm)o(unication)11 b(of)f(t)o(yp)q(ed)i(v)m (alues)g(\(e.g.,)e(with)h(datat)o(yp)q(e)g(di\013eren)o(t)g(from)21 b Fv(MPI)p 1586 1509 V 16 w(BYTE)p Fx(\),)11 b(where)189 1565 y(the)k(datat)o(yp)q(es)f(of)g(the)h(corresp)q(onding)h(en)o (tries)f(in)h(the)f(sender)g(program,)e(in)j(the)f(send)g(call,)h(in) 189 1622 y(the)f(receiv)o(e)h(call)g(and)g(in)g(the)f(receiv)o(er)h (program)e(m)o(ust)h(all)h(matc)o(h.)143 1711 y Ft(\017)23 b Fx(Comm)o(unication)16 b(of)g(un)o(t)o(yp)q(ed)h(v)m(alues)g(\(e.g.,) e(of)g(datat)o(yp)q(e)h Fv(MPI)p 1330 1711 V 16 w(BYTE)p Fx(\),)g(where)g(b)q(oth)g(sender)189 1768 y(and)g(receiv)o(er)h(use)f (the)g(datat)o(yp)q(e)g Fv(MPI)p 881 1768 V 15 w(BYTE)p Fx(.)h(In)f(this)h(case,)f(there)g(are)g(no)f(requiremen)o(ts)i(on)189 1824 y(the)d(t)o(yp)q(es)h(of)f(the)g(corresp)q(onding)h(en)o(tries)g (in)h(the)e(sender)h(and)g(the)f(receiv)o(er)h(programs,)e(nor)h(is)189 1880 y(it)h(required)h(that)f(they)g(b)q(e)h(the)f(same.)143 1970 y Ft(\017)23 b Fx(Comm)o(unication)15 b(in)o(v)o(olving)i(pac)o(k) o(ed)e(data,)f(where)h Fv(MPI)p 1195 1970 V 16 w(P)l(A)o(CKED)g Fx(is)h(used.)166 2052 y(The)f(follo)o(wing)h(examples)g(illustrate)h (the)e(\014rst)g(t)o(w)o(o)e(cases.)75 2143 y Fq(Example)18 b(3.1)k Fx(Sender)16 b(and)g(receiv)o(er)g(sp)q(ecify)g(matc)o(hing)g (t)o(yp)q(es.)75 2226 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))75 2282 y(IF\(rank.EQ.0\))f(THEN)170 2339 y(CALL)i (MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(ierr\))75 2395 y(ELSE)170 2452 y(CALL)h(MPI_RECV\(b\(1\),)e(15,)h(MPI_REAL,)g(0,) g(tag,)h(comm,)f(status,)g(ierr\))75 2508 y(END)g(IF)166 2591 y Fx(This)16 b(co)q(de)h(is)f(correct)f(if)h(b)q(oth)g Fv(a)g Fx(and)g Fv(b)g Fx(are)f(real)h(arra)o(ys)f(of)g(size)i Ft(\025)d Fx(10.)21 b(\(In)16 b(F)l(ortran,)e(it)i(migh)o(t)75 2647 y(b)q(e)e(correct)e(to)h(use)g(this)g(co)q(de)h(ev)o(en)f(if)h Fv(a)f Fx(or)f Fv(b)i Fx(ha)o(v)o(e)e(size)i Fp(<)f Fx(10:)19 b(e.g.,)12 b(when)h Fv(a\(1\))f Fx(can)h(b)q(e)h(equiv)m(alenced)75 2704 y(to)h(an)g(arra)o(y)f(with)h(ten)g(reals.\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 23 31 23 30 bop 75 -100 a Fm(3.3.)34 b(D)o(A)l(T)l(A)15 b(TYPE)g(MA)l(TCHING) g(AND)g(D)o(A)l(T)l(A)g(CONVERSION)520 b Fx(23)75 45 y Fq(Example)18 b(3.2)k Fx(Sender)16 b(and)g(receiv)o(er)g(do)f(not)g (sp)q(ecify)h(matc)o(hing)f(t)o(yp)q(es.)75 135 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 192 y(IF\(rank.EQ.0\))f (THEN)170 248 y(CALL)i(MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,) h(comm,)f(ierr\))75 305 y(ELSE)170 361 y(CALL)h(MPI_RECV\(b\(1\),)e (40,)h(MPI_BYTE,)g(0,)g(tag,)h(comm,)f(status,)g(ierr\))75 417 y(END)g(IF)166 507 y Fx(This)d(co)q(de)g(is)f(erroneous,)h(since)g (sender)g(and)g(receiv)o(er)g(do)f(not)f(pro)o(vide)i(matc)o(hing)f (datat)o(yp)q(e)75 564 y(argumen)o(ts.)75 654 y Fq(Example)f(3.3)k Fx(Sender)16 b(and)g(receiv)o(er)g(sp)q(ecify)g(comm)o(unication)g(of)f (un)o(t)o(yp)q(ed)h(v)m(alues.)75 736 y Fu(CALL)23 b (MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 793 y(IF\(rank.EQ.0\))f(THEN) 170 849 y(CALL)i(MPI_SEND\(a\(1\),)e(40,)h(MPI_BYTE,)g(1,)g(tag,)h (comm,)f(ierr\))75 906 y(ELSE)170 962 y(CALL)h(MPI_RECV\(b\(1\),)e(60,) h(MPI_BYTE,)g(0,)g(tag,)h(comm,)f(status,)g(ierr\))75 1018 y(END)g(IF)166 1101 y Fx(This)16 b(co)q(de)g(is)g(correct,)e (irresp)q(ectiv)o(e)j(of)e(the)h(t)o(yp)q(e)f(and)h(size)g(of)f Fv(a)g Fx(and)h Fv(b)f Fx(\(unless)i(this)e(results)h(in)75 1157 y(an)f(out)g(of)g(b)q(ound)h(memory)e(access\).)189 1247 y Fl(A)n(dvic)n(e)e(to)i(users.)38 b Fx(If)12 b(a)g(bu\013er)g(of) f(t)o(yp)q(e)h Fi(MPI)p 965 1247 13 2 v 15 w(BYTE)f Fx(is)i(passed)f (as)g(an)g(argumen)o(t)f(to)g Fv(MPI)p 1732 1247 14 2 v 16 w(SEND)p Fx(,)189 1304 y(then)j Fv(MPI)g Fx(will)i(send)f(the)g (data)f(stored)g(at)f(con)o(tiguous)i(lo)q(cations,)g(starting)f(from)f (the)i(address)189 1360 y(indicated)22 b(b)o(y)e(the)h Fv(buf)g Fx(argumen)o(t.)35 b(This)21 b(ma)o(y)f(ha)o(v)o(e)g(unexp)q (ected)i(results)f(when)g(the)g(data)189 1417 y(la)o(y)o(out)f(is)i (not)f(as)f(a)h(casual)h(user)f(w)o(ould)h(exp)q(ect)f(it)h(to)e(b)q (e.)39 b(F)l(or)20 b(example,)j(some)e(F)l(ortran)189 1473 y(compilers)e(implemen)o(t)g(v)m(ariables)g(of)f(t)o(yp)q(e)g Fi(CHARA)o(CTER)e Fx(as)i(a)g(structure)f(that)h(con)o(tains)g(the)189 1529 y(c)o(haracter)c(length)i(and)g(a)f(p)q(oin)o(ter)h(to)f(the)g (actual)h(string.)k(In)c(suc)o(h)g(an)f(en)o(vironmen)o(t,)h(sending) 189 1586 y(and)g(receiving)h(a)e(F)l(ortran)g Fi(CHARA)o(CTER)f Fx(v)m(ariable)j(using)g(the)e Fi(MPI)p 1378 1586 13 2 v 15 w(BYTE)g Fx(t)o(yp)q(e)h(will)h(not)e(ha)o(v)o(e)189 1642 y(the)f(an)o(ticipated)i(result)f(of)f(transferring)h(the)f(c)o (haracter)g(string.)20 b(F)l(or)14 b(this)h(reason,)f(the)h(user)f(is) 189 1699 y(advised)i(to)e(use)i(t)o(yp)q(ed)f(comm)o(unications)h (whenev)o(er)g(p)q(ossible.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)h(to)f (users.)p Fx(\))75 1817 y Fv(T)l(yp)q(e)g Fi(MPI)p 260 1817 V 15 w(CHARA)o(CTER)75 1903 y Fx(The)c(t)o(yp)q(e)g Fv(MPI)p 349 1903 14 2 v 16 w(CHARA)o(CTER)h Fx(matc)o(hes)f(one)g(c)o (haracter)f(of)g(a)h(F)l(ortran)f(v)m(ariable)i(of)f(t)o(yp)q(e)g Fu(CHARACTER)p Fx(,)75 1959 y(rather)20 b(then)g(the)h(en)o(tire)g(c)o (haracter)e(string)i(stored)f(in)h(the)f(v)m(ariable.)37 b(F)l(ortran)19 b(v)m(ariables)j(of)e(t)o(yp)q(e)75 2015 y Fi(CHARA)o(CTER)g Fx(or)i(substrings)g(are)f(transferred)h(as)f(if)i (they)f(w)o(ere)g(arra)o(ys)e(of)i(c)o(haracters.)39 b(This)22 b(is)75 2072 y(illustrated)17 b(in)f(the)f(example)h(b)q(elo) o(w.)75 2162 y Fq(Example)i(3.4)k Fx(T)l(ransfer)15 b(of)g(F)l(ortran)f Fi(CHARA)o(CTER)p Fx(s.)75 2252 y Fu(CHARACTER*10)22 b(a)75 2308 y(CHARACTER*10)g(b)75 2421 y(CALL)h(MPI_COMM_RANK\(comm,)f (rank,)h(ierr\))75 2478 y(IF\(rank.EQ.0\))f(THEN)170 2534 y(CALL)i(MPI_SEND\(a,)e(5,)i(MPI_CHARACTER,)e(1,)h(tag,)h(comm,)f (ierr\))75 2591 y(ELSE)170 2647 y(CALL)h(MPI_RECV\(b\(6:10\),)d(5,)j (MPI_CHARACTER,)e(0,)h(tag,)h(comm,)f(status,)g(ierr\))75 2704 y(END)g(IF)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 24 32 24 31 bop 75 -100 a Fx(24)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)166 45 y Fx(The)d(last)h(\014v)o (e)f(c)o(haracters)g(of)f(string)i Fi(b)f Fx(at)g(pro)q(cess)g(1)g(are) g(replaced)i(b)o(y)e(the)g(\014rst)g(\014v)o(e)h(c)o(haracters)75 102 y(of)g(string)g Fi(a)g Fx(at)g(pro)q(cess)g(0.)189 208 y Fl(R)n(ationale.)45 b Fx(The)16 b(alternativ)o(e)h(c)o(hoice)g(w) o(ould)f(b)q(e)h(for)e Fv(MPI)p 1246 208 14 2 v 16 w(CHARA)o(CTER)i Fx(to)f(matc)o(h)f(a)h(c)o(har-)189 264 y(acter)e(of)h(arbitrary)g (length.)20 b(This)c(runs)f(in)o(to)h(problems.)189 339 y(A)g(F)l(ortran)e(c)o(haracter)h(v)m(ariable)j(is)e(a)g(constan)o(t)f (length)h(string,)g(with)g(no)g(sp)q(ecial)i(termination)189 396 y(sym)o(b)q(ol.)h(There)12 b(is)h(no)g(\014xed)f(con)o(v)o(en)o (tion)h(on)f(ho)o(w)g(to)g(represen)o(t)g(c)o(haracters,)g(and)g(ho)o (w)g(to)g(store)189 452 y(their)h(length.)20 b(Some)13 b(compilers)h(pass)e(a)h(c)o(haracter)f(argumen)o(t)g(to)g(a)h(routine) g(as)f(a)h(pair)g(of)f(argu-)189 509 y(men)o(ts,)g(one)g(holding)h(the) f(address)h(of)e(the)h(string)g(and)h(the)f(other)f(holding)j(the)e (length)h(of)e(string.)189 565 y(Consider)h(the)f(case)h(of)f(an)g Fv(MPI)g Fx(comm)o(unication)h(call)h(that)e(is)h(passed)g(a)f(comm)o (unication)h(bu\013er)189 622 y(with)17 b(t)o(yp)q(e)f(de\014ned)i(b)o (y)f(a)f(deriv)o(ed)h(datat)o(yp)q(e)f(\(Section)i(3.12\).)k(If)17 b(this)g(comm)o(unicator)f(bu\013er)189 678 y(con)o(tains)e(v)m (ariables)h(of)e(t)o(yp)q(e)h Fu(CHARACTER)f Fx(then)h(the)g (information)g(on)g(their)h(length)f(will)i(not)d(b)q(e)189 735 y(passed)i(to)g(the)g Fv(MPI)g Fx(routine.)189 810 y(This)j(problem)g(forces)f(us)h(to)f(pro)o(vide)h(explicit)i (information)d(on)h(c)o(haracter)f(length)h(with)g(the)189 866 y Fv(MPI)d Fx(call.)21 b(One)c(could)f(add)g(a)f(length)h (parameter)f(to)g(the)g(t)o(yp)q(e)h Fv(MPI)p 1411 866 V 15 w(CHARA)o(CTER)p Fx(,)h(but)f(this)189 923 y(do)q(es)11 b(not)g(add)h(m)o(uc)o(h)f(con)o(v)o(enience)i(and)e(the)h(same)f (functionalit)o(y)h(can)g(b)q(e)g(ac)o(hiev)o(ed)g(b)o(y)f(de\014ning) 189 979 y(a)k(suitable)h(deriv)o(ed)g(datat)o(yp)q(e.)k(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 1085 y Fl(A)n(dvic)n(e)f(to) i(implementors.)48 b Fx(Some)16 b(compilers)i(pass)e(F)l(ortran)g Fi(CHARA)o(CTER)e Fx(argumen)o(ts)i(as)g(a)189 1142 y(structure)g(with) i(a)f(length)g(and)h(a)e(p)q(oin)o(ter)i(to)e(the)h(actual)h(string.)25 b(In)18 b(suc)o(h)f(an)g(en)o(vironmen)o(t,)189 1198 y(the)h Fv(MPI)g Fx(call)h(needs)h(to)d(dereference)j(the)e(p)q(oin)o (ter)h(in)g(order)g(to)e(reac)o(h)h(the)h(string.)29 b(\()p Fl(End)19 b(of)189 1255 y(advic)n(e)d(to)g(implementors.)p Fx(\))75 1376 y Fj(3.3.2)49 b(Data)17 b(conversion)75 1462 y Fx(One)g(of)e(the)h(goals)f(of)h Fv(MPI)f Fx(is)h(to)f(supp)q (ort)h(parallel)i(computations)d(across)g(heterogeneous)h(en)o(viron-) 75 1519 y(men)o(ts.)j(Comm)o(unication)d(in)g(a)f(heterogeneous)g(en)o (vironmen)o(t)g(ma)o(y)g(require)h(data)e(con)o(v)o(ersions.)20 b(W)l(e)75 1575 y(use)c(the)f(follo)o(wing)h(terminology)l(.)75 1669 y Fq(t)o(yp)q(e)h(con)o(v)o(ersion)22 b Fx(c)o(hanges)12 b(the)h(datat)o(yp)q(e)f(of)g(a)g(v)m(alue,)i(e.g.,)e(b)o(y)g(rounding) h(a)f Fu(REAL)g Fx(to)g(an)g Fu(INTEGER)p Fx(.)75 1763 y Fq(represen)o(tation)17 b(con)o(v)o(ersion)22 b Fx(c)o(hanges)16 b(the)g(binary)g(represen)o(tation)g(of)f(a)g(v)m(alue,)i(e.g.,)e(from) g(Hex)189 1819 y(\015oating)g(p)q(oin)o(t)g(to)g(IEEE)g(\015oating)h(p) q(oin)o(t.)166 1913 y(The)e(t)o(yp)q(e)f(matc)o(hing)h(rules)g(imply)h (that)e Fv(MPI)g Fx(comm)o(unication)h(nev)o(er)f(en)o(tails)i(t)o(yp)q (e)e(con)o(v)o(ersion.)75 1970 y(On)20 b(the)g(other)f(hand,)i Fv(MPI)e Fx(requires)h(that)f(a)g(represen)o(tation)h(con)o(v)o(ersion) f(b)q(e)h(p)q(erformed)g(when)g(a)75 2026 y(t)o(yp)q(ed)h(v)m(alue)h (is)f(transferred)f(across)g(en)o(vironmen)o(ts)h(that)f(use)h (di\013eren)o(t)f(represen)o(tations)h(for)f(the)75 2082 y(datat)o(yp)q(e)h(of)f(this)i(v)m(alue.)38 b Fv(MPI)21 b Fx(do)q(es)g(not)g(sp)q(ecify)h(rules)g(for)f(represen)o(tation)g (con)o(v)o(ersion.)37 b(Suc)o(h)75 2139 y(con)o(v)o(ersion)21 b(is)g(exp)q(ected)i(to)d(preserv)o(e)h(in)o(teger,)h(logical)g(or)e(c) o(haracter)h(v)m(alues,)i(and)e(to)f(con)o(v)o(ert)g(a)75 2195 y(\015oating)15 b(p)q(oin)o(t)h(v)m(alue)g(to)f(the)g(nearest)g(v) m(alue)h(that)f(can)g(b)q(e)h(represen)o(ted)g(on)f(the)g(target)f (system.)166 2252 y(Ov)o(er\015o)o(w)h(and)g(under\015o)o(w)g (exceptions)h(ma)o(y)f(o)q(ccur)g(during)h(\015oating)f(p)q(oin)o(t)h (con)o(v)o(ersions.)k(Con-)75 2308 y(v)o(ersion)h(of)g(in)o(tegers)g (or)g(c)o(haracters)f(ma)o(y)g(also)i(lead)f(to)g(exceptions)h(when)g (a)e(v)m(alue)j(that)d(can)h(b)q(e)75 2365 y(represen)o(ted)16 b(in)g(one)f(system)g(cannot)g(b)q(e)h(represen)o(ted)f(in)h(the)f (other)g(system.)20 b(An)15 b(exception)h(o)q(ccur-)75 2421 y(ring)i(during)g(represen)o(tation)g(con)o(v)o(ersion)f(results)h (in)h(a)e(failure)i(of)e(the)g(comm)o(unication.)28 b(An)18 b(error)75 2478 y(o)q(ccurs)e(either)f(in)h(the)g(send)g(op)q(eration,) f(or)f(the)i(receiv)o(e)g(op)q(eration,)f(or)f(b)q(oth.)166 2534 y(If)22 b(a)f(v)m(alue)i(sen)o(t)e(in)h(a)f(message)g(is)h(un)o(t) o(yp)q(ed)h(\(i.e.,)f(of)f(t)o(yp)q(e)h Fv(MPI)p 1370 2534 V 15 w(BYTE)p Fx(\),)g(then)g(the)f(binary)75 2591 y(represen)o(tation)e(of)f(the)h(b)o(yte)g(stored)f(at)g(the)h(receiv)o (er)h(is)f(iden)o(tical)i(to)d(the)h(binary)g(represen)o(tation)75 2647 y(of)d(the)h(b)o(yte)g(loaded)g(at)g(the)f(sender.)26 b(This)17 b(holds)h(true,)e(whether)h(sender)h(and)f(receiv)o(er)g(run) g(in)h(the)75 2703 y(same)i(or)f(in)i(distinct)g(en)o(vironmen)o(ts.)35 b(No)20 b(represen)o(tation)g(con)o(v)o(ersion)g(is)h(required.)35 b(\(Note)20 b(that)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 25 33 25 32 bop 75 -100 a Fm(3.4.)34 b(COMMUNICA)l(TION)16 b(MODES)1048 b Fx(25)75 45 y(represen)o(tation)15 b(con)o(v)o(ersion)g (ma)o(y)g(o)q(ccur)g(when)h(v)m(alues)g(of)e(t)o(yp)q(e)i Fv(MPI)p 1307 45 14 2 v 15 w(CHARA)o(CTER)h Fx(or)d Fv(MPI)p 1741 45 V 16 w(CHAR)75 102 y Fx(are)h(transferred,)f(for)h(example,)g (from)g(an)g(EBCDIC)g(enco)q(ding)i(to)d(an)h(ASCI)q(I)i(enco)q (ding.\))166 158 y(No)g(con)o(v)o(ersion)h(need)g(o)q(ccur)g(when)h(an) e Fv(MPI)g Fx(program)f(executes)j(in)f(a)f(homogeneous)h(system,)75 214 y(where)d(all)i(pro)q(cesses)e(run)h(in)g(the)f(same)g(en)o (vironmen)o(t.)166 271 y(Consider)d(the)g(three)g(examples,)h(3.1{3.3.) j(The)c(\014rst)g(program)e(is)j(correct,)e(assuming)h(that)f Fv(a)h Fx(and)75 327 y Fv(b)i Fx(are)g Fu(REAL)f Fx(arra)o(ys)g(of)g (size)i Ft(\025)e Fx(10.)19 b(If)14 b(the)g(sender)g(and)g(receiv)o(er) h(execute)f(in)h(di\013eren)o(t)f(en)o(vironmen)o(ts,)75 384 y(then)21 b(the)g(ten)h(real)f(v)m(alues)h(that)e(are)h(fetc)o(hed) g(from)g(the)g(send)g(bu\013er)g(will)i(b)q(e)f(con)o(v)o(erted)e(to)h (the)75 440 y(represen)o(tation)g(for)f(reals)g(on)h(the)g(receiv)o(er) g(site)g(b)q(efore)g(they)f(are)h(stored)f(in)h(the)g(receiv)o(e)g (bu\013er.)75 497 y(While)e(the)e(n)o(um)o(b)q(er)h(of)f(real)g(elemen) o(ts)h(fetc)o(hed)g(from)f(the)g(send)h(bu\013er)f(equal)i(the)e(n)o (um)o(b)q(er)h(of)f(real)75 553 y(elemen)o(ts)d(stored)e(in)i(the)f (receiv)o(e)h(bu\013er,)f(the)g(n)o(um)o(b)q(er)g(of)g(b)o(ytes)g (stored)f(need)i(not)f(equal)h(the)f(n)o(um)o(b)q(er)75 610 y(of)e(b)o(ytes)g(loaded.)20 b(F)l(or)11 b(example,)i(the)e(sender) h(ma)o(y)f(use)h(a)g(four)f(b)o(yte)g(represen)o(tation)h(and)g(the)f (receiv)o(er)75 666 y(an)k(eigh)o(t)h(b)o(yte)f(represen)o(tation)g (for)f(reals.)166 723 y(The)h(second)h(program)e(is)i(erroneous,)f(and) g(its)g(b)q(eha)o(vior)h(is)g(unde\014ned.)166 779 y(The)e(third)g (program)e(is)i(correct.)19 b(The)13 b(exact)g(same)g(sequence)i(of)e (fort)o(y)f(b)o(ytes)h(that)g(w)o(ere)g(loaded)75 835 y(from)i(the)h(send)g(bu\013er)g(will)h(b)q(e)g(stored)e(in)h(the)g (receiv)o(e)h(bu\013er,)e(ev)o(en)h(if)g(sender)h(and)f(receiv)o(er)g (run)g(in)75 892 y(a)g(di\013eren)o(t)g(en)o(vironmen)o(t.)22 b(The)17 b(message)e(sen)o(t)h(has)g(exactly)g(the)g(same)g(length)h (\(in)f(b)o(ytes\))g(and)g(the)75 948 y(same)g(binary)g(represen)o (tation)g(as)g(the)g(message)f(receiv)o(ed.)24 b(If)16 b Fv(a)f Fx(and)i Fv(b)f Fx(are)g(of)f(di\013eren)o(t)h(t)o(yp)q(es,)g (or)f(if)75 1005 y(they)h(are)f(of)h(the)g(same)f(t)o(yp)q(e)h(but)g (di\013eren)o(t)g(data)f(represen)o(tations)h(are)g(used,)g(then)g(the) g(bits)g(stored)75 1061 y(in)h(the)f(receiv)o(e)h(bu\013er)g(ma)o(y)e (enco)q(de)i(v)m(alues)h(that)d(are)h(di\013eren)o(t)g(from)g(the)g(v)m (alues)h(they)g(enco)q(ded)g(in)75 1118 y(the)e(send)h(bu\013er.)166 1174 y(Data)e(represen)o(tation)i(con)o(v)o(ersion)f(also)h(applies)h (to)e(the)g(en)o(v)o(elop)q(e)i(of)e(a)g(message:)20 b(source,)15 b(des-)75 1231 y(tination)h(and)f(tag)f(are)h(all)h(in)o (tegers)g(that)e(ma)o(y)h(need)h(to)e(b)q(e)i(con)o(v)o(erted.)189 1322 y Fl(A)n(dvic)n(e)g(to)i(implementors.)50 b Fx(The)17 b(curren)o(t)f(de\014nition)j(do)q(es)e(not)g(require)g(messages)f(to)g (carry)189 1379 y(data)h(t)o(yp)q(e)i(information.)29 b(Both)18 b(sender)h(and)f(receiv)o(er)h(pro)o(vide)g(complete)g(data)e (t)o(yp)q(e)i(infor-)189 1435 y(mation.)24 b(In)17 b(a)g(heterogeneous) f(en)o(vironmen)o(t,)h(one)g(can)g(either)h(use)f(a)f(mac)o(hine)i (indep)q(enden)o(t)189 1492 y(enco)q(ding)h(suc)o(h)f(as)g(XDR,)f(or)h (ha)o(v)o(e)f(the)h(receiv)o(er)g(con)o(v)o(ert)g(from)f(the)h(sender)g (represen)o(tation)189 1548 y(to)c(its)i(o)o(wn,)e(or)h(ev)o(en)g(ha)o (v)o(e)g(the)g(sender)h(do)f(the)g(con)o(v)o(ersion.)189 1621 y(Additional)20 b(t)o(yp)q(e)e(information)g(migh)o(t)g(b)q(e)h (added)g(to)f(messages)g(in)h(order)f(to)f(allo)o(w)i(the)f(sys-)189 1678 y(tem)g(to)f(detect)i(mismatc)o(hes)f(b)q(et)o(w)o(een)g(datat)o (yp)q(e)g(at)g(sender)g(and)h(receiv)o(er.)29 b(This)19 b(migh)o(t)f(b)q(e)189 1734 y(particularly)13 b(useful)g(in)f(a)g(slo)o (w)o(er)f(but)h(safer)g(debug)g(mo)q(de.)19 b(\()p Fl(End)13 b(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))166 1826 y Fv(MPI)19 b Fx(do)q(es)g(not)g(require)h(supp)q(ort)g(for)e(in)o (ter-language)i(comm)o(unication.)33 b(The)20 b(b)q(eha)o(vior)f(of)g (a)75 1882 y(program)12 b(is)i(unde\014ned)h(if)e(messages)g(are)g(sen) o(t)g(b)o(y)g(a)g(C)g(pro)q(cess)g(and)h(receiv)o(ed)g(b)o(y)f(a)g(F)l (ortran)f(pro)q(cess,)75 1939 y(or)j(vice-v)o(ersa.)189 2030 y Fl(R)n(ationale.)39 b Fv(MPI)15 b Fx(do)q(es)g(not)f(handle)i (in)o(ter-language)f(comm)o(unication)h(b)q(ecause)g(there)f(are)f(no) 189 2087 y(agreed)h(standards)g(for)g(the)g(corresp)q(ondence)i(b)q(et) o(w)o(een)f(C)f(t)o(yp)q(es)g(and)h(F)l(ortran)e(t)o(yp)q(es.)21 b(There-)189 2143 y(fore,)14 b Fv(MPI)h Fx(programs)f(that)g(mix)i (languages)f(w)o(ould)h(not)e(p)q(ort.)20 b(\()p Fl(End)c(of)g(r)n (ationale.)p Fx(\))189 2235 y Fl(A)n(dvic)n(e)k(to)h(implementors.)71 b Fv(MPI)20 b Fx(implemen)o(tors)h(ma)o(y)f(w)o(an)o(t)f(to)h(supp)q (ort)g(in)o(ter-language)189 2292 y(comm)o(unication)13 b(b)o(y)g(allo)o(wing)g(F)l(ortran)f(programs)f(to)h(use)h(\\C)f(MPI)h (t)o(yp)q(es,")g(suc)o(h)g(as)f Fi(MPI)p 1782 2292 13 2 v 14 w(INT,)189 2348 y(MPI)p 266 2348 V 14 w(CHAR)p Fx(,)19 b(etc.,)h(and)f(allo)o(wing)i(C)e(programs)g(to)g(use)h(F)l (ortran)e(t)o(yp)q(es.)33 b(\()p Fl(End)20 b(of)h(advic)n(e)f(to)189 2404 y(implementors.)p Fx(\))75 2546 y Fn(3.4)59 b(Communication)18 b(Mo)r(des)75 2647 y Fx(The)e(send)h(call)h(describ)q(ed)g(in)f (Section)g(3.2.1)d(is)j Fq(blo)q(c)o(king)p Fx(:)23 b(it)16 b(do)q(es)h(not)f(return)g(un)o(til)h(the)f(message)75 2704 y(data)j(and)g(en)o(v)o(elop)q(e)h(ha)o(v)o(e)f(b)q(een)h(safely)g (stored)f(a)o(w)o(a)o(y)e(so)i(that)g(the)g(sender)h(is)g(free)f(to)f (access)i(and)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 26 34 26 33 bop 75 -100 a Fx(26)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fx(o)o(v)o(erwrite)e (the)g(send)h(bu\013er.)k(The)c(message)f(migh)o(t)g(b)q(e)h(copied)g (directly)h(in)o(to)e(the)h(matc)o(hing)f(receiv)o(e)75 102 y(bu\013er,)g(or)f(it)i(migh)o(t)f(b)q(e)h(copied)g(in)o(to)f(a)g (temp)q(orary)g(system)g(bu\013er.)166 158 y(Message)c(bu\013ering)i (decouples)g(the)f(send)h(and)f(receiv)o(e)h(op)q(erations.)19 b(A)12 b(blo)q(c)o(king)h(send)g(can)f(com-)75 214 y(plete)i(as)e(so)q (on)h(as)g(the)f(message)h(w)o(as)f(bu\013ered,)h(ev)o(en)h(if)f(no)g (matc)o(hing)g(receiv)o(e)g(has)g(b)q(een)h(executed)g(b)o(y)75 271 y(the)f(receiv)o(er.)19 b(On)14 b(the)e(other)g(hand,)i(message)e (bu\013ering)h(can)g(b)q(e)g(exp)q(ensiv)o(e,)h(as)e(it)h(en)o(tails)h (additional)75 327 y(memory-to-memory)j(cop)o(ying,)h(and)g(it)g (requires)h(the)f(allo)q(cation)h(of)e(memory)h(for)f(bu\013ering.)29 b Fv(MPI)75 384 y Fx(o\013ers)12 b(the)i(c)o(hoice)g(of)f(sev)o(eral)g (comm)o(unication)i(mo)q(des)e(that)g(allo)o(w)g(one)h(to)e(con)o(trol) h(the)h(c)o(hoice)g(of)f(the)75 440 y(comm)o(unication)j(proto)q(col.) 166 497 y(The)g(send)h(call)g(describ)q(ed)h(in)e(Section)h(3.2.1)d (used)j(the)f Fq(standard)g Fx(comm)o(unication)h(mo)q(de.)22 b(In)75 553 y(this)16 b(mo)q(de,)e(it)i(is)f(up)h(to)e Fv(MPI)h Fx(to)f(decide)j(whether)e(outgoing)g(messages)f(will)j(b)q(e) f(bu\013ered.)k Fv(MPI)15 b Fx(ma)o(y)75 610 y(bu\013er)j(outgoing)h (messages.)29 b(In)19 b(suc)o(h)g(a)f(case,)h(the)f(send)h(call)h(ma)o (y)e(complete)h(b)q(efore)g(a)f(matc)o(hing)75 666 y(receiv)o(e)d(is)f (in)o(v)o(ok)o(ed.)20 b(On)14 b(the)g(other)g(hand,)g(bu\013er)g(space) g(ma)o(y)f(b)q(e)h(una)o(v)m(ailable,)i(or)e Fv(MPI)f Fx(ma)o(y)g(c)o(ho)q(ose)75 723 y(not)k(to)h(bu\013er)f(outgoing)h (messages,)g(for)f(p)q(erformance)h(reasons.)27 b(In)19 b(this)f(case,)g(the)g(send)g(call)h(will)75 779 y(not)13 b(complete)h(un)o(til)g(a)f(matc)o(hing)g(receiv)o(e)h(has)f(b)q(een)i (p)q(osted,)e(and)g(the)h(data)e(has)h(b)q(een)i(mo)o(v)o(ed)d(to)h (the)75 835 y(receiv)o(er.)166 892 y(Th)o(us,)j(a)f(send)i(in)g (standard)f(mo)q(de)g(can)g(b)q(e)h(started)e(whether)h(or)g(not)f(a)h (matc)o(hing)g(receiv)o(e)h(has)75 948 y(b)q(een)d(p)q(osted.)19 b(It)12 b(ma)o(y)g(complete)h(b)q(efore)g(a)f(matc)o(hing)g(receiv)o(e) i(is)f(p)q(osted.)19 b(The)13 b(standard)f(mo)q(de)g(send)75 1005 y(is)18 b Fq(non-lo)q(cal)p Fx(:)27 b(successful)19 b(completion)g(of)e(the)g(send)h(op)q(eration)g(ma)o(y)f(dep)q(end)i (on)e(the)h(o)q(ccurrence)75 1061 y(of)d(a)g(matc)o(hing)g(receiv)o(e.) 189 1168 y Fl(R)n(ationale.)42 b Fx(The)15 b(reluctance)i(of)e Fv(MPI)g Fx(to)g(mandate)g(whether)h(standard)f(sends)h(are)f (bu\013ering)189 1224 y(or)e(not)h(stems)g(from)f(the)i(desire)g(to)f (ac)o(hiev)o(e)g(p)q(ortable)h(programs.)j(Since)e(an)o(y)e(system)g (will)i(run)189 1280 y(out)d(of)g(bu\013er)g(resources)h(as)f(message)g (sizes)h(are)f(increased,)i(and)f(some)f(implemen)o(tations)i(ma)o(y) 189 1337 y(w)o(an)o(t)i(to)g(pro)o(vide)i(little)h(bu\013ering,)f Fv(MPI)e Fx(tak)o(es)h(the)g(p)q(osition)h(that)f(correct)g(\(and)g (therefore,)189 1393 y(p)q(ortable\))c(programs)e(do)i(not)g(rely)g(on) g(system)g(bu\013ering)g(in)h(standard)e(mo)q(de.)20 b(Bu\013ering)15 b(ma)o(y)189 1450 y(impro)o(v)o(e)g(the)h(p)q (erformance)g(of)g(a)f(correct)h(program,)e(but)i(it)g(do)q(esn't)g (a\013ect)f(the)h(result)g(of)g(the)189 1506 y(program.)33 b(If)21 b(the)f(user)g(wishes)h(to)f(guaran)o(tee)f(a)h(certain)h (amoun)o(t)e(of)h(bu\013ering,)h(the)g(user-)189 1563 y(pro)o(vided)13 b(bu\013er)g(system)f(of)g(Sec.)20 b(3.6)12 b(should)i(b)q(e)f(used,)g(along)g(with)g(the)g(bu\013ered-mo)q(de)h (send.)189 1619 y(\()p Fl(End)h(of)i(r)n(ationale.)p Fx(\))166 1725 y(There)e(are)g(three)h(additional)g(comm)o(unication)g (mo)q(des.)166 1782 y(A)j Fq(bu\013ered)f Fx(mo)q(de)h(send)g(op)q (eration)g(can)g(b)q(e)h(started)e(whether)h(or)f(not)h(a)f(matc)o (hing)h(receiv)o(e)75 1838 y(has)g(b)q(een)h(p)q(osted.)31 b(It)19 b(ma)o(y)f(complete)i(b)q(efore)f(a)f(matc)o(hing)h(receiv)o(e) h(is)g(p)q(osted.)31 b(Ho)o(w)o(ev)o(er,)18 b(unlik)o(e)75 1895 y(the)k(standard)f(send,)i(this)f(op)q(eration)g(is)g Fq(lo)q(cal)p Fx(,)j(and)d(its)g(completion)h(do)q(es)f(not)f(dep)q (end)i(on)f(the)75 1951 y(o)q(ccurrence)d(of)e(a)h(matc)o(hing)g (receiv)o(e.)28 b(Th)o(us,)18 b(if)g(a)g(send)g(is)g(executed)h(and)f (no)g(matc)o(hing)g(receiv)o(e)g(is)75 2008 y(p)q(osted,)13 b(then)f Fv(MPI)g Fx(m)o(ust)f(bu\013er)h(the)g(outgoing)g(message,)g (so)g(as)f(to)h(allo)o(w)g(the)g(send)h(call)g(to)f(complete.)75 2064 y(An)h(error)g(will)i(o)q(ccur)e(if)h(there)f(is)g(insu\016cien)o (t)i(bu\013er)e(space.)20 b(The)13 b(amoun)o(t)f(of)h(a)o(v)m(ailable)i (bu\013er)e(space)75 2121 y(is)k(con)o(trolled)f(b)o(y)g(the)g(user)h (|)f(see)g(Section)h(3.6.)22 b(Bu\013er)16 b(allo)q(cation)h(b)o(y)f (the)g(user)g(ma)o(y)f(b)q(e)i(required)75 2177 y(for)e(the)g (bu\013ered)g(mo)q(de)h(to)f(b)q(e)g(e\013ectiv)o(e.)166 2234 y(A)j(send)g(that)f(uses)g(the)h Fq(sync)o(hronous)e Fx(mo)q(de)i(can)g(b)q(e)g(started)f(whether)g(or)g(not)h(a)f(matc)o (hing)75 2290 y(receiv)o(e)k(w)o(as)e(p)q(osted.)34 b(Ho)o(w)o(ev)o (er,)19 b(the)h(send)h(will)g(complete)g(successfully)h(only)e(if)g(a)g (matc)o(hing)g(re-)75 2346 y(ceiv)o(e)f(is)h(p)q(osted,)f(and)g(the)f (receiv)o(e)i(op)q(eration)f(has)f(started)g(to)g(receiv)o(e)h(the)g (message)f(sen)o(t)g(b)o(y)h(the)75 2403 y(sync)o(hronous)c(send.)21 b(Th)o(us,)14 b(the)h(completion)i(of)d(a)h(sync)o(hronous)g(send)h (not)e(only)i(indicates)h(that)d(the)75 2459 y(send)g(bu\013er)f(can)g (b)q(e)h(reused,)f(but)g(also)g(indicates)i(that)d(the)h(receiv)o(er)h (has)f(reac)o(hed)g(a)g(certain)h(p)q(oin)o(t)f(in)75 2516 y(its)i(execution,)g(namely)g(that)e(it)i(has)f(started)g (executing)h(the)g(matc)o(hing)f(receiv)o(e.)21 b(If)14 b(b)q(oth)h(sends)g(and)75 2572 y(receiv)o(es)e(are)e(blo)q(c)o(king)j (op)q(erations)e(then)g(the)g(use)h(of)e(the)h(sync)o(hronous)g(mo)q (de)g(pro)o(vides)h(sync)o(hronous)75 2629 y(comm)o(unication)18 b(seman)o(tics:)25 b(a)18 b(comm)o(unication)g(do)q(es)g(not)g (complete)g(at)f(either)h(end)h(b)q(efore)f(b)q(oth)75 2685 y(pro)q(cesses)e(rendezv)o(ous)f(at)g(the)g(comm)o(unication.)21 b(A)15 b(send)h(executed)g(in)g(this)g(mo)q(de)f(is)h Fq(non-lo)q(cal)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 27 35 27 34 bop 75 -100 a Fm(3.4.)34 b(COMMUNICA)l(TION)16 b(MODES)1048 b Fx(27)166 45 y(A)14 b(send)h(that)e(uses)i(the)f Fq(ready)g Fx(comm)o(unication)g(mo)q(de)h(ma)o(y)e(b)q(e)i(started)e Fl(only)h Fx(if)h(the)f(matc)o(hing)75 102 y(receiv)o(e)k(is)g(already) f(p)q(osted.)27 b(Otherwise,)18 b(the)f(op)q(eration)h(is)f(erroneous)g (and)h(its)f(outcome)g(is)h(unde-)75 158 y(\014ned.)i(On)13 b(some)f(systems,)h(this)g(allo)o(ws)g(the)f(remo)o(v)m(al)h(of)f(a)h (hand-shak)o(e)g(op)q(eration)g(that)f(is)h(otherwise)75 214 y(required)k(and)f(results)g(in)h(impro)o(v)o(ed)f(p)q(erformance.) 22 b(The)16 b(completion)h(of)e(the)h(send)h(op)q(eration)f(do)q(es)75 271 y(not)g(dep)q(end)i(on)e(the)g(status)f(of)h(a)g(matc)o(hing)g (receiv)o(e,)h(and)f(merely)h(indicates)h(that)d(the)h(send)h(bu\013er) 75 327 y(can)i(b)q(e)g(reused.)30 b(A)19 b(send)g(op)q(eration)g(that)f (uses)g(the)h(ready)f(mo)q(de)h(has)g(the)f(same)g(seman)o(tics)h(as)f (a)75 384 y(standard)i(send)i(op)q(eration,)g(or)e(a)g(sync)o(hronous)h (send)g(op)q(eration;)j(it)d(is)g(merely)g(that)g(the)f(sender)75 440 y(pro)o(vides)e(additional)g(information)g(to)e(the)i(system)f (\(namely)g(that)g(a)f(matc)o(hing)i(receiv)o(e)g(is)g(already)75 497 y(p)q(osted\),)d(that)g(can)h(sa)o(v)o(e)e(some)i(o)o(v)o(erhead.)k (In)c(a)f(correct)g(program,)f(therefore,)h(a)g(ready)h(send)g(could)75 553 y(b)q(e)i(replaced)g(b)o(y)f(a)f(standard)h(send)h(with)f(no)g (e\013ect)g(on)f(the)i(b)q(eha)o(vior)f(of)g(the)g(program)e(other)i (than)75 610 y(p)q(erformance.)166 666 y(Three)g(additional)i(send)f (functions)g(are)f(pro)o(vided)h(for)e(the)h(three)h(additional)g(comm) o(unication)75 723 y(mo)q(des.)23 b(The)16 b(comm)o(unication)h(mo)q (de)g(is)f(indicated)i(b)o(y)e(a)g(one)g(letter)h(pre\014x:)22 b Fv(B)16 b Fx(for)g(bu\013ered,)g Fv(S)h Fx(for)75 779 y(sync)o(hronous,)e(and)g Fv(R)h Fx(for)e(ready)l(.)75 930 y Fv(MPI)p 160 930 14 2 v 16 w(BSEND)i(\(buf,)f(count,)h(datat)o (yp)q(e,)h(dest,)f(tag,)f(comm)m(\))117 1007 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 1081 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1154 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1227 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1301 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1374 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 1499 y Fu(int)23 b(MPI)p 245 1499 15 2 v 17 w(Bsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1499 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 1555 y(int)h(tag,)f(MPI)p 683 1555 V 17 w(Comm)g(comm\))75 1642 y(MPI)p 150 1642 V 17 w(BSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 1698 y()f(BUF\(*\))170 1755 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 1841 y Fx(Send)16 b(in)g(bu\013ered)g(mo)q(de.)75 1992 y Fv(MPI)p 160 1992 14 2 v 16 w(SSEND)g(\(buf,)f(count,)h(datat)o(yp)q (e,)g(dest,)h(tag,)e(comm)m(\))117 2069 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 2143 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 2216 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 2289 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 2363 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 2436 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 2561 y Fu(int)23 b(MPI)p 245 2561 15 2 v 17 w(Ssend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 2561 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2617 y(int)h(tag,)f(MPI)p 683 2617 V 17 w(Comm)g(comm\))75 2704 y(MPI)p 150 2704 V 17 w(SSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 28 36 28 35 bop 75 -100 a Fx(28)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Fu()23 b(BUF\(*\))170 102 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h (COMM,)f(IERROR)166 188 y Fx(Send)16 b(in)g(sync)o(hronous)f(mo)q(de.) 75 339 y Fv(MPI)p 160 339 14 2 v 16 w(RSEND)h(\(buf,)f(count,)h(datat)o (yp)q(e,)h(dest,)f(tag,)f(comm)m(\))117 416 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 491 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 566 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 640 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 715 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 790 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 914 y Fu(int)23 b(MPI)p 245 914 15 2 v 17 w(Rsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 914 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 971 y(int)h(tag,)f(MPI)p 683 971 V 17 w(Comm)g(comm\))75 1057 y(MPI)p 150 1057 V 17 w(RSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 1114 y()f(BUF\(*\))170 1170 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 1257 y Fx(Send)16 b(in)g(ready)f(mo)q(de.)166 1313 y(There)20 b(is)g(only)h(one)f(receiv)o(e)h(op)q(eration,)g(whic)o(h)g(can)f(matc) o(h)f(an)o(y)g(of)h(the)g(send)g(mo)q(des.)35 b(The)75 1369 y(receiv)o(e)15 b(op)q(eration)g(describ)q(ed)h(in)f(the)f(last)g (section)h(is)g Fq(blo)q(c)o(king)p Fx(:)20 b(it)15 b(returns)f(only)g (after)g(the)g(receiv)o(e)75 1426 y(bu\013er)k(con)o(tains)h(the)f (newly)i(receiv)o(ed)f(message.)29 b(A)19 b(receiv)o(e)g(can)g (complete)g(b)q(efore)f(the)h(matc)o(hing)75 1482 y(send)d(has)f (completed)h(\(of)e(course,)h(it)h(can)f(complete)h(only)g(after)e(the) i(matc)o(hing)f(send)h(has)f(started\).)166 1539 y(In)f(a)g(m)o (ulti-threaded)h(implemen)o(tation)h(of)d Fv(MPI)p Fx(,)g(the)h(system) g(ma)o(y)f(de-sc)o(hedule)j(a)e(thread)g(that)75 1595 y(is)h(blo)q(c)o(k)o(ed)h(on)f(a)g(send)g(or)f(receiv)o(e)i(op)q (eration,)f(and)g(sc)o(hedule)i(another)d(thread)h(for)f(execution)i (in)g(the)75 1652 y(same)g(address)g(space.)22 b(In)16 b(suc)o(h)h(a)e(case)h(it)g(is)h(the)f(user's)f(resp)q(onsibilit)o(y)k (not)c(to)g(access)h(or)g(mo)q(dify)g(a)75 1708 y(comm)o(unication)i (bu\013er)e(un)o(til)i(the)f(comm)o(unication)h(completes.)25 b(Otherwise,)18 b(the)f(outcome)f(of)h(the)75 1765 y(computation)e(is)h (unde\014ned.)189 1868 y Fl(R)n(ationale.)56 b Fx(W)l(e)18 b(prohibit)i(read)d(accesses)i(to)e(a)h(send)g(bu\013er)g(while)i(it)e (is)g(b)q(eing)i(used,)f(ev)o(en)189 1924 y(though)f(the)g(send)g(op)q (eration)h(is)f(not)g(supp)q(osed)h(to)e(alter)h(the)g(con)o(ten)o(t)g (of)f(this)i(bu\013er.)28 b(This)189 1981 y(ma)o(y)17 b(seem)h(more)f(stringen)o(t)g(than)h(necessary)l(,)g(but)g(the)g (additional)h(restriction)g(causes)f(little)189 2037 y(loss)e(of)g(functionalit)o(y)i(and)f(allo)o(ws)g(b)q(etter)f(p)q (erformance)h(on)f(some)g(systems)g(|)h(consider)h(the)189 2094 y(case)13 b(where)g(data)f(transfer)g(is)i(done)f(b)o(y)g(a)g(DMA) f(engine)j(that)d(is)h(not)g(cac)o(he-coheren)o(t)h(with)f(the)189 2150 y(main)i(pro)q(cessor.)20 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 2254 y Fl(A)n(dvic)n(e)10 b(to)j(implementors.)36 b Fx(Since)12 b(a)e(sync)o(hronous)g(send)h(cannot)f(complete)h(b)q (efore)g(a)f(matc)o(hing)189 2310 y(receiv)o(e)16 b(is)f(p)q(osted,)h (one)f(will)i(not)e(normally)g(bu\013er)h(messages)e(sen)o(t)h(b)o(y)g (suc)o(h)h(an)f(op)q(eration.)189 2385 y(It)j(is)h(recommended)g(to)e (c)o(ho)q(ose)i(bu\013ering)f(o)o(v)o(er)g(blo)q(c)o(king)h(the)g (sender,)g(whenev)o(er)g(p)q(ossible,)189 2441 y(for)13 b(standard)h(sends.)20 b(The)15 b(programmer)e(can)i(signal)g(his)g(or) f(her)g(preference)h(for)f(blo)q(c)o(king)i(the)189 2498 y(sender)f(un)o(til)i(a)e(matc)o(hing)g(receiv)o(e)h(o)q(ccurs)g(b)o(y) f(using)h(the)f(sync)o(hronous)g(send)h(mo)q(de.)189 2572 y(A)f(p)q(ossible)i(comm)o(unication)g(proto)q(col)e(for)g(the)h (v)m(arious)f(comm)o(unication)i(mo)q(des)e(is)h(outlined)189 2629 y(b)q(elo)o(w.)189 2704 y Fv(ready)f(send)p Fx(:)21 b(The)16 b(message)e(is)i(sen)o(t)f(as)g(so)q(on)g(as)g(p)q(ossible.) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 29 37 29 36 bop 75 -100 a Fm(3.5.)29 b(SEMANTICS)16 b(OF)f(POINT-TO-POINT)i (COMMUNICA)l(TION)455 b Fx(29)189 45 y Fv(synchronous)20 b(send:)28 b Fx(The)19 b(sender)g(sends)g(a)f(request-to-send)h (message.)30 b(The)18 b(receiv)o(er)i(stores)189 102 y(this)12 b(request.)19 b(When)13 b(a)e(matc)o(hing)i(receiv)o(e)g(is)f (p)q(osted,)h(the)f(receiv)o(er)h(sends)g(bac)o(k)f(a)f(p)q(ermission-) 189 158 y(to-send)k(message,)f(and)i(the)f(sender)h(no)o(w)f(sends)g (the)h(message.)189 233 y Fv(standa)o(rd)e(send:)21 b Fx(First)13 b(proto)q(col)h(ma)o(y)e(b)q(e)j(used)f(for)f(short)f (messages,)h(and)h(second)g(proto)q(col)g(for)189 289 y(long)h(messages.)189 364 y Fv(bu\013ered)j(send:)25 b Fx(The)18 b(sender)g(copies)g(the)f(message)g(in)o(to)g(a)g(bu\013er) g(and)h(then)f(sends)h(it)g(with)f(a)189 421 y(non)o(blo)q(c)o(king)f (send)g(\(using)g(the)f(same)g(proto)q(col)g(as)g(for)f(standard)h (send\).)189 495 y(Additional)i(con)o(trol)f(messages)g(migh)o(t)g(b)q (e)g(needed)i(for)d(\015o)o(w)h(con)o(trol)f(and)i(error)e(reco)o(v)o (ery)l(.)22 b(Of)189 552 y(course,)15 b(there)g(are)g(man)o(y)f(other)h (p)q(ossible)i(proto)q(cols.)189 627 y(Ready)i(send)h(can)g(b)q(e)g (implemen)o(ted)h(as)e(a)f(standard)h(send.)33 b(In)20 b(this)g(case)f(there)g(will)i(b)q(e)f(no)189 683 y(p)q(erformance)15 b(adv)m(an)o(tage)g(\(or)f(disadv)m(an)o(tage\))h(for)g(the)g(use)h(of) e(ready)h(send.)189 758 y(A)g(standard)f(send)i(can)f(b)q(e)h(implemen) o(ted)h(as)e(a)f(sync)o(hronous)i(send.)k(In)c(suc)o(h)f(a)g(case,)g (no)g(data)189 814 y(bu\013ering)g(is)h(needed.)21 b(Ho)o(w)o(ev)o(er,) 14 b(man)o(y)h(\(most?\))k(users)c(exp)q(ect)h(some)f(bu\013ering.)189 889 y(In)h(a)g(m)o(ulti-threaded)i(en)o(vironmen)o(t,)e(the)h (execution)g(of)f(a)f(blo)q(c)o(king)j(comm)o(unication)f(should)189 946 y(blo)q(c)o(k)k(only)g(the)f(executing)i(thread,)f(allo)o(wing)g (the)g(thread)f(sc)o(heduler)i(to)e(de-sc)o(hedule)j(this)189 1002 y(thread)15 b(and)g(sc)o(hedule)i(another)e(thread)g(for)f (execution.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))75 1145 y Fn(3.5)59 b(Semantics)18 b(of)i(p)r(oint-to-p)r(oint)e (communication)75 1247 y Fx(A)g(v)m(alid)i Fv(MPI)e Fx(implemen)o (tation)h(guaran)o(tees)f(certain)h(general)f(prop)q(erties)h(of)f(p)q (oin)o(t-to-p)q(oin)o(t)h(com-)75 1303 y(m)o(unication,)d(whic)o(h)g (are)f(describ)q(ed)i(in)f(this)f(section.)75 1423 y Fv(Order)46 b Fx(Messages)17 b(are)g Fl(non-overtaking)p Fx(:)25 b(If)18 b(a)f(sender)i(sends)f(t)o(w)o(o)f(messages)g(in)i (succession)g(to)e(the)75 1479 y(same)d(destination,)i(and)f(b)q(oth)g (matc)o(h)f(the)h(same)f(receiv)o(e,)h(then)h(this)f(op)q(eration)g (cannot)f(receiv)o(e)i(the)75 1536 y(second)e(message)e(if)i(the)f (\014rst)g(one)h(is)f(still)i(p)q(ending.)21 b(If)13 b(a)g(receiv)o(er)h(p)q(osts)f(t)o(w)o(o)f(receiv)o(es)i(in)g (succession,)75 1592 y(and)k(b)q(oth)g(matc)o(h)f(the)h(same)g (message,)g(then)g(the)g(second)g(receiv)o(e)h(op)q(eration)f(cannot)g (b)q(e)g(satis\014ed)75 1649 y(b)o(y)g(this)g(message,)f(if)i(the)e (\014rst)h(one)g(is)g(still)h(p)q(ending.)29 b(This)19 b(requiremen)o(t)f(facilitates)h(matc)o(hing)e(of)75 1705 y(sends)k(to)f(receiv)o(es.)36 b(It)21 b(guaran)o(tees)f(that)f (message-passing)i(co)q(de)g(is)g(deterministic,)j(if)d(pro)q(cesses)75 1761 y(are)d(single-threaded)j(and)e(the)g(wildcard)h Fi(MPI)p 913 1761 13 2 v 14 w(ANY)p 1012 1761 V 15 w(SOURCE)c Fx(is)k(not)e(used)h(in)h(receiv)o(es.)31 b(\(Some)18 b(of)75 1818 y(the)c(calls)i(describ)q(ed)g(later,)e(suc)o(h)h(as)f Fv(MPI)p 813 1818 14 2 v 15 w(CANCEL)h Fx(or)f Fv(MPI)p 1152 1818 V 16 w(W)l(AIT)l(ANY)p Fx(,)g(are)g(additional)i(sources)e (of)75 1874 y(nondeterminism.\))166 1931 y(If)j(a)g(pro)q(cess)h(has)f (a)g(single)h(thread)f(of)g(execution,)i(then)e(an)o(y)g(t)o(w)o(o)f (comm)o(unications)i(executed)75 1987 y(b)o(y)e(this)h(pro)q(cess)g (are)f(ordered.)23 b(On)17 b(the)f(other)g(hand,)h(if)g(the)f(pro)q (cess)h(is)g(m)o(ulti-threaded,)g(then)g(the)75 2044 y(seman)o(tics)e(of)f(thread)h(execution)h(ma)o(y)e(not)g(de\014ne)i(a) e(relativ)o(e)i(order)e(b)q(et)o(w)o(een)h(t)o(w)o(o)e(send)j(op)q (erations)75 2100 y(executed)23 b(b)o(y)f(t)o(w)o(o)f(distinct)i (threads.)40 b(The)23 b(op)q(erations)f(are)g(logically)h(concurren)o (t,)h(ev)o(en)e(if)h(one)75 2157 y(ph)o(ysically)f(precedes)e(the)g (other.)33 b(In)20 b(suc)o(h)g(a)f(case,)i(the)e(t)o(w)o(o)g(messages)g (sen)o(t)g(can)h(b)q(e)g(receiv)o(ed)h(in)75 2213 y(an)o(y)f(order.)37 b(Similarly)l(,)24 b(if)d(t)o(w)o(o)e(receiv)o(e)j(op)q(erations)f (that)f(are)g(logically)j(concurren)o(t)e(receiv)o(e)g(t)o(w)o(o)75 2270 y(successiv)o(ely)i(sen)o(t)d(messages,)h(then)g(the)g(t)o(w)o(o)f (messages)g(can)h(matc)o(h)f(the)h(t)o(w)o(o)e(receiv)o(es)j(in)f (either)75 2326 y(order.)75 2430 y Fq(Example)d(3.5)k Fx(An)16 b(example)g(of)f(non-o)o(v)o(ertaking)f(messages.)75 2534 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2591 y(IF)h(\(rank.EQ.0\))e(THEN)170 2647 y(CALL)i(MPI_BSEND\(buf1,)e (count,)h(MPI_REAL,)f(1,)i(tag,)f(comm,)g(ierr\))170 2704 y(CALL)h(MPI_BSEND\(buf2,)e(count,)h(MPI_REAL,)f(1,)i(tag,)f (comm,)g(ierr\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 30 38 30 37 bop 75 -100 a Fx(30)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fu(ELSE)95 b(!)24 b(rank.EQ.1)170 102 y(CALL)g(MPI_RECV\(buf1,)e(count,)h (MPI_REAL,)f(0,)i(MPI_ANY_TAG,)e(comm,)h(status,)g(ierr\))170 158 y(CALL)h(MPI_RECV\(buf2,)e(count,)h(MPI_REAL,)f(0,)i(tag,)f(comm,)g (status,)g(ierr\))75 214 y(END)g(IF)75 315 y Fx(The)17 b(message)f(sen)o(t)h(b)o(y)g(the)f(\014rst)h(send)g(m)o(ust)f(b)q(e)i (receiv)o(ed)g(b)o(y)f(the)f(\014rst)h(receiv)o(e,)h(and)f(the)f (message)75 372 y(sen)o(t)f(b)o(y)g(the)g(second)h(send)g(m)o(ust)e(b)q (e)i(receiv)o(ed)h(b)o(y)e(the)g(second)h(receiv)o(e.)75 491 y Fv(Progress)45 b Fx(If)13 b(a)g(pair)g(of)f(matc)o(hing)h(send)h (and)f(receiv)o(es)g(ha)o(v)o(e)g(b)q(een)h(initiated)g(on)f(t)o(w)o(o) e(pro)q(cesses,)j(then)75 547 y(at)k(least)h(one)h(of)e(these)h(t)o(w)o (o)f(op)q(erations)h(will)i(complete,)f(indep)q(enden)o(tly)i(of)c (other)h(actions)g(in)h(the)75 604 y(system:)e(the)13 b(send)h(op)q(eration)f(will)h(complete,)g(unless)g(the)f(receiv)o(e)h (is)f(satis\014ed)h(b)o(y)e(another)h(message,)75 660 y(and)h(completes;)g(the)g(receiv)o(e)h(op)q(eration)f(will)h (complete,)g(unless)g(the)f(message)f(sen)o(t)g(is)i(consumed)f(b)o(y) 75 717 y(another)h(matc)o(hing)g(receiv)o(e)h(that)f(w)o(as)f(p)q (osted)h(at)g(the)g(same)g(destination)h(pro)q(cess.)75 817 y Fq(Example)i(3.6)k Fx(An)16 b(example)g(of)f(t)o(w)o(o,)e(in)o (tert)o(wined)j(matc)o(hing)f(pairs.)75 907 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 964 y(IF)h(\(rank.EQ.0\))e (THEN)170 1020 y(CALL)i(MPI_BSEND\(buf1,)e(count,)h(MPI_REAL,)f(1,)i (tag1,)f(comm,)g(ierr\))170 1077 y(CALL)h(MPI_SSEND\(buf2,)e(count,)h (MPI_REAL,)f(1,)i(tag2,)f(comm,)g(ierr\))75 1133 y(ELSE)95 b(!)24 b(rank.EQ.1)170 1190 y(CALL)g(MPI_RECV\(buf1,)e(count,)h (MPI_REAL,)f(0,)i(tag2,)f(comm,)g(status,)g(ierr\))170 1246 y(CALL)h(MPI_RECV\(buf2,)e(count,)h(MPI_REAL,)f(0,)i(tag1,)f (comm,)g(status,)g(ierr\))75 1302 y(END)g(IF)75 1392 y Fx(Both)18 b(pro)q(cesses)g(in)o(v)o(ok)o(e)h(their)f(\014rst)g(comm) o(unication)h(call.)30 b(Since)19 b(the)f(\014rst)g(send)h(of)f(pro)q (cess)g(zero)75 1449 y(uses)h(the)f(bu\013ered)h(mo)q(de,)g(it)g(m)o (ust)f(complete,)i(irresp)q(ectiv)o(e)g(of)e(the)g(state)g(of)g(pro)q (cess)h(one.)29 b(Since)75 1505 y(no)15 b(matc)o(hing)h(receiv)o(e)g (is)g(p)q(osted,)g(the)g(message)f(will)i(b)q(e)f(copied)h(in)o(to)e (bu\013er)h(space.)21 b(\(If)15 b(insu\016cien)o(t)75 1562 y(bu\013er)h(space)g(is)h(a)o(v)m(ailable,)g(then)f(the)g(program) f(will)j(fail.\))k(The)16 b(second)h(send)f(is)h(then)f(in)o(v)o(ok)o (ed.)22 b(A)o(t)75 1618 y(that)16 b(p)q(oin)o(t,)g(a)h(matc)o(hing)f (pair)h(of)f(send)h(and)f(receiv)o(e)h(op)q(eration)g(is)g(enabled,)h (and)e(b)q(oth)h(op)q(erations)75 1675 y(m)o(ust)f(complete.)23 b(Pro)q(cess)16 b(one)g(next)g(in)o(v)o(ok)o(es)g(its)g(second)h (receiv)o(e)g(call,)g(whic)o(h)g(will)h(b)q(e)e(satis\014ed)h(b)o(y)75 1731 y(the)c(bu\013ered)g(message.)19 b(Note)12 b(that)h(pro)q(cess)g (one)g(receiv)o(ed)h(the)f(messages)f(in)i(the)f(rev)o(erse)f(order)h (they)75 1788 y(w)o(ere)i(sen)o(t.)75 1907 y Fv(F)o(airness)46 b(MPI)15 b Fx(mak)o(es)h(no)f(guaran)o(tee)h(of)f Fl(fairness)g Fx(in)i(the)f(handling)h(of)f(comm)o(unication.)23 b(Supp)q(ose)75 1963 y(that)17 b(a)g(send)h(is)g(p)q(osted.)26 b(Then)18 b(it)g(is)g(p)q(ossible)h(that)d(the)i(destination)g(pro)q(cess)g(rep)q (eatedly)g(p)q(osts)f(a)75 2020 y(receiv)o(e)i(that)e(matc)o(hes)g (this)h(send,)h(y)o(et)e(the)h(message)f(is)h(nev)o(er)g(receiv)o(ed,)h (b)q(ecause)g(it)f(is)g(eac)o(h)g(time)75 2076 y(o)o(v)o(ertak)o(en)c (b)o(y)i(another)f(message,)g(sen)o(t)h(from)f(another)g(source.)21 b(Similarly)l(,)d(supp)q(ose)f(that)e(a)g(receiv)o(e)75 2133 y(w)o(as)f(p)q(osted)i(b)o(y)f(a)g(m)o(ulti-threaded)h(pro)q (cess.)k(Then)c(it)f(is)h(p)q(ossible)h(that)d(messages)h(that)f(matc)o (h)h(this)75 2189 y(receiv)o(e)20 b(are)g(rep)q(eatedly)g(receiv)o(ed,) i(y)o(et)d(the)g(receiv)o(e)i(is)f(nev)o(er)f(satis\014ed,)i(b)q (ecause)f(it)g(is)g(o)o(v)o(ertak)o(en)75 2246 y(b)o(y)15 b(other)h(receiv)o(es)g(p)q(osted)g(at)e(this)i(no)q(de)h(\(b)o(y)e (other)g(executing)h(threads\).)21 b(It)15 b(is)h(the)g(programmer's)75 2302 y(resp)q(onsibilit)o(y)i(to)c(prev)o(en)o(t)h(starv)m(ation)g(in)h (suc)o(h)g(situations.)75 2421 y Fv(Resource)23 b(limitations)44 b Fx(An)o(y)22 b(p)q(ending)i(comm)o(unication)e(op)q(eration)h (consumes)f(system)g(resources)75 2478 y(that)14 b(are)g(limited.)22 b(Errors)14 b(ma)o(y)g(o)q(ccur)h(when)g(lac)o(k)g(of)f(resources)h (prev)o(en)o(t)f(the)h(execution)h(of)e(an)h Fv(MPI)75 2534 y Fx(call.)20 b(A)13 b(qualit)o(y)g(implemen)o(tation)h(will)g (use)f(a)f(\(small\))g(\014xed)h(amoun)o(t)f(of)g(resources)h(for)f (eac)o(h)g(p)q(ending)75 2591 y(send)i(in)f(the)g(ready)g(or)g(sync)o (hronous)g(mo)q(de)g(and)g(for)g(eac)o(h)g(p)q(ending)i(receiv)o(e.)20 b(Ho)o(w)o(ev)o(er,)12 b(bu\013er)h(space)75 2647 y(ma)o(y)g(b)q(e)h (consumed)g(to)e(store)h(messages)g(sen)o(t)g(in)h(standard)f(mo)q(de,) h(and)f(m)o(ust)g(b)q(e)h(consumed)g(to)f(store)75 2704 y(messages)e(sen)o(t)g(in)i(bu\013ered)f(mo)q(de,)g(when)g(no)f(matc)o (hing)h(receiv)o(e)h(is)f(a)o(v)m(ailable.)20 b(The)12 b(amoun)o(t)f(of)g(space)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 31 39 31 38 bop 75 -100 a Fm(3.5.)34 b(SEMANTICS)16 b(OF)f(POINT-TO-POINT)i (COMMUNICA)l(TION)450 b Fx(31)75 45 y(a)o(v)m(ailable)17 b(for)e(bu\013ering)i(will)g(b)q(e)f(m)o(uc)o(h)g(smaller)h(than)e (program)g(data)g(memory)g(on)h(man)o(y)f(systems.)75 102 y(Then,)g(it)h(will)h(b)q(e)e(easy)g(to)g(write)g(programs)f(that)h (o)o(v)o(errun)f(a)o(v)m(ailable)j(bu\013er)f(space.)166 158 y Fv(MPI)e Fx(allo)o(ws)h(the)g(user)g(to)f(pro)o(vide)i(bu\013er)f (memory)f(for)g(messages)g(sen)o(t)h(in)h(the)f(bu\013ered)g(mo)q(de.) 75 214 y(F)l(urthermore,)f Fv(MPI)g Fx(sp)q(eci\014es)j(a)e(detailed)h (op)q(erational)g(mo)q(del)g(for)e(the)h(use)h(of)e(this)i(bu\013er.)j (An)d Fv(MPI)75 271 y Fx(implemen)o(tation)f(is)f(required)h(to)e(do)h (no)f(w)o(orse)g(than)h(implied)i(b)o(y)d(this)h(mo)q(del.)21 b(This)14 b(allo)o(ws)g(users)g(to)75 327 y(a)o(v)o(oid)h(bu\013er)g(o) o(v)o(er\015o)o(ws)e(when)i(they)g(use)h(bu\013ered)f(sends.)20 b(Bu\013er)15 b(allo)q(cation)h(and)f(use)h(is)f(describ)q(ed)75 384 y(in)h(Section)g(3.6.)166 440 y(A)j(bu\013ered)g(send)h(op)q (eration)f(that)f(cannot)h(complete)h(b)q(ecause)f(of)g(a)g(lac)o(k)g (of)f(bu\013er)h(space)g(is)75 497 y(erroneous.)36 b(When)20 b(suc)o(h)h(a)f(situation)h(is)g(detected,)h(an)f(error)e(is)i (signalled)i(that)c(ma)o(y)h(cause)h(the)75 553 y(program)f(to)h (terminate)g(abnormally)l(.)38 b(On)22 b(the)f(other)g(hand,)i(a)e (standard)g(send)g(op)q(eration)h(that)75 610 y(cannot)16 b(complete)g(b)q(ecause)h(of)f(lac)o(k)g(of)f(bu\013er)h(space)g(will)i (merely)e(blo)q(c)o(k,)h(w)o(aiting)f(for)f(bu\013er)h(space)75 666 y(to)g(b)q(ecome)h(a)o(v)m(ailable)h(or)e(for)f(a)h(matc)o(hing)h (receiv)o(e)g(to)f(b)q(e)h(p)q(osted.)23 b(This)17 b(b)q(eha)o(vior)g (is)g(preferable)g(in)75 723 y(man)o(y)h(situations.)31 b(Consider)20 b(a)e(situation)h(where)g(a)g(pro)q(ducer)g(rep)q (eatedly)h(pro)q(duces)g(new)f(v)m(alues)75 779 y(and)j(sends)g(them)f (to)g(a)g(consumer.)39 b(Assume)22 b(that)e(the)i(pro)q(ducer)g(pro)q (duces)g(new)g(v)m(alues)h(faster)75 835 y(than)18 b(the)g(consumer)g (can)g(consume)g(them.)28 b(If)18 b(bu\013ered)g(sends)g(are)g(used,)h (then)f(a)f(bu\013er)h(o)o(v)o(er\015o)o(w)75 892 y(will)i(result.)31 b(Additional)20 b(sync)o(hronization)g(has)e(to)g(b)q(e)h(added)h(to)e (the)g(program)g(so)g(as)g(to)g(prev)o(en)o(t)75 948 y(this)h(from)e(o)q(ccurring.)29 b(If)19 b(standard)e(sends)i(are)f (used,)h(then)f(the)g(pro)q(ducer)h(will)h(b)q(e)f(automatically)75 1005 y(throttled,)c(as)f(its)i(send)g(op)q(erations)f(will)i(blo)q(c)o (k)f(when)f(bu\013er)h(space)f(is)h(una)o(v)m(ailable.)166 1061 y(In)c(some)f(situations,)h(a)f(lac)o(k)g(of)g(bu\013er)h(space)f (leads)h(to)f(deadlo)q(c)o(k)h(situations.)19 b(This)12 b(is)g(illustrated)75 1118 y(b)o(y)j(the)g(examples)h(b)q(elo)o(w.)75 1206 y Fq(Example)i(3.7)k Fx(An)16 b(exc)o(hange)f(of)g(messages.)75 1294 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1350 y(IF)h(\(rank.EQ.0\))e(THEN)170 1407 y(CALL)i(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\))170 1463 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(status,)g(ierr\))75 1519 y(ELSE)95 b(!)24 b(rank.EQ.1)170 1576 y(CALL)g(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(status,)g(ierr\))170 1632 y(CALL)h(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))75 1689 y(END)g(IF)75 1777 y Fx(This)16 b(program)f(will)i(succeed)g(ev)o(en)f (if)g(no)g(bu\013er)g(space)g(for)f(data)g(is)h(a)o(v)m(ailable.)23 b(The)16 b(standard)f(send)75 1833 y(op)q(eration)g(can)h(b)q(e)g (replaced,)g(in)g(this)f(example,)h(with)g(a)e(sync)o(hronous)i(send.) 75 1921 y Fq(Example)i(3.8)k Fx(An)16 b(attempt)e(to)h(exc)o(hange)g (messages.)75 2002 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))75 2058 y(IF)h(\(rank.EQ.0\))e(THEN)170 2115 y(CALL)i(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g (status,)g(ierr\))170 2171 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i (MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\))75 2228 y(ELSE)95 b(!)24 b(rank.EQ.1)170 2284 y(CALL)g(MPI_RECV\(recvbuf,)d(count,)i (MPI_REAL,)g(0,)h(tag,)f(comm,)g(status,)g(ierr\))170 2341 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))75 2397 y(END)g(IF)75 2478 y Fx(The)16 b(receiv)o(e)g(op)q(eration)f(of)g(the)h(\014rst)f(pro)q(cess)g(m)o (ust)g(complete)h(b)q(efore)g(its)g(send,)f(and)h(can)f(complete)75 2534 y(only)g(if)g(the)g(matc)o(hing)f(send)h(of)f(the)h(second)g(pro)q (cessor)f(is)h(executed.)21 b(The)15 b(receiv)o(e)g(op)q(eration)g(of)f (the)75 2591 y(second)j(pro)q(cess)g(m)o(ust)e(complete)i(b)q(efore)g (its)g(send)g(and)f(can)h(complete)g(only)g(if)g(the)f(matc)o(hing)g (send)75 2647 y(of)d(the)h(\014rst)f(pro)q(cess)g(is)h(executed.)21 b(This)14 b(program)e(will)j(alw)o(a)o(ys)e(deadlo)q(c)o(k.)20 b(The)14 b(same)f(holds)h(for)f(an)o(y)75 2704 y(other)i(send)h(mo)q (de.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 32 40 32 39 bop 75 -100 a Fx(32)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fq(Example)h(3.9)k Fx(An)16 b(exc)o(hange)f(that)g(relies)h(on)f(bu\013ering.)75 139 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 195 y(IF)h(\(rank.EQ.0\))e(THEN)170 252 y(CALL)i(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\))170 308 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(status,)g(ierr\))75 365 y(ELSE)95 b(!)24 b(rank.EQ.1)170 421 y(CALL)g(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))170 478 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i (MPI_REAL,)g(0,)h(tag,)f(comm,)g(status,)g(ierr\))75 534 y(END)g(IF)75 628 y Fx(The)17 b(message)g(sen)o(t)g(b)o(y)g(eac)o (h)g(pro)q(cess)g(has)g(to)g(b)q(e)h(copied)g(out)f(b)q(efore)g(the)g (send)h(op)q(eration)f(returns)75 684 y(and)e(the)g(receiv)o(e)h(op)q (eration)f(starts.)k(F)l(or)14 b(the)h(program)f(to)g(complete,)h(it)h (is)f(necessary)g(that)g(at)f(least)75 741 y(one)22 b(of)f(the)g(t)o(w) o(o)f(messages)h(sen)o(t)g(b)q(e)h(bu\013ered.)39 b(Th)o(us,)23 b(this)f(program)e(can)h(succeed)i(only)f(if)g(the)75 797 y(comm)o(unication)16 b(system)f(can)g(bu\013er)g(at)g(least)g Fv(count)i Fx(w)o(ords)d(of)h(data.)189 904 y Fl(A)n(dvic)n(e)f(to)h (users.)40 b Fx(When)14 b(standard)g(send)g(op)q(erations)g(are)g (used,)g(then)h(a)f(deadlo)q(c)o(k)h(situation)189 960 y(ma)o(y)c(o)q(ccur)h(where)h(b)q(oth)f(pro)q(cesses)g(are)g(blo)q(c)o (k)o(ed)h(b)q(ecause)g(bu\013er)f(space)g(is)h(not)e(a)o(v)m(ailable.) 21 b(The)189 1016 y(same)15 b(will)i(certainly)g(happ)q(en,)f(if)g(the) g(sync)o(hronous)f(mo)q(de)h(is)g(used.)21 b(If)16 b(the)g(bu\013ered)g (mo)q(de)f(is)189 1073 y(used,)k(and)f(not)g(enough)g(bu\013er)h(space) f(is)h(a)o(v)m(ailable,)h(then)e(the)h(program)e(will)i(not)f(complete) 189 1129 y(either.)34 b(Ho)o(w)o(ev)o(er,)19 b(rather)g(than)h(a)f (deadlo)q(c)o(k)i(situation,)g(w)o(e)e(shall)i(ha)o(v)o(e)e(a)h (bu\013er)f(o)o(v)o(er\015o)o(w)189 1186 y(error.)189 1261 y(A)c(program)g(is)h(\\safe")e(if)i(no)g(message)f(bu\013ering)h (is)g(required)h(for)e(the)g(program)g(to)g(complete.)189 1317 y(One)22 b(can)g(replace)h(all)f(sends)h(in)f(suc)o(h)g(program)f (with)h(sync)o(hronous)g(sends,)h(and)f(the)g(pro-)189 1374 y(gram)16 b(will)i(still)h(run)e(correctly)l(.)25 b(This)18 b(conserv)m(ativ)o(e)f(programming)f(st)o(yle)h(pro)o(vides)h (the)f(b)q(est)189 1430 y(p)q(ortabilit)o(y)l(,)e(since)g(program)f (completion)h(do)q(es)g(not)e(dep)q(end)j(on)f(the)f(amoun)o(t)g(of)f (bu\013er)i(space)189 1487 y(a)o(v)m(ailable)i(or)d(in)i(the)g(comm)o (unication)f(proto)q(col)h(used.)189 1562 y(Man)o(y)f(programmers)f (prefer)i(to)f(ha)o(v)o(e)h(more)f(leew)o(a)o(y)h(and)g(b)q(e)g(able)h (to)e(use)h(the)g(\\unsafe")f(pro-)189 1618 y(gramming)c(st)o(yle)g (sho)o(wn)h(in)g(example)h(3.9.)k(In)12 b(suc)o(h)g(cases,)g(the)g(use) g(of)f(standard)g(sends)h(is)g(lik)o(ely)189 1675 y(to)i(pro)o(vide)i (the)g(b)q(est)f(compromise)h(b)q(et)o(w)o(een)f(p)q(erformance)h(and)f (robustness:)20 b(qualit)o(y)c(imple-)189 1731 y(men)o(tations)e(will)j (pro)o(vide)f(su\016cien)o(t)g(bu\013ering)f(so)g(that)f(\\common)h (practice")g(programs)f(will)189 1788 y(not)j(deadlo)q(c)o(k.)28 b(The)18 b(bu\013ered)g(send)g(mo)q(de)g(can)f(b)q(e)i(used)f(for)f (programs)f(that)h(require)h(more)189 1844 y(bu\013ering,)k(or)e(in)i (situations)f(where)g(the)g(programmer)f(w)o(an)o(ts)f(more)i(con)o (trol.)36 b(This)22 b(mo)q(de)189 1901 y(migh)o(t)14 b(also)g(b)q(e)h(used)g(for)e(debugging)j(purp)q(oses,)e(as)g(bu\013er) g(o)o(v)o(er\015o)o(w)f(conditions)j(are)e(easier)h(to)189 1957 y(diagnose)g(than)g(deadlo)q(c)o(k)h(conditions.)189 2032 y(Non)o(blo)q(c)o(king)h(message-passing)f(op)q(erations,)h(as)f (describ)q(ed)i(in)f(Section)g(3.7,)e(can)i(b)q(e)g(used)g(to)189 2089 y(a)o(v)o(oid)f(the)g(need)h(for)f(bu\013ering)g(outgoing)g (messages.)23 b(This)16 b(prev)o(en)o(ts)g(deadlo)q(c)o(ks)h(due)g(to)f (lac)o(k)189 2145 y(of)h(bu\013er)h(space,)g(and)g(impro)o(v)o(es)f(p)q (erformance,)h(b)o(y)g(allo)o(wing)g(o)o(v)o(erlap)g(of)f(computation)h (and)189 2202 y(comm)o(unication,)12 b(and)f(a)o(v)o(oiding)h(the)f(o)o (v)o(erheads)g(of)f(allo)q(cating)j(bu\013ers)e(and)g(cop)o(ying)g (messages)189 2258 y(in)o(to)k(bu\013ers.)20 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fx(\))75 2401 y Fn(3.6)59 b(Bu\013er)20 b(allo)r(cation)e(and)i(usage)75 2503 y Fx(A)13 b(user)g(ma)o(y)f(sp)q(ecify)i(a)e(bu\013er)h(to)f(b)q(e)h (used)h(for)e(bu\013ering)h(messages)f(sen)o(t)h(in)g(bu\013ered)h(mo)q (de.)19 b(Bu\013er-)75 2559 y(ing)d(is)f(done)h(b)o(y)f(the)g(sender.) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 33 41 33 40 bop 75 -100 a Fm(3.6.)34 b(BUFFER)15 b(ALLOCA)l(TION)i(AND)f(USA) o(GE)843 b Fx(33)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(BUFFER)p 345 45 V 17 w(A)l(TT)l(A)o(CH\()16 b(bu\013er,)f(size\))117 122 y Fk(IN)155 b Fv(bu\013er)478 b Fk(initial)12 b(bu\013er)j(address) g(\(c)o(hoice\))117 197 y(IN)155 b Fv(size)519 b Fk(bu\013er)15 b(size,)f(in)g(b)o(ytes)g(\(in)o(teger\))75 322 y Fu(int)23 b(MPI)p 245 322 15 2 v 17 w(Buffer)p 406 322 V 17 w(attach\()f(void*)i (buffer,)f(int)g(size\))75 408 y(MPI)p 150 408 V 17 w(BUFFER)p 311 408 V 16 w(ATTACH\()g(BUFFER,)g(SIZE,)g(IERROR\))170 465 y()g(BUFFER\(*\))170 521 y(INTEGER)g(SIZE,)h(IERROR)166 608 y Fx(Pro)o(vides)16 b(to)f Fv(MPI)g Fx(a)h(bu\013er)g(in)h(the)e (user's)h(memory)f(to)h(b)q(e)g(used)h(for)e(bu\013ering)h(outgoing)g (mes-)75 664 y(sages.)22 b(The)16 b(bu\013er)g(is)h(used)g(only)f(b)o (y)g(messages)g(sen)o(t)f(in)i(bu\013ered)g(mo)q(de.)23 b(Only)17 b(one)f(bu\013er)g(can)g(b)q(e)75 721 y(attac)o(hed)f(to)f(a) h(pro)q(cess)h(at)e(a)h(time.)75 872 y Fv(MPI)p 160 872 14 2 v 16 w(BUFFER)p 345 872 V 17 w(DET)l(A)o(CH\()g(bu\013er,)g (size\))117 949 y Fk(OUT)108 b Fv(bu\013er)478 b Fk(initial)12 b(bu\013er)j(address)g(\(c)o(hoice\))117 1024 y(OUT)108 b Fv(size)519 b Fk(bu\013er)15 b(size,)f(in)g(b)o(ytes)g(\(in)o (teger\))75 1148 y Fu(int)23 b(MPI)p 245 1148 15 2 v 17 w(Buffer)p 406 1148 V 17 w(detach\()f(void**)i(buffer,)e(int*)i (size\))75 1235 y(MPI)p 150 1235 V 17 w(BUFFER)p 311 1235 V 16 w(DETACH\()f(BUFFER,)g(SIZE,)g(IERROR\))170 1291 y()g(BUFFER\(*\))170 1348 y(INTEGER)g(SIZE,)h(IERROR)166 1434 y Fx(Detac)o(h)19 b(the)h(bu\013er)f(curren)o(tly)i(asso)q(ciated) f(with)g Fv(MPI)p Fx(.)e(This)j(op)q(eration)e(will)j(blo)q(c)o(k)e(un) o(til)h(all)75 1491 y(messages)16 b(curren)o(tly)h(in)g(the)f(bu\013er) g(ha)o(v)o(e)g(b)q(een)i(transmitted.)k(Up)q(on)17 b(return)f(of)g (this)h(function,)g(the)75 1547 y(user)e(ma)o(y)g(reuse)g(or)g(deallo)q (cate)h(the)g(space)f(tak)o(en)g(b)o(y)g(the)g(bu\013er.)166 1604 y(The)j(statemen)o(ts)f(made)g(in)i(this)f(section)h(describ)q(e)g (the)f(b)q(eha)o(vior)g(of)g Fv(MPI)f Fx(for)g(bu\013ered-mo)q(de)75 1660 y(sends.)38 b(When)22 b(no)f(bu\013er)g(is)g(curren)o(tly)h(asso)q (ciated,)g Fv(MPI)f Fx(b)q(eha)o(v)o(es)g(as)g(if)g(a)g(zero-sized)h (bu\013er)f(is)75 1716 y(asso)q(ciated)15 b(with)h(the)f(pro)q(cess.) 166 1773 y Fv(MPI)20 b Fx(m)o(ust)g(pro)o(vide)h(as)f(m)o(uc)o(h)h (bu\013ering)g(for)f(outgoing)g(messages)g Fl(as)h(if)30 b Fx(outgoing)20 b(message)75 1829 y(data)d(w)o(ere)h(bu\013ered)h(b)o (y)f(the)g(sending)h(pro)q(cess,)g(in)g(the)f(sp)q(eci\014ed)i (bu\013er)e(space,)h(using)f(a)g(circular,)75 1886 y(con)o (tiguous-space)e(allo)q(cation)h(p)q(olicy)l(.)24 b(W)l(e)16 b(outline)h(b)q(elo)o(w)f(a)g(mo)q(del)h(implemen)o(tation)g(that)e (de\014nes)75 1942 y(this)i(p)q(olicy)l(.)24 b Fv(MPI)16 b Fx(ma)o(y)g(pro)o(vide)g(more)g(bu\013ering,)h(and)f(ma)o(y)g(use)g (a)g(b)q(etter)g(bu\013er)h(allo)q(cation)g(algo-)75 1999 y(rithm)i(than)g(describ)q(ed)i(b)q(elo)o(w.)31 b(On)19 b(the)g(other)g(hand,)h Fv(MPI)e Fx(ma)o(y)g(signal)i(an)e (error)h(whenev)o(er)g(the)75 2055 y(simple)14 b(bu\013ering)f(allo)q (cator)f(describ)q(ed)i(b)q(elo)o(w)f(w)o(ould)g(run)g(out)f(of)g (space.)19 b(In)13 b(particular,)g(if)g(no)f(bu\013er)75 2112 y(is)k(explicitly)i(asso)q(ciated)d(with)h(the)f(pro)q(cess,)g (then)h(an)o(y)e(bu\013ered)i(send)g(ma)o(y)e(cause)i(an)f(error.)166 2168 y Fv(MPI)10 b Fx(do)q(es)g(not)g(pro)o(vide)h(mec)o(hanisms)f(for) g(querying)h(or)f(con)o(trolling)h(bu\013ering)g(done)f(b)o(y)g (standard)75 2225 y(mo)q(de)16 b(sends.)22 b(It)15 b(is)h(exp)q(ected)h (that)e(v)o(endors)h(will)h(pro)o(vide)f(suc)o(h)g(information)g(for)f (their)h(implemen-)75 2281 y(tations.)189 2387 y Fl(R)n(ationale.)53 b Fx(There)18 b(is)g(a)f(wide)h(sp)q(ectrum)g(of)f(p)q(ossible)i (implemen)o(tations)g(of)e(bu\013ered)h(com-)189 2444 y(m)o(unication:)32 b(bu\013ering)22 b(can)f(b)q(e)h(done)g(at)e (sender,)j(at)d(receiv)o(er,)j(or)e(b)q(oth;)j(bu\013ers)d(can)g(b)q(e) 189 2500 y(dedicated)d(to)e(one)g(sender-receiv)o(er)i(pair,)f(or)g(b)q (e)g(shared)g(b)o(y)f(all)i(comm)o(unications;)f(bu\013ering)189 2557 y(can)g(b)q(e)h(done)g(in)g(real)g(or)e(in)j(virtual)f(memory;)f (it)h(can)f(use)h(dedicated)h(memory)l(,)e(or)g(memory)189 2613 y(shared)d(b)o(y)h(other)f(pro)q(cesses;)h(bu\013er)f(space)h(ma)o (y)f(b)q(e)h(allo)q(cated)h(statically)f(or)f(b)q(e)i(c)o(hanged)f(dy-) 189 2670 y(namically;)f(etc.)19 b(It)12 b(do)q(es)g(not)g(seem)g (feasible)h(to)f(pro)o(vide)g(a)g(p)q(ortable)g(mec)o(hanism)h(for)e (querying)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 34 42 34 41 bop 75 -100 a Fx(34)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Fx(or)f(con)o(trolling) h(bu\013ering)h(that)d(w)o(ould)i(b)q(e)h(compatible)f(with)g(all)h (these)f(c)o(hoices,)g(y)o(et)f(pro)o(vide)189 102 y(meaningful)g (information.)k(\()p Fl(End)c(of)g(r)n(ationale.)p Fx(\))75 223 y Fj(3.6.1)49 b(Mo)q(del)18 b(implem)n(entation)c(of)i(bu\013ered)g (mo)q(de)75 309 y Fx(The)22 b(mo)q(del)g(implemen)o(tation)h(uses)f (the)f(pac)o(king)h(and)g(unpac)o(king)g(functions)g(describ)q(ed)i(in) e(Sec-)75 366 y(tion)15 b(3.13)f(and)i(the)f(non)o(blo)q(c)o(king)i (comm)o(unication)f(functions)f(describ)q(ed)j(in)e(Section)g(3.7.)166 422 y(W)l(e)21 b(assume)f(that)g(a)g(circular)i(queue)f(of)f(p)q (ending)j(message)d(en)o(tries)h(\(PME\))e(is)i(main)o(tained.)75 478 y(Eac)o(h)14 b(en)o(try)h(con)o(tains)f(a)g(comm)o(unication)i (request)e(handle)i(that)e(iden)o(ti\014es)i(a)e(p)q(ending)j(non)o (blo)q(c)o(king)75 535 y(send,)h(a)e(p)q(oin)o(ter)i(to)e(the)h(next)g (en)o(try)f(and)i(the)f(pac)o(k)o(ed)g(message)f(data.)25 b(The)17 b(en)o(tries)g(are)g(stored)f(in)75 591 y(successiv)o(e)j(lo)q (cations)h(in)f(the)f(bu\013er.)29 b(F)l(ree)19 b(space)f(is)h(a)o(v)m (ailable)h(b)q(et)o(w)o(een)f(the)f(queue)h(tail)g(and)g(the)75 648 y(queue)d(head.)166 704 y(A)f(bu\013ered)h(send)g(call)g(results)f (in)i(the)e(execution)h(of)f(the)g(follo)o(wing)h(co)q(de.)143 798 y Ft(\017)23 b Fx(T)l(ra)o(v)o(erse)14 b(sequen)o(tially)k(the)e (PME)f(queue)h(from)f(head)h(to)o(w)o(ards)e(the)i(tail,)g(deleting)h (all)g(en)o(tries)189 855 y(for)f(comm)o(unications)h(that)f(ha)o(v)o (e)g(completed,)i(up)f(to)f(the)g(\014rst)h(en)o(try)f(with)h(an)f (uncompleted)189 911 y(request;)e(up)q(date)i(queue)g(head)g(to)e(p)q (oin)o(t)i(to)f(that)f(en)o(try)l(.)143 1005 y Ft(\017)23 b Fx(Compute)18 b(the)g(n)o(um)o(b)q(er,)h Fv(n)p Fx(,)g(of)e(b)o(ytes) h(needed)i(to)d(store)h(en)o(try)f(for)h(new)g(message)g(\(length)g(of) 189 1061 y(pac)o(k)o(ed)h(message)h(computed)g(with)g Fv(MPI)p 930 1061 14 2 v 16 w(P)l(A)o(CK)p 1061 1061 V 16 w(SIZE)g Fx(plus)h(space)f(for)f(request)h(handle)h(and)189 1118 y(p)q(oin)o(ter\).)143 1212 y Ft(\017)i Fx(Find)14 b(the)g(next)g(con)o(tiguous)f(empt)o(y)h(space)g(of)f Fv(n)h Fx(b)o(ytes)g(in)g(bu\013er)g(\(space)g(follo)o(wing)g(queue)h (tail,)189 1268 y(or)d(space)h(at)f(start)g(of)g(bu\013er)h(if)h(queue) f(tail)h(is)f(to)q(o)f(close)i(to)e(end)i(of)e(bu\013er\).)19 b(If)13 b(space)g(not)g(found)189 1324 y(then)i(raise)h(bu\013er)f(o)o (v)o(er\015o)o(w)f(error.)143 1418 y Ft(\017)23 b Fx(App)q(end)14 b(to)d(end)j(of)e(PME)g(queue)h(in)g(con)o(tiguous)g(space)f(the)h(new) g(en)o(try)f(that)f(con)o(tains)i(request)189 1475 y(handle,)j(next)f (p)q(oin)o(ter)h(and)f(pac)o(k)o(ed)g(message)g(data;)f Fv(MPI)p 1221 1475 V 16 w(P)l(A)o(CK)h Fx(is)h(used)g(to)e(pac)o(k)h (data.)143 1569 y Ft(\017)23 b Fx(P)o(ost)14 b(non)o(blo)q(c)o(king)i (send)g(\(standard)f(mo)q(de\))g(for)f(pac)o(k)o(ed)i(data.)143 1662 y Ft(\017)23 b Fx(Return)75 1806 y Fn(3.7)59 b(Nonblo)r(cking)20 b(communication)75 1907 y Fx(One)e(can)g(impro)o(v)o(e)f(p)q (erformance)g(on)h(man)o(y)e(systems)h(b)o(y)h(o)o(v)o(erlapping)f (comm)o(unication)i(and)e(com-)75 1963 y(putation.)35 b(This)21 b(is)g(esp)q(ecially)i(true)d(on)g(systems)g(where)g(comm)o (unication)h(can)g(b)q(e)g(executed)g(au-)75 2020 y(tonomously)12 b(b)o(y)h(an)g(in)o(telligen)o(t)h(comm)o(unication)g(con)o(troller.)19 b(Ligh)o(t-w)o(eigh)o(t)13 b(threads)f(are)h(one)g(mec)o(h-)75 2076 y(anism)21 b(for)g(ac)o(hieving)h(suc)o(h)g(o)o(v)o(erlap.)37 b(An)21 b(alternativ)o(e)h(mec)o(hanism)g(that)e(often)h(leads)g(to)g (b)q(etter)75 2133 y(p)q(erformance)15 b(is)g(to)g(use)g Fq(non)o(blo)q(c)o(king)j(comm)o(unication)p Fx(.)j(A)15 b(non)o(blo)q(c)o(king)h Fq(send)g(start)f Fx(call)h(ini-)75 2189 y(tiates)g(the)h(send)g(op)q(eration,)f(but)h(do)q(es)f(not)g (complete)h(it.)24 b(The)17 b(send)g(start)e(call)i(will)h(return)f(b)q (efore)75 2246 y(the)g(message)f(w)o(as)f(copied)j(out)e(of)g(the)g (send)h(bu\013er.)24 b(A)17 b(separate)e Fq(send)k(complete)e Fx(call)h(is)f(needed)75 2302 y(to)12 b(complete)i(the)f(comm)o (unication,)h(i.e.,)g(to)e(v)o(erify)h(that)g(the)g(data)f(has)h(b)q (een)h(copied)h(out)e(of)f(the)h(send)75 2359 y(bu\013er.)19 b(With)13 b(suitable)i(hardw)o(are,)d(the)h(transfer)g(of)f(data)h(out) g(of)f(the)h(sender)h(memory)f(ma)o(y)f(pro)q(ceed)75 2415 y(concurren)o(tly)g(with)h(computations)e(done)h(at)g(the)g (sender)g(after)f(the)h(send)g(w)o(as)f(initiated)j(and)e(b)q(efore)g (it)75 2472 y(completed.)21 b(Similarly)l(,)16 b(a)e(non)o(blo)q(c)o (king)i Fq(receiv)o(e)g(start)h(call)f Fx(initiates)g(the)f(receiv)o(e) g(op)q(eration,)g(but)75 2528 y(do)q(es)f(not)g(complete)h(it.)k(The)14 b(call)i(will)f(return)f(b)q(efore)g(a)g(message)g(is)g(stored)f(in)o (to)h(the)g(receiv)o(e)h(bu\013er.)75 2584 y(A)i(separate)f Fq(receiv)o(e)i(complete)f Fx(call)h(is)f(needed)h(to)e(complete)h(the) g(receiv)o(e)g(op)q(eration)g(and)g(v)o(erify)75 2641 y(that)11 b(the)i(data)e(has)h(b)q(een)h(receiv)o(ed)h(in)o(to)e(the)g (receiv)o(e)h(bu\013er.)19 b(With)12 b(suitable)i(hardw)o(are,)d(the)i (transfer)75 2697 y(of)j(data)g(in)o(to)g(the)h(receiv)o(er)g(memory)f (ma)o(y)f(pro)q(ceed)j(concurren)o(tly)f(with)g(computations)f(done)h (after)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 35 43 35 42 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fx(35)75 45 y(the)14 b(receiv)o(e)g(w)o(as)f(initiated)j(and)e(b)q (efore)g(it)g(completed.)20 b(The)14 b(use)g(of)f(non)o(blo)q(c)o(king) i(receiv)o(es)g(ma)o(y)e(also)75 102 y(a)o(v)o(oid)18 b(system)g(bu\013ering)h(and)f(memory-to-memory)f(cop)o(ying,)i(as)f (information)h(is)g(pro)o(vided)g(early)75 158 y(on)c(the)g(lo)q (cation)h(of)f(the)g(receiv)o(e)i(bu\013er.)166 214 y(Non)o(blo)q(c)o (king)f(send)f(start)f(calls)h(can)g(use)g(the)g(same)f(four)h(mo)q (des)g(as)f(blo)q(c)o(king)i(sends:)k Fv(standa)o(rd)p Fx(,)75 271 y Fv(bu\013ered)p Fx(,)c Fv(synchronous)i Fx(and)e Fv(ready)p Fx(.)22 b(These)16 b(carry)f(the)h(same)f(meaning.) 23 b(Sends)16 b(of)g(all)g(mo)q(des,)g Fv(ready)75 327 y Fx(excepted,)f(can)f(b)q(e)g(started)f(whether)h(a)g(matc)o(hing)g (receiv)o(e)g(has)g(b)q(een)h(p)q(osted)f(or)g(not;)f(a)h(non)o(blo)q (c)o(king)75 384 y Fv(ready)e Fx(send)g(can)f(b)q(e)i(started)d(only)j (if)f(a)f(matc)o(hing)h(receiv)o(e)g(is)g(p)q(osted.)19 b(In)12 b(all)h(cases,)f(the)f(send)h(start)f(call)75 440 y(is)g(lo)q(cal:)19 b(it)11 b(returns)f(immediately)l(,)j(irresp)q (ectiv)o(e)f(of)e(the)h(status)f(of)g(other)g(pro)q(cesses.)19 b(If)10 b(the)h(call)h(causes)75 497 y(some)17 b(system)f(resource)h (to)f(b)q(e)i(exhausted,)f(then)g(it)g(will)i(fail)f(and)f(return)g(an) f(error)h(co)q(de.)25 b(Qualit)o(y)75 553 y(implemen)o(tations)14 b(of)f Fv(MPI)f Fx(should)i(ensure)f(that)g(this)g(happ)q(ens)h(only)f (in)h(\\pathological")f(cases.)20 b(That)75 610 y(is,)13 b(an)f Fv(MPI)f Fx(implemen)o(tation)i(should)g(b)q(e)g(able)f(to)g (supp)q(ort)g(a)f(large)h(n)o(um)o(b)q(er)g(of)g(p)q(ending)i(non)o (blo)q(c)o(king)75 666 y(op)q(erations.)166 723 y(The)j(send-complete)i (call)f(returns)f(when)h(data)e(has)h(b)q(een)i(copied)f(out)f(of)f (the)i(send)f(bu\013er.)26 b(It)75 779 y(ma)o(y)14 b(carry)h (additional)i(meaning,)e(dep)q(ending)j(on)d(the)g(send)h(mo)q(de.)166 835 y(If)e(the)g(send)h(mo)q(de)f(is)h Fv(synchronous)p Fx(,)g(then)g(the)f(send)g(can)h(complete)f(only)h(if)f(a)g(matc)o (hing)g(receiv)o(e)75 892 y(has)j(started.)24 b(That)17 b(is,)g(a)g(receiv)o(e)h(has)f(b)q(een)h(p)q(osted,)f(and)g(has)g(b)q (een)h(matc)o(hed)f(with)g(the)h(send.)25 b(In)75 948 y(this)18 b(case,)f(the)g(send-complete)i(call)f(is)g(non-lo)q(cal.)27 b(Note)16 b(that)h(a)g(sync)o(hronous,)g(non)o(blo)q(c)o(king)h(send)75 1005 y(ma)o(y)13 b(complete,)h(if)g(matc)o(hed)g(b)o(y)f(a)g(non)o(blo) q(c)o(king)i(receiv)o(e,)g(b)q(efore)f(the)f(receiv)o(e)i(complete)f (call)h(o)q(ccurs.)75 1061 y(\(It)h(can)h(complete)g(as)f(so)q(on)g(as) g(the)h(sender)g(\\kno)o(ws")e(the)i(transfer)e(will)j(complete,)g(but) e(b)q(efore)h(the)75 1118 y(receiv)o(er)f(\\kno)o(ws")e(the)h(transfer) g(will)i(complete.\))166 1174 y(If)f(the)g(send)h(mo)q(de)f(is)h Fv(bu\013ered)g Fx(then)f(the)g(message)g(m)o(ust)g(b)q(e)g(bu\013ered) h(if)f(there)h(is)f(no)g(p)q(ending)75 1231 y(receiv)o(e.)27 b(In)18 b(this)g(case,)f(the)g(send-complete)i(call)f(is)g(lo)q(cal,)h (and)e(m)o(ust)g(succeed)h(irresp)q(ectiv)o(e)h(of)e(the)75 1287 y(status)d(of)h(a)g(matc)o(hing)g(receiv)o(e.)166 1344 y(If)f(the)g(send)g(mo)q(de)g(is)g Fv(standa)o(rd)h Fx(then)g(the)e(send-complete)j(call)f(ma)o(y)e(return)g(b)q(efore)h(a) g(matc)o(hing)75 1400 y(receiv)o(e)h(o)q(ccurred,)f(if)g(the)g(message) f(is)i(bu\013ered.)20 b(On)14 b(the)g(other)f(hand,)h(the)g (send-complete)h(ma)o(y)e(not)75 1456 y(complete)19 b(un)o(til)h(a)e (matc)o(hing)g(receiv)o(e)h(o)q(ccurred,)h(and)f(the)f(message)g(w)o (as)g(copied)h(in)o(to)g(the)f(receiv)o(e)75 1513 y(bu\013er.)166 1569 y(Non)o(blo)q(c)o(king)e(sends)g(can)f(b)q(e)h(matc)o(hed)f(with)h (blo)q(c)o(king)g(receiv)o(es,)g(and)f(vice-v)o(ersa.)189 1676 y Fl(A)n(dvic)n(e)h(to)i(users.)50 b Fx(The)17 b(completion)g(of)g (a)f(send)h(op)q(eration)g(ma)o(y)f(b)q(e)i(dela)o(y)o(ed,)f(for)f (standard)189 1732 y(mo)q(de,)d(and)g(m)o(ust)f(b)q(e)h(dela)o(y)o(ed,) h(for)e(sync)o(hronous)g(mo)q(de,)h(un)o(til)h(a)f(matc)o(hing)f (receiv)o(e)i(is)f(p)q(osted.)189 1789 y(The)k(use)g(of)f(non)o(blo)q (c)o(king)j(sends)e(in)h(these)f(t)o(w)o(o)e(cases)i(allo)o(ws)g(the)g (sender)h(to)e(pro)q(ceed)i(ahead)189 1845 y(of)c(the)g(receiv)o(er,)h (so)f(that)g(the)h(computation)f(is)h(more)f(toleran)o(t)g(of)g (\015uctuations)h(in)g(the)g(sp)q(eeds)189 1901 y(of)f(the)i(t)o(w)o(o) e(pro)q(cesses.)189 1977 y(Non)o(blo)q(c)o(king)k(sends)g(in)f(the)h (bu\013ered)f(and)h(ready)f(mo)q(des)g(ha)o(v)o(e)g(a)f(more)h(limited) i(impact.)26 b(A)189 2033 y(non)o(blo)q(c)o(king)16 b(send)g(will)h (return)e(as)g(so)q(on)g(as)g(p)q(ossible,)i(whereas)e(a)g(blo)q(c)o (king)h(send)g(will)h(return)189 2090 y(after)h(the)h(data)g(has)g(b)q (een)h(copied)h(out)e(of)f(the)i(sender)f(memory)l(.)32 b(The)19 b(use)h(of)f(non)o(blo)q(c)o(king)189 2146 y(sends)j(is)g(adv) m(an)o(tageous)f(in)h(these)g(cases)g(only)g(if)g(data)f(cop)o(ying)h (can)g(b)q(e)g(concurren)o(t)g(with)189 2202 y(computation.)189 2278 y(The)e(message-passing)g(mo)q(del)g(implies)i(that)d(comm)o (unication)i(is)f(initiated)h(b)o(y)f(the)g(sender.)189 2334 y(The)d(comm)o(unication)g(will)h(generally)g(ha)o(v)o(e)e(lo)o(w) o(er)g(o)o(v)o(erhead)h(if)g(a)f(receiv)o(e)i(is)f(already)g(p)q(osted) 189 2390 y(when)22 b(the)h(sender)g(initiates)g(the)f(comm)o(unication) h(\(data)f(can)g(b)q(e)h(mo)o(v)o(ed)f(directly)h(to)f(the)189 2447 y(receiv)o(e)e(bu\013er,)f(and)g(there)g(is)h(no)f(need)h(to)e (queue)i(a)e(p)q(ending)j(send)f(request\).)31 b(Ho)o(w)o(ev)o(er,)18 b(a)189 2503 y(receiv)o(e)g(op)q(eration)f(can)h(complete)g(only)g (after)e(the)i(matc)o(hing)f(send)h(has)f(o)q(ccurred.)27 b(The)18 b(use)189 2560 y(of)13 b(non)o(blo)q(c)o(king)i(receiv)o(es)f (allo)o(ws)g(one)g(to)f(ac)o(hiev)o(e)h(lo)o(w)o(er)f(comm)o(unication) i(o)o(v)o(erheads)e(without)189 2616 y(blo)q(c)o(king)j(the)f(receiv)o (er)h(while)h(it)f(w)o(aits)e(for)h(the)g(send.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fx(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 36 44 36 43 bop 75 -100 a Fx(36)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fj(3.7.1)49 b(Comm)o(unication)14 b(Objects)75 131 y Fx(Non)o(blo)q(c)o(king)19 b(comm)o(unications)f(use)g(opaque)g Fv(request)h Fx(ob)s(jects)e(to)g (iden)o(tify)i(comm)o(unication)g(op)q(er-)75 187 y(ations)e(and)g (matc)o(h)f(the)i(op)q(eration)f(that)f(initiates)i(the)f(comm)o (unication)h(with)f(the)g(op)q(eration)h(that)75 244 y(terminates)f(it.)27 b(These)17 b(are)g(system)g(ob)s(jects)g(that)f (are)h(accessed)h(via)g(a)f(handle.)27 b(A)17 b(request)h(ob)s(ject)75 300 y(iden)o(ti\014es)c(v)m(arious)f(prop)q(erties)h(of)e(a)g(comm)o (unication)h(op)q(eration,)g(suc)o(h)g(as)f(the)h(send)g(mo)q(de,)g (the)g(com-)75 357 y(m)o(unication)i(bu\013er)f(that)g(is)h(asso)q (ciated)f(with)h(it,)f(its)g(con)o(text,)g(the)g(tag)f(and)i (destination)g(argumen)o(ts)75 413 y(to)e(b)q(e)h(used)f(for)g(a)g (send,)h(or)e(the)i(tag)e(and)i(source)f(argumen)o(ts)f(to)h(b)q(e)h (used)g(for)e(a)h(receiv)o(e.)20 b(In)14 b(addition,)75 470 y(this)i(ob)s(ject)e(stores)h(information)g(ab)q(out)g(the)g (status)g(of)f(the)i(p)q(ending)h(comm)o(unication)f(op)q(eration.)75 591 y Fj(3.7.2)49 b(Comm)o(unication)14 b(initiation)75 677 y Fx(W)l(e)j(use)h(the)g(same)f(naming)g(con)o(v)o(en)o(tions)h(as) f(for)g(blo)q(c)o(king)h(comm)o(unication:)25 b(a)17 b(pre\014x)h(of)f Fv(B)p Fx(,)g Fv(S)p Fx(,)g(or)75 734 y Fv(R)f Fx(is)h(used)f(for)g Fv(bu\013ered)p Fx(,)g Fv(synchronous)i Fx(or)e Fv(ready)g Fx(mo)q(de.)22 b(In)17 b(addition)g(a)f(pre\014x)g(of)g Fv(I)f Fx(\(for)g Fv(imm)m(ediate)p Fx(\))75 790 y(indicates)i(that)d(the)h(call)i(is)e(non)o(blo)q(c)o (king.)75 941 y Fv(MPI)p 160 941 14 2 v 16 w(ISEND\(buf,)g(count,)h (datat)o(yp)q(e,)h(dest,)f(tag,)f(comm)m(,)d(request\))117 1018 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 1093 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 1169 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e (\(handle\))117 1244 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1319 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1394 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1469 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1593 y Fu(int)23 b(MPI)p 245 1593 15 2 v 17 w(Isend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1593 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 1650 y(int)h(tag,)f(MPI)p 683 1650 V 17 w(Comm)g(comm,)g(MPI)p 1034 1650 V 17 w(Request)g(*request\))75 1736 y(MPI)p 150 1736 V 17 w(ISEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g (COMM,)h(REQUEST,)e(IERROR\))170 1793 y()h(BUF\(*\))170 1849 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1936 y Fx(Start)14 b(a)h(standard)g(mo)q(de,)g(non)o(blo)q (c)o(king)h(send.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 37 45 37 44 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fx(37)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(IBSEND\(buf,)15 b(count,)i(datat)o(yp)q(e,)f(dest,)g(tag,)f(comm)m(,)d(request\))117 122 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 197 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 273 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e (\(handle\))117 348 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 423 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 498 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Fu(int)23 b(MPI)p 245 698 15 2 v 17 w(Ibsend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 698 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(IBSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR\))170 897 y()g(BUF\(*\))170 953 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1040 y Fx(Start)14 b(a)h(bu\013ered)h(mo)q(de,)f(non)o(blo) q(c)o(king)h(send.)75 1191 y Fv(MPI)p 160 1191 14 2 v 16 w(ISSEND\(buf,)f(count,)i(datat)o(yp)q(e,)f(dest,)g(tag,)f(comm)m(,) d(request\))117 1268 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1343 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1418 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1493 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1569 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1644 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1719 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1843 y Fu(int)23 b(MPI)p 245 1843 15 2 v 17 w(Issend\(void*)f(buf,)i (int)f(count,)g(MPI)p 1026 1843 V 17 w(Datatype)g(datatype,)f(int)i (dest,)393 1900 y(int)g(tag,)f(MPI)p 683 1900 V 17 w(Comm)g(comm,)g (MPI)p 1034 1900 V 17 w(Request)g(*request\))75 1986 y(MPI)p 150 1986 V 17 w(ISSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2043 y()g(BUF\(*\))170 2099 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 2186 y Fx(Start)14 b(a)h(sync)o(hronous)g(mo)q(de,)g(non)o (blo)q(c)o(king)i(send.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 38 46 38 45 bop 75 -100 a Fx(38)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(IRSEND\(buf,)f(count,)g(datat)o(yp)q(e,)g(dest,)g (tag,)f(comm)m(,)d(request\))117 122 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 197 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 273 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h (bu\013er)g(elemen)o(t)e(\(handle\))117 348 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 423 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 498 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Fu(int)23 b(MPI)p 245 698 15 2 v 17 w(Irsend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 698 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(IRSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR\))170 897 y()g(BUF\(*\))170 953 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1040 y Fx(Start)14 b(a)h(ready)g(mo)q(de)h(non)o(blo)q(c)o (king)g(send.)75 1191 y Fv(MPI)p 160 1191 14 2 v 16 w(IRECV)f(\(buf,)h (count,)g(datat)o(yp)q(e,)g(source,)g(tag,)e(comm)n(,)d(request\))117 1268 y Fk(OUT)108 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(receiv) o(e)h(bu\013er)g(\(c)o(hoice\))117 1343 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 1418 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen) o(t)e(\(handle\))117 1493 y(IN)155 b Fv(source)468 b Fk(rank)14 b(of)f(source)j(\(in)o(teger\))117 1569 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1644 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1719 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1843 y Fu(int)23 b(MPI)p 245 1843 15 2 v 17 w(Irecv\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1843 V 17 w(Datatype)g(datatype,)g(int)g(source,)393 1900 y(int)h(tag,)f(MPI)p 683 1900 V 17 w(Comm)g(comm,)g(MPI)p 1034 1900 V 17 w(Request)g(*request\))75 1986 y(MPI)p 150 1986 V 17 w(IRECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g (COMM,)g(REQUEST,)g(IERROR\))170 2043 y()g(BUF\(*\))170 2099 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR)166 2186 y Fx(Start)14 b(a)h(non)o(blo)q(c)o(king)i (receiv)o(e.)166 2242 y(These)f(calls)h(allo)q(cate)g(a)e(comm)o (unication)i(request)f(ob)s(ject)f(and)h(asso)q(ciate)g(it)g(with)g (the)g(request)75 2298 y(handle)j(\(the)f(argumen)o(t)g Fv(request)p Fx(\).)30 b(The)18 b(request)h(can)f(b)q(e)h(used)f(later) h(to)e(query)i(the)f(status)f(of)h(the)75 2355 y(comm)o(unication)e(or) f(w)o(ait)f(for)h(its)g(completion.)166 2411 y(A)i(non)o(blo)q(c)o (king)i(send)f(call)g(indicates)h(that)e(the)g(system)g(ma)o(y)g(start) f(cop)o(ying)h(data)g(out)g(of)g(the)75 2468 y(send)g(bu\013er.)24 b(The)17 b(sender)g(should)g(not)g(access)f(an)o(y)g(part)g(of)g(the)h (send)g(bu\013er)g(after)e(a)i(non)o(blo)q(c)o(king)75 2524 y(send)f(op)q(eration)f(is)h(called,)g(un)o(til)h(the)e(send)h (completes.)166 2581 y(A)d(non)o(blo)q(c)o(king)h(receiv)o(e)g(call)g (indicates)g(that)e(the)h(system)f(ma)o(y)g(start)g(writing)h(data)f (in)o(to)h(the)g(re-)75 2637 y(ceiv)o(e)f(bu\013er.)19 b(The)11 b(receiv)o(er)h(should)h(not)e(access)g(an)o(y)g(part)g(of)g (the)g(receiv)o(e)i(bu\013er)e(after)g(a)g(non)o(blo)q(c)o(king)75 2694 y(receiv)o(e)16 b(op)q(eration)g(is)f(called,)i(un)o(til)f(the)f (receiv)o(e)h(completes.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 39 47 39 46 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fx(39)75 45 y Fj(3.7.3)49 b(Comm)o(unication)14 b(Completion)75 131 y Fx(The)j(functions)g Fv(MPI)p 452 131 14 2 v 16 w(W)l(AIT)f Fx(and)g Fv(MPI)p 768 131 V 16 w(TEST)h Fx(are)f(used)h(to) f(complete)h(a)f(non)o(blo)q(c)o(king)h(comm)o(unica-)75 187 y(tion.)i(The)13 b(completion)h(of)e(a)g(send)h(op)q(eration)g (indicates)h(that)e(the)g(sender)h(is)g(no)o(w)f(free)h(to)f(up)q(date) h(the)75 244 y(lo)q(cations)k(in)f(the)h(send)f(bu\013er)g(\(the)g (send)g(op)q(eration)g(itself)h(lea)o(v)o(es)f(the)g(con)o(ten)o(t)g (of)f(the)h(send)h(bu\013er)75 300 y(unc)o(hanged\).)25 b(It)17 b(do)q(es)g(not)f(indicate)i(that)e(the)h(message)f(has)h(b)q (een)g(receiv)o(ed,)h(rather,)e(it)h(ma)o(y)f(ha)o(v)o(e)75 357 y(b)q(een)g(bu\013ered)g(b)o(y)f(the)h(comm)o(unication)g (subsystem.)k(Ho)o(w)o(ev)o(er,)14 b(if)i(a)f Fv(synchronous)i Fx(mo)q(de)f(send)g(w)o(as)75 413 y(used,)f(the)g(completion)h(of)f (the)f(send)i(op)q(eration)f(indicates)h(that)e(a)h(matc)o(hing)g (receiv)o(e)h(w)o(as)e(initiated,)75 470 y(and)h(that)g(the)g(message)g (will)i(ev)o(en)o(tually)f(b)q(e)g(receiv)o(ed)g(b)o(y)f(this)h(matc)o (hing)f(receiv)o(e.)166 526 y(The)20 b(completion)h(of)e(a)g(receiv)o (e)i(op)q(eration)f(indicates)h(that)e(the)h(receiv)o(e)g(bu\013er)g (con)o(tains)g(the)75 583 y(receiv)o(ed)d(message,)e(the)h(receiv)o(er) h(is)f(no)o(w)f(free)h(to)f(access)h(it,)g(and)g(that)f(the)h(status)f (ob)s(ject)g(is)h(set.)22 b(It)75 639 y(do)q(es)13 b(not)g(indicate)i (that)d(the)h(matc)o(hing)g(send)h(op)q(eration)f(has)g(completed)h (\(but)f(indicates,)i(of)d(course,)75 695 y(that)i(the)i(send)g(w)o(as) e(initiated\).)166 752 y(W)l(e)27 b(shall)h(use)g(the)f(follo)o(wing)h (terminology)l(.)56 b(A)27 b Fq(n)o(ull)g Fx(handle)i(is)e(a)g(handle)h (with)g(v)m(alue)75 808 y Fi(MPI)p 152 808 13 2 v 14 w(REQUEST)p 354 808 V 14 w(NULL)p Fx(.)17 b(A)g(p)q(ersisten)o(t)g (request)f(and)h(the)g(handle)h(to)e(it)g(are)h Fq(inactiv)o(e)g Fx(if)g(the)g(request)75 865 y(is)f(not)f(asso)q(ciated)h(with)g(an)o (y)f(ongoing)h(comm)o(unication)g(\(see)g(Section)g(3.9\).)k(A)15 b(handle)i(is)f Fq(activ)o(e)g Fx(if)75 921 y(it)f(is)h(neither)g(n)o (ull)h(nor)e(inactiv)o(e.)75 1072 y Fv(MPI)p 160 1072 14 2 v 16 w(W)l(AIT\(request,)h(status\))117 1149 y Fk(INOUT)62 b Fv(request)452 b Fk(request)16 b(\(handle\))117 1223 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1347 y Fu(int)23 b(MPI)p 245 1347 15 2 v 17 w(Wait\(MPI)p 454 1347 V 16 w(Request)g(*request,)g(MPI)p 972 1347 V 17 w(Status)g(*status\))75 1434 y(MPI)p 150 1434 V 17 w(WAIT\(REQUEST,)f(STATUS,)h(IERROR\))170 1490 y(INTEGER)g(REQUEST,) g(STATUS\(MPI)p 819 1490 V 16 w(STATUS)p 979 1490 V 16 w(SIZE\),)g(IERROR)166 1577 y Fx(A)12 b(call)i(to)d Fv(MPI)p 430 1577 14 2 v 16 w(W)l(AIT)h Fx(returns)g(when)h(the)f(op)q(eration)h (iden)o(ti\014ed)h(b)o(y)e Fv(request)i Fx(is)f(complete.)19 b(If)13 b(the)75 1633 y(comm)o(unication)19 b(ob)s(ject)e(asso)q (ciated)i(with)f(this)g(request)g(w)o(as)g(created)g(b)o(y)g(a)f(non)o (blo)q(c)o(king)j(send)e(or)75 1690 y(receiv)o(e)d(call,)h(then)e(the)h (ob)s(ject)f(is)h(deallo)q(cated)h(b)o(y)e(the)h(call)g(to)f Fv(MPI)p 1281 1690 V 16 w(W)l(AIT)g Fx(and)h(the)f(request)h(handle)75 1746 y(is)h(set)f(to)f Fi(MPI)p 324 1746 13 2 v 15 w(REQUEST)p 527 1746 V 13 w(NULL)p Fx(.)i Fv(MPI)p 754 1746 14 2 v 16 w(W)l(AIT)f Fx(is)h(a)f(non-lo)q(cal)h(op)q(eration.)166 1802 y(The)j(call)h(returns,)f(in)g Fv(status)p Fx(,)j(information)c (on)h(the)g(completed)h(op)q(eration.)30 b(The)19 b(con)o(ten)o(t)f(of) 75 1859 y(the)d(status)f(ob)s(ject)h(for)f(a)h(receiv)o(e)h(op)q (eration)f(can)h(b)q(e)f(accessed)h(as)f(describ)q(ed)i(in)f(section)f (3.2.5.)k(The)75 1915 y(status)13 b(ob)s(ject)g(for)f(a)i(send)g(op)q (eration)f(ma)o(y)g(b)q(e)h(queried)h(b)o(y)f(a)f(call)h(to)f Fv(MPI)p 1385 1915 V 16 w(TEST)p 1515 1915 V 17 w(CANCELLED)g Fx(\(see)75 1972 y(Section)j(3.8\).)166 2028 y(One)f(is)g(allo)o(w)o (ed)f(to)g(call)h Fv(MPI)p 688 2028 V 16 w(W)l(AIT)f Fx(with)h(a)e(n)o(ull)j(or)e(inactiv)o(e)h Fv(request)h Fx(argumen)o(t.)j(In)14 b(this)h(case)75 2085 y(the)10 b(op)q(eration)h(returns)f(immediately)l(.)20 b(The)10 b Fv(status)i Fx(argumen)o(t)e(is)g(set)g(to)g(return)g Fv(tag)g(=)g(MPI)p 1672 2085 V 16 w(ANY)p 1780 2085 V 17 w(T)l(A)o(G)p Fx(,)75 2141 y Fv(source)19 b(=)g(MPI)p 352 2141 V 16 w(ANY)p 460 2141 V 17 w(SOURCE)p Fx(,)h(and)f(is)g(also)g (in)o(ternally)h(con\014gured)f(so)g(that)f(calls)i(to)e Fv(MPI)p 1759 2141 V 15 w(GET-)p 75 2198 V 89 2198 a(COUNT)e Fx(and)f Fv(MPI)p 434 2198 V 16 w(GET)p 538 2198 V 17 w(ELEMENTS)g Fx(return)g Fv(count)i(=)e(0)p Fx(.)189 2291 y Fl(R)n(ationale.)52 b Fx(This)18 b(mak)o(es)e Fv(MPI)p 772 2291 V 16 w(W)l(AIT)h Fx(functionally)i(equiv)m(alen)o(t)g (to)e Fv(MPI)p 1528 2291 V 16 w(W)l(AIT)l(ALL)g Fx(with)g(a)189 2348 y(list)g(of)f(length)h(one)g(and)f(adds)h(some)f(elegance.)25 b(Status)16 b(is)g(set)h(in)g(this)g(w)o(a)o(y)e(so)h(as)g(to)g(prev)o (en)o(t)189 2404 y(errors)e(due)i(to)e(accesses)i(of)f(stale)g (information.)189 2478 y(Successful)h(return)e(of)g Fv(MPI)p 676 2478 V 16 w(W)l(AIT)g Fx(after)g(a)g Fv(MPI)p 1045 2478 V 16 w(IBSEND)h Fx(implies)h(that)e(the)g(user)h(send)g(bu\013er) 189 2534 y(can)j(b)q(e)h(reused)h(|)f(i.e.,)g(data)e(has)i(b)q(een)g (sen)o(t)g(out)f(or)g(copied)i(in)o(to)e(a)g(bu\013er)h(attac)o(hed)f (with)189 2591 y Fv(MPI)p 274 2591 V 15 w(BUFFER)p 458 2591 V 18 w(A)l(TT)l(A)o(CH)p Fx(.)h(Note)g(that,)g(at)g(this)g(p)q (oin)o(t,)i(w)o(e)e(can)g(no)g(longer)h(cancel)g(the)f(send)189 2647 y(\(see)c(Sec.)20 b(3.8\).)f(If)c(a)g(matc)o(hing)g(receiv)o(e)h (is)g(nev)o(er)f(p)q(osted,)g(then)h(the)f(bu\013er)g(cannot)g(b)q(e)h (freed.)189 2704 y(This)e(runs)g(somewhat)f(coun)o(ter)g(to)g(the)h (stated)f(goal)h(of)f Fv(MPI)p 1259 2704 V 16 w(CANCEL)h Fx(\(alw)o(a)o(ys)e(b)q(eing)j(able)g(to)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 40 48 40 47 bop 75 -100 a Fx(40)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Fx(free)g(program)f (space)i(that)e(w)o(as)h(committed)g(to)f(the)i(comm)o(unication)g (subsystem\).)25 b(\()p Fl(End)18 b(of)189 102 y(r)n(ationale.)p Fx(\))189 207 y Fl(A)n(dvic)n(e)i(to)h(implementors.)72 b Fx(In)21 b(a)f(m)o(ulti-threaded)i(en)o(vironmen)o(t,)f(a)f(call)i (to)40 b Fv(MPI)p 1749 207 14 2 v 16 w(W)l(AIT)189 263 y Fx(should)12 b(blo)q(c)o(k)g(only)g(the)g(calling)h(thread,)f(allo)o (wing)g(the)g(thread)f(sc)o(heduler)i(to)e(sc)o(hedule)i(another)189 320 y(thread)i(for)f(execution.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)g (to)h(implementors.)p Fx(\))75 520 y Fv(MPI)p 160 520 V 16 w(TEST\(request,)f(\015ag,)f(status\))117 597 y Fk(INOUT)62 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))117 672 y(OUT)108 b Fv(\015ag)518 b Fi(true)15 b Fk(if)e(op)q(eration)h(completed)f(\(logical\))117 747 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\)) 75 872 y Fu(int)23 b(MPI)p 245 872 15 2 v 17 w(Test\(MPI)p 454 872 V 16 w(Request)g(*request,)g(int)g(*flag,)g(MPI)p 1234 872 V 17 w(Status)g(*status\))75 958 y(MPI)p 150 958 V 17 w(TEST\(REQUEST,)f(FLAG,)h(STATUS,)g(IERROR\))170 1014 y(LOGICAL)g(FLAG)170 1071 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 1071 V 16 w(STATUS)p 979 1071 V 16 w(SIZE\),)g(IERROR)166 1157 y Fx(A)17 b(call)i(to)e Fv(MPI)p 446 1157 14 2 v 15 w(TEST)h Fx(returns)f Fv(\015ag)h(=)f(true)h Fx(if)g(the)g(op)q (eration)f(iden)o(ti\014ed)j(b)o(y)d Fv(request)i Fx(is)f(com-)75 1214 y(plete.)37 b(In)21 b(suc)o(h)g(a)f(case,)i(the)e(status)g(ob)s (ject)g(is)h(set)f(to)g(con)o(tain)h(information)g(on)f(the)h (completed)75 1270 y(op)q(eration;)15 b(if)h(the)f(comm)o(unication)h (ob)s(ject)e(w)o(as)h(created)g(b)o(y)g(a)g(non)o(blo)q(c)o(king)h (send)g(or)f(receiv)o(e,)h(then)75 1327 y(it)g(is)f(deallo)q(cated)i (and)f(the)f(request)h(handle)g(is)g(set)f(to)g Fi(MPI)p 1127 1327 13 2 v 14 w(REQUEST)p 1329 1327 V 14 w(NULL)p Fx(.)h(The)f(call)i(returns)e Fv(\015ag)75 1383 y(=)g(false)p Fx(,)g(otherwise.)20 b(In)c(this)g(case,)f(the)g(v)m(alue)h(of)f(the)g (status)f(ob)s(ject)h(is)h(unde\014ned.)21 b Fv(MPI)p 1664 1383 14 2 v 16 w(TEST)16 b Fx(is)f(a)75 1440 y(lo)q(cal)h(op)q (eration.)166 1496 y(The)11 b(return)g(status)f(ob)s(ject)h(for)f(a)h (receiv)o(e)h(op)q(eration)f(carries)g(information)g(that)g(can)g(b)q (e)g(accessed)75 1553 y(as)19 b(describ)q(ed)h(in)g(section)g(3.2.5.)29 b(The)19 b(status)g(ob)s(ject)f(for)g(a)h(send)g(op)q(eration)h (carries)f(information)75 1609 y(that)14 b(can)i(b)q(e)g(accessed)f(b)o (y)h(a)e(call)j(to)d Fv(MPI)p 823 1609 V 16 w(TEST)p 953 1609 V 17 w(CANCELLED)h Fx(\(see)g(Section)h(3.8\).)166 1666 y(One)i(is)f(allo)o(w)o(ed)g(to)g(call)h Fv(MPI)p 702 1666 V 15 w(TEST)f Fx(with)h(a)e(n)o(ull)j(or)d(inactiv)o(e)i Fv(request)g Fx(argumen)o(t.)25 b(In)17 b(suc)o(h)g(a)75 1722 y(case)e(the)g(op)q(eration)h(returns)f Fv(\015ag)g(=)h(false)p Fx(.)166 1778 y(The)22 b(functions)h Fv(MPI)p 554 1778 V 15 w(W)l(AIT)f Fx(and)g Fv(MPI)p 881 1778 V 16 w(TEST)g Fx(can)g(b)q(e)g(used)h(to)e(complete)h(b)q(oth)g(sends)h(and)75 1835 y(receiv)o(es.)189 1940 y Fl(A)n(dvic)n(e)e(to)h(users.)79 b Fx(The)22 b(use)g(of)f(the)h(non)o(blo)q(c)o(king)h Fv(MPI)p 1267 1940 V 16 w(TEST)f Fx(call)g(allo)o(ws)g(the)g(user)g(to) 189 1997 y(sc)o(hedule)e(alternativ)o(e)f(activities)h(within)g(a)f (single)h(thread)f(of)f(execution.)32 b(An)19 b(ev)o(en)o(t-driv)o(en) 189 2053 y(thread)14 b(sc)o(heduler)h(can)f(b)q(e)h(em)o(ulated)g(with) f(p)q(erio)q(dic)i(calls)f(to)f Fv(MPI)p 1382 2053 V 16 w(TEST)p Fx(.)g(\()p Fl(End)g(of)i(advic)n(e)f(to)189 2110 y(users.)p Fx(\))75 2215 y Fq(Example)j(3.10)23 b Fx(Simple)16 b(usage)f(of)g(non)o(blo)q(c)o(king)i(op)q(erations)e (and)g Fv(MPI)p 1378 2215 V 16 w(W)l(AIT)p Fx(.)75 2308 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2365 y(IF\(rank.EQ.0\))f(THEN)170 2421 y(CALL)i(MPI_ISEND\(a\(1\),)e (10,)h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(request,)g(ierr\))170 2478 y(****)h(do)f(some)h(computation)e(to)i(mask)f(latency)g(****)170 2534 y(CALL)h(MPI_WAIT\(request,)d(status,)i(ierr\))75 2591 y(ELSE)170 2647 y(CALL)h(MPI_IRECV\(a\(1\),)e(15,)h(MPI_REAL,)g (0,)g(tag,)h(comm,)f(request,)g(ierr\))170 2704 y(****)h(do)f(some)h (computation)e(to)i(mask)f(latency)g(****)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 41 49 41 48 bop 75 -100 a Fm(3.7.)29 b(NONBLOCKING)17 b(COMMUNICA)l(TION)868 b Fx(41)170 45 y Fu(CALL)24 b(MPI_WAIT\(request,)d(status,)i(ierr\))75 102 y(END)g(IF)166 208 y Fx(A)14 b(request)g(ob)s(ject)f(can)h(b)q(e)g (deallo)q(cated)h(without)f(w)o(aiting)g(for)f(the)h(asso)q(ciated)g (comm)o(unication)75 264 y(to)h(complete,)g(b)o(y)g(using)h(the)f (follo)o(wing)h(op)q(eration.)75 415 y Fv(MPI)p 160 415 14 2 v 16 w(REQUEST)p 379 415 V 18 w(FREE\(request\))117 492 y Fk(INOUT)62 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 617 y Fu(int)23 b(MPI)p 245 617 15 2 v 17 w(Request)p 430 617 V 16 w(free\(MPI)p 638 617 V 17 w(Request)f(*request\))75 703 y(MPI)p 150 703 V 17 w(REQUEST)p 335 703 V 16 w(FREE\(REQUEST,)g(IERROR\))170 760 y(INTEGER)h(REQUEST,)g(IERROR)166 846 y Fx(Mark)18 b(the)h(request)g(ob)s(ject)g(for)f(deallo)q(cation)j(and)e(set)g Fv(request)i Fx(to)d Fi(MPI)p 1467 846 13 2 v 14 w(REQUEST)p 1669 846 V 14 w(NULL)p Fx(.)i(An)75 903 y(ongoing)g(comm)o(unication)h (that)e(is)i(asso)q(ciated)f(with)h(the)f(request)g(will)i(b)q(e)f (allo)o(w)o(ed)f(to)g(complete.)75 959 y(The)15 b(request)h(will)h(b)q (e)e(deallo)q(cated)i(only)f(after)e(its)i(completion.)189 1065 y Fl(R)n(ationale.)51 b Fx(The)17 b Fv(MPI)p 622 1065 14 2 v 16 w(REQUEST)p 841 1065 V 18 w(FREE)g Fx(mec)o(hanism)h(is) f(pro)o(vided)h(for)f(reasons)f(of)h(p)q(erfor-)189 1122 y(mance)e(and)g(con)o(v)o(enience)i(on)e(the)h(sending)g(side.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 1228 y Fl(A)n(dvic)n(e)i(to)i(users.)68 b Fx(Once)21 b(a)f(request)f(is)i (freed)f(b)o(y)g(a)f(call)i(to)f Fv(MPI)p 1419 1228 V 15 w(REQUEST)p 1637 1228 V 18 w(FREE)p Fx(,)g(it)g(is)189 1285 y(not)c(p)q(ossible)j(to)e(c)o(hec)o(k)g(for)g(the)g(successful)i (completion)f(of)f(the)g(asso)q(ciated)g(comm)o(unication)189 1341 y(with)h(calls)h(to)e Fv(MPI)p 542 1341 V 16 w(W)l(AIT)h Fx(or)g Fv(MPI)p 830 1341 V 15 w(TEST)p Fx(.)g(Also,)h(if)f(an)g(error) g(o)q(ccurs)g(subsequen)o(tly)h(during)189 1398 y(the)f(comm)o (unication,)i(an)f(error)f(co)q(de)i(cannot)e(b)q(e)i(returned)f(to)f (the)h(user)g(|)g(suc)o(h)g(an)g(error)189 1454 y(m)o(ust)d(b)q(e)i (treated)f(as)f(fatal.)26 b(Questions)18 b(arise)f(as)g(to)f(ho)o(w)h (one)g(kno)o(ws)g(when)h(the)f(op)q(erations)189 1510 y(ha)o(v)o(e)f(completed)i(when)g(using)f Fv(MPI)p 840 1510 V 16 w(REQUEST)p 1059 1510 V 18 w(FREE)p Fx(.)g(Dep)q(ending)i(on) e(the)g(program)e(logic,)189 1567 y(there)j(ma)o(y)g(b)q(e)g(other)g(w) o(a)o(ys)f(in)j(whic)o(h)f(the)f(program)f(kno)o(ws)h(that)f(certain)i (op)q(erations)f(ha)o(v)o(e)189 1623 y(completed)f(and)f(this)h(mak)o (es)e(usage)h(of)g Fv(MPI)p 986 1623 V 16 w(REQUEST)p 1205 1623 V 18 w(FREE)g Fx(practical.)24 b(F)l(or)15 b(example,)i(an)189 1680 y(activ)o(e)j(send)g(request)g(could)h(b)q(e)g (freed)g(when)f(the)g(logic)h(of)f(the)g(program)f(is)h(suc)o(h)h(that) e(the)189 1736 y(receiv)o(er)i(sends)h(a)f(reply)h(to)e(the)h(message)g (sen)o(t)g(|)g(the)g(arriv)m(al)h(of)f(the)g(reply)h(informs)f(the)189 1793 y(sender)f(that)f(the)g(send)h(has)g(completed)g(and)g(the)g(send) g(bu\013er)f(can)h(b)q(e)g(reused.)34 b(An)20 b(activ)o(e)189 1849 y(receiv)o(e)e(request)f(should)h(nev)o(er)f(b)q(e)g(freed)h(as)e (the)h(receiv)o(er)h(will)h(ha)o(v)o(e)d(no)h(w)o(a)o(y)f(to)g(v)o (erify)i(that)189 1906 y(the)h(receiv)o(e)i(has)e(completed)h(and)g (the)g(receiv)o(e)g(bu\013er)g(can)f(b)q(e)h(reused.)34 b(\()p Fl(End)19 b(of)i(advic)n(e)f(to)189 1962 y(users.)p Fx(\))75 2068 y Fq(Example)e(3.11)23 b Fx(An)15 b(example)h(using)g Fv(MPI)p 863 2068 V 16 w(REQUEST)p 1082 2068 V 18 w(FREE)p Fx(.)75 2175 y Fu(CALL)23 b(MPI_COMM_RANK\(MPI_COMM_WORLD)o(,)e(rank\)) 75 2231 y(IF\(rank.EQ.0\))h(THEN)170 2288 y(DO)i(i=1,)f(n)218 2344 y(CALL)g(MPI_ISEND\(outval,)f(1,)i(MPI_REAL,)e(1,)i(0,)f(req,)h (ierr\))218 2400 y(CALL)f(MPI_REQUEST_FREE\(req,)e(ierr\))218 2457 y(CALL)i(MPI_IRECV\(inval,)f(1,)i(MPI_REAL,)e(1,)i(0,)g(req,)f (ierr\))218 2513 y(CALL)g(MPI_WAIT\(req,)g(status,)f(ierr\))170 2570 y(END)i(DO)75 2626 y(ELSE)95 b(!)24 b(rank.EQ.1)170 2683 y(CALL)g(MPI_IRECV\(inval,)d(1,)j(MPI_REAL,)f(0,)g(0,)h(req,)f (ierr\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 42 50 42 49 bop 75 -100 a Fx(42)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Fu(CALL)24 b(MPI_WAIT\(req,)e(status\))170 102 y(DO)i(I=1,)f(n-1)242 158 y(CALL)g(MPI_ISEND\(outval,)f(1,)h(MPI_REAL,)g(0,)h(0,)f(req,)h (ierr\))242 214 y(CALL)f(MPI_REQUEST_FREE\(req,)e(ierr\))242 271 y(CALL)i(MPI_IRECV\(inval,)f(1,)i(MPI_REAL,)e(0,)i(0,)f(req,)h (ierr\))242 327 y(CALL)f(MPI_WAIT\(req,)f(status,)h(ierr\))170 384 y(END)h(DO)170 440 y(CALL)g(MPI_ISEND\(outval,)d(1,)j(MPI_REAL,)f (0,)g(0,)h(req,)f(ierr\))170 497 y(CALL)h(MPI_WAIT\(req,)e(status\))75 553 y(END)h(IF)75 675 y Fj(3.7.4)49 b(Semantics)13 b(of)k(Nonblo)q (cking)h(Comm)o(unications)75 761 y Fx(The)11 b(seman)o(tics)f(of)h (non)o(blo)q(c)o(king)g(comm)o(unication)h(is)f(de\014ned)h(b)o(y)e (suitably)i(extending)g(the)e(de\014nitions)75 817 y(in)16 b(Section)g(3.5.)75 937 y Fv(Order)46 b Fx(Non)o(blo)q(c)o(king)12 b(comm)o(unication)f(op)q(erations)g(are)g(ordered)g(according)g(to)f (the)h(execution)h(order)75 994 y(of)h(the)g(calls)h(that)e(initiate)i (the)f(comm)o(unication.)20 b(The)13 b(non-o)o(v)o(ertaking)g (requiremen)o(t)g(of)g(Section)h(3.5)75 1050 y(is)i(extended)g(to)e (non)o(blo)q(c)o(king)j(comm)o(unication,)f(with)f(this)h(de\014nition) h(of)d(order)h(b)q(eing)i(used.)75 1156 y Fq(Example)h(3.12)23 b Fx(Message)14 b(ordering)i(for)e(non)o(blo)q(c)o(king)j(op)q (erations.)75 1250 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))75 1307 y(IF)h(\(RANK.EQ.0\))e(THEN)218 1363 y(CALL)h(MPI_ISEND\(a,)g(1,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(r1,)h (ierr\))218 1420 y(CALL)f(MPI_ISEND\(b,)g(1,)g(MPI_REAL,)g(1,)h(0,)f (comm,)g(r2,)h(ierr\))75 1476 y(ELSE)95 b(!)24 b(rank.EQ.1)218 1533 y(CALL)f(MPI_IRECV\(a,)g(1,)g(MPI_REAL,)g(0,)h(MPI_ANY_TAG,)e (comm,)h(r1,)g(ierr\))218 1589 y(CALL)g(MPI_IRECV\(b,)g(1,)g(MPI_REAL,) g(0,)h(0,)f(comm,)g(r2,)h(ierr\))75 1645 y(END)f(IF)75 1702 y(CALL)g(MPI_WAIT\(r1,status\))75 1758 y(CALL)g (MPI_WAIT\(r2,status\))75 1852 y Fx(The)11 b(\014rst)f(send)h(of)f(pro) q(cess)h(zero)f(will)j(matc)o(h)d(the)g(\014rst)g(receiv)o(e)i(of)e (pro)q(cess)h(one,)g(ev)o(en)g(if)g(b)q(oth)f(messages)75 1909 y(are)15 b(sen)o(t)g(b)q(efore)g(pro)q(cess)h(one)f(executes)h (either)g(receiv)o(e.)75 2029 y Fv(Progress)45 b Fx(A)12 b(call)h(to)f Fv(MPI)p 545 2029 14 2 v 15 w(W)l(AIT)g Fx(that)g(completes)g(a)g(receiv)o(e)h(will)h(ev)o(en)o(tually)f (terminate)f(and)g(return)75 2085 y(if)21 b(a)f(matc)o(hing)g(send)h (has)f(b)q(een)h(started,)g(unless)g(the)g(send)g(is)f(satis\014ed)h(b) o(y)f(another)g(receiv)o(e.)36 b(In)75 2142 y(particular,)20 b(if)f(the)g(matc)o(hing)g(send)g(is)g(non)o(blo)q(c)o(king,)i(then)e (the)g(receiv)o(e)g(should)h(complete)f(ev)o(en)g(if)75 2198 y(no)c(call)i(is)e(executed)i(b)o(y)e(the)g(sender)h(to)f (complete)h(the)f(send.)20 b(Similarly)l(,)e(a)d(call)h(to)f Fv(MPI)p 1651 2198 V 15 w(W)l(AIT)h Fx(that)75 2254 y(completes)i(a)e (send)i(will)h(ev)o(en)o(tually)f(return)f(if)g(a)g(matc)o(hing)g (receiv)o(e)h(has)f(b)q(een)h(started,)f(unless)h(the)75 2311 y(receiv)o(e)e(is)g(satis\014ed)f(b)o(y)g(another)g(send,)g(and)g (ev)o(en)h(if)f(no)g(call)h(is)g(executed)g(to)e(complete)i(the)f (receiv)o(e.)75 2417 y Fq(Example)j(3.13)23 b Fx(An)15 b(illustration)i(of)d(progress)h(seman)o(tics.)75 2511 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2567 y(IF)h(\(RANK.EQ.0\))e(THEN)218 2624 y(CALL)h(MPI_SSEND\(a,)g(1,)g (MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))218 2680 y(CALL)g(MPI_SEND\(b,)g (1,)g(MPI_REAL,)g(1,)h(1,)f(comm,)g(ierr\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 43 51 43 50 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fx(43)75 45 y Fu(ELSE)95 b(!)24 b(rank.EQ.1)218 102 y(CALL)f(MPI_IRECV\(a,)g(1,)g(MPI_REAL,)g(0,)h(0,)f(comm,)g(r,)h (ierr\))218 158 y(CALL)f(MPI_RECV\(b,)g(1,)g(MPI_REAL,)g(0,)h(1,)f (comm,)g(ierr\))218 214 y(CALL)g(MPI_WAIT\(r,)g(status,)g(ierr\))75 271 y(END)g(IF)166 357 y Fx(This)14 b(co)q(de)f(should)h(not)f(deadlo)q (c)o(k)h(in)g(a)e(correct)h Fv(MPI)g Fx(implemen)o(tation.)20 b(The)13 b(\014rst)g(sync)o(hronous)75 414 y(send)22 b(of)e(pro)q(cess)h(zero)g(m)o(ust)f(complete)i(after)e(pro)q(cess)i (one)f(p)q(osts)f(the)h(matc)o(hing)g(\(non)o(blo)q(c)o(king\))75 470 y(receiv)o(e)15 b(ev)o(en)f(if)h(pro)q(cess)f(one)g(has)g(not)f(y)o (et)h(reac)o(hed)g(the)g(completing)h(w)o(ait)f(call.)20 b(Th)o(us,)14 b(pro)q(cess)g(zero)75 526 y(will)j(con)o(tin)o(ue)f(and) f(execute)h(the)f(second)h(send,)f(allo)o(wing)h(pro)q(cess)g(one)f(to) g(complete)h(execution.)166 622 y(If)d(an)f Fv(MPI)p 354 622 14 2 v 16 w(TEST)h Fx(that)f(completes)h(a)g(receiv)o(e)g(is)g (rep)q(eatedly)h(called)g(with)f(the)g(same)f(argumen)o(ts,)75 679 y(and)g(a)g(matc)o(hing)g(send)h(has)f(b)q(een)h(started,)f(then)g (the)g(call)i(will)f(ev)o(en)o(tually)g(return)f Fv(\015ag)g(=)h(true)p Fx(,)g(unless)75 735 y(the)g(send)g(is)g(satis\014ed)g(b)o(y)f(another) h(receiv)o(e.)19 b(If)13 b(an)g Fv(MPI)p 1046 735 V 16 w(TEST)f Fx(that)g(completes)h(a)g(send)g(is)g(rep)q(eatedly)75 791 y(called)i(with)f(the)g(same)f(argumen)o(ts,)g(and)h(a)f(matc)o (hing)g(receiv)o(e)i(has)e(b)q(een)i(started,)e(then)h(the)g(call)g (will)75 848 y(ev)o(en)o(tually)i(return)f Fv(\015ag)g(=)h(true)p Fx(,)f(unless)i(the)e(receiv)o(e)h(is)g(satis\014ed)g(b)o(y)f(another)f (send.)75 968 y Fj(3.7.5)49 b(Multiple)17 b(Completions)75 1054 y Fx(It)h(is)h(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)g(to)f(w)o (ait)g(for)f(the)i(completion)g(of)f(an)o(y)l(,)h(some,)f(or)g(all)h (the)g(op)q(erations)75 1110 y(in)h(a)f(list,)i(rather)d(than)i(ha)o (ving)f(to)g(w)o(ait)g(for)f(a)h(sp)q(eci\014c)i(message.)32 b(A)19 b(call)i(to)d Fv(MPI)p 1600 1110 V 16 w(W)l(AIT)l(ANY)i Fx(or)75 1167 y Fv(MPI)p 160 1167 V 16 w(TEST)l(ANY)d Fx(can)f(b)q(e)g(used)g(to)f(w)o(ait)h(for)f(the)h(completion)h(of)e (one)h(out)f(of)g(sev)o(eral)h(op)q(erations.)22 b(A)75 1223 y(call)d(to)d Fv(MPI)p 303 1223 V 16 w(W)l(AIT)l(ALL)h Fx(or)g Fv(MPI)p 665 1223 V 16 w(TEST)l(ALL)h Fx(can)f(b)q(e)h(used)g (to)f(w)o(ait)g(for)g(all)h(p)q(ending)i(op)q(erations)d(in)75 1280 y(a)e(list.)23 b(A)16 b(call)h(to)e Fv(MPI)p 484 1280 V 15 w(W)l(AITSOME)i Fx(or)e Fv(MPI)p 892 1280 V 16 w(TESTSOME)h Fx(can)g(b)q(e)h(used)f(to)f(complete)i(all)f(enabled) 75 1336 y(op)q(erations)f(in)h(a)f(list.)75 1487 y Fv(MPI)p 160 1487 V 16 w(W)l(AIT)l(ANY)h(\(count,)g(a)o(rra)o(y)p 640 1487 V 14 w(of)p 691 1487 V 16 w(requests,)h(index,)f(status\))117 1564 y Fk(IN)155 b Fv(count)482 b Fk(list)14 b(length)f(\(in)o(teger\)) 117 1638 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 1638 V 15 w(of)p 468 1638 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 1712 y(OUT)108 b Fv(index)489 b Fk(index)12 b(of)f(handle)g(for)g(op)q (eration)h(that)f(completed)g(\(in)o(teger\))117 1785 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1910 y Fu(int)23 b(MPI)p 245 1910 15 2 v 17 w(Waitany\(int)g(count,)g (MPI)p 788 1910 V 16 w(Request)g(*array)p 1139 1910 V 17 w(of)p 1204 1910 V 17 w(requests,)f(int)i(*index,)393 1966 y(MPI)p 468 1966 V 17 w(Status)f(*status\))75 2053 y(MPI)p 150 2053 V 17 w(WAITANY\(COUNT,)f(ARRAY)p 645 2053 V 16 w(OF)p 709 2053 V 17 w(REQUESTS,)h(INDEX,)g(STATUS,)g (IERROR\))170 2109 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2109 V 17 w(OF)p 716 2109 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 2109 V 16 w(STATUS)p 1610 2109 V 16 w(SIZE\),)170 2165 y(IERROR)166 2252 y Fx(Blo)q(c)o(ks)16 b(un)o(til)g(one)f(of)g (the)h(op)q(erations)f(asso)q(ciated)g(with)h(the)f(activ)o(e)h (requests)f(in)h(the)f(arra)o(y)f(has)75 2308 y(completed.)36 b(If)21 b(more)f(then)h(one)f(op)q(eration)h(is)g(enabled)h(and)e(can)h (terminate,)g(one)f(is)h(arbitrarily)75 2365 y(c)o(hosen.)28 b(Returns)18 b(in)h Fv(index)g Fx(the)f(index)h(of)e(that)h(request)f (in)i(the)f(arra)o(y)f(and)h(returns)f(in)i Fv(status)h Fx(the)75 2421 y(status)c(of)h(the)f(completing)j(comm)o(unication.)25 b(\(The)17 b(arra)o(y)f(is)h(indexed)i(from)d(zero)h(in)g(C,)g(and)g (from)75 2478 y(one)g(in)i(F)l(ortran.\))24 b(If)18 b(the)f(request)g (w)o(as)g(allo)q(cated)h(b)o(y)f(a)g(non)o(blo)q(c)o(king)i(comm)o (unication)f(op)q(eration,)75 2534 y(then)e(it)f(is)h(deallo)q(cated)g (and)g(the)f(request)g(handle)i(is)e(set)g(to)g Fi(MPI)p 1229 2534 13 2 v 14 w(REQUEST)p 1431 2534 V 14 w(NULL)p Fx(.)166 2591 y(The)i Fv(a)o(rra)o(y)p 359 2591 14 2 v 15 w(of)p 411 2591 V 16 w(requests)i Fx(list)e(ma)o(y)g(con)o(tain)g (n)o(ull)h(or)f(inactiv)o(e)h(handles.)26 b(If)17 b(the)g(list)h(con)o (tains)f(no)75 2647 y(activ)o(e)e(handles)i(\(list)f(has)f(length)h (zero)f(or)g(all)h(en)o(tries)g(are)f(n)o(ull)i(or)e(inactiv)o(e\),)g (then)h(the)g(call)g(returns)75 2704 y(immediately)h(with)e Fv(index)i(=)e(MPI)p 687 2704 V 16 w(UNDEFINED)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 44 52 44 51 bop 75 -100 a Fx(44)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)166 45 y Fx(The)i(execution)h (of)e Fv(MPI)p 610 45 14 2 v 16 w(W)l(AIT)l(ANY\(count,)j(a)o(rra)o(y)p 1079 45 V 14 w(of)p 1130 45 V 16 w(requests,)h(index,)e(status\))g Fx(has)f(the)f(same)75 102 y(e\013ect)k(as)g(the)g(execution)i(of)44 b Fv(MPI)p 727 102 V 16 w(W)l(AIT\(&a)o(rra)o(y)p 1003 102 V 14 w(of)p 1054 102 V 16 w(requests[i],)26 b(status\))p Fx(,)g(where)c Fv(i)g Fx(is)h(the)g(v)m(alue)75 158 y(returned)17 b(b)o(y)f Fv(index)p Fx(.)25 b Fv(MPI)p 546 158 V 16 w(W)l(AIT)l(ANY)18 b Fx(with)f(an)f(arra)o(y)f(con)o(taining)j(one)e (activ)o(e)h(en)o(try)f(is)h(equiv)m(alen)o(t)75 214 y(to)e Fv(MPI)p 216 214 V 15 w(W)l(AIT)p Fx(.)75 365 y Fv(MPI)p 160 365 V 16 w(TEST)l(ANY\(count,)i(a)o(rra)o(y)p 626 365 V 14 w(of)p 677 365 V 16 w(requests,)g(index,)f(\015ag,)f (status\))117 443 y Fk(IN)155 b Fv(count)482 b Fk(list)14 b(length)f(\(in)o(teger\))117 518 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 518 V 15 w(of)p 468 518 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 593 y(OUT)108 b Fv(index)489 b Fk(index)23 b(of)g(op)q(eration)g(that)g (completed,)i(or)e Fi(MPI)p 1736 593 13 2 v 14 w(UNDE-)905 649 y(FINED)13 b Fk(if)g(none)i(completed)e(\(in)o(teger\))117 724 y(OUT)108 b Fv(\015ag)518 b Fi(true)15 b Fk(if)e(one)h(of)g(the)g (op)q(erations)g(is)g(complete)f(\(logical\))117 800 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 924 y Fu(int)23 b(MPI)p 245 924 15 2 v 17 w(Testany\(int)g(count,)g (MPI)p 788 924 V 16 w(Request)g(*array)p 1139 924 V 17 w(of)p 1204 924 V 17 w(requests,)f(int)i(*index,)393 981 y(int)g(*flag,)f(MPI)p 731 981 V 16 w(Status)g(*status\))75 1067 y(MPI)p 150 1067 V 17 w(TESTANY\(COUNT,)f(ARRAY)p 645 1067 V 16 w(OF)p 709 1067 V 17 w(REQUESTS,)h(INDEX,)g(FLAG,)g (STATUS,)g(IERROR\))170 1123 y(LOGICAL)g(FLAG)170 1180 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1180 V 17 w(OF)p 716 1180 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 1180 V 16 w(STATUS)p 1610 1180 V 16 w(SIZE\),)170 1236 y(IERROR)166 1323 y Fx(T)l(ests)d(for)h(completion)h(of)e(either)i(one) f(or)f(none)h(of)g(the)g(op)q(erations)g(asso)q(ciated)g(with)g(activ)o (e)75 1379 y(handles.)g(In)14 b(the)g(former)f(case,)h(it)g(returns)g Fv(\015ag)g(=)g(true)p Fx(,)h(returns)e(in)i Fv(index)g Fx(the)f(index)h(of)f(this)g(request)75 1436 y(in)h(the)g(arra)o(y)l(,) f(and)g(returns)h(in)g Fv(status)i Fx(the)e(status)f(of)g(that)g(op)q (eration;)h(if)g(the)f(request)h(w)o(as)f(allo)q(cated)75 1492 y(b)o(y)i(a)f(non)o(blo)q(c)o(king)i(comm)o(unication)g(call)f (then)g(the)g(request)g(is)g(deallo)q(cated)h(and)f(the)g(handle)h(is)f (set)75 1549 y(to)e Fi(MPI)p 207 1549 13 2 v 14 w(REQUEST)p 409 1549 V 14 w(NULL)p Fx(.)h(\(The)f(arra)o(y)f(is)i(indexed)h(from)d (zero)i(in)g(C,)f(and)g(from)g(one)g(in)h(F)l(ortran.\))j(In)75 1605 y(the)13 b(latter)g(case,)h(it)f(returns)g Fv(\015ag)h(=)f(false)p Fx(,)h(returns)f(a)g(v)m(alue)i(of)d Fi(MPI)p 1254 1605 V 15 w(UNDEFINED)g Fx(in)i Fv(index)g Fx(and)g Fv(status)75 1662 y Fx(is)f(unde\014ned.)21 b(The)12 b(arra)o(y)f(ma)o(y)h(con)o (tain)h(n)o(ull)g(or)f(inactiv)o(e)i(handles.)20 b(If)12 b(the)h(arra)o(y)e(con)o(tains)i(no)f(activ)o(e)75 1718 y(handles)18 b(then)f(the)f(call)i(returns)e(immediately)i(with)f Fv(\015ag)g(=)f(false)p Fx(,)h Fv(index)g Fx(=)g Fi(MPI)p 1528 1718 V 15 w(UNDEFINED)p Fx(,)e(and)75 1774 y Fv(status)i Fx(unde\014ned.)166 1831 y(The)i(execution)g(of)f Fv(MPI)p 609 1831 14 2 v 16 w(TEST)l(ANY\(count,)j(a)o(rra)o(y)p 1079 1831 V 15 w(of)p 1131 1831 V 16 w(requests,)g(index,)f(status\))g Fx(has)f(the)f(same)75 1887 y(e\013ect)c(as)f(the)h(execution)h(of)f Fv(MPI)p 663 1887 V 16 w(TEST\(&a)o(rra)o(y)p 940 1887 V 14 w(of)p 991 1887 V 16 w(requests[i],)j(\015ag,)c(status\))p Fx(,)j(for)d Fv(i=0,)h(1)f(,...,)g(count-)75 1944 y(1)p Fx(,)21 b(in)h(some)e(arbitrary)h(order,)g(un)o(til)h(one)f(call)h (returns)e Fv(\015ag)h(=)g(true)p Fx(,)h(or)f(all)g(fail.)38 b(In)21 b(the)g(former)75 2000 y(case,)c Fv(index)g Fx(is)g(set)g(to)f (the)h(last)f(v)m(alue)i(of)e Fv(i)p Fx(,)h(and)f(in)i(the)f(latter)f (case,)h(it)f(is)i(set)e(to)g Fi(MPI)p 1618 2000 13 2 v 14 w(UNDEFINED)p Fx(.)75 2057 y Fv(MPI)p 160 2057 14 2 v 16 w(TEST)l(ANY)g Fx(with)g(an)f(arra)o(y)f(con)o(taining)i(one)f (activ)o(e)g(en)o(try)g(is)h(equiv)m(alen)o(t)h(to)d Fv(MPI)p 1633 2057 V 16 w(TEST)p Fx(.)75 2208 y Fv(MPI)p 160 2208 V 16 w(W)l(AIT)l(ALL\()g(count,)i(a)o(rra)o(y)p 626 2208 V 15 w(of)p 678 2208 V 16 w(requests,)h(a)o(rra)o(y)p 971 2208 V 15 w(of)p 1023 2208 V 15 w(statuses\))117 2285 y Fk(IN)155 b Fv(count)482 b Fk(lists)14 b(length)g(\(in)o (teger\))117 2360 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 2360 V 15 w(of)p 468 2360 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 2435 y(OUT)108 b Fv(a)o(rra)o(y)p 416 2435 V 15 w(of)p 468 2435 V 16 w(statuses)276 b Fk(arra)o(y)14 b(of)f(status)i(ob)r(jects)g (\(arra)o(y)f(of)f(Status\))75 2560 y Fu(int)23 b(MPI)p 245 2560 15 2 v 17 w(Waitall\(int)g(count,)g(MPI)p 788 2560 V 16 w(Request)g(*array)p 1139 2560 V 17 w(of)p 1204 2560 V 17 w(requests,)393 2616 y(MPI)p 468 2616 V 17 w(Status)g(*array)p 796 2616 V 16 w(of)p 860 2616 V 17 w(statuses\))75 2703 y(MPI)p 150 2703 V 17 w(WAITALL\(COUNT,)f (ARRAY)p 645 2703 V 16 w(OF)p 709 2703 V 17 w(REQUESTS,)h(ARRAY)p 1085 2703 V 16 w(OF)p 1149 2703 V 17 w(STATUSES,)g(IERROR\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 45 53 45 52 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fx(45)170 45 y Fu(INTEGER)23 b(COUNT,)g(ARRAY)p 651 45 15 2 v 17 w(OF)p 716 45 V 17 w(REQUESTS\(*\))170 102 y(INTEGER)g(ARRAY)p 484 102 V 17 w(OF)p 549 102 V 17 w(STATUSES\(MPI)p 854 102 V 15 w(STATUS)p 1013 102 V 17 w(SIZE,*\),)f(IERROR)166 188 y Fx(Blo)q(c)o(ks)e(un)o(til)h(all)f (comm)o(unication)g(op)q(erations)g(asso)q(ciated)g(with)g(activ)o(e)f (handles)i(in)g(the)e(list)75 244 y(complete,)j(and)e(return)g(the)g (status)f(of)g(all)i(these)f(op)q(erations)g(\(this)h(includes)h(the)e (case)g(where)g(no)75 301 y(handle)j(in)f(the)f(list)i(is)f(activ)o (e\).)38 b(Both)21 b(arra)o(ys)f(ha)o(v)o(e)h(the)h(same)f(n)o(um)o(b)q (er)h(of)f(v)m(alid)i(en)o(tries.)39 b(The)75 357 y Fv(i)p Fx(-th)20 b(en)o(try)g(in)h Fv(a)o(rra)o(y)p 444 357 14 2 v 14 w(of)p 495 357 V 16 w(statuses)i Fx(is)e(set)f(to)f(the)i (return)f(status)f(of)h(the)g Fv(i)p Fx(-th)g(op)q(eration.)35 b(Requests)75 414 y(that)13 b(w)o(ere)h(created)g(b)o(y)h(non)o(blo)q (c)o(king)g(comm)o(unication)g(op)q(erations)f(are)g(deallo)q(cated)i (and)e(the)g(corre-)75 470 y(sp)q(onding)j(handles)g(in)g(the)f(arra)o (y)e(are)i(set)g(to)f Fi(MPI)p 968 470 13 2 v 14 w(REQUEST)p 1170 470 V 14 w(NULL)p Fx(.)h(The)g(list)h(ma)o(y)e(con)o(tain)h(n)o (ull)h(or)75 527 y(inactiv)o(e)i(handles.)29 b(The)18 b(call)g(returns)g(in)h(the)f(status)e(of)i(eac)o(h)g(suc)o(h)g(en)o (try)f Fv(tag)h(=)g(MPI)p 1654 527 14 2 v 16 w(ANY)p 1762 527 V 17 w(T)l(A)o(G)p Fx(,)75 583 y Fv(source)24 b(=)g(MPI)p 362 583 V 16 w(ANY)p 470 583 V 17 w(SOURCE)p Fx(,)g(and)g(eac)o(h)g(status)e(en)o(try)h(is)h(also)g(con\014gured)g (so)f(that)g(calls)i(to)75 640 y Fv(MPI)p 160 640 V 16 w(GET)p 264 640 V 17 w(COUNT)16 b Fx(and)f Fv(MPI)p 623 640 V 16 w(GET)p 727 640 V 17 w(ELEMENTS)g Fx(return)g Fv(count)h(=)g(0)p Fx(.)166 696 y(The)c(execution)h(of)e Fv(MPI)p 589 696 V 16 w(W)l(AIT)l(ALL\(count,)i(a)o(rra)o(y)p 1038 696 V 14 w(of)p 1089 696 V 16 w(requests,)h(a)o(rra)o(y)p 1379 696 V 15 w(of)p 1431 696 V 16 w(statuses\))g Fx(has)d(the)h(same) 75 753 y(e\013ect)17 b(as)f(the)h(execution)h(of)f Fv(MPI)p 678 753 V 16 w(W)l(AIT\(&a)o(rra)o(y)p 954 753 V 14 w(of)p 1005 753 V 16 w(request[i],)i(&a)o(rra)o(y)p 1354 753 V 15 w(of)p 1406 753 V 16 w(statuses[i]\))p Fx(,)h(for)c Fv(i=0)h(,...,)75 809 y(count-1)p Fx(,)h(in)f(some)g(arbitrary)f (order.)24 b Fv(MPI)p 840 809 V 16 w(W)l(AIT)l(ALL)16 b Fx(with)h(an)g(arra)o(y)e(of)i(length)g(one)g(is)g(equiv)m(alen)o(t) 75 865 y(to)e Fv(MPI)p 216 865 V 15 w(W)l(AIT)p Fx(.)75 1016 y Fv(MPI)p 160 1016 V 16 w(TEST)l(ALL\(count,)h(a)o(rra)o(y)p 613 1016 V 14 w(of)p 664 1016 V 16 w(requests,)h(\015ag,)e(a)o(rra)o(y) p 1054 1016 V 14 w(of)p 1105 1016 V 16 w(statuses\))117 1094 y Fk(IN)155 b Fv(count)482 b Fk(lists)14 b(length)g(\(in)o (teger\))117 1169 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 1169 V 15 w(of)p 468 1169 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 1244 y(OUT)108 b Fv(\015ag)518 b Fk(\(logical\))117 1319 y(OUT)108 b Fv(a)o(rra)o(y)p 416 1319 V 15 w(of)p 468 1319 V 16 w(statuses)276 b Fk(arra)o(y)14 b(of)f(status)i(ob)r(jects)g(\(arra)o (y)f(of)f(Status\))75 1443 y Fu(int)23 b(MPI)p 245 1443 15 2 v 17 w(Testall\(int)g(count,)g(MPI)p 788 1443 V 16 w(Request)g(*array)p 1139 1443 V 17 w(of)p 1204 1443 V 17 w(requests,)f(int)i(*flag,)393 1500 y(MPI)p 468 1500 V 17 w(Status)f(*array)p 796 1500 V 16 w(of)p 860 1500 V 17 w(statuses\))75 1586 y(MPI)p 150 1586 V 17 w(TESTALL\(COUNT,)f(ARRAY)p 645 1586 V 16 w(OF)p 709 1586 V 17 w(REQUESTS,)h(FLAG,)g(ARRAY)p 1228 1586 V 16 w(OF)p 1292 1586 V 17 w(STATUSES,)g(IERROR\))170 1643 y(LOGICAL)g(FLAG)170 1699 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1699 V 17 w(OF)p 716 1699 V 17 w(REQUESTS\(*\),)170 1756 y(ARRAY)p 293 1756 V 17 w(OF)p 358 1756 V 17 w(STATUSES\(MPI)p 663 1756 V 15 w(STATUS)p 822 1756 V 17 w(SIZE,*\),)g(IERROR)166 1842 y Fx(Returns)17 b Fv(\015ag)g(=)g(true)g Fx(if)h(all)f(comm)o (unications)h(asso)q(ciated)f(with)g(activ)o(e)g(handles)h(in)f(the)g (arra)o(y)75 1899 y(ha)o(v)o(e)f(completed)i(\(this)e(includes)j(the)d (case)h(where)g(no)f(handle)i(in)f(the)g(list)g(is)g(activ)o(e\).)23 b(In)17 b(this)g(case,)75 1955 y(eac)o(h)h(status)f(en)o(try)g(that)g (corresp)q(onds)i(to)e(an)h(activ)o(e)f(handle)j(request)d(is)i(set)e (to)h(the)f(status)g(of)h(the)75 2012 y(corresp)q(onding)h(comm)o (unication;)g(if)g(the)e(request)h(w)o(as)f(allo)q(cated)i(b)o(y)f(a)f (non)o(blo)q(c)o(king)i(comm)o(unica-)75 2068 y(tion)f(call)h(then)f (it)g(is)h(deallo)q(cated,)g(and)f(the)g(handle)i(is)e(set)g(to)f Fi(MPI)p 1293 2068 13 2 v 14 w(REQUEST)p 1495 2068 V 14 w(NULL)p Fx(.)h(Eac)o(h)g(status)75 2124 y(en)o(try)f(that)g (corresp)q(onds)h(to)f(a)g(n)o(ull)i(or)e(inactiv)o(e)h(handle)h(is)f (set)f(to)g(return)h Fv(tag)f(=)h(MPI)p 1654 2124 14 2 v 16 w(ANY)p 1762 2124 V 17 w(T)l(A)o(G)p Fx(,)75 2181 y Fv(source)e(=)f(MPI)p 345 2181 V 16 w(ANY)p 453 2181 V 17 w(SOURCE)p Fx(,)i(and)e(is)h(also)f(con\014gured)h(so)f(that)f (calls)j(to)d Fv(MPI)p 1512 2181 V 16 w(GET)p 1616 2181 V 17 w(COUNT)i Fx(and)75 2237 y Fv(MPI)p 160 2237 V 16 w(GET)p 264 2237 V 17 w(ELEMENTS)f Fx(return)g Fv(count)i(=)e(0)p Fx(.)166 2294 y(Otherwise,)h Fv(\015ag)f(=)g(false)h Fx(is)f(returned,)h(no)f(request)g(is)h(mo)q(di\014ed)h(and)e(the)g(v)m (alues)i(of)d(the)i(status)75 2350 y(en)o(tries)g(are)e(unde\014ned.)22 b(This)16 b(is)g(a)f(lo)q(cal)h(op)q(eration.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 46 54 46 53 bop 75 -100 a Fx(46)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(W)l(AITSOME\(incount,)f(a)o(rra)o(y)p 695 45 V 15 w(of)p 747 45 V 16 w(requests,)h(outcount,)f(a)o(rra)o(y)p 1236 45 V 15 w(of)p 1288 45 V 16 w(indices,)g(a)o(rra)o(y)p 1553 45 V 14 w(of)p 1604 45 V 16 w(statuses\))117 122 y Fk(IN)155 b Fv(incount)448 b Fk(length)14 b(of)f(arra)o(y)p 1176 122 13 2 v 15 w(of)p 1225 122 V 14 w(requests)j(\(in)o(teger\))117 197 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 197 14 2 v 15 w(of)p 468 197 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o (y)e(of)f(handles\))117 273 y(OUT)108 b Fv(outcount)420 b Fk(n)o(um)o(b)q(er)13 b(of)h(completed)f(requests)j(\(in)o(teger\)) 117 348 y(OUT)108 b Fv(a)o(rra)o(y)p 416 348 V 15 w(of)p 468 348 V 16 w(indices)299 b Fk(arra)o(y)12 b(of)f(indices)h(of)f(op)q (erations)g(that)h(completed)f(\(arra)o(y)h(of)905 404 y(in)o(tegers\))117 479 y(OUT)108 b Fv(a)o(rra)o(y)p 416 479 V 15 w(of)p 468 479 V 16 w(statuses)276 b Fk(arra)o(y)15 b(of)g(status)h(ob)r(jects)g(for)f(op)q(erations)g(that)h(completed)905 536 y(\(arra)o(y)e(of)f(Status\))75 660 y Fu(int)23 b(MPI)p 245 660 15 2 v 17 w(Waitsome\(int)f(incount,)h(MPI)p 859 660 V 17 w(Request)g(*array)p 1211 660 V 16 w(of)p 1275 660 V 17 w(requests,)g(int)g(*outcount,)393 717 y(int)h(*array)p 636 717 V 16 w(of)p 700 717 V 17 w(indices,)f(MPI)p 1004 717 V 16 w(Status)g(*array)p 1331 717 V 17 w(of)p 1396 717 V 17 w(statuses\))75 803 y(MPI)p 150 803 V 17 w(WAITSOME\(INCOUNT,)e(ARRAY)p 716 803 V 17 w(OF)p 781 803 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 803 V 16 w(OF)p 1459 803 V 17 w(INDICES,)393 860 y(ARRAY)p 516 860 V 17 w(OF)p 581 860 V 17 w(STATUSES,)f(IERROR\))170 916 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 916 V 17 w(OF)p 764 916 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 916 V 17 w(OF)p 1514 916 V 17 w(INDICES\(*\),)170 972 y(ARRAY)p 293 972 V 17 w(OF)p 358 972 V 17 w(STATUSES\(MPI)p 663 972 V 15 w(STATUS)p 822 972 V 17 w(SIZE,*\),)h(IERROR)166 1059 y Fx(W)l(aits)d(un)o(til)h(at)e(least)i(one)f(of)g(the)g(op)q (erations)g(asso)q(ciated)g(with)h(activ)o(e)f(handles)h(in)g(the)f (list)75 1115 y(ha)o(v)o(e)15 b(completed.)21 b(Returns)15 b(in)i Fv(outcount)g Fx(the)e(n)o(um)o(b)q(er)h(of)e(requests)h(from)g (the)g(list)h Fv(a)o(rra)o(y)p 1656 1115 14 2 v 15 w(of)p 1708 1115 V 16 w(requests)75 1172 y Fx(that)c(ha)o(v)o(e)h(completed.) 20 b(Returns)13 b(in)h(the)f(\014rst)g Fv(outcount)i Fx(lo)q(cations)f(of)e(the)h(arra)o(y)f Fv(a)o(rra)o(y)p 1608 1172 V 14 w(of)p 1659 1172 V 16 w(indices)j Fx(the)75 1228 y(indices)21 b(of)d(these)h(op)q(erations)f(\(index)i(within)g (the)e(arra)o(y)g Fv(a)o(rra)o(y)p 1218 1228 V 14 w(of)p 1269 1228 V 16 w(requests)p Fx(;)k(the)d(arra)o(y)e(is)i(indexed)75 1285 y(from)12 b(zero)g(in)i(C)e(and)h(from)e(one)i(in)g(F)l(ortran\).) 18 b(Returns)13 b(in)g(the)g(\014rst)f Fv(outcount)i Fx(lo)q(cations)g(of)e(the)g(arra)o(y)75 1341 y Fv(a)o(rra)o(y)p 173 1341 V 15 w(of)p 225 1341 V 16 w(status)18 b Fx(the)g(status)e(for) g(these)h(completed)h(op)q(erations.)26 b(If)17 b(a)g(request)g(that)f (completed)i(w)o(as)75 1398 y(allo)q(cated)g(b)o(y)g(a)f(non)o(blo)q(c) o(king)h(comm)o(unication)g(call,)h(then)f(it)f(is)h(deallo)q(cated,)h (and)e(the)h(asso)q(ciated)75 1454 y(handle)e(is)g(set)f(to)g Fi(MPI)p 471 1454 13 2 v 14 w(REQUEST)p 673 1454 V 14 w(NULL)p Fx(.)166 1511 y(If)h(the)g(list)g(con)o(tains)g(no)f(activ)o (e)h(handles,)h(then)f(the)f(call)i(returns)f(immediately)h(with)f Fv(outcount)75 1567 y(=)g(0)p Fx(.)75 1718 y Fv(MPI)p 160 1718 14 2 v 16 w(TESTSOME\(incount,)h(a)o(rra)o(y)p 697 1718 V 14 w(of)p 748 1718 V 16 w(requests,)g(outcount,)f(a)o(rra)o (y)p 1237 1718 V 15 w(of)p 1289 1718 V 16 w(indices,)g(a)o(rra)o(y)p 1554 1718 V 14 w(of)p 1605 1718 V 16 w(statuses\))117 1795 y Fk(IN)155 b Fv(incount)448 b Fk(length)14 b(of)f(arra)o(y)p 1176 1795 13 2 v 15 w(of)p 1225 1795 V 14 w(requests)j(\(in)o(teger\)) 117 1870 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 1870 14 2 v 15 w(of)p 468 1870 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 1945 y(OUT)108 b Fv(outcount)420 b Fk(n)o(um)o(b)q(er)13 b(of)h(completed)f (requests)j(\(in)o(teger\))117 2021 y(OUT)108 b Fv(a)o(rra)o(y)p 416 2021 V 15 w(of)p 468 2021 V 16 w(indices)299 b Fk(arra)o(y)12 b(of)f(indices)h(of)f(op)q(erations)g(that)h(completed)f(\(arra)o(y)h (of)905 2077 y(in)o(tegers\))117 2152 y(OUT)108 b Fv(a)o(rra)o(y)p 416 2152 V 15 w(of)p 468 2152 V 16 w(statuses)276 b Fk(arra)o(y)15 b(of)g(status)h(ob)r(jects)g(for)f(op)q(erations)g(that)h(completed)905 2209 y(\(arra)o(y)e(of)f(Status\))75 2333 y Fu(int)23 b(MPI)p 245 2333 15 2 v 17 w(Testsome\(int)f(incount,)h(MPI)p 859 2333 V 17 w(Request)g(*array)p 1211 2333 V 16 w(of)p 1275 2333 V 17 w(requests,)g(int)g(*outcount,)393 2390 y(int)h(*array)p 636 2390 V 16 w(of)p 700 2390 V 17 w(indices,)f(MPI)p 1004 2390 V 16 w(Status)g(*array)p 1331 2390 V 17 w(of)p 1396 2390 V 17 w(statuses\))75 2476 y(MPI)p 150 2476 V 17 w(TESTSOME\(INCOUNT,)e(ARRAY)p 716 2476 V 17 w(OF)p 781 2476 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 2476 V 16 w(OF)p 1459 2476 V 17 w(INDICES,)393 2532 y(ARRAY)p 516 2532 V 17 w(OF)p 581 2532 V 17 w(STATUSES,)f(IERROR\))170 2589 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 2589 V 17 w(OF)p 764 2589 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 2589 V 17 w(OF)p 1514 2589 V 17 w(INDICES\(*\),)170 2645 y(ARRAY)p 293 2645 V 17 w(OF)p 358 2645 V 17 w(STATUSES\(MPI)p 663 2645 V 15 w(STATUS)p 822 2645 V 17 w(SIZE,*\),)h(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 47 55 47 54 bop 75 -100 a Fm(3.7.)29 b(NONBLOCKING)17 b(COMMUNICA)l(TION)868 b Fx(47)166 45 y(Beha)o(v)o(es)14 b(lik)o(e)i Fv(MPI)p 508 45 14 2 v 16 w(W)l(AITSOME)p Fx(,)e(except)h(that)f(it)h(returns)f (immediately)l(.)21 b(If)15 b(no)f(op)q(eration)h(has)75 102 y(completed)h(it)g(returns)f Fv(outcount)i(=)e(0)p Fx(.)166 158 y Fv(MPI)p 251 158 V 16 w(TESTSOME)g Fx(is)h(a)e(lo)q(cal) i(op)q(eration,)f(whic)o(h)g(returns)g(immediately)l(,)h(whereas)f Fv(MPI)p 1734 158 V 16 w(W)l(AIT-)75 214 y(SOME)i Fx(will)h(blo)q(c)o (k)f(un)o(til)h(a)e(comm)o(unication)h(completes,)g(if)g(it)g(w)o(as)e (passed)i(a)f(list)h(that)f(con)o(tains)h(at)75 271 y(least)h(one)g (activ)o(e)g(handle.)29 b(Both)18 b(calls)h(ful\014l)h(a)d Fv(fairness)i Fx(requiremen)o(t:)26 b(If)18 b(a)f(request)h(for)f(a)h (receiv)o(e)75 327 y(rep)q(eatedly)c(app)q(ears)e(in)i(a)e(list)h(of)g (requests)f(passed)h(to)f Fv(MPI)p 1122 327 V 16 w(W)l(AITSOME)h Fx(or)f Fv(MPI)p 1524 327 V 16 w(TESTSOME)p Fx(,)h(and)75 384 y(a)i(matc)o(hing)g(send)h(has)f(b)q(een)i(p)q(osted,)e(then)h(the) f(receiv)o(e)h(will)h(ev)o(en)o(tually)f(succeed,)h(unless)f(the)f (send)75 440 y(is)h(satis\014ed)f(b)o(y)h(another)e(receiv)o(e;)i(and)g (similarly)g(for)f(send)h(requests.)189 547 y Fl(A)n(dvic)n(e)11 b(to)i(users.)37 b Fx(The)12 b(use)f(of)g Fv(MPI)p 831 547 V 16 w(TESTSOME)h Fx(is)f(lik)o(ely)i(to)e(b)q(e)h(more)e (e\016cien)o(t)i(than)f(the)h(use)189 603 y(of)h Fv(MPI)p 324 603 V 16 w(TEST)l(ANY)p Fx(.)i(The)f(former)f(returns)g (information)h(on)g(all)h(completed)g(comm)o(unications,)189 659 y(with)g(the)g(latter,)g(a)g(new)g(call)i(is)e(required)i(for)d (eac)o(h)h(comm)o(unication)h(that)f(completes.)189 735 y(A)24 b(serv)o(er)g(with)g(m)o(ultiple)i(clien)o(ts)g(can)e(use)h Fv(MPI)p 1102 735 V 15 w(W)l(AITSOME)g Fx(so)f(as)g(not)g(to)f(starv)o (e)g(an)o(y)189 791 y(clien)o(t.)35 b(Clien)o(ts)20 b(send)h(messages)e (to)g(the)h(serv)o(er)g(with)g(service)h(requests.)33 b(The)20 b(serv)o(er)g(calls)189 848 y Fv(MPI)p 274 848 V 15 w(W)l(AITSOME)h Fx(with)g(one)f(receiv)o(e)h(request)f(for)g(eac)o (h)g(clien)o(t,)i(and)f(then)f(handles)i(all)f(re-)189 904 y(ceiv)o(es)g(that)e(completed.)35 b(If)20 b(a)g(call)h(to)f Fv(MPI)p 1002 904 V 16 w(W)l(AIT)l(ANY)h Fx(is)f(used)h(instead,)g (then)g(one)f(clien)o(t)189 960 y(could)d(starv)o(e)e(while)j(requests) e(from)g(another)g(clien)o(t)h(alw)o(a)o(ys)f(sneak)g(in)h(\014rst.)23 b(\()p Fl(End)17 b(of)g(advic)n(e)189 1017 y(to)f(users.)p Fx(\))189 1123 y Fl(A)n(dvic)n(e)h(to)h(implementors.)52 b Fv(MPI)p 810 1123 V 16 w(TESTSOME)18 b Fx(should)g(complete)g(as)f (man)o(y)g(p)q(ending)i(com-)189 1180 y(m)o(unications)d(as)e(p)q (ossible.)22 b(\()p Fl(End)16 b(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))75 1286 y Fq(Example)h(3.14)23 b Fx(Clien)o(t-serv)o(er)15 b(co)q(de)h(\(starv)m(ation)f(can)g(o)q(ccur\).)75 1436 y Fu(CALL)23 b(MPI_COMM_SIZE\(comm,)f(size,)h(ierr\))75 1493 y(CALL)g(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1549 y(IF\(rank)g(>)h(0\))f(THEN)214 b(!)24 b(client)f(code)170 1606 y(DO)h(WHILE\(.TRUE.\))242 1662 y(CALL)f(MPI_ISEND\(a,)g(n,)g (MPI_REAL,)g(0,)g(tag,)h(comm,)f(request,)g(ierr\))242 1718 y(CALL)g(MPI_WAIT\(request,)f(status,)h(ierr\))170 1775 y(END)h(DO)75 1831 y(ELSE)214 b(!)24 b(rank=0)f(--)h(server)f (code)242 1888 y(DO)h(i=1,)f(size-1)314 1944 y(CALL)g (MPI_IRECV\(a\(1,i\),)e(n,)j(MPI_REAL,)f(0,)g(tag,)528 2001 y(comm,)h(request_list\(i\),)d(ierr\))242 2057 y(END)i(DO)242 2114 y(DO)h(WHILE\(.TRUE.\))314 2170 y(CALL)f(MPI_WAITANY\(size-1,)e (request_list,)h(index,)h(status,)g(ierr\))314 2227 y(CALL)g (DO_SERVICE\(a\(1,index\)\))45 b(!)23 b(handle)g(one)h(message)314 2283 y(CALL)f(MPI_IRECV\(a\(1,)f(index\),)h(n,)g(MPI_REAL,)g(0,)h(tag,) 552 2339 y(comm,)f(request_list\(index\),)f(ierr\))242 2396 y(END)h(DO)75 2452 y(END)g(IF)75 2608 y Fq(Example)18 b(3.15)23 b Fx(Same)15 b(co)q(de,)g(using)h Fv(MPI)p 849 2608 V 16 w(W)l(AITSOME)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 48 56 48 55 bop 75 -100 a Fx(48)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 102 y Fu(CALL)23 b(MPI_COMM_SIZE\(comm,)f(size,)h(ierr\))75 158 y(CALL)g (MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 214 y(IF\(rank)g(>)h(0\))f (THEN)214 b(!)24 b(client)f(code)170 271 y(DO)h(WHILE\(.TRUE.\))242 327 y(CALL)f(MPI_ISEND\(a,)g(n,)g(MPI_REAL,)g(0,)g(tag,)h(comm,)f (request,)g(ierr\))242 384 y(CALL)g(MPI_WAIT\(request,)f(status,)h (ierr\))170 440 y(END)h(DO)75 497 y(ELSE)214 b(!)24 b(rank=0)f(--)h (server)f(code)170 553 y(DO)h(i=1,)f(size-1)242 610 y(CALL)g (MPI_IRECV\(a\(1,i\),)f(n,)h(MPI_REAL,)g(0,)h(tag,)600 666 y(comm,)f(request_list\(i\),)f(ierr\))170 723 y(END)i(DO)170 779 y(DO)g(WHILE\(.TRUE.\))242 835 y(CALL)f(MPI_WAITSOME\(size,)f (request_list,)g(numdone,)648 892 y(index_list,)g(status_list,)g (ierr\))242 948 y(DO)i(i=1,)f(numdone)314 1005 y(CALL)g (DO_SERVICE\(a\(1,)f(index_list\(i\)\)\))314 1061 y(CALL)h (MPI_IRECV\(a\(1,)f(index_list\(i\)\),)g(n,)h(MPI_REAL,)g(0,)h(tag,)624 1118 y(comm,)f(request_list\(i\),)f(ierr\))242 1174 y(END)h(DO)170 1231 y(END)h(DO)75 1287 y(END)f(IF)75 1430 y Fn(3.8)59 b(Prob)r(e)20 b(and)f(Cancel)75 1532 y Fx(The)d Fv(MPI)p 254 1532 14 2 v 16 w(PROBE)g Fx(and)g Fv(MPI)p 605 1532 V 16 w(IPROBE)g Fx(op)q(erations)g(allo)o(w)g(incoming)h(messages)e(to) g(b)q(e)h(c)o(hec)o(k)o(ed)g(for,)75 1588 y(without)g(actually)h (receiving)h(them.)k(The)17 b(user)f(can)g(then)h(decide)g(ho)o(w)f(to) g(receiv)o(e)h(them,)f(based)g(on)75 1645 y(the)i(information)f (returned)h(b)o(y)f(the)h(prob)q(e)g(\(basically)l(,)h(the)e (information)h(returned)g(b)o(y)f Fv(status)p Fx(\).)28 b(In)75 1701 y(particular,)16 b(the)g(user)g(ma)o(y)f(allo)q(cate)h (memory)f(for)g(the)h(receiv)o(e)h(bu\013er,)e(according)h(to)f(the)h (length)g(of)75 1758 y(the)f(prob)q(ed)h(message.)166 1814 y(The)i Fv(MPI)p 347 1814 V 16 w(CANCEL)g Fx(op)q(eration)g(allo)o (ws)g(p)q(ending)i(comm)o(unications)e(to)g(b)q(e)g(canceled.)30 b(This)18 b(is)75 1871 y(required)i(for)f(clean)o(up.)33 b(P)o(osting)19 b(a)g(send)h(or)f(a)g(receiv)o(e)h(ties)f(up)h(user)f (resources)h(\(send)f(or)g(receiv)o(e)75 1927 y(bu\013ers\),)14 b(and)i(a)f(cancel)h(ma)o(y)f(b)q(e)g(needed)i(to)d(free)i(these)f (resources)g(gracefully)l(.)75 2078 y Fv(MPI)p 160 2078 V 16 w(IPROBE\(source,)h(tag,)f(comm)m(,)c(\015ag,)k(status\))117 2155 y Fk(IN)155 b Fv(source)468 b Fk(source)15 b(rank,)f(or)27 b Fi(MPI)p 1281 2155 13 2 v 15 w(ANY)p 1381 2155 V 14 w(SOURCE)13 b Fk(\(in)o(teger\))117 2230 y(IN)155 b Fv(tag)526 b Fk(tag)14 b(v)n(alue)f(or)28 b Fi(MPI)p 1226 2230 V 14 w(ANY)p 1325 2230 V 15 w(T)m(A)o(G)12 b Fk(\(in)o(teger\))117 2305 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 2381 y(OUT)108 b Fv(\015ag)518 b Fk(\(logical\))117 2456 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 2580 y Fu(int)23 b(MPI)p 245 2580 15 2 v 17 w(Iprobe\(int)g(source,)g (int)g(tag,)g(MPI)p 1002 2580 V 17 w(Comm)g(comm,)h(int)f(*flag,)393 2637 y(MPI)p 468 2637 V 17 w(Status)g(*status\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 49 57 49 56 bop 75 -100 a Fm(3.8.)29 b(PR)o(OBE)16 b(AND)f(CANCEL)1166 b Fx(49)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(IPROBE\(SOURCE,)22 b(TAG,)h(COMM,)g(FLAG,)g(STATUS,)g(IERROR\))170 102 y(LOGICAL)g(FLAG) 170 158 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 158 V 16 w(STATUS)p 1218 158 V 16 w(SIZE\),)g(IERROR)166 244 y Fv(MPI)p 251 244 14 2 v 16 w(IPROBE\(source,)18 b(tag,)f(comm)m(,)d(\015ag,)k(status\))g Fx(returns)f Fv(\015ag)g(=)h(true)f Fx(if)h(there)f(is)h(a)e(message)75 301 y(that)f(can)i(b)q(e)f(receiv)o(ed)h(and)g(that)e(matc)o(hes)h(the) g(pattern)f(sp)q(eci\014ed)k(b)o(y)d(the)g(argumen)o(ts)f Fv(source)p Fx(,)h Fv(tag)p Fx(,)75 357 y(and)f Fv(comm)n Fx(.)i(The)e(call)i(matc)o(hes)e(the)g(same)g(message)g(that)f(w)o (ould)i(ha)o(v)o(e)f(b)q(een)h(receiv)o(ed)h(b)o(y)e(a)g(call)i(to)75 414 y Fv(MPI)p 160 414 V 16 w(RECV\(...,)f(source,)i(tag,)f(comm)m(,)d (status\))k Fx(executed)g(at)e(the)h(same)g(p)q(oin)o(t)g(in)h(the)f (program,)f(and)75 470 y(returns)c(in)i Fv(status)g Fx(the)f(same)f(v)m (alue)i(that)e(w)o(ould)g(ha)o(v)o(e)h(b)q(een)g(returned)g(b)o(y)g Fv(MPI)p 1467 470 V 15 w(RECV\(\))p Fx(.)19 b(Otherwise,)75 527 y(the)c(call)i(returns)e Fv(\015ag)g(=)g(false)p Fx(,)g(and)h(lea)o(v)o(es)f Fv(status)i Fx(unde\014ned.)166 583 y(If)g Fv(MPI)p 298 583 V 15 w(IPROBE)h Fx(returns)e Fv(\015ag)h(=)f(true)p Fx(,)h(then)g(the)g(con)o(ten)o(t)f(of)g(the)g (status)g(ob)s(ject)g(can)g(b)q(e)h(sub-)75 640 y(sequen)o(tly)i (accessed)h(as)e(describ)q(ed)i(in)g(section)f(3.2.5)e(to)h(\014nd)h (the)g(source,)g(tag)f(and)g(length)i(of)e(the)75 696 y(prob)q(ed)e(message.)166 753 y(A)d(subsequen)o(t)i(receiv)o(e)f (executed)g(with)g(the)g(same)f(con)o(text,)g(and)g(the)h(source)f(and) h(tag)f(returned)75 809 y(in)19 b Fv(status)i Fx(b)o(y)e Fv(MPI)p 415 809 V 15 w(IPROBE)h Fx(will)g(receiv)o(e)g(the)f(message)f (that)g(w)o(as)g(matc)o(hed)g(b)o(y)h(the)f(prob)q(e,)i(if)f(no)75 865 y(other)c(in)o(terv)o(ening)h(receiv)o(e)g(o)q(ccurs)f(after)g(the) g(prob)q(e.)20 b(If)15 b(the)g(receiving)i(pro)q(cess)e(is)h(m)o (ulti-threaded,)75 922 y(it)f(is)h(the)f(user's)g(resp)q(onsibilit)o(y) j(to)d(ensure)g(that)g(the)g(last)g(condition)i(holds.)166 978 y(The)f Fv(source)h Fx(argumen)o(t)e(of)h Fv(MPI)p 738 978 V 16 w(PROBE)h Fx(can)f(b)q(e)h Fi(MPI)p 1140 978 13 2 v 14 w(ANY)p 1239 978 V 15 w(SOURCE)p Fx(,)d(and)i(the)h Fv(tag)f Fx(argumen)o(t)75 1035 y(can)g(b)q(e)g Fi(MPI)p 298 1035 V 15 w(ANY)p 398 1035 V 14 w(T)m(A)o(G)p Fx(,)f(so)g(that)g (one)h(can)g(prob)q(e)g(for)f(messages)h(from)f(an)h(arbitrary)f (source)h(and/or)75 1091 y(with)h(an)g(arbitrary)f(tag.)24 b(Ho)o(w)o(ev)o(er,)16 b(a)h(sp)q(eci\014c)h(comm)o(unication)g(con)o (text)e(m)o(ust)g(b)q(e)i(pro)o(vided)g(with)75 1148 y(the)d Fv(comm)10 b Fx(argumen)o(t.)166 1204 y(It)16 b(is)g(not)f(necessary)h(to)f(receiv)o(e)h(a)f(message)h(immediately)h (after)e(it)h(has)f(b)q(een)i(prob)q(ed)f(for,)f(and)75 1261 y(the)g(same)g(message)g(ma)o(y)f(b)q(e)i(prob)q(ed)g(for)f(sev)o (eral)g(times)h(b)q(efore)f(it)h(is)f(receiv)o(ed.)75 1412 y Fv(MPI)p 160 1412 14 2 v 16 w(PROBE\(source,)h(tag,)f(comm)m(,)d (status\))117 1489 y Fk(IN)155 b Fv(source)468 b Fk(source)15 b(rank,)f(or)27 b Fi(MPI)p 1281 1489 13 2 v 15 w(ANY)p 1381 1489 V 14 w(SOURCE)13 b Fk(\(in)o(teger\))117 1564 y(IN)155 b Fv(tag)526 b Fk(tag)14 b(v)n(alue,)e(or)28 b Fi(MPI)p 1237 1564 V 14 w(ANY)p 1336 1564 V 15 w(T)m(A)o(G)13 b Fk(\(in)o(teger\))117 1639 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1714 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1839 y Fu(int)23 b(MPI)p 245 1839 15 2 v 17 w(Probe\(int)g(source,)g (int)g(tag,)h(MPI)p 979 1839 V 16 w(Comm)g(comm,)f(MPI)p 1330 1839 V 17 w(Status)g(*status\))75 1925 y(MPI)p 150 1925 V 17 w(PROBE\(SOURCE,)f(TAG,)h(COMM,)g(STATUS,)g(IERROR\))170 1982 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 1982 V 16 w(STATUS)p 1218 1982 V 16 w(SIZE\),)g(IERROR)166 2068 y Fv(MPI)p 251 2068 14 2 v 16 w(PROBE)d Fx(b)q(eha)o(v)o(es)f(lik) o(e)h Fv(MPI)p 777 2068 V 16 w(IPROBE)g Fx(except)f(that)f(it)i(is)f(a) g(blo)q(c)o(king)h(call)g(that)e(returns)75 2124 y(only)e(after)e(a)h (matc)o(hing)g(message)g(has)g(b)q(een)h(found.)166 2181 y(The)11 b Fv(MPI)f Fx(implemen)o(tation)i(of)e Fv(MPI)p 799 2181 V 16 w(PROBE)i Fx(and)e Fv(MPI)p 1140 2181 V 16 w(IPROBE)i Fx(needs)f(to)f(guaran)o(tee)g(progress:)75 2237 y(if)17 b(a)g(call)h(to)e Fv(MPI)p 385 2237 V 16 w(PROBE)i Fx(has)f(b)q(een)h(issued)g(b)o(y)f(a)f(pro)q(cess,)i(and)f (a)f(send)i(that)e(matc)o(hes)h(the)f(prob)q(e)75 2294 y(has)j(b)q(een)i(initiated)g(b)o(y)f(some)f(pro)q(cess,)i(then)f(the)f (call)i(to)e Fv(MPI)p 1244 2294 V 16 w(PROBE)i Fx(will)g(return,)f (unless)h(the)75 2350 y(message)c(is)i(receiv)o(ed)g(b)o(y)f(another)f (concurren)o(t)h(receiv)o(e)h(op)q(eration)f(\(that)e(is)j(executed)g (b)o(y)e(another)75 2407 y(thread)e(at)g(the)h(probing)g(pro)q(cess\).) 21 b(Similarly)l(,)d(if)e(a)f(pro)q(cess)h(busy)g(w)o(aits)f(with)h Fv(MPI)p 1573 2407 V 16 w(IPROBE)g Fx(and)g(a)75 2463 y(matc)o(hing)e(message)g(has)g(b)q(een)h(issued,)g(then)f(the)g(call)h (to)f Fv(MPI)p 1179 2463 V 16 w(IPROBE)g Fx(will)i(ev)o(en)o(tually)f (return)f Fv(\015ag)75 2520 y(=)i(true)f Fx(unless)i(the)e(message)g (is)g(receiv)o(ed)i(b)o(y)e(another)g(concurren)o(t)g(receiv)o(e)h(op)q (eration.)75 2626 y Fq(Example)i(3.16)23 b Fx(Use)15 b(blo)q(c)o(king)h(prob)q(e)g(to)f(w)o(ait)f(for)h(an)g(incoming)i (message.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 50 58 50 57 bop 75 -100 a Fx(50)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)242 45 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))242 102 y(IF)h(\(rank.EQ.0\))e (THEN)361 158 y(CALL)i(MPI_SEND\(i,)e(1,)i(MPI_INTEGER,)e(2,)h(0,)h (comm,)f(ierr\))242 214 y(ELSE)g(IF\(rank.EQ.1\))f(THEN)361 271 y(CALL)i(MPI_SEND\(x,)e(1,)i(MPI_REAL,)e(2,)i(0,)f(comm,)h(ierr\)) 242 327 y(ELSE)71 b(!)24 b(rank.EQ.2)337 384 y(DO)g(i=1,)f(2)409 440 y(CALL)g(MPI_PROBE\(MPI_ANY_SOURCE,)e(0,)791 497 y(comm,)i(status,)g(ierr\))409 553 y(IF)h(\(status\(MPI_SOURCE\))d(=)j (0\))f(THEN)75 610 y(100)381 b(CALL)24 b(MPI_RECV\(i,)e(1,)i (MPI_INTEGER,)e(0,)h(0,)h(status,)f(ierr\))409 666 y(ELSE)75 723 y(200)381 b(CALL)24 b(MPI_RECV\(x,)e(1,)i(MPI_REAL,)e(1,)i(0,)f (status,)g(ierr\))409 779 y(END)h(IF)337 835 y(END)g(DO)242 892 y(END)f(IF)75 998 y Fx(Eac)o(h)15 b(message)g(is)g(receiv)o(ed)i (with)e(the)h(righ)o(t)f(t)o(yp)q(e.)75 1104 y Fq(Example)j(3.17)23 b Fx(A)15 b(similar)h(program)e(to)h(the)g(previous)h(example,)g(but)f (no)o(w)g(it)g(has)g(a)g(problem.)242 1198 y Fu(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))242 1255 y(IF)h(\(rank.EQ.0\))e (THEN)361 1311 y(CALL)i(MPI_SEND\(i,)e(1,)i(MPI_INTEGER,)e(2,)h(0,)h (comm,)f(ierr\))242 1368 y(ELSE)g(IF\(rank.EQ.1\))f(THEN)361 1424 y(CALL)i(MPI_SEND\(x,)e(1,)i(MPI_REAL,)e(2,)i(0,)f(comm,)h(ierr\)) 242 1481 y(ELSE)337 1537 y(DO)g(i=1,)f(2)409 1593 y(CALL)g (MPI_PROBE\(MPI_ANY_SOURCE,)e(0,)791 1650 y(comm,)i(status,)g(ierr\)) 409 1706 y(IF)h(\(status\(MPI_SOURCE\))d(=)j(0\))f(THEN)75 1763 y(100)381 b(CALL)24 b(MPI_RECV\(i,)e(1,)i(MPI_INTEGER,)e (MPI_ANY_SOURCE,)862 1819 y(0,)i(status,)f(ierr\))409 1876 y(ELSE)75 1932 y(200)381 b(CALL)24 b(MPI_RECV\(x,)e(1,)i (MPI_REAL,)e(MPI_ANY_SOURCE,)862 1989 y(0,)i(status,)f(ierr\))409 2045 y(END)h(IF)337 2102 y(END)g(DO)242 2158 y(END)f(IF)166 2252 y Fx(W)l(e)16 b(sligh)o(tly)h(mo)q(di\014ed)g(example)f(3.16,)e (using)j Fi(MPI)p 1075 2252 13 2 v 14 w(ANY)p 1174 2252 V 15 w(SOURCE)d Fx(as)h(the)h Fv(source)g Fx(argumen)o(t)f(in)75 2308 y(the)f(t)o(w)o(o)g(receiv)o(e)h(calls)g(in)h(statemen)o(ts)d(lab) q(eled)j(100)e(and)g(200.)19 b(The)c(program)e(is)i(no)o(w)f (incorrect:)20 b(the)75 2365 y(receiv)o(e)f(op)q(eration)f(ma)o(y)f (receiv)o(e)i(a)f(message)f(that)h(is)g(distinct)h(from)f(the)g (message)f(prob)q(ed)i(b)o(y)f(the)75 2421 y(preceding)f(call)f(to)f Fv(MPI)p 504 2421 14 2 v 15 w(PROBE)p Fx(.)189 2527 y Fl(A)n(dvic)n(e)e(to)i(implementors.)38 b Fx(A)13 b(call)i(to)d Fv(MPI)p 971 2527 V 16 w(PROBE\(source,)j(tag,)e(comm)m(,)d(status\))15 b Fx(will)g(matc)o(h)189 2584 y(the)k(message)g(that)g(w)o(ould)h(ha)o (v)o(e)f(b)q(een)i(receiv)o(ed)f(b)o(y)g(a)f(call)h(to)f Fv(MPI)p 1431 2584 V 16 w(RECV\(...,)h(source,)g(tag,)189 2640 y(comm)m(,)9 b(status\))k Fx(executed)g(at)e(the)h(same)g(p)q(oin) o(t.)19 b(Supp)q(ose)13 b(that)e(this)h(message)g(has)f(source)h Fv(s)p Fx(,)h(tag)189 2697 y Fv(t)i Fx(and)h(comm)o(unicator)f Fv(c)p Fx(.)20 b(If)c(the)f(tag)g(argumen)o(t)f(in)j(the)e(prob)q(e)h (call)g(has)f(v)m(alue)i Fi(MPI)p 1684 2697 13 2 v 14 w(ANY)p 1783 2697 V 15 w(T)m(A)o(G)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 51 59 51 58 bop 75 -100 a Fm(3.8.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1161 b Fx(51)189 45 y(then)13 b(the)g(message)f(prob)q(ed)i(will)g(b)q(e)g (the)f(earliest)g(p)q(ending)i(message)e(from)f(source)h Fv(s)g Fx(with)g(com-)189 102 y(m)o(unicator)g Fv(c)h Fx(and)f(an)o(y)g(tag;)g(in)i(an)o(y)e(case,)g(the)h(message)f(prob)q (ed)h(will)h(b)q(e)f(the)g(earliest)g(p)q(ending)189 158 y(message)f(from)g(source)h Fv(s)g Fx(with)g(tag)e Fv(t)j Fx(and)e(comm)o(unicator)h Fv(c)g Fx(\(this)f(is)i(the)e (message)h(that)f(w)o(ould)189 214 y(ha)o(v)o(e)j(b)q(een)h(receiv)o (ed,)g(so)f(as)g(to)g(preserv)o(e)g(message)g(order\).)22 b(This)17 b(message)f(con)o(tin)o(ues)h(as)f(the)189 271 y(earliest)g(p)q(ending)h(message)e(from)g(source)g Fv(s)h Fx(with)g(tag)f Fv(t)g Fx(and)h(comm)o(unicator)f Fv(c)p Fx(,)h(un)o(til)g(it)g(is)g(re-)189 327 y(ceiv)o(ed.)k(A)12 b(receiv)o(e)h(op)q(eration)g(subsequen)o(t)g(to)e(the)i(prob)q(e)g (that)e(uses)i(the)f(same)g(comm)o(unicator)189 384 y(as)j(the)h(prob)q (e)g(and)g(uses)g(the)f(tag)g(and)h(source)g(v)m(alues)h(returned)f(b)o (y)f(the)h(prob)q(e,)g(m)o(ust)f(receiv)o(e)189 440 y(this)h(message,)f (unless)i(it)f(has)g(already)g(b)q(een)h(receiv)o(ed)g(b)o(y)f(another) g(receiv)o(e)h(op)q(eration.)22 b(\()p Fl(End)189 497 y(of)16 b(advic)n(e)g(to)h(implementors.)p Fx(\))75 697 y Fv(MPI)p 160 697 14 2 v 16 w(CANCEL\(request\))117 775 y Fk(IN)155 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 899 y Fu(int)23 b(MPI)p 245 899 15 2 v 17 w(Cancel\(MPI)p 502 899 V 16 w(Request)g(*request\))75 986 y(MPI)p 150 986 V 17 w(CANCEL\(REQUEST,)f(IERROR\))170 1042 y(INTEGER)h(REQUEST,)g(IERROR)166 1129 y Fx(A)15 b(call)i(to)e Fv(MPI)p 440 1129 14 2 v 16 w(CANCEL)g Fx(marks)g(for)g(cancellation)i(a)e(p)q(ending,)i(non)o(blo)q(c)o(king) g(comm)o(unication)75 1185 y(op)q(eration)d(\(send)g(or)f(receiv)o (e\).)20 b(The)13 b(cancel)i(call)g(is)f(lo)q(cal.)20 b(It)14 b(returns)f(immediately)l(,)j(p)q(ossibly)f(b)q(efore)75 1241 y(the)h(comm)o(unication)h(is)g(actually)g(canceled.)25 b(It)16 b(is)h(still)g(necessary)g(to)e(complete)i(a)f(comm)o (unication)75 1298 y(that)f(has)g(b)q(een)h(mark)o(ed)f(for)g (cancellation,)h(using)g(a)f(call)i(to)d Fv(MPI)p 1237 1298 V 16 w(REQUEST)p 1456 1298 V 18 w(FREE)p Fx(,)h Fv(MPI)p 1693 1298 V 16 w(W)l(AIT)g Fx(or)75 1354 y Fv(MPI)p 160 1354 V 16 w(TEST)g Fx(\(or)g(an)o(y)f(of)h(the)g(deriv)o(ed)i(op)q (erations\).)166 1411 y(If)i(a)g(comm)o(unication)h(is)g(mark)o(ed)f (for)g(cancellation,)j(then)d(a)g Fv(MPI)p 1370 1411 V 16 w(W)l(AIT)g Fx(call)i(for)d(that)h(com-)75 1467 y(m)o(unication)i(is)f(guaran)o(teed)f(to)h(return,)g(irresp)q(ectiv)o (e)h(of)f(the)g(activities)h(of)e(other)g(pro)q(cesses)i(\(i.e.,)75 1524 y Fv(MPI)p 160 1524 V 16 w(W)l(AIT)f Fx(b)q(eha)o(v)o(es)g(as)g(a) g(lo)q(cal)i(function\);)g(similarly)g(if)f Fv(MPI)p 1235 1524 V 16 w(TEST)f Fx(is)h(rep)q(eatedly)g(called)h(in)f(a)75 1580 y(busy)13 b(w)o(ait)g(lo)q(op)h(for)e(a)h(canceled)h(comm)o (unication,)g(then)g Fv(MPI)p 1173 1580 V 15 w(TEST)g Fx(will)g(ev)o(en)o(tually)g(b)q(e)g(successful.)166 1637 y Fv(MPI)p 251 1637 V 16 w(CANCEL)e Fx(can)f(b)q(e)i(used)f(to)f (cancel)i(a)e(comm)o(unication)h(that)f(uses)h(a)f(p)q(ersisten)o(t)h (request)g(\(see)75 1693 y(Sec.)26 b(3.9\),)15 b(in)j(the)f(same)f(w)o (a)o(y)g(it)h(is)h(used)f(for)g(nonp)q(ersisten)o(t)g(requests.)25 b(A)17 b(successful)h(cancellation)75 1750 y(cancels)c(the)f(activ)o(e) g(comm)o(unication,)h(but)f(not)f(the)h(request)g(itself.)20 b(After)13 b(the)g(call)h(to)e Fv(MPI)p 1690 1750 V 16 w(CANCEL)75 1806 y Fx(and)j(the)g(subsequen)o(t)h(call)g(to)e Fv(MPI)p 697 1806 V 16 w(W)l(AIT)h Fx(or)f Fv(MPI)p 978 1806 V 16 w(TEST)p Fx(,)h(the)g(request)g(b)q(ecomes)h(inactiv)o(e)g (and)f(can)75 1862 y(b)q(e)h(activ)m(ated)g(for)e(a)h(new)g(comm)o (unication.)166 1919 y(The)j(successful)i(cancellation)g(of)e(a)g (bu\013ered)h(send)g(frees)f(the)g(bu\013er)h(space)f(o)q(ccupied)j(b)o (y)d(the)75 1975 y(p)q(ending)f(message.)166 2032 y(Either)i(the)f (cancellation)i(succeeds,)f(or)f(the)g(comm)o(unication)h(succeeds,)h (but)e(not)g(b)q(oth.)29 b(If)18 b(a)75 2088 y(send)h(is)h(mark)o(ed)e (for)g(cancellation,)j(then)e(it)g(m)o(ust)f(b)q(e)i(the)f(case)g(that) f(either)h(the)g(send)g(completes)75 2145 y(normally)l(,)f(in)h(whic)o (h)f(case)f(the)h(message)f(sen)o(t)g(w)o(as)g(receiv)o(ed)h(at)f(the)h (destination)g(pro)q(cess,)g(or)f(that)75 2201 y(the)f(send)g(is)g (successfully)i(canceled,)f(in)g(whic)o(h)f(case)g(no)g(part)f(of)g (the)h(message)f(w)o(as)g(receiv)o(ed)i(at)e(the)75 2258 y(destination.)21 b(Then,)15 b(an)o(y)f(matc)o(hing)h(receiv)o(e)h(has) e(to)h(b)q(e)g(satis\014ed)h(b)o(y)e(another)h(send.)20 b(If)15 b(a)g(receiv)o(e)g(is)75 2314 y(mark)o(ed)d(for)f (cancellation,)k(then)d(it)h(m)o(ust)e(b)q(e)i(the)g(case)f(that)f (either)i(the)g(receiv)o(e)g(completes)g(normally)l(,)75 2371 y(or)18 b(that)g(the)g(receiv)o(e)i(is)f(successfully)h(canceled,) h(in)e(whic)o(h)h(case)e(no)h(part)f(of)g(the)g(receiv)o(e)i(bu\013er)e (is)75 2427 y(altered.)i(Then,)c(an)o(y)e(matc)o(hing)i(send)g(has)f (to)f(b)q(e)i(satis\014ed)g(b)o(y)f(another)g(receiv)o(e.)166 2483 y(If)h(the)f(op)q(eration)h(has)g(b)q(een)g(canceled,)h(then)f (information)g(to)f(that)f(e\013ect)i(will)h(b)q(e)f(returned)g(in)75 2540 y(the)f(status)g(argumen)o(t)f(of)h(the)g(op)q(eration)g(that)g (completes)h(the)f(comm)o(unication.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 52 60 52 59 bop 75 -100 a Fx(52)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(TEST)p 290 45 V 16 w(CANCELLED\(status,)g(\015ag\)) 117 122 y Fk(IN)155 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g (\(Status\))117 197 y(OUT)108 b Fv(\015ag)518 b Fk(\(logical\))75 322 y Fu(int)23 b(MPI)p 245 322 15 2 v 17 w(Test)p 358 322 V 17 w(cancelled\(MPI)p 687 322 V 15 w(Status)g(status,)g(int)h (*flag\))75 408 y(MPI)p 150 408 V 17 w(TEST)p 263 408 V 16 w(CANCELLED\(STATUS,)e(FLAG,)h(IERROR\))170 465 y(LOGICAL)g(FLAG)170 521 y(INTEGER)g(STATUS\(MPI)p 604 521 V 16 w(STATUS)p 764 521 V 17 w(SIZE\),)g(IERROR)166 608 y Fx(Returns)12 b Fv(\015ag)f(=)h(true)g Fx(if)f(the)h(comm)o (unication)g(asso)q(ciated)f(with)h(the)g(status)e(ob)s(ject)h(w)o(as)f (canceled)75 664 y(successfully)l(.)27 b(In)18 b(suc)o(h)f(a)g(case,)g (all)g(other)g(\014elds)h(of)e Fv(status)j Fx(\(suc)o(h)e(as)g Fv(count)h Fx(or)e Fv(tag)p Fx(\))h(are)f(unde\014ned.)75 721 y(Returns)c Fv(\015ag)g(=)g(false)p Fx(,)g(otherwise.)19 b(If)12 b(a)g(receiv)o(e)g(op)q(eration)g(migh)o(t)g(b)q(e)h(canceled)g (then)f(one)g(should)h(call)75 777 y Fv(MPI)p 160 777 14 2 v 16 w(TEST)p 290 777 V 16 w(CANCELLED)g Fx(\014rst,)f(to)g(c)o (hec)o(k)g(whether)h(the)f(op)q(eration)h(w)o(as)e(canceled,)j(b)q (efore)f(c)o(hec)o(king)75 834 y(on)i(the)g(other)g(\014elds)i(of)d (the)i(return)f(status.)189 940 y Fl(A)n(dvic)n(e)k(to)h(users.)64 b Fx(Cancel)20 b(can)g(b)q(e)f(an)h(exp)q(ensiv)o(e)g(op)q(eration)g (that)e(should)j(b)q(e)f(used)f(only)189 996 y(exceptionally)l(.)j(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fx(\))189 1103 y Fl(A)n(dvic)n(e)e(to)i(implementors.)39 b Fx(If)15 b(a)f(send)h(op)q(eration)g(uses)f(an)g(\\eager")g(proto)q(col)g (\(data)g(is)g(trans-)189 1159 y(ferred)e(to)f(the)h(receiv)o(er)h(b)q (efore)f(a)g(matc)o(hing)f(receiv)o(e)i(is)g(p)q(osted\),)f(then)g(the) g(cancellation)i(of)e(this)189 1215 y(send)i(ma)o(y)g(require)h(comm)o (unication)g(with)f(the)g(in)o(tended)i(receiv)o(er)f(in)f(order)g(to)g (free)g(allo)q(cated)189 1272 y(bu\013ers.)k(On)13 b(some)f(systems)g (this)h(ma)o(y)f(require)h(an)f(in)o(terrupt)h(to)e(the)i(in)o(tended)h (receiv)o(er.)19 b(Note)189 1328 y(that,)14 b(while)j(comm)o(unication) g(ma)o(y)e(b)q(e)h(needed)h(to)e(implemen)o(t)i Fv(MPI)p 1415 1328 V 16 w(CANCEL)p Fx(,)e(this)h(is)g(still)h(a)189 1385 y(lo)q(cal)g(op)q(eration,)f(since)i(its)e(completion)h(do)q(es)g (not)f(dep)q(end)i(on)e(the)g(co)q(de)h(executed)g(b)o(y)f(other)189 1441 y(pro)q(cesses.)21 b(If)15 b(pro)q(cessing)i(is)f(required)g(on)f (another)g(pro)q(cess,)h(this)g(should)g(b)q(e)g(transparen)o(t)f(to) 189 1498 y(the)h(application)j(\(hence)e(the)g(need)h(for)e(an)h(in)o (terrupt)g(and)g(an)f(in)o(terrupt)h(handler\).)26 b(\()p Fl(End)17 b(of)189 1554 y(advic)n(e)f(to)g(implementors.)p Fx(\))75 1697 y Fn(3.9)59 b(P)n(ersistent)19 b(communication)f (requests)75 1799 y Fx(Often)f(a)f(comm)o(unication)g(with)h(the)f (same)g(argumen)o(t)g(list)h(is)g(rep)q(eatedly)g(executed)g(within)h (the)e(in-)75 1855 y(ner)k(lo)q(op)h(of)f(a)g(parallel)i(computation.) 35 b(In)20 b(suc)o(h)h(a)f(situation,)h(it)g(ma)o(y)e(b)q(e)i(p)q (ossible)h(to)e(optimize)75 1912 y(the)c(comm)o(unication)g(b)o(y)g (binding)h(the)f(list)g(of)f(comm)o(unication)i(argumen)o(ts)d(to)h(a)h Fq(p)q(ersisten)o(t)f Fx(com-)75 1968 y(m)o(unication)k(request)e(once) h(and,)g(then,)g(rep)q(eatedly)h(using)f(the)g(request)g(to)e(initiate) j(and)f(complete)75 2025 y(messages.)g(The)12 b(p)q(ersisten)o(t)g (request)g(th)o(us)f(created)h(can)g(b)q(e)g(though)o(t)f(of)g(as)g(a)h (comm)o(unication)g(p)q(ort)f(or)75 2081 y(a)k(\\half-c)o(hannel.")21 b(It)15 b(do)q(es)h(not)e(pro)o(vide)i(the)f(full)i(functionalit)o(y)f (of)f(a)f(con)o(v)o(en)o(tional)i(c)o(hannel,)g(since)75 2138 y(there)i(is)h(no)f(binding)i(of)e(the)g(send)h(p)q(ort)f(to)f (the)i(receiv)o(e)g(p)q(ort.)28 b(This)19 b(construct)f(allo)o(ws)g (reduction)75 2194 y(of)c(the)g(o)o(v)o(erhead)g(for)g(comm)o (unication)h(b)q(et)o(w)o(een)g(the)f(pro)q(cess)h(and)f(comm)o (unication)h(con)o(troller,)g(but)75 2251 y(not)e(of)g(the)g(o)o(v)o (erhead)g(for)g(comm)o(unication)h(b)q(et)o(w)o(een)g(one)f(comm)o (unication)h(con)o(troller)g(and)g(another.)75 2307 y(It)19 b(is)h(not)e(necessary)i(that)e(messages)h(sen)o(t)f(with)i(a)f(p)q (ersisten)o(t)g(request)g(b)q(e)h(receiv)o(ed)g(b)o(y)f(a)g(receiv)o(e) 75 2363 y(op)q(eration)c(using)h(a)f(p)q(ersisten)o(t)h(request,)f(or)f (vice)j(v)o(ersa.)166 2420 y(A)j(p)q(ersisten)o(t)g(comm)o(unication)g (request)g(is)g(created)g(using)h(one)e(of)h(the)f(four)h(follo)o(wing) g(calls.)75 2476 y(These)c(calls)g(in)o(v)o(olv)o(e)g(no)f(comm)o (unication.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 53 61 53 60 bop 75 -100 a Fm(3.9.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (REQUESTS)661 b Fx(53)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(SEND)p 293 45 V 17 w(INIT\(buf,)14 b(count,)i(datat)o(yp)q(e,)h (dest,)f(tag,)f(comm)m(,)d(request\))117 122 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 197 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 273 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)g(\(handle\))117 348 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 423 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 498 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Fu(int)23 b(MPI)p 245 698 15 2 v 17 w(Send)p 358 698 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1091 698 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(SEND)p 263 840 V 16 w(INIT\(BUF,)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 897 y()g (BUF\(*\))170 953 y(INTEGER)g(REQUEST,)g(COUNT,)g(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 1040 y Fx(Creates)15 b(a)h(p)q(ersisten)o(t)h(comm)o(unication)g(request)f(for)g(a)f (standard)h(mo)q(de)h(send)f(op)q(eration,)h(and)75 1096 y(binds)f(to)f(it)g(all)i(the)e(argumen)o(ts)f(of)h(a)g(send)h(op)q (eration.)75 1247 y Fv(MPI)p 160 1247 14 2 v 16 w(BSEND)p 323 1247 V 17 w(INIT\(buf,)f(count,)h(datat)o(yp)q(e,)g(dest,)g(tag,)f (comm)m(,)d(request\))117 1324 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1400 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 1475 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)g(\(handle\))117 1550 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1625 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1700 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1775 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1900 y Fu(int)23 b(MPI)p 245 1900 15 2 v 17 w(Bsend)p 382 1900 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 1900 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 1956 y(int)g(tag,)f(MPI)p 683 1956 V 17 w(Comm)g(comm,)g(MPI)p 1034 1956 V 17 w(Request)g(*request\))75 2043 y(MPI)p 150 2043 V 17 w(BSEND)p 287 2043 V 16 w(INIT\(BUF,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2099 y()g(BUF\(*\))170 2156 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 2242 y Fx(Creates)14 b(a)h(p)q(ersisten)o(t)h(comm)o(unication)g (request)f(for)g(a)g(bu\013ered)g(mo)q(de)h(send.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 54 62 54 61 bop 75 -100 a Fx(54)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(SSEND)p 318 45 V 17 w(INIT\(buf,)e(count,)h(datat)o (yp)q(e,)g(dest,)g(tag,)f(comm)m(,)d(request\))117 122 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 197 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\)) 117 273 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 348 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 423 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 498 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Fu(int)23 b(MPI)p 245 698 15 2 v 17 w(Ssend)p 382 698 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 698 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(SSEND)p 287 840 V 16 w(INIT\(BUF,)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 897 y()g (BUF\(*\))170 953 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h (COMM,)f(REQUEST,)f(IERROR)166 1040 y Fx(Creates)14 b(a)h(p)q(ersisten) o(t)h(comm)o(unication)g(ob)s(ject)f(for)f(a)h(sync)o(hronous)g(mo)q (de)h(send)f(op)q(eration.)75 1191 y Fv(MPI)p 160 1191 14 2 v 16 w(RSEND)p 322 1191 V 17 w(INIT\(buf,)g(count,)h(datat)o(yp)q (e,)g(dest,)g(tag,)f(comm)m(,)d(request\))117 1268 y Fk(IN)155 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 1343 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\)) 117 1418 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 1493 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1569 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1644 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1719 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1843 y Fu(int)23 b(MPI)p 245 1843 15 2 v 17 w(Rsend)p 382 1843 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 1843 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 1900 y(int)g(tag,)f(MPI)p 683 1900 V 17 w(Comm)g(comm,)g(MPI)p 1034 1900 V 17 w(Request)g(*request\))75 1986 y(MPI)p 150 1986 V 17 w(RSEND)p 287 1986 V 16 w(INIT\(BUF,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2043 y()g(BUF\(*\))170 2099 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)166 2186 y Fx(Creates)14 b(a)h(p)q(ersisten)o(t)h(comm)o(unication)g(ob)s(ject)f (for)f(a)h(ready)g(mo)q(de)h(send)f(op)q(eration.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 55 63 55 62 bop 75 -100 a Fm(3.9.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (REQUESTS)661 b Fx(55)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(RECV)p 291 45 V 17 w(INIT\(buf,)15 b(count,)h(datat)o(yp)q(e,)g (source,)g(tag,)e(comm)m(,)e(request\))117 122 y Fk(OUT)108 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g (\(c)o(hoice\))117 197 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(receiv)o(ed)j(\(in)o(teger\))117 273 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 348 y(IN)155 b Fv(source)468 b Fk(rank)14 b(of)f(source)j(or)d(MPI)p 1309 348 13 2 v 16 w(ANY)p 1418 348 V 15 w(SOUR)o(CE)h(\(in)o(teger\)) 117 423 y(IN)155 b Fv(tag)526 b Fk(message)14 b(tag)f(or)h(MPI)p 1270 423 V 15 w(ANY)p 1378 423 V 16 w(T)m(A)o(G)f(\(in)o(teger\))117 498 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Fu(int)23 b(MPI)p 245 698 15 2 v 17 w(Recv)p 358 698 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1091 698 V 17 w(Datatype)g(datatype,)f(int)i(source,) 393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(RECV)p 263 840 V 16 w(INIT\(BUF,)g(COUNT,)g(DATATYPE,)g (SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 897 y()g(BUF\(*\))170 953 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 1040 y Fx(Creates)13 b(a)h(p)q(ersisten)o(t)h(comm)o(unication)f(request)h(for)e(a)h(receiv) o(e)h(op)q(eration.)20 b(The)14 b(argumen)o(t)f Fv(buf)75 1096 y Fx(is)f(mark)o(ed)g(as)f Fv(OUT)h Fx(b)q(ecause)h(the)f(user)g (giv)o(es)g(p)q(ermission)h(to)e(write)h(on)g(the)f(receiv)o(e)i (bu\013er)f(b)o(y)g(passing)75 1153 y(the)j(argumen)o(t)g(to)f Fv(MPI)p 497 1153 14 2 v 16 w(RECV)p 628 1153 V 17 w(INIT)p Fx(.)166 1209 y(A)i(p)q(ersisten)o(t)h(comm)o(unication)f(request)h(is) f(inactiv)o(e)h(after)f(it)g(w)o(as)f(created)h(|)h(no)f(activ)o(e)g (com-)75 1266 y(m)o(unication)g(is)g(attac)o(hed)e(to)h(the)g(request.) 166 1322 y(A)k(comm)o(unication)g(\(send)g(or)g(receiv)o(e\))g(that)f (uses)h(a)f(p)q(ersisten)o(t)i(request)e(is)i(initiated)g(b)o(y)f(the) 75 1379 y(function)d Fv(MPI)p 338 1379 V 16 w(ST)l(ART)p Fx(.)75 1530 y Fv(MPI)p 160 1530 V 16 w(ST)l(ART\(request\))117 1607 y Fk(INOUT)62 b Fv(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1731 y Fu(int)23 b(MPI)p 245 1731 15 2 v 17 w(Start\(MPI)p 478 1731 V 16 w(Request)g(*request\))75 1818 y(MPI)p 150 1818 V 17 w(START\(REQUEST,)f(IERROR\))170 1874 y(INTEGER)h(REQUEST,)g(IERROR)166 1961 y Fx(The)d(argumen)o(t,)h Fv(request)p Fx(,)h(is)f(a)f(handle)h(returned)g(b)o(y)f(one)g(of)g (the)h(previous)f(\014v)o(e)h(calls.)36 b(The)75 2017 y(asso)q(ciated)15 b(request)h(should)g(b)q(e)g(inactiv)o(e.)21 b(The)15 b(request)g(b)q(ecomes)h(activ)o(e)f(once)h(the)f(call)h(is)g (made.)166 2073 y(If)e(the)g(request)g(is)h(for)e(a)h(send)h(with)f (ready)g(mo)q(de,)g(then)g(a)g(matc)o(hing)g(receiv)o(e)h(should)g(b)q (e)g(p)q(osted)75 2130 y(b)q(efore)j(the)g(call)h(is)g(made.)28 b(The)18 b(comm)o(unication)h(bu\013er)e(should)i(not)f(b)q(e)g (accessed)h(after)e(the)h(call,)75 2186 y(and)d(un)o(til)i(the)e(op)q (eration)g(completes.)166 2243 y(The)22 b(call)i(is)e(lo)q(cal,)j(with) e(similar)g(seman)o(tics)f(to)g(the)g(non)o(blo)q(c)o(king)i(comm)o (unication)f(op)q(era-)75 2299 y(tions)d(describ)q(ed)i(in)e(section)h (3.7.)33 b(That)19 b(is,)i(a)f(call)h(to)e Fv(MPI)p 1174 2299 14 2 v 16 w(ST)l(ART)i Fx(with)f(a)f(request)h(created)g(b)o(y)75 2356 y Fv(MPI)p 160 2356 V 16 w(SEND)p 293 2356 V 17 w(INIT)13 b Fx(starts)f(a)h(comm)o(unication)i(in)g(the)e(same)h (manner)g(as)f(a)g(call)i(to)e Fv(MPI)p 1601 2356 V 16 w(ISEND)p Fx(;)g(a)h(call)75 2412 y(to)h Fv(MPI)p 216 2412 V 16 w(ST)l(ART)h Fx(with)g(a)f(request)g(created)h(b)o(y)f Fv(MPI)p 995 2412 V 16 w(BSEND)p 1158 2412 V 17 w(INIT)g Fx(starts)f(a)h(comm)o(unication)i(in)f(the)75 2469 y(same)f(manner)g (as)g(a)g(call)h(to)f Fv(MPI)p 671 2469 V 16 w(IBSEND)p Fx(;)g(and)g(so)g(on.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 56 64 56 63 bop 75 -100 a Fx(56)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(ST)l(ART)l(ALL\(count,)f(a)o(rra)o(y)p 641 45 V 15 w(of)p 693 45 V 16 w(requests\))117 122 y Fk(IN)155 b Fv(count)482 b Fk(list)14 b(length)f(\(in)o(teger\))117 197 y(INOUT)62 b Fv(a)o(rra)o(y)p 416 197 V 15 w(of)p 468 197 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o (y)e(of)f(handle\))75 322 y Fu(int)23 b(MPI)p 245 322 15 2 v 17 w(Startall\(int)f(count,)h(MPI)p 811 322 V 17 w(Request)g(*array)p 1163 322 V 17 w(of)p 1228 322 V 16 w(requests\))75 408 y(MPI)p 150 408 V 17 w(STARTALL\(COUNT,)f (ARRAY)p 669 408 V 16 w(OF)p 733 408 V 17 w(REQUESTS,)h(IERROR\))170 465 y(INTEGER)g(COUNT,)g(ARRAY)p 651 465 V 17 w(OF)p 716 465 V 17 w(REQUESTS\(*\),)f(IERROR)166 551 y Fx(Start)i(all)j(comm) o(unications)f(asso)q(ciated)g(with)f(requests)h(in)g Fv(a)o(rra)o(y)p 1399 551 14 2 v 15 w(of)p 1451 551 V 16 w(requests)p Fx(.)52 b(A)26 b(call)h(to)75 608 y Fv(MPI)p 160 608 V 16 w(ST)l(ART)l(ALL\(count,)18 b(a)o(rra)o(y)p 643 608 V 15 w(of)p 695 608 V 16 w(requests\))h Fx(has)d(the)h(same)g (e\013ect)f(as)h(calls)h(to)e Fv(MPI)p 1599 608 V 16 w(ST)l(ART)i(\(&a)o(r-)75 664 y(ra)o(y)p 136 664 V 15 w(of)p 188 664 V 16 w(requests[i]\))p Fx(,)f(executed)f(for)f Fv(i=0)g(,...,)e(count-1)p Fx(,)j(in)g(some)f(arbitrary)g(order.)166 721 y(A)i(comm)o(unication)g(started)f(with)h(a)g(call)h(to)e Fv(MPI)p 1063 721 V 16 w(ST)l(ART)h Fx(or)f Fv(MPI)p 1377 721 V 16 w(ST)l(ART)l(ALL)i Fx(is)f(completed)75 777 y(b)o(y)11 b(a)h(call)g(to)f Fv(MPI)p 385 777 V 16 w(W)l(AIT)p Fx(,)g Fv(MPI)p 620 777 V 16 w(TEST)p Fx(,)h(or)f(one)g(of) g(the)h(deriv)o(ed)h(functions)f(describ)q(ed)h(in)g(section)f(3.7.5.) 75 834 y(The)18 b(request)f(b)q(ecomes)h(inactiv)o(e)h(after)d (successful)j(completion)g(of)e(suc)o(h)h(call.)27 b(The)18 b(request)f(is)h(not)75 890 y(deallo)q(cated)f(and)e(it)g(can)h(b)q(e)g (activ)m(ated)f(anew)h(b)o(y)f(an)g Fv(MPI)p 1116 890 V 16 w(ST)l(ART)h Fx(or)e Fv(MPI)p 1427 890 V 16 w(ST)l(ART)l(ALL)i Fx(call.)166 946 y(A)f(p)q(ersisten)o(t)h(request)f(is)h(deallo)q (cated)g(b)o(y)g(a)e(call)j(to)d Fv(MPI)p 1190 946 V 16 w(REQUEST)p 1409 946 V 18 w(FREE)i Fx(\(Section)f(3.7.3\).)166 1003 y(The)g(call)g(to)f Fv(MPI)p 481 1003 V 16 w(REQUEST)p 700 1003 V 17 w(FREE)h Fx(can)g(o)q(ccur)g(at)e(an)o(y)h(p)q(oin)o(t)h (in)h(the)e(program)f(after)h(the)g(p)q(er-)75 1059 y(sisten)o(t)h (request)g(w)o(as)f(created.)20 b(Ho)o(w)o(ev)o(er,)14 b(the)h(request)g(will)h(b)q(e)g(deallo)q(cated)g(only)g(after)e(it)h (b)q(ecomes)75 1116 y(inactiv)o(e.)33 b(Activ)o(e)20 b(receiv)o(e)g(requests)f(should)i(not)e(b)q(e)h(freed.)32 b(Otherwise,)21 b(it)e(will)i(not)e(b)q(e)h(p)q(ossible)75 1172 y(to)c(c)o(hec)o(k)g(that)f(the)i(receiv)o(e)g(has)f(completed.)23 b(It)17 b(is)f(preferable,)h(in)g(general,)g(to)e(free)h(requests)g (when)75 1229 y(they)h(are)g(inactiv)o(e.)27 b(If)17 b(this)g(rule)h(is)g(follo)o(w)o(ed,)g(then)f(the)g(functions)h (describ)q(ed)h(in)f(this)g(section)f(will)75 1285 y(b)q(e)f(in)o(v)o (ok)o(ed)f(in)h(a)f(sequence)i(of)d(the)i(form,)166 1401 y Fq(Create)h Fx(\()p Fq(Start)g(Complete)p Fx(\))724 1384 y Fh(\003)760 1401 y Fq(F)l(ree)f Fp(;)e Fx(where)h Ft(\003)f Fx(indicates)i(zero)f(or)f(more)g(rep)q(etitions.)21 b(If)15 b(the)75 1516 y(same)g(comm)o(unication)i(ob)s(ject)e(is)h (used)g(in)h(sev)o(eral)f(concurren)o(t)f(threads,)h(it)g(is)g(the)f (user's)h(resp)q(onsi-)75 1573 y(bilit)o(y)h(to)d(co)q(ordinate)i (calls)g(so)f(that)f(the)i(correct)e(sequence)j(is)f(ob)q(ey)o(ed.)166 1629 y(A)c(send)g(op)q(eration)f(initiated)j(with)d Fv(MPI)p 874 1629 V 16 w(ST)l(ART)i Fx(can)e(b)q(e)i(matc)o(hed)e(with)h(an)o(y) f(receiv)o(e)i(op)q(eration)75 1686 y(and,)g(lik)o(ewise,)h(a)e(receiv) o(e)i(op)q(eration)e(initiated)i(with)f Fv(MPI)p 1098 1686 V 16 w(ST)l(ART)g Fx(can)g(receiv)o(e)g(messages)f(generated)75 1742 y(b)o(y)j(an)o(y)g(send)h(op)q(eration.)75 1885 y Fn(3.10)59 b(Send-receive)75 1987 y Fx(The)18 b Fq(send-receiv)o(e)g Fx(op)q(erations)g(com)o(bine)h(in)g(one)f(call)h(the)f(sending)i(of)d (a)h(message)g(to)f(one)h(desti-)75 2043 y(nation)g(and)f(the)h (receiving)h(of)e(another)g(message,)h(from)e(another)h(pro)q(cess.)28 b(The)17 b(t)o(w)o(o)g(\(source)g(and)75 2100 y(destination\))i(are)f (p)q(ossibly)j(the)d(same.)30 b(A)19 b(send-receiv)o(e)h(op)q(eration)f (is)g(v)o(ery)f(useful)i(for)e(executing)75 2156 y(a)g(shift)i(op)q (eration)f(across)f(a)g(c)o(hain)i(of)e(pro)q(cesses.)31 b(If)19 b(blo)q(c)o(king)h(sends)f(and)g(receiv)o(es)h(are)e(used)i (for)75 2213 y(suc)o(h)f(a)f(shift,)h(then)g(one)f(needs)h(to)f(order)g (the)g(sends)h(and)g(receiv)o(es)g(correctly)g(\(for)e(example,)j(ev)o (en)75 2269 y(pro)q(cesses)d(send,)f(then)h(receiv)o(e,)g(o)q(dd)f(pro) q(cesses)h(receiv)o(e)g(\014rst,)f(then)g(send\))h(so)f(as)g(to)f(prev) o(en)o(t)h(cyclic)75 2325 y(dep)q(endencies)j(that)c(ma)o(y)g(lead)h (to)f(deadlo)q(c)o(k.)23 b(When)16 b(a)f(send-receiv)o(e)j(op)q (eration)e(is)g(used,)g(the)g(com-)75 2382 y(m)o(unication)j(subsystem) e(tak)o(es)g(care)h(of)f(these)h(issues.)29 b(The)18 b(send-receiv)o(e)h(op)q(eration)f(can)g(b)q(e)g(used)75 2438 y(in)e(conjunction)f(with)h(the)e(functions)i(describ)q(ed)h(in)e (Chapter)g(6)f(in)i(order)e(to)h(p)q(erform)f(shifts)h(on)g(v)m(ar-)75 2495 y(ious)20 b(logical)h(top)q(ologies.)33 b(Also,)21 b(a)e(send-receiv)o(e)j(op)q(eration)d(is)h(useful)h(for)e(implemen)o (ting)j(remote)75 2551 y(pro)q(cedure)16 b(calls.)166 2608 y(A)g(message)f(sen)o(t)g(b)o(y)h(a)f(send-receiv)o(e)j(op)q (eration)d(can)h(b)q(e)h(receiv)o(ed)f(b)o(y)g(a)f(regular)h(receiv)o (e)h(op)q(er-)75 2664 y(ation)f(or)f(prob)q(ed)i(b)o(y)f(a)g(prob)q(e)g (op)q(eration;)g(a)g(send-receiv)o(e)i(op)q(eration)e(can)g(receiv)o(e) h(a)f(message)f(sen)o(t)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 57 65 57 64 bop 75 -100 a Fm(3.10.)34 b(SEND-RECEIVE)1277 b Fx(57)75 45 y(b)o(y)15 b(a)g(regular)g(send)h(op)q(eration.)75 196 y Fv(MPI)p 160 196 14 2 v 16 w(SENDRECV\(sendbuf,)24 b(sendcount,)f(sendt)o(yp)q(e,)g(dest,)f(sendtag,)g(recvbuf,)g (recvcount,)f(recvt)o(yp)q(e,)75 252 y(source,)15 b(recvtag,)g(comm)m (,)d(status\))117 330 y Fk(IN)155 b Fv(sendbuf)444 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 405 y(IN)155 b Fv(sendcount)399 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 480 y(IN)155 b Fv(sendt)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(bu\013er)g(\(handle\))117 555 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 630 y(IN)155 b Fv(sendtag)443 b Fk(send)15 b(tag)e(\(in)o(teger\))117 705 y(OUT)108 b Fv(recvbuf)450 b Fk(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\)) 117 781 y(IN)155 b Fv(recvcount)405 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 856 y(IN)155 b Fv(recvt)o(yp)q(e)430 b Fk(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(receiv)o(e)i(bu\013er)g(\(handle\))117 931 y(IN)155 b Fv(source)468 b Fk(rank)14 b(of)f(source)j(\(in)o (teger\))117 1006 y(IN)155 b Fv(recvtag)449 b Fk(receiv)o(e)15 b(tag)f(\(in)o(teger\))117 1081 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1156 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1281 y Fu(int)23 b(MPI)p 245 1281 15 2 v 17 w(Sendrecv\(void)f (*sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1281 V 17 w(Datatype)f(sendtype,)393 1337 y(int)i(dest,)f(int)g(sendtag,)g(void)g (*recvbuf,)g(int)g(recvcount,)393 1394 y(MPI)p 468 1394 V 17 w(Datatype)g(recvtype,)f(int)i(source,)f(MPI)p 1297 1394 V 16 w(Datatype)g(recvtag,)393 1450 y(MPI)p 468 1450 V 17 w(Comm)g(comm,)g(MPI)p 819 1450 V 17 w(Status)g(*status\))75 1536 y(MPI)p 150 1536 V 17 w(SENDRECV\(SENDBUF,)e(SENDCOUNT,)i (SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVBUF,)393 1593 y(RECVCOUNT,)h (RECVTYPE,)f(SOURCE,)h(RECVTAG,)g(COMM,)g(STATUS,)g(IERROR\))170 1649 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1706 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVCOUNT,)h (RECVTYPE,)170 1762 y(SOURCE,)g(RECVTAG,)g(COMM,)g(STATUS\(MPI)p 962 1762 V 16 w(STATUS)p 1122 1762 V 16 w(SIZE\),)h(IERROR)166 1849 y Fx(Execute)19 b(a)e(blo)q(c)o(king)j(send)f(and)f(receiv)o(e)h (op)q(eration.)29 b(Both)18 b(send)h(and)f(receiv)o(e)h(use)g(the)f (same)75 1905 y(comm)o(unicator,)i(but)g(p)q(ossibly)i(di\013eren)o(t)e (tags.)33 b(The)20 b(send)g(bu\013er)g(and)g(receiv)o(e)h(bu\013ers)f (m)o(ust)f(b)q(e)75 1962 y(disjoin)o(t,)c(and)h(ma)o(y)e(ha)o(v)o(e)h (di\013eren)o(t)h(lengths)f(and)h(datat)o(yp)q(es.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 58 66 58 65 bop 75 -100 a Fx(58)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(SENDRECV)p 408 45 V 17 w(REPLA)o(CE\(buf,)i(count,)f (datat)o(yp)q(e,)h(dest,)f(sendtag,)h(source,)f(recvtag,)f(comm)m(,)d (sta-)75 102 y(tus\))117 179 y Fk(INOUT)62 b Fv(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(and)g(receiv)o(e)h(bu\013er)g (\(c)o(hoice\))117 254 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)9 b(of)g(elemen)o(ts)h(in)f(send)i(and)e(receiv)o(e)i(bu\013er)g(\(in)o (teger\))117 329 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(and)f(receiv)o(e)h(bu\013er)g (\(handle\))117 404 y(IN)155 b Fv(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 479 y(IN)155 b Fv(sendtag)443 b Fk(send)15 b(message)f(tag)f(\(in)o(teger\))117 554 y(IN)155 b Fv(source)468 b Fk(rank)14 b(of)f(source)j(\(in)o (teger\))117 630 y(IN)155 b Fv(recvtag)449 b Fk(receiv)o(e)15 b(message)f(tag)f(\(in)o(teger\))117 705 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 780 y(OUT)108 b Fv(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 904 y Fu(int)23 b(MPI)p 245 904 15 2 v 17 w(Sendrecv)p 454 904 V 16 w(replace\(void*)f(buf,)i(int)f(count,)g(MPI)p 1258 904 V 17 w(Datatype)g(datatype,)393 961 y(int)h(dest,)f(int)g (sendtag,)g(int)g(source,)g(int)h(recvtag,)f(MPI)p 1614 961 V 16 w(Comm)h(comm,)393 1017 y(MPI)p 468 1017 V 17 w(Status)f(*status\))75 1104 y(MPI)p 150 1104 V 17 w(SENDRECV)p 359 1104 V 16 w(REPLACE\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(SENDTAG,)g (SOURCE,)g(RECVTAG,)393 1160 y(COMM,)g(STATUS,)g(IERROR\))170 1217 y()g(BUF\(*\))170 1273 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)g(COMM,)170 1329 y(STATUS\(MPI)p 413 1329 V 16 w(STATUS)p 573 1329 V 17 w(SIZE\),)g(IERROR)166 1416 y Fx(Execute)18 b(a)f(blo)q(c)o(king)i (send)f(and)g(receiv)o(e.)27 b(The)18 b(same)f(bu\013er)h(is)g(used)g (b)q(oth)f(for)g(the)h(send)g(and)75 1472 y(for)d(the)g(receiv)o(e,)h (so)e(that)h(the)g(message)g(sen)o(t)g(is)h(replaced)g(b)o(y)f(the)g (message)g(receiv)o(ed.)166 1529 y(The)d(seman)o(tics)h(of)f(a)g (send-receiv)o(e)i(op)q(eration)e(is)h(what)e(w)o(ould)i(b)q(e)g (obtained)g(if)g(the)f(caller)h(fork)o(ed)75 1585 y(t)o(w)o(o)j (concurren)o(t)h(threads,)g(one)g(to)f(execute)i(the)f(send,)g(and)h (one)f(to)f(execute)i(the)f(receiv)o(e,)h(follo)o(w)o(ed)75 1642 y(b)o(y)d(a)g(join)h(of)e(these)i(t)o(w)o(o)e(threads.)189 1748 y Fl(A)n(dvic)n(e)d(to)h(implementors.)37 b Fx(Additional)12 b(in)o(termediate)f(bu\013ering)g(is)g(needed)h(for)e(the)h(\\replace") 189 1804 y(v)m(arian)o(t.)19 b(\()p Fl(End)d(of)g(advic)n(e)h(to)f (implementors.)p Fx(\))75 1948 y Fn(3.11)59 b(Null)21 b(p)n(ro)r(cesses)75 2049 y Fx(In)15 b(man)o(y)e(instances,)i(it)f(is)h (con)o(v)o(enien)o(t)g(to)e(sp)q(ecify)j(a)d(\\dumm)o(y")h(source)g(or) g(destination)h(for)e(comm)o(u-)75 2106 y(nication.)23 b(This)16 b(simpli\014es)i(the)e(co)q(de)g(that)g(is)g(needed)h(for)e (dealing)i(with)f(b)q(oundaries,)h(for)e(example,)75 2162 y(in)h(the)f(case)h(of)e(a)h(non-circular)i(shift)e(done)h(with)g (calls)g(to)e(send-receiv)o(e.)166 2219 y(The)f(sp)q(ecial)h(v)m(alue) 25 b Fi(MPI)p 605 2219 13 2 v 14 w(PROC)p 731 2219 V 13 w(NULL)14 b Fx(can)e(b)q(e)h(used)g(instead)g(of)f(a)g(rank)g (wherev)o(er)h(a)f(source)g(or)g(a)75 2275 y(destination)j(argumen)o(t) e(is)h(required)i(in)e(a)g(call.)21 b(A)14 b(comm)o(unication)g(with)h (pro)q(cess)26 b Fi(MPI)p 1634 2275 V 15 w(PROC)p 1761 2275 V 13 w(NULL)75 2331 y Fx(has)11 b(no)h(e\013ect.)18 b(A)12 b(send)g(to)21 b Fi(MPI)p 633 2331 V 15 w(PROC)p 760 2331 V 13 w(NULL)13 b Fx(succeeds)f(and)g(returns)f(as)g(so)q(on)h (as)f(p)q(ossible.)20 b(A)12 b(receiv)o(e)75 2388 y(from)24 b Fi(MPI)p 269 2388 V 14 w(PROC)p 395 2388 V 13 w(NULL)14 b Fx(succeeds)g(and)f(returns)f(as)h(so)q(on)f(as)h(p)q(ossible)h(with) f(no)g(mo)q(di\014cations)h(to)e(the)75 2444 y(receiv)o(e)17 b(bu\013er.)22 b(When)17 b(a)e(receiv)o(e)i(with)g Fv(source)f Fx(=)31 b Fi(MPI)p 1080 2444 V 15 w(PROC)p 1207 2444 V 13 w(NULL)17 b Fx(is)f(executed)h(then)g(the)f(status)75 2501 y(ob)s(ject)f(returns)30 b Fv(source)16 b Fx(=)29 b Fi(MPI)p 661 2501 V 14 w(PROC)p 787 2501 V 14 w(NULL)p Fx(,)15 b Fv(tag)g Fx(=)30 b Fi(MPI)p 1147 2501 V 14 w(ANY)p 1246 2501 V 15 w(T)m(A)o(G)14 b Fx(and)h Fv(count)i(=)e(0)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 59 67 59 66 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fx(59)75 45 y Fn(3.12)59 b(Derived)19 b(datat)n(yp)r(es)75 147 y Fx(Up)13 b(to)f(here,)h(all)h(p)q(oin)o(t)f(to)f(p)q(oin)o(t)i (comm)o(unication)f(ha)o(v)o(e)f(in)o(v)o(olv)o(ed)i(only)f(con)o (tiguous)g(bu\013ers)g(con)o(tain-)75 203 y(ing)h(a)f(sequence)i(of)e (elemen)o(ts)i(of)e(the)g(same)h(t)o(yp)q(e.)19 b(This)14 b(is)g(to)q(o)f(constraining)h(on)g(t)o(w)o(o)e(accoun)o(ts.)19 b(One)75 259 y(often)d(w)o(an)o(ts)f(to)g(pass)h(messages)f(that)h(con) o(tain)g(v)m(alues)h(with)f(di\013eren)o(t)h(datat)o(yp)q(es)e(\(e.g.,) g(an)h(in)o(teger)75 316 y(coun)o(t,)e(follo)o(w)o(ed)g(b)o(y)g(a)g (sequence)h(of)e(real)i(n)o(um)o(b)q(ers\);)f(and)g(one)g(often)g(w)o (an)o(ts)f(to)g(send)i(noncon)o(tiguous)75 372 y(data)k(\(e.g.,)g(a)g (sub-blo)q(c)o(k)i(of)e(a)h(matrix\).)32 b(One)20 b(solution)h(is)f(to) f(pac)o(k)g(noncon)o(tiguous)h(data)f(in)o(to)g(a)75 429 y(con)o(tiguous)e(bu\013er)h(at)e(the)i(sender)g(site)f(and)h (unpac)o(k)f(it)h(bac)o(k)f(at)g(the)g(receiv)o(er)h(site.)27 b(This)18 b(has)f(the)75 485 y(disadv)m(an)o(tage)12 b(of)g(requiring)h(additional)h(memory-to-memory)d(cop)o(y)h(op)q (erations)g(at)g(b)q(oth)g(sites,)h(ev)o(en)75 542 y(when)j(the)g(comm) o(unication)g(subsystem)g(has)f(scatter-gather)g(capabilities.)24 b(Instead,)15 b Fv(MPI)g Fx(pro)o(vides)75 598 y(mec)o(hanisms)h(to)f (sp)q(ecify)i(more)f(general,)g(mixed,)g(and)g(noncon)o(tiguous)g(comm) o(unication)h(bu\013ers.)k(It)75 655 y(is)16 b(up)g(to)f(the)h (implemen)o(tation)h(to)e(decide)i(whether)f(data)f(should)h(b)q(e)g (\014rst)g(pac)o(k)o(ed)f(in)i(a)e(con)o(tiguous)75 711 y(bu\013er)c(b)q(efore)h(b)q(eing)h(transmitted,)e(or)g(whether)h(it)g (can)f(b)q(e)h(collected)i(directly)e(from)f(where)h(it)f(resides.)166 768 y(The)j(general)h(mec)o(hanisms)g(pro)o(vided)g(here)f(allo)o(w)h (one)f(to)g(transfer)f(directly)l(,)j(without)e(cop)o(ying,)75 824 y(ob)s(jects)j(of)h(v)m(arious)h(shap)q(e)f(and)h(size.)29 b(It)18 b(is)h(not)f(assumed)g(that)f(the)h Fv(MPI)g Fx(library)h(is)f(cognizan)o(t)g(of)75 880 y(the)d(ob)s(jects)f (declared)i(in)g(the)f(host)f(language.)20 b(Th)o(us,)15 b(if)g(one)g(w)o(an)o(ts)f(to)g(transfer)g(a)g(structure,)h(or)f(an)75 937 y(arra)o(y)g(section,)h(it)g(will)h(b)q(e)f(necessary)g(to)g(pro)o (vide)g(in)g Fv(MPI)g Fx(a)f(de\014nition)j(of)d(a)g(comm)o(unication)i (bu\013er)75 993 y(that)d(mimics)j(the)e(de\014nition)i(of)e(the)g (structure)g(or)f(arra)o(y)g(section)i(in)g(question.)20 b(These)15 b(facilities)h(can)75 1050 y(b)q(e)j(used)g(b)o(y)f(library) h(designers)h(to)d(de\014ne)j(comm)o(unication)f(functions)g(that)f (can)g(transfer)g(ob)s(jects)75 1106 y(de\014ned)e(in)g(the)f(host)f (language)h(|)g(b)o(y)g(deco)q(ding)h(their)f(de\014nitions)i(as)d(a)o (v)m(ailable)i(in)g(a)e(sym)o(b)q(ol)i(table)75 1163 y(or)f(a)g(dop)q(e)g(v)o(ector.)20 b(Suc)o(h)15 b(higher-lev)o(el)j (comm)o(unication)e(functions)g(are)f(not)g(part)f(of)h Fv(MPI)p Fx(.)166 1219 y(More)d(general)g(comm)o(unication)i(bu\013ers) e(are)g(sp)q(eci\014ed)i(b)o(y)f(replacing)g(the)g(basic)g(datat)o(yp)q (es)f(that)75 1276 y(ha)o(v)o(e)e(b)q(een)i(used)f(so)f(far)g(with)h (deriv)o(ed)g(datat)o(yp)q(es)f(that)g(are)g(constructed)g(from)g (basic)h(datat)o(yp)q(es)f(using)75 1332 y(the)j(constructors)e (describ)q(ed)j(in)g(this)f(section.)19 b(These)13 b(metho)q(ds)f(of)g (constructing)h(deriv)o(ed)h(datat)o(yp)q(es)75 1389 y(can)h(b)q(e)h(applied)h(recursiv)o(ely)l(.)166 1445 y(A)e Fq(general)j(datat)o(yp)q(e)e Fx(is)g(an)f(opaque)h(ob)s(ject)e (that)h(sp)q(eci\014es)i(t)o(w)o(o)d(things:)143 1536 y Ft(\017)23 b Fx(A)15 b(sequence)h(of)f(basic)h(datat)o(yp)q(es)143 1628 y Ft(\017)23 b Fx(A)15 b(sequence)h(of)f(in)o(teger)g(\(b)o(yte\)) g(displacemen)o(ts)166 1719 y(The)22 b(displacemen)o(ts)g(are)g(not)f (required)h(to)f(b)q(e)h(p)q(ositiv)o(e,)i(distinct,)g(or)d(in)h (increasing)h(order.)75 1776 y(Therefore,)15 b(the)g(order)h(of)e (items)i(need)g(not)f(coincide)j(with)d(their)h(order)g(in)g(store,)e (and)i(an)f(item)h(ma)o(y)75 1832 y(app)q(ear)j(more)g(than)g(once.)33 b(W)l(e)19 b(call)i(suc)o(h)e(a)g(pair)h(of)f(sequences)h(\(or)f (sequence)h(of)f(pairs\))g(a)g Fq(t)o(yp)q(e)75 1889 y(map)p Fx(.)24 b(The)17 b(sequence)h(of)f(basic)g(datat)o(yp)q(es)f (\(displacemen)o(ts)i(ignored\))f(is)g(the)g Fq(t)o(yp)q(e)i(signature) e Fx(of)75 1945 y(the)e(datat)o(yp)q(e.)166 2002 y(Let)189 2092 y Fp(T)6 b(y)r(pemap)12 b Fx(=)h Ft(f)p Fx(\()p Fp(ty)r(pe)562 2099 y Fg(0)581 2092 y Fp(;)8 b(disp)686 2099 y Fg(0)705 2092 y Fx(\))p Fp(;)g(:::;)g Fx(\()o Fp(ty)r(pe)905 2099 y Ff(n)p Fh(\000)p Fg(1)971 2092 y Fp(;)g(disp)1076 2099 y Ff(n)p Fh(\000)p Fg(1)1144 2092 y Fx(\))p Ft(g)p Fp(;)75 2183 y Fx(b)q(e)16 b(suc)o(h)f(a)g(t)o (yp)q(e)h(map,)e(where)i Fp(ty)r(pe)709 2190 y Ff(i)738 2183 y Fx(are)f(basic)h(t)o(yp)q(es,)f(and)g Fp(disp)1233 2190 y Ff(i)1262 2183 y Fx(are)g(displacemen)o(ts.)21 b(Let)189 2274 y Fp(T)6 b(y)r(pesig)13 b Fx(=)g Ft(f)p Fp(ty)r(pe)517 2281 y Fg(0)537 2274 y Fp(;)8 b(:::;)g(ty)r(pe)702 2281 y Ff(n)p Fh(\000)p Fg(1)768 2274 y Ft(g)75 2365 y Fx(b)q(e)14 b(the)f(asso)q(ciated)h(t)o(yp)q(e)f(signature.)19 b(This)14 b(t)o(yp)q(e)f(map,)g(together)g(with)g(a)g(base)g(address)h Fl(buf)p Fx(,)f(sp)q(eci\014es)75 2421 y(a)19 b(comm)o(unication)i (bu\013er:)28 b(the)20 b(comm)o(unication)h(bu\013er)e(that)g(consists) h(of)f Fp(n)h Fx(en)o(tries,)h(where)f(the)75 2478 y Fp(i)p Fx(-th)c(en)o(try)f(is)i(at)e(address)h Fp(buf)f Fx(+)c Fp(disp)763 2485 y Ff(i)793 2478 y Fx(and)16 b(has)g(t)o(yp)q(e) g Fp(ty)r(pe)1151 2485 y Ff(i)1165 2478 y Fx(.)23 b(A)16 b(message)f(assem)o(bled)i(from)e(suc)o(h)h(a)75 2534 y(comm)o(unication)g(bu\013er)f(will)i(consist)e(of)g Fp(n)h Fx(v)m(alues,)g(of)e(the)i(t)o(yp)q(es)f(de\014ned)h(b)o(y)g Fp(T)6 b(y)r(pesig)r Fx(.)166 2591 y(W)l(e)11 b(can)f(use)h(a)f(handle) i(to)e(a)g(general)h(datat)o(yp)q(e)f(as)h(an)f(argumen)o(t)g(in)h(a)f (send)i(or)e(receiv)o(e)h(op)q(eration,)75 2647 y(instead)h(of)e(a)h (basic)g(datat)o(yp)q(e)g(argumen)o(t.)18 b(The)11 b(op)q(eration)g Fv(MPI)p 1198 2647 14 2 v 16 w(SEND\(buf,)h(1,)f(datat)o(yp)q(e,...\)) 19 b Fx(will)13 b(use)75 2704 y(the)18 b(send)g(bu\013er)f(de\014ned)i (b)o(y)e(the)h(base)f(address)h Fv(buf)g Fx(and)g(the)f(general)h (datat)o(yp)q(e)f(asso)q(ciated)h(with)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 60 68 60 67 bop 75 -100 a Fx(60)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(datat)o(yp)q(e)p Fx(;)23 b(it)c(will)i(generate)f(a)f(message)g(with)g(the)h(t)o(yp)q(e) f(signature)h(determined)h(b)o(y)e(the)g Fv(datat)o(yp)q(e)75 102 y Fx(argumen)o(t.)28 b Fv(MPI)p 390 102 14 2 v 16 w(RECV\(buf,)20 b(1,)e(datat)o(yp)q(e,...\))29 b Fx(will)20 b(use)e(the)h(receiv)o(e)g(bu\013er)f(de\014ned)h(b)o(y)g(the)f(base)75 158 y(address)d Fv(buf)h Fx(and)g(the)f(general)h(datat)o(yp)q(e)e (asso)q(ciated)i(with)f Fv(datat)o(yp)q(e)p Fx(.)166 214 y(General)h(datat)o(yp)q(es)f(can)h(b)q(e)h(used)f(in)h(all)g(send) f(and)h(receiv)o(e)f(op)q(erations.)22 b(W)l(e)16 b(discuss,)h(in)g (Sec.)75 271 y(3.12.5,)c(the)i(case)g(where)h(the)f(second)h(argumen)o (t)e Fv(count)j Fx(has)e(v)m(alue)h Fp(>)d Fx(1.)166 327 y(The)e(basic)g(datat)o(yp)q(es)f(presen)o(ted)i(in)f(section)g (3.2.2)f(are)g(particular)h(cases)g(of)f(a)h(general)g(datat)o(yp)q(e,) 75 384 y(and)19 b(are)f(prede\014ned.)32 b(Th)o(us,)19 b Fv(MPI)p 711 384 V 16 w(INT)f Fx(is)h(a)g(prede\014ned)h(handle)g(to) e(a)g(datat)o(yp)q(e)g(with)h(t)o(yp)q(e)g(map)75 440 y Ft(f)p Fx(\()p Fv(int)p Fp(;)8 b Fx(0\))p Ft(g)p Fx(,)13 b(with)j(one)g(en)o(try)f(of)g(t)o(yp)q(e)g Fv(int)h Fx(and)g(displacemen)o(t)h(zero.)j(The)15 b(other)g(basic)i(datat)o(yp) q(es)d(are)75 497 y(similar.)166 553 y(The)h Fq(exten)o(t)f Fx(of)g(a)g(datat)o(yp)q(e)g(is)h(de\014ned)g(to)f(b)q(e)h(the)g(span)f (from)g(the)g(\014rst)g(b)o(yte)h(to)e(the)i(last)f(b)o(yte)75 610 y(o)q(ccupied)19 b(b)o(y)d(en)o(tries)h(in)h(this)f(datat)o(yp)q (e,)f(rounded)h(up)g(to)f(satisfy)h(alignmen)o(t)g(requiremen)o(ts.)25 b(That)75 666 y(is,)15 b(if)189 752 y Fp(T)6 b(y)r(pemap)12 b Fx(=)h Ft(f)p Fx(\()p Fp(ty)r(pe)562 759 y Fg(0)581 752 y Fp(;)8 b(disp)686 759 y Fg(0)705 752 y Fx(\))p Fp(;)g(:::;)g Fx(\()o Fp(ty)r(pe)905 759 y Ff(n)p Fh(\000)p Fg(1)971 752 y Fp(;)g(disp)1076 759 y Ff(n)p Fh(\000)p Fg(1)1144 752 y Fx(\))p Ft(g)p Fp(;)75 837 y Fx(then)283 922 y Fp(l)q(b)p Fx(\()p Fp(T)e(y)r(pemap)p Fx(\))40 b(=)h(min)688 951 y Ff(j)742 922 y Fp(disp)826 929 y Ff(j)844 922 y Fp(;)272 1013 y(ub)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))40 b(=)h(max)693 1041 y Ff(j)743 1013 y Fx(\()p Fp(disp)845 1020 y Ff(j)873 1013 y Fx(+)10 b Fp(siz)r(eof)5 b Fx(\()p Fp(ty)r(pe)1150 1020 y Ff(j)1169 1013 y Fx(\)\))p Fp(;)21 b Fx(and)189 1103 y Fp(extent)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))42 b(=)f Fp(ub)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))j Ft(\000)h Fp(l)q(b)p Fx(\()p Fp(T)c(y)r(pemap)p Fx(\))j(+)h Fp(\017:)456 b Fx(\(3.1\))75 1188 y(If)17 b Fp(ty)r(pe)206 1195 y Ff(i)237 1188 y Fx(requires)g(alignmen)o(t)g (to)f(a)h(b)o(yte)f(address)h(that)f(is)h(is)g(a)f(m)o(ultiple)j(of)d Fp(k)1474 1195 y Ff(i)1488 1188 y Fx(,)g(then)h Fp(\017)g Fx(is)g(the)g(least)75 1245 y(nonnegativ)o(e)f(incremen)o(t)g(needed)g (to)f(round)g Fp(extent)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))16 b(to)e(the)h(next)h(m)o(ultiple)h(of)d(max)1794 1252 y Ff(i)1816 1245 y Fp(k)1840 1252 y Ff(i)1854 1245 y Fx(.)75 1339 y Fq(Example)k(3.18)23 b Fx(Assume)12 b(that)g Fp(T)6 b(y)r(pe)13 b Fx(=)g Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Ft(g)j Fx(\(a)h Fv(double)i Fx(at)e(displacemen)o(t)i(zero,)75 1396 y(follo)o(w)o(ed)k(b)o(y)g(a)f Fv(cha)o(r)g Fx(at)g(displacemen)o (t)j(eigh)o(t\).)27 b(Assume,)18 b(furthermore,)f(that)g(doubles)i(ha)o (v)o(e)e(to)g(b)q(e)75 1452 y(strictly)d(aligned)g(at)e(addresses)i (that)e(are)h(m)o(ultiples)h(of)f(eigh)o(t.)19 b(Then,)14 b(the)f(exten)o(t)g(of)f(this)i(datat)o(yp)q(e)e(is)75 1508 y(16)e(\(9)g(rounded)h(to)f(the)h(next)g(m)o(ultiple)h(of)f(8\).) 17 b(A)11 b(datat)o(yp)q(e)f(that)g(consists)h(of)f(a)g(c)o(haracter)g (immediately)75 1565 y(follo)o(w)o(ed)16 b(b)o(y)f(a)g(double)h(will)h (also)e(ha)o(v)o(e)g(an)g(exten)o(t)g(of)g(16.)189 1659 y Fl(R)n(ationale.)39 b Fx(The)15 b(de\014nition)h(of)e(exten)o(t)g(is) h(motiv)m(ated)f(b)o(y)h(the)f(assumption)h(that)e(the)i(amoun)o(t)189 1716 y(of)i(padding)i(added)g(at)e(the)h(end)g(of)g(eac)o(h)g (structure)f(in)i(an)f(arra)o(y)e(of)i(structures)f(is)i(the)f(least) 189 1772 y(needed)d(to)e(ful\014ll)j(alignmen)o(t)e(constrain)o(ts.)19 b(More)14 b(explicit)i(con)o(trol)d(of)g(the)h(exten)o(t)g(is)g(pro)o (vided)189 1829 y(in)i(section)g(3.12.3.)k(Suc)o(h)c(explicit)i(con)o (trol)e(is)g(needed)h(in)f(cases)g(where)g(the)g(assumption)g(do)q(es) 189 1885 y(not)e(hold,)i(for)f(example,)g(where)h(union)g(t)o(yp)q(es)f (are)g(used.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))75 2005 y Fj(3.12.1)49 b(Datat)o(yp)q(e)17 b(constructo)o(rs)75 2091 y Fv(Contiguous)47 b Fx(The)16 b(simplest)i(datat)o(yp)q(e)e (constructor)f(is)i Fv(MPI)p 1171 2091 V 16 w(TYPE)p 1304 2091 V 17 w(CONTIGUOUS)h Fx(whic)o(h)f(allo)o(ws)75 2148 y(replication)g(of)e(a)g(datat)o(yp)q(e)f(in)o(to)h(con)o(tiguous) h(lo)q(cations.)75 2299 y Fv(MPI)p 160 2299 V 16 w(TYPE)p 293 2299 V 17 w(CONTIGUOUS\(count,)h(oldt)o(yp)q(e,)f(newt)o(yp)q(e\)) 117 2376 y Fk(IN)155 b Fv(count)482 b Fk(replication)14 b(coun)o(t)g(\(nonnegativ)o(e)g(in)o(teger\))117 2449 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i (\(handle\))117 2523 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 2647 y Fu(int)23 b(MPI)p 245 2647 15 2 v 17 w(Type)p 358 2647 V 17 w(contiguous\(int)f (count,)h(MPI)p 972 2647 V 17 w(Datatype)f(oldtype,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)h(*newtype\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 61 69 61 68 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fx(61)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(TYPE)p 263 45 V 16 w(CONTIGUOUS\(COUNT,)22 b(OLDTYPE,)h(NEWTYPE,)f(IERROR\)) 170 102 y(INTEGER)h(COUNT,)g(OLDTYPE,)g(NEWTYPE,)g(IERROR)166 188 y Fv(newt)o(yp)q(e)c Fx(is)f(the)f(datat)o(yp)q(e)g(obtained)h(b)o (y)f(concatenating)h Fv(count)h Fx(copies)f(of)f Fv(oldt)o(yp)q(e)p Fx(.)27 b(Concate-)75 244 y(nation)15 b(is)h(de\014ned)h(using)f Fl(extent)e Fx(as)h(the)h(size)g(of)e(the)i(concatenated)f(copies.)75 339 y Fq(Example)j(3.19)23 b Fx(Let)17 b Fv(oldt)o(yp)q(e)g Fx(ha)o(v)o(e)g(t)o(yp)q(e)g(map)f Ft(f)p Fx(\()p Fv(double)q Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Ft(g)p Fp(;)15 b Fx(with)i(exten)o(t)f(16,)h(and)f(let)75 396 y Fv(count)e Fx(=)f(3.)19 b(The)d(t)o(yp)q(e)f(map)g(of)g(the)g (datat)o(yp)q(e)g(returned)g(b)o(y)h Fv(newt)o(yp)q(e)h Fx(is)189 482 y Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Fx(16\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(24\))p Fp(;)g Fx(\()p Fv(double)n Fp(;)g Fx(32\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(40\))p Ft(g)p Fx(;)75 567 y(i.e.,)15 b(alternating)g Fv(double)i Fx(and)e Fv(cha)o(r)g Fx(elemen)o(ts,)h(with)f(displacemen)o(ts)i(0)p Fp(;)8 b Fx(8)p Fp(;)g Fx(16)p Fp(;)f Fx(24)p Fp(;)g Fx(32)o Fp(;)h Fx(4)o(0.)166 745 y(In)16 b(general,)f(assume)g(that)g(the)g(t)o (yp)q(e)g(map)g(of)g Fv(oldt)o(yp)q(e)h Fx(is)189 831 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 838 y Fg(0)333 831 y Fp(;)8 b(disp)438 838 y Fg(0)457 831 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 838 y Ff(n)p Fh(\000)p Fg(1)723 831 y Fp(;)h(disp)828 838 y Ff(n)p Fh(\000)p Fg(1)895 831 y Fx(\))p Ft(g)p Fp(;)75 917 y Fx(with)16 b(exten)o(t)e Fp(ex)p Fx(.)20 b(Then)c Fv(newt)o(yp)q(e)h Fx(has)e(a)g(t)o(yp)q(e)g (map)g(with)h Fv(count)11 b Ft(\001)f Fv(n)16 b Fx(en)o(tries)f (de\014ned)i(b)o(y:)189 1003 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 1010 y Fg(0)333 1003 y Fp(;)8 b(disp)438 1010 y Fg(0)457 1003 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 1010 y Ff(n)p Fh(\000)p Fg(1)723 1003 y Fp(;)h(disp)828 1010 y Ff(n)p Fh(\000)p Fg(1)895 1003 y Fx(\))p Fp(;)g Fx(\()p Fp(ty)r(pe)1036 1010 y Fg(0)1055 1003 y Fp(;)g(disp)1160 1010 y Fg(0)1189 1003 y Fx(+)i Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)1482 1010 y Ff(n)p Fh(\000)p Fg(1)1548 1003 y Fp(;)g(disp)1653 1010 y Ff(n)p Fh(\000)p Fg(1)1730 1003 y Fx(+)j Fp(ex)p Fx(\))p Fp(;)189 1098 y(:::;)d Fx(\()p Fp(ty)r(pe)351 1105 y Fg(0)368 1098 y Fp(;)g(disp)473 1105 y Fg(0)502 1098 y Fx(+)i Fp(ex)h Ft(\001)e Fx(\()p Fv(count)j Ft(\000)e Fx(1\)\))p Fp(;)e(:::;)f Fx(\()p Fp(ty)r(pe)1048 1105 y Ff(n)p Fh(\000)p Fg(1)1114 1098 y Fp(;)h(disp)1219 1105 y Ff(n)p Fh(\000)p Fg(1)1296 1098 y Fx(+)j Fp(ex)f Ft(\001)g Fx(\()p Fv(count)h Ft(\000)f Fx(1\)\))p Ft(g)p Fp(:)75 1337 y Fv(V)o(ecto)o(r)45 b Fx(The)16 b(function)g Fv(MPI)p 600 1337 14 2 v 16 w(TYPE)p 733 1337 V 17 w(VECTOR)g Fx(is)g(a)f(more)g(general)h(constructor)e (that)h(allo)o(ws)g(repli-)75 1394 y(cation)20 b(of)g(a)g(datat)o(yp)q (e)f(in)o(to)h(lo)q(cations)h(that)e(consist)i(of)e(equally)j(spaced)f (blo)q(c)o(ks.)35 b(Eac)o(h)20 b(blo)q(c)o(k)g(is)75 1450 y(obtained)g(b)o(y)f(concatenating)h(the)f(same)g(n)o(um)o(b)q(er) h(of)f(copies)h(of)f(the)g(old)h(datat)o(yp)q(e.)32 b(The)19 b(spacing)75 1507 y(b)q(et)o(w)o(een)d(blo)q(c)o(ks)f(is)h(a)f(m)o (ultiple)i(of)e(the)g(exten)o(t)g(of)g(the)g(old)h(datat)o(yp)q(e.)75 1658 y Fv(MPI)p 160 1658 V 16 w(TYPE)p 293 1658 V 17 w(VECTOR\()g(count,)g(blo)q(cklength,)g(stride,)g(oldt)o(yp)q(e,)g (newt)o(yp)q(e\))117 1735 y Fk(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(nonnegativ)o(e)f(in)o (teger\))117 1808 y(IN)155 b Fv(blo)q(cklength)371 b Fk(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g(in)f(eac)o(h)i(blo)q(c)o(k)e (\(nonnegativ)o(e)h(in)o(te-)905 1865 y(ger\))117 1938 y(IN)155 b Fv(stride)484 b Fk(n)o(um)o(b)q(er)13 b(of)f(elemen)o(ts)h (b)q(et)o(w)o(een)h(start)g(of)f(eac)o(h)g(blo)q(c)o(k)g(\(in)o(te-)905 1995 y(ger\))117 2068 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i(\(handle\))117 2142 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 2266 y Fu(int)23 b(MPI)p 245 2266 15 2 v 17 w(Type)p 358 2266 V 17 w(vector\(int)f(count,)h(int)h(blocklength,)e(int)i (stride,)393 2323 y(MPI)p 468 2323 V 17 w(Datatype)f(oldtype,)f(MPI)p 986 2323 V 17 w(Datatype)h(*newtype\))75 2409 y(MPI)p 150 2409 V 17 w(TYPE)p 263 2409 V 16 w(VECTOR\(COUNT,)f(BLOCKLENGTH,)h (STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR\))170 2466 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)75 2591 y Fq(Example)18 b(3.20)23 b Fx(Assume,)15 b(again,)f(that)h Fv(oldt)o(yp)q(e)h Fx(has)f(t)o(yp)q(e)g(map)g Ft(f)p Fx(\()p Fv(double)q Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Ft(g)p Fp(;)13 b Fx(with)j(ex-)75 2647 y(ten)o(t)g(16.)21 b(A)c(call)g(to)e Fv(MPI)p 527 2647 14 2 v 16 w(TYPE)p 660 2647 V 17 w(VECTOR\()i(2,)e(3,)h(4,)f(oldt) o(yp)q(e,)i(newt)o(yp)q(e\))h Fx(will)g(create)d(the)h(datat)o(yp)q(e) 75 2704 y(with)g(t)o(yp)q(e)f(map,)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 62 70 62 69 bop 75 -100 a Fx(62)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Fx(16\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(24\))p Fp(;)g Fx(\()p Fv(double)n Fp(;)g Fx(32\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(40\))p Fp(;)189 151 y Fx(\()p Fv(double)q Fp(;)g Fx(64\))p Fp(;)g Fx(\()p Fv(cha)o(r)m Fp(;)g Fx(72\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Fx(80\))p Fp(;)g Fx(\()p Fv(cha)o(r)m Fp(;)g Fx(88\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Fx(96\))p Fp(;)g Fx(\()p Fv(cha)o(r)m Fp(;)g Fx(104\))p Ft(g)p Fp(:)75 258 y Fx(That)13 b(is,)h(t)o(w)o(o)e (blo)q(c)o(ks)j(with)e(three)h(copies)h(eac)o(h)e(of)g(the)h(old)g(t)o (yp)q(e,)g(with)g(a)f(stride)h(of)f(4)h(elemen)o(ts)g(\(4)7 b Ft(\001)g Fx(16)75 314 y(b)o(ytes\))15 b(b)q(et)o(w)o(een)g(the)g (blo)q(c)o(ks.)75 420 y Fq(Example)j(3.21)23 b Fx(A)c(call)i(to)d Fv(MPI)p 692 420 14 2 v 16 w(TYPE)p 825 420 V 17 w(VECTOR\(3,)j(1,)e (-2,)h(oldt)o(yp)q(e,)i(newt)o(yp)q(e\))f Fx(will)g(create)e(the)75 477 y(datat)o(yp)q(e,)189 571 y Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Ft(\000)p Fx(32\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Ft(\000)p Fx(24\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Ft(\000)p Fx(64\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Ft(\000)p Fx(56\))p Ft(g)p Fp(:)166 760 y Fx(In)16 b(general,)f(assume)g(that)g Fv(oldt)o(yp)q(e)h Fx(has)f(t)o(yp)q(e)g(map,)189 853 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 860 y Fg(0)333 853 y Fp(;)8 b(disp)438 860 y Fg(0)457 853 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 860 y Ff(n)p Fh(\000)p Fg(1)723 853 y Fp(;)h(disp)828 860 y Ff(n)p Fh(\000)p Fg(1)895 853 y Fx(\))p Ft(g)p Fp(;)75 947 y Fx(with)14 b(exten)o(t)f Fp(ex)p Fx(.)19 b(Let)13 b Fv(bl)h Fx(b)q(e)g(the)f Fv(blo)q(cklength)p Fx(.)21 b(The)14 b(newly)g(created)f(datat)o(yp)q(e)g(has)g(a)g(t)o(yp) q(e)h(map)f(with)75 1004 y Fv(count)e Ft(\001)f Fv(bl)h Ft(\001)e Fp(n)16 b Fx(en)o(tries:)189 1097 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 1104 y Fg(0)333 1097 y Fp(;)8 b(disp)438 1104 y Fg(0)457 1097 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 1104 y Ff(n)p Fh(\000)p Fg(1)723 1097 y Fp(;)h(disp)828 1104 y Ff(n)p Fh(\000)p Fg(1)895 1097 y Fx(\))p Fp(;)189 1204 y Fx(\()p Fp(ty)r(pe)291 1211 y Fg(0)310 1204 y Fp(;)g(disp)415 1211 y Fg(0)444 1204 y Fx(+)i Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)737 1211 y Ff(n)p Fh(\000)p Fg(1)803 1204 y Fp(;)g(disp)908 1211 y Ff(n)p Fh(\000)p Fg(1)985 1204 y Fx(+)j Fp(ex)p Fx(\))p Fp(;)d(:::;)189 1310 y Fx(\()p Fp(ty)r(pe)291 1317 y Fg(0)310 1310 y Fp(;)g(disp)415 1317 y Fg(0)444 1310 y Fx(+)i(\()p Fv(bl)g Ft(\000)h Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)918 1317 y Ff(n)p Fh(\000)p Fg(1)984 1310 y Fp(;)g(disp)1089 1317 y Ff(n)p Fh(\000)p Fg(1)1166 1310 y Fx(+)j(\()p Fv(bl)f Ft(\000)g Fx(1\))g Ft(\001)g Fp(ex)p Fx(\))p Fp(;)189 1416 y Fx(\()p Fp(ty)r(pe)291 1423 y Fg(0)310 1416 y Fp(;)e(disp)415 1423 y Fg(0)444 1416 y Fx(+)i Fv(stride)h Ft(\001)f Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)874 1423 y Ff(n)p Fh(\000)p Fg(1)940 1416 y Fp(;)g(disp)1045 1423 y Ff(n)p Fh(\000)p Fg(1)1122 1416 y Fx(+)j Fv(stride)g Ft(\001)e Fp(ex)p Fx(\))p Fp(;)f(:::;)189 1523 y Fx(\()p Fp(ty)r(pe)291 1530 y Fg(0)310 1523 y Fp(;)g(disp)415 1530 y Fg(0)444 1523 y Fx(+)i(\()p Fv(stride)h Fx(+)g Fv(bl)f Ft(\000)g Fx(1\))g Ft(\001)g Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()o Fp(ty)r(pe)1077 1530 y Ff(n)p Fh(\000)p Fg(1)1143 1523 y Fp(;)g(disp)1248 1530 y Ff(n)p Fh(\000)p Fg(1)1326 1523 y Fx(+)i(\()p Fv(stride)h Fx(+)f Fv(bl)h Ft(\000)f Fx(1\))g Ft(\001)f Fp(ex)p Fx(\))p Fp(;)f(::::;)189 1629 y Fx(\()p Fp(ty)r(pe)291 1636 y Fg(0)310 1629 y Fp(;)g(disp)415 1636 y Fg(0)444 1629 y Fx(+)i Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)g Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)189 1735 y Fx(\()p Fp(ty)r(pe)291 1742 y Ff(n)p Fh(\000)p Fg(1)359 1735 y Fp(;)g(disp)464 1742 y Ff(n)p Fh(\000)p Fg(1)542 1735 y Fx(+)i Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)f Fx(1\))g Ft(\001)g Fp(ex)p Fx(\))p Fp(;)e(:::;)189 1841 y Fx(\()p Fp(ty)r(pe)291 1848 y Fg(0)310 1841 y Fp(;)g(disp)415 1848 y Fg(0)444 1841 y Fx(+)i(\()p Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)f Fx(1\))g(+)g Fv(bl)h Ft(\000)f Fx(1\))g Ft(\001)f Fp(ex)p Fx(\))p Fp(;)f(:::;)189 1948 y Fx(\()p Fp(ty)r(pe)291 1955 y Ff(n)p Fh(\000)p Fg(1)359 1948 y Fp(;)g(disp)464 1955 y Ff(n)p Fh(\000)p Fg(1)542 1948 y Fx(+)i(\()p Fv(stride)h Ft(\001)e Fx(\()p Fv(count)j Ft(\000)e Fx(1\))g(+)g Fv(bl)g Ft(\000)h Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Ft(g)p Fp(:)166 2137 y Fx(A)16 b(call)i(to)e Fv(MPI)p 443 2137 V 16 w(TYPE)p 576 2137 V 16 w(CONTIGUOUS\(count,)k(oldt)o(yp)q(e,)d(newt)o(yp)q(e\))h Fx(is)f(equiv)m(alen)o(t)h(to)e(a)g(call)i(to)75 2193 y Fv(MPI)p 160 2193 V 16 w(TYPE)p 293 2193 V 17 w(VECTOR\(count,)g(1,)f (1,)f(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))p Fx(,)h(or)e(to)g(a)h(call)g (to)f Fv(MPI)p 1495 2193 V 16 w(TYPE)p 1628 2193 V 17 w(VECTOR\(1,)75 2249 y(count,)g(n,)f(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))p Fx(,)h Fv(n)e Fx(arbitrary)l(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 63 71 63 70 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fx(63)75 45 y Fv(Hvecto)o(r)45 b Fx(The)19 b(function)h Fv(MPI)p 631 45 14 2 v 16 w(TYPE)p 764 45 V 17 w(HVECTOR)g Fx(is)f(iden)o(tical)i(to)d Fv(MPI)p 1389 45 V 16 w(TYPE)p 1522 45 V 17 w(VECTOR)p Fx(,)h(except)75 102 y(that)h Fv(stride)h Fx(is)f(giv)o(en)h(in)g(b)o(ytes,)g(rather)f(than)g(in)h (elemen)o(ts.)36 b(The)20 b(use)h(for)e(b)q(oth)i(t)o(yp)q(es)f(of)g(v) o(ector)75 158 y(constructors)14 b(is)i(illustrated)h(in)f(Sec.)k (3.12.7.)e(\()p Fv(H)d Fx(stands)g(for)g(\\heterogeneous"\).)75 309 y Fv(MPI)p 160 309 V 16 w(TYPE)p 293 309 V 17 w(HVECTOR\()h(count,) g(blo)q(cklength,)h(stride,)e(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))117 386 y Fk(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(nonnegativ)o(e)f(in)o(teger\))117 461 y(IN)155 b Fv(blo)q(cklength)371 b Fk(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g(in)f(eac)o(h)i(blo)q(c)o(k)e(\(nonnegativ)o(e)h (in)o(te-)905 518 y(ger\))117 593 y(IN)155 b Fv(stride)484 b Fk(n)o(um)o(b)q(er)13 b(of)g(b)o(ytes)h(b)q(et)o(w)o(een)h(start)f (of)f(eac)o(h)h(blo)q(c)o(k)f(\(in)o(teger\))117 668 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i (\(handle\))117 743 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 868 y Fu(int)23 b(MPI)p 245 868 15 2 v 17 w(Type)p 358 868 V 17 w(hvector\(int)f (count,)h(int)h(blocklength,)e(MPI)p 1306 868 V 17 w(Aint)h(stride,)393 924 y(MPI)p 468 924 V 17 w(Datatype)g(oldtype,)f(MPI)p 986 924 V 17 w(Datatype)h(*newtype\))75 1011 y(MPI)p 150 1011 V 17 w(TYPE)p 263 1011 V 16 w(HVECTOR\(COUNT,)f(BLOCKLENGTH,)h (STRIDE,)f(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1067 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)166 1236 y Fx(Assume)15 b(that)g Fv(oldt)o(yp)q(e)h Fx(has)f(t)o(yp)q(e)g (map,)189 1330 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 1337 y Fg(0)333 1330 y Fp(;)8 b(disp)438 1337 y Fg(0)457 1330 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 1337 y Ff(n)p Fh(\000)p Fg(1)723 1330 y Fp(;)h(disp)828 1337 y Ff(n)p Fh(\000)p Fg(1)895 1330 y Fx(\))p Ft(g)p Fp(;)75 1424 y Fx(with)14 b(exten)o(t)f Fp(ex)p Fx(.)19 b(Let)13 b Fv(bl)h Fx(b)q(e)g(the)f Fv(blo)q(cklength)p Fx(.)21 b(The)14 b(newly)g(created)f(datat)o(yp)q(e)g(has)g(a)g(t)o(yp)q(e)h (map)f(with)75 1480 y Fv(count)e Ft(\001)f Fv(bl)h Ft(\001)e Fp(n)16 b Fx(en)o(tries:)189 1574 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 1581 y Fg(0)333 1574 y Fp(;)8 b(disp)438 1581 y Fg(0)457 1574 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 1581 y Ff(n)p Fh(\000)p Fg(1)723 1574 y Fp(;)h(disp)828 1581 y Ff(n)p Fh(\000)p Fg(1)895 1574 y Fx(\))p Fp(;)189 1680 y Fx(\()p Fp(ty)r(pe)291 1687 y Fg(0)310 1680 y Fp(;)g(disp)415 1687 y Fg(0)444 1680 y Fx(+)i Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)737 1687 y Ff(n)p Fh(\000)p Fg(1)803 1680 y Fp(;)g(disp)908 1687 y Ff(n)p Fh(\000)p Fg(1)985 1680 y Fx(+)j Fp(ex)p Fx(\))p Fp(;)d(:::;)189 1787 y Fx(\()p Fp(ty)r(pe)291 1794 y Fg(0)310 1787 y Fp(;)g(disp)415 1794 y Fg(0)444 1787 y Fx(+)i(\()p Fv(bl)g Ft(\000)h Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)918 1794 y Ff(n)p Fh(\000)p Fg(1)984 1787 y Fp(;)g(disp)1089 1794 y Ff(n)p Fh(\000)p Fg(1)1166 1787 y Fx(+)j(\()p Fv(bl)f Ft(\000)g Fx(1\))g Ft(\001)g Fp(ex)p Fx(\))p Fp(;)189 1893 y Fx(\()p Fp(ty)r(pe)291 1900 y Fg(0)310 1893 y Fp(;)e(disp)415 1900 y Fg(0)444 1893 y Fx(+)i Fv(stride)q Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)794 1900 y Ff(n)p Fh(\000)p Fg(1)860 1893 y Fp(;)g(disp)965 1900 y Ff(n)p Fh(\000)p Fg(1)1042 1893 y Fx(+)j Fv(stride)q Fx(\))p Fp(;)d(:::;)189 1999 y Fx(\()p Fp(ty)r(pe)291 2006 y Fg(0)310 1999 y Fp(;)g(disp)415 2006 y Fg(0)444 1999 y Fx(+)i Fv(stride)h Fx(+)g(\()p Fv(bl)f Ft(\000)g Fx(1\))g Ft(\001)g Fp(ex)p Fx(\))p Fp(;)e(:::;)189 2105 y Fx(\()p Fp(ty)r(pe)291 2112 y Ff(n)p Fh(\000)p Fg(1)359 2105 y Fp(;)g(disp)464 2112 y Ff(n)p Fh(\000)p Fg(1)542 2105 y Fx(+)i Fv(stride)h Fx(+)g(\()p Fv(bl)f Ft(\000)g Fx(1\))g Ft(\001)f Fp(ex)p Fx(\))p Fp(;)f(::::;)189 2212 y Fx(\()p Fp(ty)r(pe)291 2219 y Fg(0)310 2212 y Fp(;)g(disp)415 2219 y Fg(0)444 2212 y Fx(+)i Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)g Fx(1\)\))p Fp(;)d(:::;)f Fx(\()p Fp(ty)r(pe)1047 2219 y Ff(n)p Fh(\000)p Fg(1)1113 2212 y Fp(;)h(disp)1218 2219 y Ff(n)p Fh(\000)p Fg(1)1295 2212 y Fx(+)i Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)g Fx(1\)\))p Fp(;)d(:::;)189 2318 y Fx(\()p Fp(ty)r(pe)291 2325 y Fg(0)310 2318 y Fp(;)g(disp)415 2325 y Fg(0)444 2318 y Fx(+)i Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)g Fx(1\))e(+)i(\()p Fv(bl)f Ft(\000)g Fx(1\))g Ft(\001)f Fp(ex)p Fx(\))p Fp(;)f(:::;)189 2424 y Fx(\()p Fp(ty)r(pe)291 2431 y Ff(n)p Fh(\000)p Fg(1)359 2424 y Fp(;)g(disp)464 2431 y Ff(n)p Fh(\000)p Fg(1)542 2424 y Fx(+)i Fv(stride)h Ft(\001)f Fx(\()p Fv(count)h Ft(\000)f Fx(1\))g(+)g(\()p Fv(bl)g Ft(\000)h Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Ft(g)p Fp(:)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 64 72 64 71 bop 75 -100 a Fx(64)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(Indexed)47 b Fx(The)19 b(function)h Fv(MPI)p 629 45 14 2 v 16 w(TYPE)p 762 45 V 17 w(INDEXED)f Fx(allo)o(ws)h(replication)h(of)d(an)i(old)f (datat)o(yp)q(e)g(in)o(to)g(a)75 102 y(sequence)g(of)f(blo)q(c)o(ks)h (\(eac)o(h)f(blo)q(c)o(k)h(is)g(a)f(concatenation)g(of)g(the)g(old)h (datat)o(yp)q(e\),)e(where)i(eac)o(h)f(blo)q(c)o(k)75 158 y(can)23 b(con)o(tain)f(a)g(di\013eren)o(t)h(n)o(um)o(b)q(er)g(of)f (copies)h(and)g(ha)o(v)o(e)f(a)g(di\013eren)o(t)g(displacemen)o(t.)43 b(All)24 b(blo)q(c)o(k)75 214 y(displacemen)o(ts)17 b(are)e(m)o (ultiples)h(of)f(the)h(old)f(t)o(yp)q(e)h(exten)o(t.)75 365 y Fv(MPI)p 160 365 V 16 w(TYPE)p 293 365 V 17 w(INDEXED\()c(count,) i(a)o(rra)o(y)p 762 365 V 14 w(of)p 813 365 V 16 w(blo)q(cklengths,)h (a)o(rra)o(y)p 1185 365 V 15 w(of)p 1237 365 V 16 w(displacements,)e (oldt)o(yp)q(e,)h(newt)o(yp)q(e\))117 499 y Fk(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)34 b(of)f(blo)q(c)o(ks)h({)g(also)f (n)o(um)o(b)q(er)h(of)f(en)o(tries)i(in)905 556 y Fi(a)o(rra)o(y)p 992 556 13 2 v 15 w(of)p 1041 556 V 15 w(displacements)17 b Fk(and)h Fi(a)o(rra)o(y)p 1483 556 V 15 w(of)p 1532 556 V 15 w(blo)q(cklengths)i Fk(\(non-)905 612 y(negativ)o(e)14 b(in)o(teger\))117 687 y(IN)155 b Fv(a)o(rra)o(y)p 416 687 14 2 v 15 w(of)p 468 687 V 16 w(blo)q(cklengths)191 b Fk(n)o(um)o(b)q(er)18 b(of)f(elemen)o(ts)h(p)q(er)i(blo)q(c)o(k)d (\(arra)o(y)i(of)e(nonnegativ)o(e)905 744 y(in)o(tegers\))117 819 y(IN)155 b Fv(a)o(rra)o(y)p 416 819 V 15 w(of)p 468 819 V 16 w(displacements)162 b Fk(displacemen)o(t)19 b(for)h(eac)o(h)g(blo)q(c)o(k,)g(in)g(m)o(ultiples)e(of)h Fi(oldt)o(yp)q(e)905 875 y Fk(exten)o(t)c(\(arra)o(y)f(of)f(in)o (teger\))117 950 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i(\(handle\))117 1025 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 1150 y Fu(int)23 b(MPI)p 245 1150 15 2 v 17 w(Type)p 358 1150 V 17 w(indexed\(int)f (count,)h(int)h(*array)p 1068 1150 V 16 w(of)p 1132 1150 V 17 w(blocklengths,)393 1206 y(int)g(*array)p 636 1206 V 16 w(of)p 700 1206 V 17 w(displacements,)e(MPI)p 1147 1206 V 17 w(Datatype)g(oldtype,)393 1263 y(MPI)p 468 1263 V 17 w(Datatype)h(*newtype\))75 1349 y(MPI)p 150 1349 V 17 w(TYPE)p 263 1349 V 16 w(INDEXED\(COUNT,)f(ARRAY)p 757 1349 V 17 w(OF)p 822 1349 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1293 1349 V 16 w(OF)p 1357 1349 V 17 w(DISPLACEMENTS,)393 1406 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1462 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1462 V 17 w(OF)p 716 1462 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1462 V 17 w(OF)p 1323 1462 V 17 w(DISPLACEMENTS\(*\),)170 1519 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 1655 y Fq(Example)18 b(3.22)23 b Fx(Let)14 b Fv(oldt)o(yp)q(e)h Fx(ha)o(v)o(e)f(t)o(yp)q(e)g (map)g Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Ft(g)p Fp(;)k Fx(with)i(exten)o(t) g(16.)19 b(Let)14 b Fv(B)h(=)75 1711 y(\(3,)e(1\))h Fx(and)g(let)h Fv(D)f(=)g(\(4,)f(0\))p Fx(.)19 b(A)14 b(call)i(to)d Fv(MPI)p 855 1711 14 2 v 16 w(TYPE)p 988 1711 V 17 w(INDEXED\(2,)g(B,)h (D,)f(oldt)o(yp)q(e,)j(newt)o(yp)q(e\))f Fx(returns)75 1768 y(a)g(datat)o(yp)q(e)g(with)g(t)o(yp)q(e)g(map,)189 1862 y Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(64\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(72\))p Fp(;)g Fx(\()p Fv(double)n Fp(;)g Fx(80\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(88\))p Fp(;)g Fx(\()p Fv(double)n Fp(;)g Fx(96\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(104\))p Fp(;)189 1968 y Fx(\()p Fv(double)q Fp(;)g Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)m Fp(;)g Fx(8\))p Ft(g)p Fp(:)75 2074 y Fx(That)15 b(is,)i(three)f(copies)g(of)g(the)g(old)g(t)o(yp)q(e)g(starting)g(at)f (displacemen)o(t)i(64,)e(and)h(one)h(cop)o(y)e(starting)h(at)75 2131 y(displacemen)o(t)h(0.)166 2320 y(In)f(general,)f(assume)g(that)g Fv(oldt)o(yp)q(e)h Fx(has)f(t)o(yp)q(e)g(map,)189 2413 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 2420 y Fg(0)333 2413 y Fp(;)8 b(disp)438 2420 y Fg(0)457 2413 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 2420 y Ff(n)p Fh(\000)p Fg(1)723 2413 y Fp(;)h(disp)828 2420 y Ff(n)p Fh(\000)p Fg(1)895 2413 y Fx(\))p Ft(g)p Fp(;)75 2507 y Fx(with)41 b(exten)o(t)f Fl(ex)p Fx(.)96 b(Let)41 b Fv(B)g Fx(b)q(e)g(the)g Fv(a)o(rra)o(y)p 986 2507 V 14 w(of)p 1037 2507 V 16 w(blo)q(cklength)i Fx(argumen)o(t)d(and)g Fv(D)h Fx(b)q(e)g(the)75 2564 y Fv(a)o(rra)o(y)p 173 2564 V 15 w(of)p 225 2564 V 16 w(displacements)15 b Fx(argumen)o(t.)k(The)c(newly)h(created)f(datat)o (yp)q(e)g(has)g Fp(n)c Ft(\001)1450 2532 y Fe(P)1494 2545 y Fi(count)q Fh(\000)p Fg(1)1494 2577 y Ff(i)p Fg(=0)1646 2564 y Fv(B)p Fx([)p Fv(i)p Fx(])j(en)o(tries:)189 2657 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 2664 y Fg(0)333 2657 y Fp(;)8 b(disp)438 2664 y Fg(0)467 2657 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])f Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)874 2664 y Ff(n)p Fh(\000)p Fg(1)939 2657 y Fp(;)g(disp)1044 2664 y Ff(n)p Fh(\000)p Fg(1)1122 2657 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])f Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 65 73 65 72 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fx(65)189 45 y(\()p Fp(ty)r(pe)291 52 y Fg(0)310 45 y Fp(;)8 b(disp)415 52 y Fg(0)444 45 y Fx(+)i(\()p Fv(D)p Fx([)p Fv(0)p Fx(])f(+)h Fv(B)p Fx([)p Fv(0)p Fx(])g Ft(\000)g Fx(1\))g Ft(\001)f Fp(ex)p Fx(\))p Fp(;)f(:::;)g Fx(\()p Fp(ty)r(pe)1098 52 y Ff(n)p Fh(\000)p Fg(1)1164 45 y Fp(;)g(disp)1269 52 y Ff(n)p Fh(\000)p Fg(1)1347 45 y Fx(+)i(\()p Fv(D)p Fx([)p Fv(0)p Fx(])f(+)h Fv(B)p Fx([)p Fv(0)p Fx(])f Ft(\000)i Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)189 151 y Fx(\()p Fp(ty)r(pe)291 158 y Fg(0)310 151 y Fp(;)g(disp)415 158 y Fg(0)444 151 y Fx(+)i Fv(D)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)1013 158 y Ff(n)p Fh(\000)p Fg(1)1079 151 y Fp(;)g(disp)1184 158 y Ff(n)p Fh(\000)p Fg(1)1261 151 y Fx(+)i Fv(D)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)189 258 y Fx(\()p Fp(ty)r(pe)291 265 y Fg(0)310 258 y Fp(;)g(disp)415 265 y Fg(0)444 258 y Fx(+)i(\()p Fv(D)p Fx([)p Fv(count)h Ft(\000)f Fv(1)p Fx(])g(+)g Fv(B)p Fx([)p Fv(count)i Ft(\000)e Fv(1)p Fx(])g Ft(\000)g Fx(1\))f Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)189 364 y Fx(\()p Fp(ty)r(pe)291 371 y Ff(n)p Fh(\000)p Fg(1)359 364 y Fp(;)g(disp)464 371 y Ff(n)p Fh(\000)p Fg(1)542 364 y Fx(+)i(\()p Fv(D)p Fx([)p Fv(count)h Ft(\000)f Fv(1)p Fx(])g(+)g Fv(B)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])e Ft(\000)i Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Ft(g)p Fp(:)166 553 y Fx(A)j(call)g(to)f Fv(MPI)p 431 553 14 2 v 16 w(TYPE)p 564 553 V 17 w(VECTOR\(count,)j(blo)q(cklength,) g(stride,)e(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))g Fx(is)f(equiv)m(alen)o (t)75 609 y(to)i(a)f(call)j(to)d Fv(MPI)p 392 609 V 16 w(TYPE)p 525 609 V 17 w(INDEXED\(count,)i(B,)f(D,)f(oldt)o(yp)q(e,)i (newt)o(yp)q(e\))h Fx(where)189 703 y Fv(D)p Fx([)p Fv(j)p Fx(])11 b(=)i Fp(j)g Ft(\001)d Fv(stride)q Fp(;)22 b(j)15 b Fx(=)e(0)p Fp(;)8 b(:::;)g Fv(count)g Ft(\000)j Fx(1)p Fp(;)75 797 y Fx(and)189 891 y Fv(B)p Fx([)p Fv(j)p Fx(])h(=)h Fv(blo)q(cklength)r Fp(;)22 b(j)15 b Fx(=)e(0)p Fp(;)8 b(:::;)g Fv(count)h Ft(\000)h Fx(1)p Fp(:)75 1011 y Fv(Hindexed)47 b Fx(The)11 b(function)h Fv(MPI)p 643 1011 V 15 w(TYPE)p 775 1011 V 17 w(HINDEXED)f Fx(is)g(iden)o(tical)i(to)d Fv(MPI)p 1383 1011 V 15 w(TYPE)p 1515 1011 V 17 w(INDEXED)p Fx(,)g(except)75 1067 y(that)21 b(blo)q(c)o(k)i(displacemen)o(ts)g(in)g Fv(a)o(rra)o(y)p 761 1067 V 14 w(of)p 812 1067 V 16 w(displacements)f Fx(are)g(sp)q(eci\014ed)i(in)e(b)o(ytes,)h(rather)e(than)h(in)75 1124 y(m)o(ultiples)17 b(of)e(the)g Fv(oldt)o(yp)q(e)h Fx(exten)o(t.)75 1275 y Fv(MPI)p 160 1275 V 16 w(TYPE)p 293 1275 V 17 w(HINDEXED\()21 b(count,)k(a)o(rra)o(y)p 814 1275 V 14 w(of)p 865 1275 V 16 w(blo)q(cklengths,)h(a)o(rra)o(y)p 1248 1275 V 15 w(of)p 1300 1275 V 16 w(displacements,)d(oldt)o(yp)q(e,) h(new-)75 1331 y(t)o(yp)q(e\))117 1408 y Fk(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)34 b(of)f(blo)q(c)o(ks)h({)g(also)f (n)o(um)o(b)q(er)h(of)f(en)o(tries)i(in)905 1465 y Fi(a)o(rra)o(y)p 992 1465 13 2 v 15 w(of)p 1041 1465 V 15 w(displacements)17 b Fk(and)h Fi(a)o(rra)o(y)p 1483 1465 V 15 w(of)p 1532 1465 V 14 w(blo)q(cklengths)i Fk(\(in)o(te-)905 1521 y(ger\))117 1596 y(IN)155 b Fv(a)o(rra)o(y)p 416 1596 14 2 v 15 w(of)p 468 1596 V 16 w(blo)q(cklengths)191 b Fk(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts)h(in)f(eac)o(h)h(blo)q(c)o (k)g(\(arra)o(y)f(of)g(nonnega-)905 1653 y(tiv)o(e)f(in)o(tegers\))117 1728 y(IN)155 b Fv(a)o(rra)o(y)p 416 1728 V 15 w(of)p 468 1728 V 16 w(displacements)162 b Fk(b)o(yte)14 b(displacemen)o(t)g (of)f(eac)o(h)h(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1803 y(IN)155 b Fv(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i (\(handle\))117 1878 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 2003 y Fu(int)23 b(MPI)p 245 2003 15 2 v 17 w(Type)p 358 2003 V 17 w(hindexed\(int)f (count,)h(int)h(*array)p 1092 2003 V 16 w(of)p 1156 2003 V 17 w(blocklengths,)393 2059 y(MPI)p 468 2059 V 17 w(Aint)f(*array)p 748 2059 V 17 w(of)p 813 2059 V 17 w(displacements,)e(MPI)p 1259 2059 V 17 w(Datatype)i(oldtype,)393 2116 y(MPI)p 468 2116 V 17 w(Datatype)g(*newtype\))75 2202 y(MPI)p 150 2202 V 17 w(TYPE)p 263 2202 V 16 w(HINDEXED\(COUNT,)f(ARRAY)p 781 2202 V 17 w(OF)p 846 2202 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1317 2202 V 16 w(OF)p 1381 2202 V 17 w(DISPLACEMENTS,)393 2259 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 2315 y(INTEGER)g(COUNT,)g (ARRAY)p 651 2315 V 17 w(OF)p 716 2315 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 2315 V 17 w(OF)p 1323 2315 V 17 w(DISPLACEMENTS\(*\),)170 2371 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)166 2541 y Fx(Assume)15 b(that)g Fv(oldt)o(yp)q(e)h Fx(has)f(t)o(yp)q(e)g(map,)189 2634 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 2641 y Fg(0)333 2634 y Fp(;)8 b(disp)438 2641 y Fg(0)457 2634 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 2641 y Ff(n)p Fh(\000)p Fg(1)723 2634 y Fp(;)h(disp)828 2641 y Ff(n)p Fh(\000)p Fg(1)895 2634 y Fx(\))p Ft(g)p Fp(;)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 66 74 66 73 bop 75 -100 a Fx(66)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fx(with)41 b(exten)o(t)e Fp(ex)p Fx(.)95 b(Let)41 b Fv(B)f Fx(b)q(e)h(the)g Fv(a)o(rra)o(y)p 988 45 14 2 v 14 w(of)p 1039 45 V 16 w(blo)q(cklength)h Fx(argumen)o(t)e(and)g Fv(D)g Fx(b)q(e)h(the)75 102 y Fv(a)o(rra)o(y)p 173 102 V 15 w(of)p 225 102 V 16 w(displacements)21 b Fx(argumen)o(t.)39 b(The)22 b(newly)h(created)f (datat)o(yp)q(e)f(has)h(a)f(t)o(yp)q(e)h(map)f(with)i Fp(n)14 b Ft(\001)75 126 y Fe(P)119 139 y Fi(count)q Fh(\000)p Fg(1)119 171 y Ff(i)p Fg(=0)271 158 y Fv(B)p Fx([)p Fv(i)p Fx(])g(en)o(tries:)189 251 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 258 y Fg(0)333 251 y Fp(;)8 b(disp)438 258 y Fg(0)467 251 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])o(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)794 258 y Ff(n)p Fh(\000)p Fg(1)859 251 y Fp(;)g(disp)964 258 y Ff(n)p Fh(\000)p Fg(1)1042 251 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])o(\))p Fp(;)e(:::;)189 355 y Fx(\()p Fp(ty)r(pe)291 362 y Fg(0)310 355 y Fp(;)g(disp)415 362 y Fg(0)444 355 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])f(+)i(\()p Fv(B)p Fx([)p Fv(0)p Fx(])e Ft(\000)h Fx(1\))g Ft(\001)f Fp(ex)p Fx(\))p Fp(;)f(:::;)189 460 y Fx(\()p Fp(ty)r(pe)291 467 y Ff(n)p Fh(\000)p Fg(1)359 460 y Fp(;)g(disp)464 467 y Ff(n)p Fh(\000)p Fg(1)542 460 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])f(+)h(\()p Fv(B)p Fx([)p Fv(0)p Fx(])f Ft(\000)i Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)189 564 y Fx(\()p Fp(ty)r(pe)291 571 y Fg(0)310 564 y Fp(;)g(disp)415 571 y Fg(0)444 564 y Fx(+)i Fv(D)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])o(\))p Fp(;)d(:::;)g Fx(\()p Fp(ty)r(pe)933 571 y Ff(n)p Fh(\000)p Fg(1)999 564 y Fp(;)g(disp)1104 571 y Ff(n)p Fh(\000)p Fg(1)1181 564 y Fx(+)i Fv(D)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])o(\))p Fp(;)d(:::;)189 669 y Fx(\()p Fp(ty)r(pe)291 676 y Fg(0)310 669 y Fp(;)g(disp)415 676 y Fg(0)444 669 y Fx(+)i Fv(D)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])e(+)i(\()p Fv(B)p Fx([)p Fv(count)g Ft(\000)f Fv(1)p Fx(])g Ft(\000)g Fx(1\))f Ft(\001)h Fp(ex)p Fx(\))p Fp(;)e(:::;)189 774 y Fx(\()p Fp(ty)r(pe)291 781 y Ff(n)p Fh(\000)p Fg(1)359 774 y Fp(;)g(disp)464 781 y Ff(n)p Fh(\000)p Fg(1)542 774 y Fx(+)i Fv(D)p Fx([)p Fv(count)h Ft(\000)f Fv(1)p Fx(])g(+)g(\()p Fv(B)p Fx([)p Fv(count)h Ft(\000)g Fv(1)p Fx(])e Ft(\000)i Fx(1\))e Ft(\001)h Fp(ex)p Fx(\))p Ft(g)p Fp(:)75 1024 y Fv(Struct)47 b(MPI)p 323 1024 V 16 w(TYPE)p 456 1024 V 16 w(STRUCT)20 b Fx(is)g(the)g(most)f(general)g(t)o(yp)q(e)h (constructor.)32 b(It)19 b(further)g(generalizes)75 1081 y(the)13 b(previous)h(one)f(in)h(that)e(it)h(allo)o(ws)g(eac)o(h)g(blo) q(c)o(k)h(to)e(consist)h(of)g(replications)h(of)f(di\013eren)o(t)g (datat)o(yp)q(es.)75 1232 y Fv(MPI)p 160 1232 V 16 w(TYPE)p 293 1232 V 17 w(STRUCT\(count,)f(a)o(rra)o(y)p 729 1232 V 15 w(of)p 781 1232 V 16 w(blo)q(cklengths,)h(a)o(rra)o(y)p 1151 1232 V 14 w(of)p 1202 1232 V 16 w(displacements,)e(a)o(rra)o(y)p 1599 1232 V 15 w(of)p 1651 1232 V 15 w(t)o(yp)q(es,)i(new-)75 1288 y(t)o(yp)q(e\))117 1365 y Fk(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)18 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))g({)g (also)f(n)o(um)o(b)q(er)h(of)f(en)o(tries)905 1422 y(in)c(arra)o(ys)h Fi(a)o(rra)o(y)p 1167 1422 13 2 v 15 w(of)p 1216 1422 V 14 w(t)o(yp)q(es)p Fk(,)h Fi(a)o(rra)o(y)p 1432 1422 V 15 w(of)p 1481 1422 V 14 w(displacements)e Fk(and)g Fi(a)o(r-)905 1478 y(ra)o(y)p 959 1478 V 15 w(of)p 1008 1478 V 15 w(blo)q(cklengths)117 1553 y Fk(IN)155 b Fv(a)o(rra)o(y)p 416 1553 14 2 v 15 w(of)p 468 1553 V 16 w(blo)q(cklength)208 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(eac)o(h)g(blo)q(c)o (k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1628 y(IN)155 b Fv(a)o(rra)o(y)p 416 1628 V 15 w(of)p 468 1628 V 16 w(displacements)162 b Fk(b)o(yte)14 b(displacemen)o(t)g(of)f(eac)o(h)h (blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1703 y(IN)155 b Fv(a)o(rra)o(y)p 416 1703 V 15 w(of)p 468 1703 V 16 w(t)o(yp)q(es)327 b Fk(t)o(yp)q(e)20 b(of)f(elemen)o(ts)g(in) g(eac)o(h)h(blo)q(c)o(k)f(\(arra)o(y)g(of)g(handles)g(to)905 1759 y(datat)o(yp)q(e)14 b(ob)r(jects\))117 1834 y(OUT)108 b Fv(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 1959 y Fu(int)23 b(MPI)p 245 1959 15 2 v 17 w(Type)p 358 1959 V 17 w(struct\(int)f(count,)h(int)h(*array)p 1044 1959 V 16 w(of)p 1108 1959 V 17 w(blocklengths,)393 2015 y(MPI)p 468 2015 V 17 w(Aint)f(*array)p 748 2015 V 17 w(of)p 813 2015 V 17 w(displacements,)e(MPI)p 1259 2015 V 17 w(Datatype)i(*array)p 1635 2015 V 16 w(of)p 1699 2015 V 17 w(types,)393 2072 y(MPI)p 468 2072 V 17 w(Datatype)g(*newtype\))75 2158 y(MPI)p 150 2158 V 17 w(TYPE)p 263 2158 V 16 w(STRUCT\(COUNT,)f(ARRAY)p 733 2158 V 17 w(OF)p 798 2158 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1269 2158 V 16 w(OF)p 1333 2158 V 17 w(DISPLACEMENTS,)393 2214 y(ARRAY)p 516 2214 V 17 w(OF)p 581 2214 V 17 w(TYPES,)h(NEWTYPE,)f (IERROR\))170 2271 y(INTEGER)h(COUNT,)g(ARRAY)p 651 2271 V 17 w(OF)p 716 2271 V 17 w(BLOCKLENGTHS\(*\),)e(ARRAY)p 1258 2271 V 17 w(OF)p 1323 2271 V 17 w(DISPLACEMENTS\(*\),)170 2327 y(ARRAY)p 293 2327 V 17 w(OF)p 358 2327 V 17 w(TYPES\(*\),)h (NEWTYPE,)h(IERROR)75 2462 y Fq(Example)18 b(3.23)23 b Fx(Let)15 b Fv(t)o(yp)q(e1)h Fx(ha)o(v)o(e)f(t)o(yp)q(e)g(map,)189 2555 y Ft(f)p Fx(\()p Fv(double)p Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(8\))p Ft(g)p Fp(;)75 2647 y Fx(with)k(exten)o(t)f(16.)18 b(Let)11 b Fv(B)h(=)g(\(2,)f(1,)g(3\))p Fx(,)g Fv(D)h(=)f(\(0,)g(16,)h(26\))p Fx(,)e(and)i Fv(T)f(=)h(\(MPI)p 1321 2647 14 2 v 15 w(FLO)o(A)l(T,)f(t)o(yp)q(e1,)i(MPI)p 1711 2647 V 16 w(CHAR\))p Fx(.)75 2704 y(Then)h(a)f(call)h(to)e Fv(MPI)p 447 2704 V 16 w(TYPE)p 580 2704 V 17 w(STRUCT\(3,)h(B,)g(D,)g (T,)g(newt)o(yp)q(e\))h Fx(returns)f(a)g(datat)o(yp)q(e)g(with)g(t)o (yp)q(e)h(map,)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 67 75 67 74 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(67)189 45 y Ft(f)p Fx(\()p Fv(\015oat)o Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fv(\015oat)n Fp(;)g Fx(4\))p Fp(;)g Fx(\()p Fv(double)o Fp(;)g Fx(16\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(24\))p Fp(;)g Fx(\()p Fv(cha)o(r)m Fp(;)g Fx(26\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(27\))p Fp(;)g Fx(\()p Fv(cha)o(r)n Fp(;)g Fx(28\))p Ft(g)p Fp(:)75 128 y Fx(That)16 b(is,)h(t)o(w)o(o)e(copies)i(of)f Fv(MPI)p 610 128 14 2 v 15 w(FLO)o(A)l(T)h Fx(starting)f(at)f(0,)h(follo)o(w)o (ed)h(b)o(y)f(one)h(cop)o(y)f(of)g Fv(t)o(yp)q(e1)h Fx(starting)f(at)75 185 y(16,)d(follo)o(w)o(ed)h(b)o(y)f(three)h(copies)h(of)e Fv(MPI)p 765 185 V 16 w(CHAR)p Fx(,)h(starting)f(at)g(26.)18 b(\(W)l(e)c(assume)f(that)g(a)g(\015oat)g(o)q(ccupies)75 241 y(four)i(b)o(ytes.\))166 416 y(In)h(general,)f(let)h Fv(T)f Fx(b)q(e)h(the)f Fv(a)o(rra)o(y)p 742 416 V 14 w(of)p 793 416 V 16 w(t)o(yp)q(es)i Fx(argumen)o(t,)d(where)i Fv(T[i])f Fx(is)g(a)g(handle)i(to,)189 499 y Fp(ty)r(pemap)360 506 y Ff(i)387 499 y Fx(=)c Ft(f)p Fx(\()p Fp(ty)r(pe)560 480 y Ff(i)560 510 y Fg(0)579 499 y Fp(;)8 b(disp)684 480 y Ff(i)684 510 y Fg(0)702 499 y Fx(\))p Fp(;)g(:::;)g Fx(\()p Fp(ty)r(pe)903 480 y Ff(i)903 510 y(n)924 515 y Fd(i)935 510 y Fh(\000)p Fg(1)982 499 y Fp(;)g(disp)1087 480 y Ff(i)1087 510 y(n)1108 515 y Fd(i)1120 510 y Fh(\000)p Fg(1)1168 499 y Fx(\))p Ft(g)p Fp(;)75 582 y Fx(with)15 b(exten)o(t)g Fp(ex)364 589 y Ff(i)378 582 y Fx(.)20 b(Let)15 b Fv(B)g Fx(b)q(e)g(the)g Fv(a)o(rra)o(y)p 774 582 V 14 w(of)p 825 582 V 16 w(blo)q(cklength)i Fx(argumen)o(t)d(and)h Fv(D)g Fx(b)q(e)g(the)g Fv(a)o(rra)o(y)p 1647 582 V 15 w(of)p 1699 582 V 15 w(displace-)75 639 y(ments)c Fx(argumen)o(t.)19 b(Let)13 b Fv(c)g Fx(b)q(e)g(the)g Fv(count)h Fx(argumen)o(t.)19 b(Then)13 b(the)g(newly)h(created)e(datat)o(yp)q(e)h(has)f(a)h(t)o(yp)q (e)75 695 y(map)i(with)280 663 y Fe(P)323 676 y Fi(c)p Fh(\000)p Fg(1)323 708 y Ff(i)p Fg(=0)397 695 y Fv(B)p Fx([)p Fv(i)p Fx(])9 b Ft(\001)h Fp(n)523 702 y Ff(i)552 695 y Fx(en)o(tries:)189 779 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 760 y Fg(0)314 790 y(0)333 779 y Fp(;)e(disp)438 760 y Fg(0)438 790 y(0)467 779 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])o(\))p Fp(;)e(:::;)g Fx(\()p Fp(ty)r(pe)794 760 y Fg(0)794 790 y Ff(n)815 795 y Fs(0)832 779 y Fp(;)g(disp)937 760 y Fg(0)937 790 y Ff(n)958 795 y Fs(0)986 779 y Fx(+)j Fv(D)p Fx([)p Fv(0)p Fx(])o(\))p Fp(;)d(:::;)189 870 y Fx(\()p Fp(ty)r(pe)291 851 y Fg(0)291 881 y(0)310 870 y Fp(;)g(disp)415 851 y Fg(0)415 881 y(0)444 870 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])f(+)i(\()p Fv(B)p Fx([)p Fv(0)p Fx(])e Ft(\000)h Fx(1\))g Ft(\001)f Fp(ex)897 877 y Fg(0)917 870 y Fx(\))p Fp(;)f(:::;)g Fx(\()p Fp(ty)r(pe)1118 851 y Fg(0)1118 881 y Ff(n)1139 886 y Fs(0)1156 870 y Fp(;)g(disp)1261 851 y Fg(0)1261 881 y Ff(n)1282 886 y Fs(0)1311 870 y Fx(+)i Fv(D)p Fx([)p Fv(0)p Fx(])f(+)i(\()p Fv(B)p Fx([)p Fv(0)p Fx(])e Ft(\000)h Fv(1)p Fx(\))g Ft(\001)f Fp(ex)1764 877 y Fg(0)1784 870 y Fx(\))p Fp(;)f(:::;)189 962 y Fx(\()p Fp(ty)r(pe)291 943 y Fi(c)p Fh(\000)p Fg(1)291 974 y(0)356 962 y Fp(;)g(disp)461 943 y Fi(c)p Fh(\000)p Fg(1)461 974 y(0)536 962 y Fx(+)i Fv(D)p Fx([)p Fv(c)g Ft(\000)g Fv(1)p Fx(]\))p Fp(;)e(:::;)g Fx(\()o Fp(ty)r(pe)938 943 y Fi(c)p Fh(\000)p Fg(1)938 974 y Ff(n)959 981 y Fi(c)p Fc(\000)p Fs(1)1017 974 y Fh(\000)p Fg(1)1064 962 y Fp(;)g(disp)1169 943 y Fi(c)p Fh(\000)p Fg(1)1169 974 y Ff(n)1190 981 y Fi(c)p Fc(\000)p Fs(1)1249 974 y Fh(\000)p Fg(1)1306 962 y Fx(+)j Fv(D)p Fx([)p Fv(c)e Ft(\000)i Fv(1)p Fx(])o(\))p Fp(;)d(:::;)189 1064 y Fx(\()p Fp(ty)r(pe)291 1045 y Fi(c)p Fh(\000)p Fg(1)291 1077 y(0)356 1064 y Fp(;)g(disp)461 1045 y Fi(c)p Fh(\000)p Fg(1)461 1077 y(0)536 1064 y Fx(+)i Fv(D)p Fx([)p Fv(c)g Ft(\000)g Fv(1)p Fx(])g(+)g(\()p Fv(B)p Fx([)p Fv(c)g Ft(\000)h Fv(1)p Fx(])e Ft(\000)i Fx(1\))e Ft(\001)h Fp(ex)1141 1071 y Fi(c)p Fh(\000)p Fg(1)1207 1064 y Fx(\))p Fp(;)e(:::;)189 1156 y Fx(\()p Fp(ty)r(pe)291 1137 y Fi(c)p Fh(\000)p Fg(1)291 1168 y Ff(n)312 1175 y Fi(c)p Fc(\000)p Fs(1)372 1168 y Fh(\000)p Fg(1)419 1156 y Fp(;)g(disp)524 1137 y Fi(c)p Fh(\000)p Fg(1)524 1168 y Ff(n)545 1175 y Fi(c)p Fc(\000)p Fs(1)604 1168 y Fh(\000)p Fg(1)661 1156 y Fx(+)j Fv(D)p Fx([)p Fv(c)e Ft(\000)i Fv(1)p Fx(])e(+)i(\()p Fv(B)p Fx([)p Fv(c)f Ft(\000)g Fv(1)p Fx(])f Ft(\000)i Fv(1)p Fx(\))e Ft(\001)h Fp(ex)1266 1163 y Fi(c)p Fh(\000)p Fg(1)1332 1156 y Fx(\))p Ft(g)p Fp(:)166 1335 y Fx(A)15 b(call)h(to)f Fv(MPI)p 439 1335 V 16 w(TYPE)p 572 1335 V 16 w(HINDEXED\()g(count,)h(B,)f(D,)g(oldt)o(yp)q(e,)h(newt)o(yp)q (e\))g Fx(is)g(equiv)m(alen)o(t)h(to)d(a)h(call)75 1391 y(to)d Fv(MPI)p 213 1391 V 16 w(TYPE)p 346 1391 V 17 w(STRUCT\()h(count,)i(B,)e(D,)f(T,)h(newt)o(yp)q(e\))p Fx(,)h(where)f(eac)o(h)g(en)o(try)g(of)g Fv(T)g Fx(is)g(equal)h(to)e Fv(oldt)o(yp)q(e)p Fx(.)75 1511 y Fj(3.12.2)49 b(Address)16 b(and)g(extent)f(functions)75 1597 y Fx(The)g(displacemen)o(ts)i(in)e (a)g(general)h(datat)o(yp)q(e)e(are)h(relativ)o(e)g(to)g(some)f (initial)j(bu\013er)e(address.)20 b Fq(Abso-)75 1653 y(lute)15 b(addresses)c Fx(can)h(b)q(e)h(substituted)f(for)g(these)g (displacemen)o(ts:)20 b(w)o(e)12 b(treat)f(them)h(as)g(displacemen)o (ts)75 1710 y(relativ)o(e)17 b(to)e(\\address)h(zero,")g(the)g(start)f (of)h(the)g(address)h(space.)23 b(This)17 b(initial)h(address)e(zero)g (is)h(indi-)75 1766 y(cated)f(b)o(y)g(the)g(constan)o(t)f Fi(MPI)p 601 1766 13 2 v 14 w(BOTTOM)p Fx(.)g(Th)o(us,)g(a)h(datat)o (yp)q(e)f(can)h(sp)q(ecify)h(the)f(absolute)g(address)g(of)75 1823 y(the)e(en)o(tries)g(in)g(the)g(comm)o(unication)g(bu\013er,)g(in) h(whic)o(h)f(case)g(the)g Fv(buf)g Fx(argumen)o(t)f(is)h(passed)g(the)g (v)m(alue)75 1879 y Fi(MPI)p 152 1879 V 14 w(BOTTOM)p Fx(.)166 1936 y(The)31 b(address)g(of)f(a)h(lo)q(cation)g(in)h(memory)e (can)h(b)q(e)h(found)f(b)o(y)g(in)o(v)o(oking)g(the)g(function)75 1992 y Fv(MPI)p 160 1992 14 2 v 16 w(ADDRESS)p Fx(.)75 2143 y Fv(MPI)p 160 2143 V 16 w(ADDRESS\(lo)q(cation,)16 b(address\))117 2220 y Fk(IN)155 b Fv(lo)q(cation)437 b Fk(lo)q(cation)13 b(in)h(caller)f(memory)f(\(c)o(hoice\))117 2293 y(OUT)108 b Fv(address)449 b Fk(address)15 b(of)f(lo)q(cation)f (\(in)o(teger\))75 2418 y Fu(int)23 b(MPI)p 245 2418 15 2 v 17 w(Address\(void*)f(location,)h(MPI)p 907 2418 V 17 w(Aint)g(*address\))75 2504 y(MPI)p 150 2504 V 17 w(ADDRESS\(LOCATION,)e(ADDRESS,)i(IERROR\))170 2561 y()g (LOCATION\(*\))170 2617 y(INTEGER)g(ADDRESS,)g(IERROR)166 2704 y Fx(Returns)16 b(the)f(\(b)o(yte\))f(address)i(of)e Fv(lo)q(cation)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 68 76 68 75 bop 75 -100 a Fx(68)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fq(Example)h(3.24)23 b Fx(Using)15 b Fv(MPI)p 619 45 14 2 v 16 w(ADDRESS)h Fx(for)f(an)g(arra)o(y)l(.)147 151 y Fu(REAL)23 b(A\(100,100\))147 208 y(INTEGER)f(I1,)i(I2,)f(DIFF)147 264 y(CALL)g (MPI_ADDRESS\(A\(1,1\),)e(I1,)j(IERROR\))147 321 y(CALL)f (MPI_ADDRESS\(A\(10,10\),)e(I2,)i(IERROR\))147 377 y(DIFF)g(=)h(I2)f(-) h(I1)75 434 y(!)g(The)f(value)g(of)h(DIFF)f(is)h(909*sizeofreal;)e(the) h(values)g(of)h(I1)f(and)h(I2)f(are)75 490 y(!)h(implementation)e (dependent.)189 596 y Fl(A)n(dvic)n(e)e(to)i(users.)73 b Fx(C)20 b(users)h(ma)o(y)f(b)q(e)h(tempted)g(to)f(a)o(v)o(oid)h(the)f (usage)h(of)f Fv(MPI)p 1659 596 V 16 w(ADDRESS)189 653 y Fx(and)e(rely)g(on)g(the)g(a)o(v)m(ailabilit)o(y)i(of)d(the)h (address)g(op)q(erator)f(&.)29 b(Note,)17 b(ho)o(w)o(ev)o(er,)h(that)f Fi(&)h Fl(c)n(ast-)189 709 y(expr)n(ession)c Fx(is)j(a)e(p)q(oin)o (ter,)i(not)e(an)h(address.)22 b(ANSI)17 b(C)f(do)q(es)g(not)g(require) g(that)g(the)g(v)m(alue)h(of)e(a)189 766 y(p)q(oin)o(ter)f(\(or)f(the)h (p)q(oin)o(ter)g(cast)g(to)f Fi(int)p Fx(\))h(b)q(e)g(the)g(absolute)g (address)g(of)f(the)h(ob)s(ject)g(p)q(oin)o(ted)g(at)g(|)189 822 y(although)g(this)h(is)g(commonly)f(the)h(case.)k(F)l(urthermore,) 14 b(referencing)h(ma)o(y)f(not)g(ha)o(v)o(e)f(a)h(unique)189 879 y(de\014nition)19 b(on)f(mac)o(hines)h(with)g(a)e(segmen)o(ted)h (address)g(space.)29 b(The)18 b(use)h(of)e Fv(MPI)p 1659 879 V 16 w(ADDRESS)189 935 y Fx(to)g(\\reference")h(C)f(v)m(ariables)i (guaran)o(tees)e(p)q(ortabilit)o(y)i(to)e(suc)o(h)h(mac)o(hines)g(as)g (w)o(ell.)28 b(\()p Fl(End)18 b(of)189 992 y(advic)n(e)e(to)g(users.)p Fx(\))166 1098 y(The)f(follo)o(wing)h(auxiliary)h(functions)f(pro)o (vide)f(useful)i(information)e(on)g(deriv)o(ed)h(datat)o(yp)q(es.)75 1249 y Fv(MPI)p 160 1249 V 16 w(TYPE)p 293 1249 V 17 w(EXTENT\(datat)o(yp)q(e,)g(extent\))117 1326 y Fk(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 1401 y(OUT)108 b Fv(extent)471 b Fk(datat)o(yp)q(e)14 b(exten)o(t)h(\(in)o(teger\))75 1526 y Fu(int)23 b(MPI)p 245 1526 15 2 v 17 w(Type)p 358 1526 V 17 w(extent\(MPI)p 615 1526 V 16 w(Datatype)f(datatype,)h(int)h(*extent\))75 1612 y(MPI)p 150 1612 V 17 w(TYPE)p 263 1612 V 16 w(EXTENT\(DATATYPE,)e (EXTENT,)h(IERROR\))170 1668 y(INTEGER)g(DATATYPE,)g(EXTENT,)g(IERROR) 166 1755 y Fx(Returns)16 b(the)f(exten)o(t)g(of)g(a)f(datat)o(yp)q(e,)h (where)g(exten)o(t)g(is)h(as)f(de\014ned)h(in)g(Eq.)f(3.1)f(on)h(page)g (60.)75 1906 y Fv(MPI)p 160 1906 14 2 v 16 w(TYPE)p 293 1906 V 17 w(SIZE\(datat)o(yp)q(e,)g(size\))117 1983 y Fk(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 2058 y(OUT)108 b Fv(size)519 b Fk(datat)o(yp)q(e)14 b(size)h(\(in)o(teger\))75 2183 y Fu(int)23 b(MPI)p 245 2183 15 2 v 17 w(Type)p 358 2183 V 17 w(size\(MPI)p 567 2183 V 16 w(Datatype)g(datatype,)f(int)i(*size\))75 2269 y(MPI)p 150 2269 V 17 w(TYPE)p 263 2269 V 16 w(SIZE\(DATATYPE,)e(SIZE,) i(IERROR\))170 2326 y(INTEGER)f(DATATYPE,)g(SIZE,)g(IERROR)166 2412 y Fv(MPI)p 251 2412 14 2 v 16 w(TYPE)p 384 2412 V 17 w(SIZE)18 b Fx(returns)h(the)g(total)f(size,)j(in)f(b)o(ytes,)f (of)g(the)g(en)o(tries)g(in)h(the)f(t)o(yp)q(e)g(signature)75 2468 y(asso)q(ciated)c(with)g Fv(datat)o(yp)q(e)p Fx(;)h(i.e.,)f(the)f (total)g(size)i(of)e(the)h(data)f(in)i(a)e(message)g(that)g(w)o(ould)i (b)q(e)f(created)75 2525 y(with)20 b(this)g(datat)o(yp)q(e.)32 b(En)o(tries)20 b(that)f(o)q(ccur)h(m)o(ultiple)h(times)f(in)h(the)e (datat)o(yp)q(e)g(are)h(coun)o(ted)g(with)75 2581 y(their)c(m)o (ultiplicit)o(y)l(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 69 77 69 76 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fx(69)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(COUNT\(datat)o(yp)q(e,)17 b(count\))117 122 y Fk(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 197 y(OUT)108 b Fv(count)482 b Fk(datat)o(yp)q(e)14 b(coun)o(t)g(\(in)o(teger\))75 322 y Fu(int)23 b(MPI)p 245 322 15 2 v 17 w(Type)p 358 322 V 17 w(count\(MPI)p 591 322 V 16 w(Datatype)g(datatype,)f(int)i(*count\))75 408 y(MPI)p 150 408 V 17 w(TYPE)p 263 408 V 16 w(COUNT\(DATATYPE,)e (COUNT,)h(IERROR\))170 465 y(INTEGER)g(DATATYPE,)g(COUNT,)g(IERROR)166 551 y Fx(Returns)16 b(the)f(n)o(um)o(b)q(er)g(of)g(\\top-lev)o(el")h (en)o(tries)g(in)g(the)f(datat)o(yp)q(e.)75 673 y Fj(3.12.3)49 b(Lo)o(w)o(er-b)q(ound)17 b(and)g(upp)q(er-b)q(ound)g(ma)o(rk)n(ers)75 759 y Fx(It)e(is)g(often)g(con)o(v)o(enien)o(t)g(to)g(de\014ne)h (explicitly)h(the)e(lo)o(w)o(er)g(b)q(ound)h(and)f(upp)q(er)h(b)q(ound) f(of)g(a)g(t)o(yp)q(e)g(map,)75 815 y(and)i(o)o(v)o(erride)g(the)f (de\014nition)j(giv)o(en)e(b)o(y)g(Equation)g(3.1)e(on)i(page)g(60.)23 b(This)18 b(allo)o(ws)f(one)f(to)g(de\014ne)i(a)75 872 y(datat)o(yp)q(e)12 b(that)g(has)h(\\holes")g(at)f(its)h(b)q(eginning)i (or)d(its)h(end,)g(or)g(a)f(datat)o(yp)q(e)g(with)h(en)o(tries)g(that)f (extend)75 928 y(ab)q(o)o(v)o(e)18 b(the)g(upp)q(er)i(b)q(ound)f(or)f (b)q(elo)o(w)h(the)f(lo)o(w)o(er)g(b)q(ound.)31 b(Examples)18 b(of)g(suc)o(h)h(usage)f(are)g(pro)o(vided)75 985 y(in)i(Sec.)31 b(3.12.7.)d(T)l(o)19 b(ac)o(hiev)o(e)g(this,)h(w)o(e)e(add)h(t)o(w)o(o) e(additional)k(\\pseudo-datat)o(yp)q(es,")d Fv(MPI)p 1715 985 14 2 v 16 w(LB)h Fx(and)75 1041 y Fv(MPI)p 160 1041 V 16 w(UB)p Fx(,)d(that)e(can)i(b)q(e)h(used,)e(resp)q(ectiv)o (ely)l(,)j(to)d(mark)g(the)g(lo)o(w)o(er)h(b)q(ound)g(or)f(the)h(upp)q (er)h(b)q(ound)f(of)f(a)75 1098 y(datat)o(yp)q(e.)k(These)c (pseudo-datat)o(yp)q(es)g(o)q(ccup)o(y)g(no)f(space)h(\()p Fp(extent)p Fx(\()p Fv(MPI)p 1357 1098 V 17 w(LB)p Fx(\))d(=)h Fp(extent)p Fx(\()p Fv(MPI)p 1734 1098 V 17 w(UB)p Fx(\))g(=)75 1154 y(0\).)18 b(They)c(do)f(not)f(a\013ect)g(the)h(size)h(or)f(coun)o (t)g(of)f(a)h(datat)o(yp)q(e,)f(and)h(do)g(not)g(a\013ect)f(the)h(the)g (con)o(ten)o(t)g(of)f(a)75 1210 y(message)i(created)h(with)h(this)f (datat)o(yp)q(e.)k(Ho)o(w)o(ev)o(er,)14 b(they)h(do)g(a\013ect)f(the)h (de\014nition)h(of)f(the)g(exten)o(t)f(of)75 1267 y(a)e(datat)o(yp)q(e) g(and,)h(therefore,)g(a\013ect)f(the)g(outcome)h(of)f(a)g(replication)i (of)f(this)g(datat)o(yp)q(e)f(b)o(y)g(a)g(datat)o(yp)q(e)75 1323 y(constructor.)75 1430 y Fq(Example)18 b(3.25)23 b Fx(Let)17 b Fv(D)f(=)h(\(-3,)g(0,)g(6\))p Fx(;)f Fv(T)h(=)g(\(MPI)p 1003 1430 V 16 w(LB,)f(MPI)p 1185 1430 V 16 w(INT,)g(MPI)p 1388 1430 V 16 w(UB\))p Fx(,)h(and)g Fv(B)g(=)g(\(1,)f(1,)h(1\))p Fx(.)75 1486 y(Then)g(a)e(call)i(to)f Fv(MPI)p 459 1486 V 16 w(TYPE)p 592 1486 V 16 w(STRUCT\(3,)g(B,)g(D,)g(T,)f(t)o(yp)q (e1\))i Fx(creates)e(a)h(new)g(datat)o(yp)q(e)g(that)f(has)h(an)75 1543 y(exten)o(t)h(of)g(9)g(\(from)f(-3)h(to)g(5,)g(5)g(included\),)i (and)f(con)o(tains)f(an)g(in)o(teger)h(at)e(displacemen)o(t)j(0.)26 b(This)17 b(is)75 1599 y(the)f(datat)o(yp)q(e)f(de\014ned)j(b)o(y)d (the)h(sequence)i Ft(f)p Fv(\(lb,)d(-3\),)g(\(int,)h(0\),)f(\(ub,)i (6\))p Ft(g)e Fx(.)22 b(If)16 b(this)g(t)o(yp)q(e)g(is)g(replicated)75 1655 y(t)o(wice)i(b)o(y)f(a)g(call)h(to)f Fv(MPI)p 528 1655 V 16 w(TYPE)p 661 1655 V 17 w(CONTIGUOUS\(2,)h(t)o(yp)q(e1,)g(t)o (yp)q(e2\))g Fx(then)g(the)f(newly)i(created)e(t)o(yp)q(e)75 1712 y(can)d(b)q(e)h(describ)q(ed)h(b)o(y)e(the)g(sequence)h Ft(f)p Fv(\(lb,)e(-3\),)h(\(int,)g(0\),)f(\(int,9\),)g(\(ub,)i(15\))p Ft(g)d Fx(.)20 b(\(En)o(tries)14 b(of)f(t)o(yp)q(e)h Fv(lb)g Fx(or)75 1768 y Fv(ub)i Fx(can)f(b)q(e)h(deleted)h(if)f(they)f (are)g(not)g(at)f(the)h(end-p)q(oin)o(ts)i(of)e(the)g(datat)o(yp)q (e.\))166 1875 y(In)h(general,)f(if)189 1968 y Fp(T)6 b(y)r(pemap)12 b Fx(=)h Ft(f)p Fx(\()p Fp(ty)r(pe)562 1975 y Fg(0)581 1968 y Fp(;)8 b(disp)686 1975 y Fg(0)705 1968 y Fx(\))p Fp(;)g(:::;)g Fx(\()o Fp(ty)r(pe)905 1975 y Ff(n)p Fh(\000)p Fg(1)971 1968 y Fp(;)g(disp)1076 1975 y Ff(n)p Fh(\000)p Fg(1)1144 1968 y Fx(\))p Ft(g)p Fp(;)75 2062 y Fx(then)16 b(the)f Fq(lo)o(w)o(er)h(b)q(ound)g Fx(of)f Fp(T)6 b(y)r(pemap)15 b Fx(is)g(de\014ned)i(to)e(b)q(e)189 2186 y Fp(l)q(b)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))k(=)506 2114 y Fe(\()560 2158 y Fx(min)636 2165 y Ff(j)662 2158 y Fp(disp)746 2165 y Ff(j)1257 2158 y Fx(if)16 b(no)f(en)o(try)f(has)i (basic)f(t)o(yp)q(e)h Fv(lb)560 2214 y Fx(min)636 2221 y Ff(j)654 2214 y Ft(f)p Fp(disp)761 2221 y Ff(j)794 2214 y Fx(suc)o(h)g(that)e Fp(ty)r(pe)1079 2221 y Ff(j)1110 2214 y Fx(=)f Fv(lb)q Ft(g)41 b Fx(otherwise)166 2310 y(Similarly)l(,)17 b(the)e Fq(upp)q(er)j(b)q(ound)e Fx(of)e Fp(T)6 b(y)r(pemap)15 b Fx(is)h(de\014ned)h(to)d(b)q(e)189 2434 y Fp(ub)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))11 b(=)518 2362 y Fe(\()572 2406 y Fx(max)657 2413 y Ff(j)682 2406 y Fp(disp)766 2413 y Ff(j)794 2406 y Fx(+)g Fp(siz)r(eof)5 b Fx(\()p Fp(ty)r(pe)1072 2413 y Ff(j)1090 2406 y Fx(\))182 b(if)16 b(no)f(en)o(try)f(has)i(basic)f(t)o(yp)q(e)h Fv(ub)572 2462 y Fx(max)657 2469 y Ff(j)675 2462 y Ft(f)p Fp(disp)782 2469 y Ff(j)814 2462 y Fx(suc)o(h)g(that)e Fp(ty)r(pe)1099 2469 y Ff(j)1131 2462 y Fx(=)f Fv(ub)q Ft(g)41 b Fx(otherwise)166 2558 y(Then)189 2652 y Fp(extent)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))13 b(=)g Fp(ub)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))i Ft(\000)j Fp(l)q(b)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))i(+)i Fp(\017)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 70 78 70 77 bop 75 -100 a Fx(70)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fx(If)i Fp(ty)r(pe)208 52 y Ff(i)242 45 y Fx(requires)h(alignmen)o(t)g(to)e(a)h (b)o(yte)g(address)g(that)g(is)h(a)e(m)o(ultiple)k(of)c Fp(k)1460 52 y Ff(i)1474 45 y Fx(,)i(then)f Fp(\017)h Fx(is)g(the)f(least)75 102 y(nonnegativ)o(e)d(incremen)o(t)g(needed)g (to)f(round)g Fp(extent)p Fx(\()p Fp(T)6 b(y)r(pemap)p Fx(\))16 b(to)e(the)h(next)h(m)o(ultiple)h(of)d(max)1794 109 y Ff(i)1816 102 y Fp(k)1840 109 y Ff(i)1854 102 y Fx(.)166 158 y(The)g(formal)f(de\014nitions)j(giv)o(en)e(for)f(the)h(v) m(arious)g(datat)o(yp)q(e)f(constructors)g(apply)i(no)o(w,)e(with)h (the)75 214 y(amended)i(de\014nition)h(of)e Fq(exten)o(t)p Fx(.)166 271 y(The)f(t)o(w)o(o)e(functions)i(b)q(elo)o(w)h(can)e(b)q(e) i(used)f(for)f(\014nding)i(the)e(lo)o(w)o(er)g(b)q(ound)i(and)f(the)f (upp)q(er)i(b)q(ound)75 327 y(of)g(a)g(datat)o(yp)q(e.)75 478 y Fv(MPI)p 160 478 14 2 v 16 w(TYPE)p 293 478 V 17 w(LB\()f(datat)o(yp)q(e,)j(displacement\))117 556 y Fk(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 631 y(OUT)108 b Fv(displacement)342 b Fk(displacemen)o(t)12 b(of)g(lo)o(w)o(er)g(b)q(ound)g(from)f(origin,)g(in)h(b)o(ytes)h(\(in-) 905 687 y(teger\))75 812 y Fu(int)23 b(MPI)p 245 812 15 2 v 17 w(Type)p 358 812 V 17 w(lb\(MPI)p 519 812 V 16 w(Datatype)g(datatype,)g(int*)g(displacement\))75 898 y(MPI)p 150 898 V 17 w(TYPE)p 263 898 V 16 w(LB\()h(DATATYPE,)e (DISPLACEMENT,)h(IERROR\))170 954 y(INTEGER)g(DATATYPE,)g (DISPLACEMENT,)f(IERROR)75 1135 y Fv(MPI)p 160 1135 14 2 v 16 w(TYPE)p 293 1135 V 17 w(UB\()15 b(datat)o(yp)q(e,)h (displacement\))117 1213 y Fk(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 1288 y(OUT)108 b Fv(displacement)342 b Fk(displacemen)o(t)11 b(of)f(upp)q(er)i(b)q (ound)f(from)f(origin,)g(in)h(b)o(ytes)g(\(in-)905 1344 y(teger\))75 1469 y Fu(int)23 b(MPI)p 245 1469 15 2 v 17 w(Type)p 358 1469 V 17 w(ub\(MPI)p 519 1469 V 16 w(Datatype)g (datatype,)g(int*)g(displacement\))75 1555 y(MPI)p 150 1555 V 17 w(TYPE)p 263 1555 V 16 w(UB\()h(DATATYPE,)e(DISPLACEMENT,)h (IERROR\))170 1612 y(INTEGER)g(DATATYPE,)g(DISPLACEMENT,)f(IERROR)189 1748 y Fl(R)n(ationale.)40 b Fx(Note)15 b(that)f(the)i(rules)g(giv)o (en)g(in)g(Sec.)k(3.12.6)14 b(imply)i(that)f(it)g(is)h(erroneous)f(to)g (call)189 1804 y Fv(MPI)p 274 1804 14 2 v 15 w(TYPE)p 406 1804 V 17 w(EXTENT)p Fx(,)j Fv(MPI)p 714 1804 V 16 w(TYPE)p 847 1804 V 17 w(LB)p Fx(,)f(and)h Fv(MPI)p 1122 1804 V 15 w(TYPE)p 1254 1804 V 17 w(UB)g Fx(with)g(a)g(datat)o(yp)q(e)f (argumen)o(t)189 1861 y(that)11 b(con)o(tains)h(absolute)h(addresses,)f (unless)h(all)g(these)g(addreses)f(are)g(within)h(the)f(same)g(sequen-) 189 1917 y(tial)17 b(storage.)k(F)l(or)16 b(this)h(reason,)f(the)g Fv(displacement)g Fx(for)f(the)i(C)f(binding)i(in)f Fv(MPI)p 1620 1917 V 16 w(TYPE)p 1753 1917 V 17 w(UB)f Fx(is)189 1974 y(an)f Fu(int)g Fx(and)g(not)g(of)f(t)o(yp)q(e)i Fu(MPI)p 737 1974 15 2 v 17 w(Aint)p Fx(.)j(\()p Fl(End)c(of)i(r)n (ationale.)p Fx(\))75 2095 y Fj(3.12.4)49 b(Comm)o(it)13 b(and)k(free)75 2181 y Fx(A)j(datat)o(yp)q(e)f(ob)s(ject)g(has)g(to)g (b)q(e)h Fq(committed)g Fx(b)q(efore)g(it)g(can)f(b)q(e)i(used)f(in)g (a)f(comm)o(unication.)34 b(A)75 2238 y(committed)15 b(datat)o(yp)q(e)g(can)h(still)h(b)q(e)f(used)g(as)e(a)i(argumen)o(t)e (in)i(datat)o(yp)q(e)f(constructors.)20 b(There)15 b(is)h(no)75 2294 y(need)g(to)f(commit)g(basic)h(datat)o(yp)q(es.)j(They)c(are)g (\\pre-committed.")75 2445 y Fv(MPI)p 160 2445 14 2 v 16 w(TYPE)p 293 2445 V 17 w(COMMIT\(datat)o(yp)q(e\))117 2522 y Fk(INOUT)62 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(that)g(is)g(committed)e(\(handle\))75 2647 y Fu(int)23 b(MPI)p 245 2647 15 2 v 17 w(Type)p 358 2647 V 17 w(commit\(MPI)p 615 2647 V 16 w(Datatype)f(*datatype\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 71 79 71 78 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fx(71)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(TYPE)p 263 45 V 16 w(COMMIT\(DATATYPE,)22 b(IERROR\))170 102 y(INTEGER)h(DATATYPE,)g(IERROR)166 188 y Fx(The)14 b(commit)h(op)q (eration)f(commits)g(the)h(datat)o(yp)q(e,)e(that)h(is,)g(the)h(formal) e(description)j(of)e(a)g(com-)75 244 y(m)o(unication)h(bu\013er,)f(not) g(the)g(con)o(ten)o(t)f(of)h(that)g(bu\013er.)19 b(Th)o(us,)14 b(after)f(a)h(datat)o(yp)q(e)g(has)g(b)q(een)h(commit-)75 301 y(ted,)e(it)g(can)f(b)q(e)i(rep)q(eatedly)f(reused)h(to)e(comm)o (unicate)h(the)f(c)o(hanging)h(con)o(ten)o(t)f(of)g(a)h(bu\013er)f(or,) h(indeed,)75 357 y(the)i(con)o(ten)o(t)g(of)g(di\013eren)o(t)g (bu\013ers,)g(with)g(di\013eren)o(t)h(starting)e(addresses.)189 464 y Fl(A)n(dvic)n(e)19 b(to)i(implementors.)67 b Fx(The)20 b(system)f(ma)o(y)g(\\compile")i(at)e(commit)g(time)h(an)g(in)o(ternal) 189 520 y(represen)o(tation)f(for)f(the)h(datat)o(yp)q(e)g(that)f (facilitates)i(comm)o(unication,)h(e.g.)31 b(c)o(hange)19 b(from)f(a)189 577 y(compacted)12 b(represen)o(tation)h(to)f(a)g (\015at)h(represen)o(tation)f(of)h(the)f(datat)o(yp)q(e,)h(and)g (select)g(the)g(most)189 633 y(con)o(v)o(enien)o(t)i(transfer)g(mec)o (hanism.)20 b(\()p Fl(End)c(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))75 834 y Fv(MPI)p 160 834 14 2 v 16 w(TYPE)p 293 834 V 17 w(FREE\(datat)o(yp)q(e\))117 911 y Fk(INOUT)62 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(that)g(is)g(freed)h (\(handle\))75 1035 y Fu(int)23 b(MPI)p 245 1035 15 2 v 17 w(Type)p 358 1035 V 17 w(free\(MPI)p 567 1035 V 16 w(Datatype)g(*datatype\))75 1122 y(MPI)p 150 1122 V 17 w(TYPE)p 263 1122 V 16 w(FREE\(DATATYPE,)f(IERROR\))170 1178 y(INTEGER)h(DATATYPE,)g(IERROR)166 1265 y Fx(Marks)14 b(the)h(datat)o(yp)q(e)g(ob)s(ject)f(asso)q(ciated)h(with)h Fv(datat)o(yp)q(e)g Fx(for)f(deallo)q(cation)h(and)g(sets)e Fv(datat)o(yp)q(e)75 1321 y Fx(to)e Fi(MPI)p 205 1321 13 2 v 14 w(D)o(A)m(T)m(A)m(TYPE)p 431 1321 V 12 w(NULL)p Fx(.)h(An)o(y)f(comm)o(unication)h(that)e(is)i(curren)o(tly)g(using)g (this)g(datat)o(yp)q(e)e(will)j(com-)75 1378 y(plete)19 b(normally)l(.)31 b(Deriv)o(ed)19 b(datat)o(yp)q(es)f(that)g(w)o(ere)g (de\014ned)i(from)e(the)g(freed)h(datat)o(yp)q(e)f(are)g(not)g(af-)75 1434 y(fected.)75 1540 y Fq(Example)g(3.26)23 b Fx(The)12 b(follo)o(wing)h(co)q(de)h(fragmen)o(t)d(giv)o(es)i(examples)g(of)f (using)h Fv(MPI)p 1530 1540 14 2 v 16 w(TYPE)p 1663 1540 V 17 w(COMMIT)p Fx(.)75 1634 y Fu(INTEGER)23 b(type1,)g(type2)75 1691 y(CALL)g(MPI_TYPE_CONTIGUOUS\(5,)e(MPI_REAL,)i(type1,)g(ierr\))409 1747 y(!)h(new)f(type)h(object)f(created)75 1804 y(CALL)g (MPI_TYPE_COMMIT\(type1,)e(ierr\))409 1860 y(!)j(now)f(type1)g(can)h (be)f(used)h(for)f(communication)75 1917 y(type2)g(=)h(type1)409 1973 y(!)g(type2)f(can)g(be)h(used)f(for)h(communication)409 2029 y(!)g(\(it)f(is)h(a)g(handle)f(to)g(same)h(object)f(as)g(type1\)) 75 2086 y(CALL)g(MPI_TYPE_VECTOR\(3,)f(5,)h(4,)h(MPI_REAL,)f(type1,)g (ierr\))409 2142 y(!)h(new)f(uncommitted)g(type)g(object)g(created)75 2199 y(CALL)g(MPI_TYPE_COMMIT\(type1,)e(ierr\))409 2255 y(!)j(now)f(type1)g(can)h(be)f(used)h(anew)f(for)h(communication)166 2362 y Fx(F)l(reeing)19 b(a)f(datat)o(yp)q(e)f(do)q(es)i(not)f (a\013ect)f(an)o(y)h(other)g(datat)o(yp)q(e)f(that)h(w)o(as)f(built)j (from)d(the)i(freed)75 2418 y(datat)o(yp)q(e.)32 b(The)20 b(system)e(b)q(eha)o(v)o(es)i(as)f(if)h(input)g(datat)o(yp)q(e)f (argumen)o(ts)g(to)f(deriv)o(ed)j(datat)o(yp)q(e)e(con-)75 2474 y(structors)14 b(are)h(passed)g(b)o(y)h(v)m(alue.)189 2581 y Fl(A)n(dvic)n(e)g(to)i(implementors.)49 b Fx(The)17 b(implemen)o(tation)h(ma)o(y)e(k)o(eep)h(a)g(reference)g(coun)o(t)g(of) f(activ)o(e)189 2637 y(comm)o(unications)h(that)e(use)i(the)f(datat)o (yp)q(e,)g(in)h(order)f(to)g(decide)i(when)f(to)e(free)i(it.)23 b(Also,)17 b(one)189 2694 y(ma)o(y)d(implemen)o(t)j(constructors)e(of)g (deriv)o(ed)h(datat)o(yp)q(es)f(so)g(that)g(they)g(k)o(eep)h(p)q(oin)o (ters)g(to)e(their)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 72 80 72 79 bop 75 -100 a Fx(72)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Fx(datat)o(yp)q(e)d (argumen)o(ts,)g(rather)h(then)g(cop)o(ying)h(them.)k(In)15 b(this)h(case,)f(one)g(needs)h(to)e(k)o(eep)i(trac)o(k)189 102 y(of)f(activ)o(e)g(datat)o(yp)q(e)g(de\014nition)i(references)f(in) g(order)g(to)e(kno)o(w)h(when)h(a)f(datat)o(yp)q(e)g(ob)s(ject)g(can) 189 158 y(b)q(e)h(freed.)k(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g (implementors.)p Fx(\))75 280 y Fj(3.12.5)49 b(Use)16 b(of)h(general)f(datat)o(yp)q(es)g(in)h(comm)n(unication)75 365 y Fx(Handles)g(to)f(deriv)o(ed)h(datat)o(yp)q(es)f(can)h(b)q(e)f (passed)h(to)f(a)g(comm)o(unication)h(call)g(wherev)o(er)f(a)g(datat)o (yp)q(e)75 422 y(argumen)o(t)11 b(is)h(required.)19 b(A)12 b(call)g(of)f(the)h(form)e Fv(MPI)p 947 422 14 2 v 16 w(SEND\(buf,)j(count,)g(datat)o(yp)q(e)g(,)f(...\))p Fx(,)e(where)i Fv(count)i Fp(>)75 478 y Fv(1)p Fx(,)e(is)h(in)o (terpreted)g(as)g(if)g(the)f(call)i(w)o(as)d(passed)i(a)f(new)h(datat)o (yp)q(e)f(whic)o(h)h(is)g(the)g(concatenation)g(of)f Fv(count)75 535 y Fx(copies)j(of)e Fv(datat)o(yp)q(e)p Fx(.)21 b(Th)o(us,)13 b Fv(MPI)p 666 535 V 16 w(SEND\(buf,)h(count,)h (datat)o(yp)q(e,)h(dest,)f(tag,)e(comm)m(\))d Fx(is)15 b(equiv)m(alen)o(t)g(to,)75 628 y Fu(MPI_TYPE_CONTIGUOUS\(count,)20 b(datatype,)j(newtype\))75 684 y(MPI_TYPE_COMMIT\(newtype\))75 741 y(MPI_SEND\(buf,)f(1,)i(newtype,)e(dest,)i(tag,)f(comm\).)75 834 y Fx(Similar)i(statemen)o(ts)e(apply)h(to)g(all)g(other)g(comm)o (unication)g(functions)h(that)e(ha)o(v)o(e)g(a)h Fv(count)h Fx(and)75 890 y Fv(datat)o(yp)q(e)17 b Fx(argumen)o(t.)166 947 y(Supp)q(ose)22 b(that)e(a)g(send)h(op)q(eration)g Fv(MPI)p 900 947 V 16 w(SEND\(buf,)h(count,)h(datat)o(yp)q(e,)g(dest,)g (tag,)f(comm)m(\))17 b Fx(is)75 1003 y(executed,)f(where)f Fv(datat)o(yp)q(e)i Fx(has)e(t)o(yp)q(e)g(map,)189 1097 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 1104 y Fg(0)333 1097 y Fp(;)8 b(disp)438 1104 y Fg(0)457 1097 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 1104 y Ff(n)p Fh(\000)p Fg(1)723 1097 y Fp(;)h(disp)828 1104 y Ff(n)p Fh(\000)p Fg(1)895 1097 y Fx(\))p Ft(g)p Fp(;)75 1190 y Fx(and)19 b(exten)o(t)f Fp(extent)p Fx(.)31 b(\(Empt)o(y)18 b(en)o(tries)h(of)f(\\pseudo-t)o (yp)q(e")36 b Fi(MPI)p 1252 1190 13 2 v 14 w(UB)18 b Fx(and)35 b Fi(MPI)p 1524 1190 V 15 w(LB)18 b Fx(are)h(not)f(listed)75 1246 y(in)g(the)e(t)o(yp)q(e)h(map,)g(but)g(they)g(a\013ect)f(the)h(v)m (alue)h(of)e Fp(extent)p Fx(.\))26 b(The)17 b(send)g(op)q(eration)g (sends)g Fp(n)12 b Ft(\001)f Fv(count)75 1303 y Fx(en)o(tries,)17 b(where)g(en)o(try)g Fp(i)10 b Ft(\001)h Fp(n)g Fx(+)h Fp(j)19 b Fx(is)e(at)f(lo)q(cation)i Fp(addr)1034 1310 y Ff(i;j)1088 1303 y Fx(=)e Fv(buf)f Fx(+)c Fp(extent)i Ft(\001)e Fp(i)f Fx(+)i Fp(disp)1581 1310 y Ff(j)1615 1303 y Fx(and)17 b(has)g(t)o(yp)q(e)75 1359 y Fp(ty)r(pe)159 1366 y Ff(j)178 1359 y Fx(,)d(for)f Fp(i)g Fx(=)g(0)p Fp(;)8 b(:::;)g Fv(co)o(unt)f Ft(\000)i Fx(1)k(and)i Fp(j)g Fx(=)e(0)p Fp(;)8 b(:::;)g(n)d Ft(\000)k Fx(1.)19 b(These)c(en)o(tries)f(need)h(not)f(b)q(e)h(con)o(tiguous,)f(nor)75 1415 y(distinct;)i(their)g(order)f(can)g(b)q(e)h(arbitrary)l(.)166 1472 y(The)g(v)m(ariable)h(stored)e(at)g(address)g Fp(addr)879 1479 y Ff(i;j)933 1472 y Fx(in)i(the)e(calling)j(program)c(should)j(b)q (e)f(of)f(a)g(t)o(yp)q(e)h(that)75 1528 y(matc)o(hes)11 b Fp(ty)r(pe)331 1535 y Ff(j)349 1528 y Fx(,)h(where)f(t)o(yp)q(e)g (matc)o(hing)h(is)f(de\014ned)i(as)d(in)i(section)g(3.3.1.)17 b(The)11 b(message)g(sen)o(t)g(con)o(tains)75 1585 y Fp(n)f Ft(\001)g Fv(count)17 b Fx(en)o(tries,)e(where)g(en)o(try)g Fp(i)10 b Ft(\001)g Fp(n)g Fx(+)g Fp(j)18 b Fx(has)d(t)o(yp)q(e)g Fp(ty)r(pe)1100 1592 y Ff(j)1119 1585 y Fx(.)166 1641 y(Similarly)l(,)g(supp)q(ose)f(that)e(a)h(receiv)o(e)h(op)q(eration)f Fv(MPI)p 1100 1641 14 2 v 16 w(RECV\(buf,)h(count,)g(datat)o(yp)q(e,)h (source,)e(tag,)75 1698 y(comm)m(,)f(status\))17 b Fx(is)e(executed,)h (where)g Fv(datat)o(yp)q(e)g Fx(has)g(t)o(yp)q(e)f(map,)189 1791 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 1798 y Fg(0)333 1791 y Fp(;)8 b(disp)438 1798 y Fg(0)457 1791 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 1798 y Ff(n)p Fh(\000)p Fg(1)723 1791 y Fp(;)h(disp)828 1798 y Ff(n)p Fh(\000)p Fg(1)895 1791 y Fx(\))p Ft(g)p Fp(;)75 1884 y Fx(with)17 b(exten)o(t)g Fp(extent)p Fx(.)25 b(\(Again,)17 b(empt)o(y)g(en)o (tries)g(of)f(\\pseudo-t)o(yp)q(e")32 b Fi(MPI)p 1385 1884 13 2 v 15 w(UB)16 b Fx(and)32 b Fi(MPI)p 1653 1884 V 14 w(LB)17 b Fx(are)f(not)75 1941 y(listed)g(in)f(the)f(t)o(yp)q(e)h (map,)f(but)g(they)h(a\013ect)e(the)i(v)m(alue)h(of)e Fp(extent)p Fx(.\))20 b(This)15 b(receiv)o(e)h(op)q(eration)e(receiv)o (es)75 1997 y Fp(n)e Ft(\001)f Fv(count)19 b Fx(en)o(tries,)f(where)f (en)o(try)g Fp(i)12 b Ft(\001)f Fp(n)h Fx(+)g Fp(j)19 b Fx(is)f(at)f(lo)q(cation)h Fv(buf)e Fx(+)c Fp(extent)h Ft(\001)e Fp(i)g Fx(+)h Fp(disp)1579 2004 y Ff(j)1614 1997 y Fx(and)18 b(has)f(t)o(yp)q(e)75 2053 y Fp(ty)r(pe)159 2060 y Ff(j)178 2053 y Fx(.)i(If)c(the)f(incoming)h(message)f(consists) g(of)g Fp(k)h Fx(elemen)o(ts,)g(then)f(w)o(e)g(m)o(ust)g(ha)o(v)o(e)g Fp(k)f Ft(\024)g Fp(n)c Ft(\001)e Fv(count)p Fx(;)16 b(the)75 2110 y Fp(i)10 b Ft(\001)f Fp(n)i Fx(+)f Fp(j)s Fx(-th)15 b(elemen)o(t)h(of)f(the)g(message)g(should)h(ha)o(v)o(e)f(a)f (t)o(yp)q(e)i(that)e(matc)o(hes)h Fp(ty)r(pe)1521 2117 y Ff(j)1540 2110 y Fx(.)166 2166 y(T)o(yp)q(e)10 b(matc)o(hing)g(is)h (de\014ned)h(according)e(to)g(the)g(t)o(yp)q(e)g(signature)h(of)e(the)h (corresp)q(onding)i(datat)o(yp)q(es,)75 2223 y(that)k(is,)h(the)f (sequence)i(of)e(basic)i(t)o(yp)q(e)e(comp)q(onen)o(ts.)24 b(T)o(yp)q(e)17 b(matc)o(hing)g(do)q(es)f(not)h(dep)q(end)h(on)e(some) 75 2279 y(asp)q(ects)k(of)f(the)h(datat)o(yp)q(e)f(de\014nition,)j(suc) o(h)e(as)g(the)f(displacemen)o(ts)j(\(la)o(y)o(out)c(in)j(memory\))e (or)g(the)75 2336 y(in)o(termediate)d(t)o(yp)q(es)f(used.)75 2441 y Fq(Example)j(3.27)23 b Fx(This)c(example)h(sho)o(ws)e(that)h(t)o (yp)q(e)g(matc)o(hing)g(is)h(de\014ned)g(in)g(terms)e(of)h(the)g(basic) 75 2498 y(t)o(yp)q(es)c(that)g(a)g(deriv)o(ed)h(t)o(yp)q(e)f(consists)g (of.)75 2591 y Fu(...)75 2647 y(CALL)23 b(MPI_TYPE_CONTIGUOUS\()e(2,)j (MPI_REAL,)f(type2,)g(...\))75 2704 y(CALL)g(MPI_TYPE_CONTIGUOUS\()e (4,)j(MPI_REAL,)f(type4,)g(...\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 73 81 73 80 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(73)75 45 y Fu(CALL)23 b(MPI_TYPE_CONTIGUOUS\()e(2,)j(type2,)f (type22,)g(...\))75 102 y(...)75 158 y(CALL)g(MPI_SEND\()g(a,)h(4,)f (MPI_REAL,)g(...\))75 214 y(CALL)g(MPI_SEND\()g(a,)h(2,)f(type2,)g (...\))75 271 y(CALL)g(MPI_SEND\()g(a,)h(1,)f(type22,)g(...\))75 327 y(CALL)g(MPI_SEND\()g(a,)h(1,)f(type4,)g(...\))75 384 y(...)75 440 y(CALL)g(MPI_RECV\()g(a,)h(4,)f(MPI_REAL,)g(...\))75 497 y(CALL)g(MPI_RECV\()g(a,)h(2,)f(type2,)g(...\))75 553 y(CALL)g(MPI_RECV\()g(a,)h(1,)f(type22,)g(...\))75 610 y(CALL)g(MPI_RECV\()g(a,)h(1,)f(type4,)g(...\))75 701 y Fx(Eac)o(h)15 b(of)g(the)g(sends)h(matc)o(hes)f(an)o(y)f(of)h (the)g(receiv)o(es.)166 804 y(A)k(datat)o(yp)q(e)g(ma)o(y)f(sp)q(ecify) i(o)o(v)o(erlapping)g(en)o(tries.)32 b(If)19 b(suc)o(h)h(a)f(datat)o (yp)q(e)f(is)i(used)f(in)h(a)f(receiv)o(e)75 861 y(op)q(eration,)13 b(that)f(is,)i(if)f(some)f(part)h(of)f(the)h(receiv)o(e)g(bu\013er)g (is)h(written)e(more)h(than)f(once)h(b)o(y)g(the)g(receiv)o(e)75 917 y(op)q(eration,)i(then)h(the)f(call)h(is)g(erroneous.)166 974 y(Supp)q(ose)22 b(that)f Fv(MPI)p 540 974 14 2 v 16 w(RECV\(buf,)j(count,)g(datat)o(yp)q(e,)g(dest,)g(tag,)f(comm)m(,)c (status\))k Fx(is)f(executed,)75 1030 y(where)15 b Fv(datat)o(yp)q(e)i Fx(has)e(t)o(yp)q(e)h(map,)189 1122 y Ft(f)p Fx(\()p Fp(ty)r(pe)314 1129 y Fg(0)333 1122 y Fp(;)8 b(disp)438 1129 y Fg(0)457 1122 y Fx(\))p Fp(;)g(:::;)f Fx(\()p Fp(ty)r(pe)657 1129 y Ff(n)p Fh(\000)p Fg(1)723 1122 y Fp(;)h(disp)828 1129 y Ff(n)p Fh(\000)p Fg(1)895 1122 y Fx(\))p Ft(g)p Fp(:)75 1213 y Fx(The)14 b(receiv)o(ed)h(message)e (need)i(not)f(\014ll)h(all)g(the)e(receiv)o(e)i(bu\013er,)f(nor)f(do)q (es)i(it)f(need)g(to)f(\014ll)j(a)d(n)o(um)o(b)q(er)h(of)75 1270 y(lo)q(cations)g(whic)o(h)h(is)f(a)f(m)o(ultiple)j(of)d Fp(n)p Fx(.)20 b(An)o(y)13 b(n)o(um)o(b)q(er,)h Fp(k)q Fx(,)g(of)f(basic)i(elemen)o(ts)f(can)g(b)q(e)g(receiv)o(ed,)h(where)75 1326 y(0)d Ft(\024)h Fp(k)h Ft(\024)f Fv(count)8 b Ft(\001)f Fp(n)p Fx(.)19 b(The)13 b(n)o(um)o(b)q(er)h(of)f(basic)h(elemen)o(ts)g (receiv)o(ed)h(can)e(b)q(e)h(retriev)o(ed)g(from)f Fv(status)i Fx(using)75 1382 y(the)g(query)h(function)g Fv(MPI)p 543 1382 V 16 w(GET)p 647 1382 V 16 w(ELEMENTS)p Fx(.)75 1533 y Fv(MPI)p 160 1533 V 16 w(GET)p 264 1533 V 17 w(ELEMENTS\()f (status,)h(datat)o(yp)q(e,)h(count\))117 1611 y Fk(IN)155 b Fv(status)476 b Fk(return)15 b(status)g(of)e(receiv)o(e)i(op)q (eration)f(\(Status\))117 1685 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(used)h(b)o(y)f(receiv)o(e)h(op)q(eration)f (\(handle\))117 1760 y(OUT)108 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(basic)f(elemen)o(ts)g(\(in)o(teger\))75 1884 y Fu(int)23 b(MPI)p 245 1884 15 2 v 17 w(Get)p 334 1884 V 17 w(elements\(MPI)p 639 1884 V 16 w(Status)g(status,)f(MPI)p 1084 1884 V 17 w(Datatype)h(datatype,)g(int)g(*count\))75 1971 y(MPI)p 150 1971 V 17 w(GET)p 239 1971 V 17 w(ELEMENTS\(STATUS,)e (DATATYPE,)i(COUNT,)g(IERROR\))170 2027 y(INTEGER)g(STATUS\(MPI)p 604 2027 V 16 w(STATUS)p 764 2027 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h (IERROR)166 2114 y Fx(The)d(previously)i(de\014ned)f(function,)h Fv(MPI)p 930 2114 14 2 v 16 w(GET)p 1034 2114 V 16 w(COUNT)f Fx(\(Sec.)35 b(3.2.5\),)19 b(has)h(a)g(di\013eren)o(t)g(b)q(e-)75 2170 y(ha)o(vior.)28 b(It)17 b(returns)h(the)g(n)o(um)o(b)q(er)g(of)g (\\top-lev)o(el)g(elemen)o(ts")h(receiv)o(ed.)29 b(In)18 b(the)g(previous)g(example,)75 2227 y Fv(MPI)p 160 2227 V 16 w(GET)p 264 2227 V 17 w(COUNT)11 b Fx(ma)o(y)e(return)h(an)o(y)g (in)o(teger)g(v)m(alue)i Fp(k)q Fx(,)e(where)h(0)h Ft(\024)h Fp(k)h Ft(\024)f Fv(count)q Fx(.)18 b(If)11 b Fv(MPI)p 1618 2227 V 15 w(GET)p 1721 2227 V 17 w(COUNT)75 2283 y Fx(returns)29 b Fp(k)q Fx(,)j(then)d(the)g(n)o(um)o(b)q(er)h(of)e (basic)i(elemen)o(ts)f(receiv)o(ed)i(\(and)d(the)h(v)m(alue)i(returned) e(b)o(y)75 2340 y Fv(MPI)p 160 2340 V 16 w(GET)p 264 2340 V 17 w(ELEMENTS)p Fx(\))19 b(is)h Fp(n)13 b Ft(\001)f Fp(k)q Fx(.)32 b(If)20 b(the)f(n)o(um)o(b)q(er)h(of)f(basic)h(elemen)o (ts)g(receiv)o(ed)g(is)g(not)f(a)g(m)o(ulti-)75 2396 y(ple)g(of)e Fp(n)p Fx(,)i(that)e(is,)h(if)h(the)f(receiv)o(e)h(op)q (eration)f(has)f(not)h(receiv)o(ed)h(an)f(in)o(tegral)g(n)o(um)o(b)q (er)g(of)f Fv(datat)o(yp)q(e)75 2453 y Fx(\\copies,")e(then)h Fv(MPI)p 456 2453 V 16 w(GET)p 560 2453 V 16 w(COUNT)g Fx(returns)f(the)h(v)m(alue)g Fi(MPI)p 1175 2453 13 2 v 14 w(UNDEFINED)p Fx(.)75 2556 y Fq(Example)i(3.28)23 b Fx(Usage)14 b(of)h Fv(MPI)p 676 2556 14 2 v 16 w(GET)p 780 2556 V 17 w(COUNT)h Fx(and)f Fv(MPI)p 1139 2556 V 16 w(GET)p 1243 2556 V 17 w(ELEMENT)p Fx(.)75 2647 y Fu(...)75 2704 y(CALL)23 b(MPI_TYPE_CONTIGUOUS\(2,)e(MPI_REAL,)i (Type2,)g(ierr\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 74 82 74 81 bop 75 -100 a Fx(74)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fu(CALL)23 b(MPI_TYPE_COMMIT\(Type2,)e(ierr\))75 102 y(...)75 158 y(CALL)i(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 214 y(IF\(rank.EQ.0\))f(THEN)218 271 y(CALL)h(MPI_SEND\(a,)g(2,)g (MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))218 327 y(CALL)g(MPI_SEND\(a,)g (3,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))75 384 y(ELSE)218 440 y(CALL)g(MPI_RECV\(a,)g(2,)g(Type2,)g(0,)h(0,)g(comm,)f(stat,)g (ierr\))218 497 y(CALL)g(MPI_GET_COUNT\(stat,)f(Type2,)h(i,)g(ierr\)) 119 b(!)24 b(returns)f(i=1)218 553 y(CALL)g(MPI_GET_ELEMENTS\(stat,)e (Type2,)i(i,)h(ierr\))47 b(!)24 b(returns)f(i=2)218 610 y(CALL)g(MPI_RECV\(a,)g(2,)g(Type2,)g(0,)h(0,)g(comm,)f(stat,)g(ierr\)) 218 666 y(CALL)g(MPI_GET_COUNT\(stat,)f(Type2,)h(i,)g(ierr\))119 b(!)24 b(returns)f(i=MPI_UNDEFINED)218 723 y(CALL)g (MPI_GET_ELEMENTS\(stat,)e(Type2,)i(i,)h(ierr\))47 b(!)24 b(returns)f(i=3)75 779 y(END)g(IF)166 885 y Fx(The)15 b(function)h Fv(MPI)p 522 885 14 2 v 16 w(GET)p 626 885 V 17 w(ELEMENTS)f Fx(can)g(also)g(b)q(e)g(used)h(after)e(a)h(prob)q(e)g (to)g(\014nd)h(the)f(n)o(um)o(b)q(er)75 942 y(of)20 b(elemen)o(ts)h(in) g(the)f(prob)q(ed)h(message.)34 b(Note)20 b(that)f(the)h(t)o(w)o(o)f (functions)i Fv(MPI)p 1507 942 V 16 w(GET)p 1611 942 V 17 w(COUNT)g Fx(and)75 998 y Fv(MPI)p 160 998 V 16 w(GET)p 264 998 V 17 w(ELEMENTS)15 b Fx(return)g(the)g(same)g(v)m (alues)i(when)e(they)h(are)e(used)i(with)g(basic)g(datat)o(yp)q(es.)189 1104 y Fl(R)n(ationale.)38 b Fx(The)14 b(extension)g(giv)o(en)g(to)f (the)h(de\014nition)h(of)e Fv(MPI)p 1303 1104 V 16 w(GET)p 1407 1104 V 16 w(COUNT)i Fx(seems)e(natural:)189 1161 y(one)22 b(w)o(ould)h(exp)q(ect)h(this)f(function)g(to)f(return)h(the)g (v)m(alue)g(of)g(the)f Fv(count)i Fx(argumen)o(t,)f(when)189 1217 y(the)c(receiv)o(e)h(bu\013er)g(is)f(\014lled.)34 b(Sometimes)20 b Fv(datat)o(yp)q(e)h Fx(represen)o(ts)e(a)g(basic)h (unit)g(of)f(data)g(one)189 1274 y(w)o(an)o(ts)h(to)g(transfer,)i(for)e (example,)j(a)e(record)g(in)h(an)g(arra)o(y)e(of)g(records)h (\(structures\).)37 b(One)189 1330 y(should)15 b(b)q(e)h(able)f(to)f (\014nd)i(out)e(ho)o(w)g(man)o(y)g(comp)q(onen)o(ts)h(w)o(ere)g(receiv) o(ed)g(without)g(b)q(othering)h(to)189 1387 y(divide)j(b)o(y)e(the)g(n) o(um)o(b)q(er)g(of)g(elemen)o(ts)h(in)g(eac)o(h)f(comp)q(onen)o(t.)26 b(Ho)o(w)o(ev)o(er,)16 b(on)h(other)g(o)q(ccasions,)189 1443 y Fv(datat)o(yp)q(e)g Fx(is)f(used)g(to)e(de\014ne)j(a)e(complex)h (la)o(y)o(out)f(of)g(data)g(in)h(the)f(receiv)o(er)h(memory)l(,)f(and)h (do)q(es)189 1500 y(not)h(represen)o(t)g(a)g(basic)i(unit)f(of)f(data)g (for)g(transfers.)26 b(In)18 b(suc)o(h)g(cases,)f(one)h(needs)g(to)f (use)h(the)189 1556 y(function)e Fv(MPI)p 452 1556 V 15 w(GET)p 555 1556 V 17 w(ELEMENTS)p Fx(.)f(\()p Fl(End)h(of)g(r)n (ationale.)p Fx(\))189 1662 y Fl(A)n(dvic)n(e)j(to)h(implementors.)66 b Fx(The)20 b(de\014nition)h(implies)h(that)c(a)i(receiv)o(e)g(cannot)f (c)o(hange)h(the)189 1719 y(v)m(alue)c(of)f(storage)f(outside)h(the)h (en)o(tries)f(de\014ned)i(to)d(comp)q(ose)h(the)h(comm)o(unication)f (bu\013er.)20 b(In)189 1775 y(particular,)13 b(the)f(de\014nition)i (implies)g(that)d(padding)i(space)g(in)g(a)e(structure)h(should)h(not)f (b)q(e)h(mo)q(d-)189 1832 y(i\014ed)h(when)g(suc)o(h)f(a)g(structure)g (is)h(copied)g(from)f(one)g(pro)q(cess)g(to)g(another.)19 b(This)14 b(w)o(ould)f(prev)o(en)o(t)189 1888 y(the)i(ob)o(vious)h (optimization)g(of)f(cop)o(ying)h(the)f(structure,)g(together)g(with)g (the)h(padding,)g(as)f(one)189 1945 y(con)o(tiguous)d(blo)q(c)o(k.)20 b(The)13 b(implemen)o(tation)g(is)g(free)g(to)f(do)g(this)h (optimization)h(when)f(it)g(do)q(es)f(not)189 2001 y(impact)17 b(the)h(outcome)f(of)g(the)h(computation.)26 b(The)18 b(user)f(can)h(\\force")f(this)h(optimization)g(b)o(y)189 2058 y(explicitly)e(including)h(padding)e(as)e(part)g(of)h(the)f (message.)19 b(\()p Fl(End)c(of)g(advic)n(e)g(to)g(implementors.)p Fx(\))75 2179 y Fj(3.12.6)49 b(Co)o(rrect)15 b(use)h(of)g(addresses)75 2265 y Fx(Successiv)o(ely)23 b(declared)f(v)m(ariables)h(in)e(C)g(or)g (F)l(ortran)e(are)i(not)g(necessarily)h(stored)f(at)f(con)o(tiguous)75 2322 y(lo)q(cations.)i(Th)o(us,)16 b(care)g(m)o(ust)f(b)q(e)h (exercised)i(that)d(displacemen)o(ts)i(do)f(not)f(cross)h(from)f(one)h (v)m(ariable)75 2378 y(to)i(another.)28 b(Also,)19 b(in)g(mac)o(hines)g (with)g(a)f(segmen)o(ted)g(address)g(space,)h(addresses)g(are)f(not)f (unique)75 2434 y(and)f(address)g(arithmetic)g(has)g(some)f(p)q (eculiar)j(prop)q(erties.)k(Th)o(us,)16 b(the)g(use)g(of)f Fq(addresses)p Fx(,)g(that)g(is,)75 2491 y(displacemen)o(ts)i(relativ)o (e)e(to)g(the)g(start)f(address)h Fi(MPI)p 1008 2491 13 2 v 15 w(BOTTOM)p Fx(,)f(has)h(to)g(b)q(e)h(restricted.)166 2547 y(V)l(ariables)g(b)q(elong)f(to)f(the)g(same)g Fq(sequen)o(tial)k (storage)c Fx(if)h(they)f(b)q(elong)i(to)e(the)g(same)g(arra)o(y)l(,)g (to)75 2604 y(the)k(same)g Fv(COMMON)g Fx(blo)q(c)o(k)h(in)g(F)l (ortran,)e(or)h(to)f(the)h(same)g(structure)f(in)i(C.)f(V)l(alid)h (addresses)f(are)75 2660 y(de\014ned)f(recursiv)o(ely)f(as)f(follo)o (ws:)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 75 83 75 82 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(75)131 45 y(1.)22 b(The)g(function)g Fv(MPI)p 558 45 14 2 v 16 w(ADDRESS)h Fx(returns)e(a)h(v)m(alid)h(address,)h (when)e(passed)g(as)f(argumen)o(t)g(a)189 102 y(v)m(ariable)16 b(of)f(the)g(calling)i(program.)131 195 y(2.)22 b(The)d Fv(buf)h Fx(argumen)o(t)f(of)g(a)g(comm)o(unication)h(function)g(ev)m (aluates)h(to)d(a)h(v)m(alid)i(address,)f(when)189 252 y(passed)15 b(as)g(argumen)o(t)f(a)h(v)m(ariable)i(of)e(the)g(calling)i (program.)131 346 y(3.)22 b(If)15 b Fv(v)g Fx(is)h(a)f(v)m(alid)i (address,)d(and)i Fv(i)f Fx(is)g(an)h(in)o(teger,)f(then)g Fv(v+i)g Fx(is)h(a)f(v)m(alid)i(address,)e(pro)o(vided)h Fv(v)f Fx(and)189 402 y Fv(v+i)g Fx(are)g(in)h(the)f(same)g(sequen)o (tial)h(storage.)131 496 y(4.)22 b(If)15 b Fv(v)g Fx(is)h(a)f(v)m(alid) i(address)e(then)g Fi(MPI)p 806 496 13 2 v 15 w(BOTTOM)e(+)h(v)i Fx(is)f(a)g(v)m(alid)i(address.)166 602 y(A)22 b(correct)g(program)g (uses)g(only)h(v)m(alid)i(addresses)d(to)g(iden)o(tify)i(the)e(lo)q (cations)h(of)f(en)o(tries)h(in)75 659 y(comm)o(unication)16 b(bu\013ers.)k(F)l(urthermore,)14 b(if)h Fv(u)h Fx(and)f Fv(v)g Fx(are)g(t)o(w)o(o)f(v)m(alid)j(addresses,)e(then)g(the)g(\(in)o (teger\))75 715 y(di\013erence)j Fv(u)g(-)g(v)f Fx(can)g(b)q(e)h (computed)g(only)f(if)h(b)q(oth)g Fv(u)f Fx(and)h Fv(v)f Fx(are)g(in)h(the)f(same)g(sequen)o(tial)i(storage.)75 772 y(No)c(other)g(arithmetic)h(op)q(erations)f(can)g(b)q(e)h (meaningfully)h(executed)f(on)g(addresses.)166 828 y(The)21 b(rules)h(ab)q(o)o(v)o(e)f(imp)q(ose)g(no)g(constrain)o(ts)g(on)g(the)g (use)g(of)g(deriv)o(ed)h(datat)o(yp)q(es,)f(as)g(long)g(as)75 884 y(they)d(are)f(used)h(to)f(de\014ne)h(a)g(comm)o(unication)g (bu\013er)f(that)g(is)h(wholly)h(con)o(tained)f(within)h(the)e(same)75 941 y(sequen)o(tial)23 b(storage.)38 b(Ho)o(w)o(ev)o(er,)22 b(the)g(construction)g(of)f(a)h(comm)o(unication)g(bu\013er)g(that)f (con)o(tains)75 997 y(v)m(ariables)f(that)f(are)g(not)g(within)h(the)f (same)g(sequen)o(tial)h(storage)e(m)o(ust)h(ob)q(ey)g(certain)h (restrictions.)75 1054 y(Basically)l(,)g(a)e(comm)o(unication)h (bu\013er)f(with)g(v)m(ariables)i(that)d(are)h(not)g(within)h(the)f (same)g(sequen)o(tial)75 1110 y(storage)g(can)h(b)q(e)h(used)g(only)g (b)o(y)f(sp)q(ecifying)i(in)f(the)g(comm)o(unication)g(call)g Fv(buf)g(=)g(MPI)p 1651 1110 14 2 v 16 w(BOTTOM)p Fx(,)75 1167 y Fv(count)i(=)g(1)p Fx(,)g(and)f(using)h(a)f Fv(datat)o(yp)q(e)i Fx(argumen)o(t)d(where)i(all)g(displacemen)o(ts)h(are)d(v)m(alid)j (\(absolute\))75 1223 y(addresses.)189 1329 y Fl(A)n(dvic)n(e)15 b(to)i(users.)43 b Fx(It)15 b(is)h(not)g(exp)q(ected)g(that)f Fv(MPI)g Fx(implemen)o(tations)i(will)g(b)q(e)f(able)h(to)e(detect)189 1386 y(erroneous,)20 b(\\out)f(of)g(b)q(ound")h(displacemen)o(ts)h(|)f (unless)g(those)f(o)o(v)o(er\015o)o(w)g(the)g(user)h(address)189 1442 y(space)15 b(|)g(since)g(the)g Fv(MPI)f Fx(call)i(ma)o(y)e(not)g (kno)o(w)h(the)f(exten)o(t)h(of)f(the)h(arra)o(ys)e(and)i(records)f(in) i(the)189 1499 y(host)e(program.)19 b(\()p Fl(End)d(of)g(advic)n(e)g (to)h(users.)p Fx(\))189 1605 y Fl(A)n(dvic)n(e)h(to)h(implementors.)57 b Fx(There)18 b(is)g(no)g(need)h(to)f(distinguish)i(\(absolute\))e (addresses)g(and)189 1662 y(\(relativ)o(e\))12 b(displacemen)o(ts)i(on) e(a)g(mac)o(hine)h(with)g(con)o(tiguous)f(address)h(space:)18 b Fi(MPI)p 1638 1662 13 2 v 15 w(BOTTOM)11 b Fx(is)189 1718 y(zero,)j(and)g(b)q(oth)g(addresses)h(and)f(displacemen)o(ts)i (are)e(in)o(tegers.)19 b(On)c(mac)o(hines)g(where)g(the)f(dis-)189 1774 y(tinction)d(is)g(required,)h(addresses)f(are)f(recognized)i(as)e (expressions)i(that)d(in)o(v)o(olv)o(e)i Fi(MPI)p 1668 1774 V 15 w(BOTTOM)p Fx(.)189 1831 y(\()p Fl(End)k(of)i(advic)n(e)f(to) g(implementors.)p Fx(\))75 1953 y Fj(3.12.7)49 b(Examples)75 2038 y Fx(The)15 b(follo)o(wing)h(examples)g(illustrate)h(the)e(use)g (of)g(deriv)o(ed)h(datat)o(yp)q(es.)75 2145 y Fq(Example)i(3.29)23 b Fx(Send)16 b(and)f(receiv)o(e)h(a)f(section)h(of)f(a)f(3D)h(arra)o(y) l(.)218 2251 y Fu(REAL)23 b(a\(100,100,100\),)f(e\(9,9,9\))218 2307 y(INTEGER)h(oneslice,)g(twoslice,)f(threeslice,)h(sizeofreal,)f (myrank,)h(ierr)218 2364 y(INTEGER)g(status\(MPI_STATUS_SIZE\))75 2477 y(C)143 b(extract)23 b(the)g(section)g(a\(1:17:2,)g(3:11,)g (2:10\))75 2533 y(C)143 b(and)23 b(store)h(it)f(in)h(e\(:,:,:\).)218 2646 y(CALL)f(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 76 84 76 83 bop 75 -100 a Fx(76)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)218 45 y Fu(CALL)23 b(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 158 y(C)119 b(create)23 b(datatype)g(for)g(a)h(1D)g(section)218 214 y(CALL)f(MPI_TYPE_VECTOR\()f(9,)i(1,)f(2,)h(MPI_REAL,)f(oneslice,)f (ierr\))75 327 y(C)119 b(create)23 b(datatype)g(for)g(a)h(2D)g(section) 218 384 y(CALL)f(MPI_TYPE_HVECTOR\(9,)f(1,)h(100*sizeofreal,)f (oneslice,)h(twoslice,)f(ierr\))75 497 y(C)119 b(create)23 b(datatype)g(for)g(the)h(entire)f(section)218 553 y(CALL)g (MPI_TYPE_HVECTOR\()f(9,)i(1,)f(100*100*sizeofreal,)e(twoslice,)i(1,) 767 610 y(threeslice,)f(ierr\))218 723 y(CALL)h(MPI_TYPE_COMMIT\()f (threeslice,)h(ierr\))218 779 y(CALL)g(MPI_SENDRECV\(a\(1,3,2\),)e(1,)j (threeslice,)e(myrank,)h(0,)h(e,)f(9*9*9,)648 835 y(MPI_REAL,)f (myrank,)h(0,)h(MPI_COMM_WORLD,)e(status,)h(ierr\))75 936 y Fq(Example)18 b(3.30)23 b Fx(Cop)o(y)14 b(the)h(\(strictly\))h (lo)o(w)o(er)e(triangular)i(part)e(of)h(a)g(matrix.)218 1036 y Fu(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 1092 y(INTEGER)47 b(disp\(100\),)22 b(blocklen\(100\),)g(ltype,)h (myrank,)g(ierr)218 1149 y(INTEGER)g(status\(MPI_STATUS_SIZE\))75 1261 y(C)119 b(copy)23 b(lower)h(triangular)e(part)h(of)h(array)f(a)75 1318 y(C)119 b(onto)23 b(lower)h(triangular)e(part)h(of)h(array)f(b)218 1431 y(CALL)g(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))75 1544 y(C)119 b(compute)23 b(start)g(and)h(size)f(of)h(each)f(column)218 1600 y(DO)h(i=1,)f(100)266 1657 y(disp\(i\))g(=)h(100*\(i-1\))e(+)i(i) 266 1713 y(block\(i\))f(=)g(100-i)218 1770 y(END)h(DO)75 1882 y(C)119 b(create)23 b(datatype)g(for)g(lower)h(triangular)e(part) 218 1939 y(CALL)h(MPI_TYPE_INDEXED\()f(100,)h(block,)g(disp,)h (MPI_REAL,)e(ltype,)h(ierr\))218 2052 y(CALL)g(MPI_TYPE_COMMIT\(ltype,) e(ierr\))218 2108 y(CALL)i(MPI_SENDRECV\()g(a,)g(1,)h(ltype,)f(myrank,) g(0,)g(b,)h(1,)552 2165 y(ltype,)f(myrank,)g(0,)h(MPI_COMM_WORLD,)e (status,)h(ierr\))75 2265 y Fq(Example)18 b(3.31)23 b Fx(T)l(ransp)q(ose)15 b(a)g(matrix.)218 2365 y Fu(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 2421 y(INTEGER)g(row,)g(xpose,)g (sizeofreal,)g(myrank,)g(ierr)218 2478 y(INTEGER)g (status\(MPI_STATUS_SIZE\))75 2591 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 2704 y(CALL)g (MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 77 85 77 84 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(77)218 102 y Fu(CALL)23 b(MPI_TYPE_EXTENT\()f(MPI_REAL,)h (sizeofreal,)f(ierr\))75 214 y(C)119 b(create)23 b(datatype)g(for)g (one)h(row)218 271 y(CALL)f(MPI_TYPE_VECTOR\()f(100,)h(1,)h(100,)f (MPI_REAL,)g(row,)g(ierr\))75 384 y(C)119 b(create)23 b(datatype)g(for)g(matrix)g(in)h(row-major)f(order)218 440 y(CALL)g(MPI_TYPE_HVECTOR\()f(100,)h(1,)h(sizeofreal,)e(row,)i (xpose,)f(ierr\))218 553 y(CALL)g(MPI_TYPE_COMMIT\()f(xpose,)h(ierr\)) 75 666 y(C)119 b(send)23 b(matrix)h(in)f(row-major)g(order)g(and)g (receive)g(in)h(column)f(major)g(order)218 723 y(CALL)g(MPI_SENDRECV\() g(a,)g(1,)h(xpose,)f(myrank,)g(0,)g(b,)h(100*100,)457 779 y(MPI_REAL,)e(myrank,)h(0,)h(MPI_COMM_WORLD,)e(status,)h(ierr\))75 882 y Fq(Example)18 b(3.32)23 b Fx(Another)15 b(approac)o(h)f(to)h(the) g(transp)q(ose)g(problem:)218 973 y Fu(REAL)23 b(a\(100,100\),)g (b\(100,100\))218 1030 y(INTEGER)47 b(disp\(2\),)23 b(blocklen\(2\),)f (type\(2\),)h(row,)g(row1,)g(sizeofreal)218 1086 y(INTEGER)47 b(myrank,)23 b(ierr)218 1143 y(INTEGER)g(status\(MPI_STATUS_SIZE\))218 1256 y(CALL)g(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))75 1369 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 1482 y(CALL)g(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 1594 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 1651 y(CALL)f(MPI_TYPE_VECTOR\()f(100,)h(1,)h(100,)f(MPI_REAL,)g(row,)g (ierr\))75 1764 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row,)f (with)h(the)f(extent)g(of)h(one)f(real)g(number)218 1820 y(disp\(1\))g(=)h(0)218 1877 y(disp\(2\))f(=)h(sizeofreal)218 1933 y(type\(1\))47 b(=)24 b(row)218 1990 y(type\(2\))47 b(=)24 b(MPI_UB)218 2046 y(blocklen\(1\))46 b(=)24 b(1)218 2103 y(blocklen\(2\))46 b(=)24 b(1)218 2159 y(CALL)f(MPI_TYPE_STRUCT\() f(2,)i(blocklen,)e(disp,)i(type,)f(row1,)g(ierr\))218 2272 y(CALL)g(MPI_TYPE_COMMIT\()f(row1,)h(ierr\))75 2385 y(C)119 b(send)23 b(100)h(rows)f(and)h(receive)f(in)g(column)g(major)g (order)218 2441 y(CALL)g(MPI_SENDRECV\()g(a,)g(100,)g(row1,)h(myrank,)f (0,)g(b,)h(100*100,)457 2498 y(MPI_REAL,)e(myrank,)h(0,)h (MPI_COMM_WORLD,)e(status,)h(ierr\))75 2601 y Fq(Example)18 b(3.33)23 b Fx(W)l(e)15 b(manipulate)h(an)f(arra)o(y)f(of)h (structures.)75 2704 y Fu(struct)23 b(Partstruct)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 78 86 78 85 bop 75 -100 a Fx(78)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)147 45 y Fu({)147 102 y(int)95 b(class;)47 b(/*)23 b(particle)g(class)g(*/)147 158 y(double)g(d[6];)71 b(/*)23 b(particle)g(coordinates)f(*/)147 214 y(char)71 b(b[7];)g(/*)23 b(some)g(additional)g(information)f(*/) 147 271 y(};)75 384 y(struct)h(Partstruct)94 b(particle[1000];)75 497 y(int)429 b(i,)24 b(dest,)f(rank;)75 553 y(MPI_Comm)118 b(comm;)75 723 y(/*)24 b(build)f(datatype)g(describing)f(structure)h (*/)75 835 y(MPI_Datatype)f(Particletype;)75 892 y(MPI_Datatype)g (type[3])h(=)h({MPI_INT,)e(MPI_DOUBLE,)h(MPI_CHAR};)75 948 y(int)238 b(blocklen[3])23 b(=)g({1,)h(6,)f(7};)75 1005 y(MPI_Aint)118 b(disp[3];)75 1061 y(int)238 b(base;)75 1231 y(/*)24 b(compute)e(displacements)h(of)g(structure)g(components)f (*/)75 1344 y(MPI_Address\()g(particle,)h(disp\);)75 1400 y(MPI_Address\()f(particle[0].d,)g(disp+1\);)75 1456 y(MPI_Address\()g(particle[0].b,)g(disp+2\);)75 1513 y(base)h(=)h(disp[0];)75 1569 y(for)f(\(i=0;)h(i)f(<3;)h(i++\))f (disp[i])g(-=)h(base;)75 1682 y(MPI_Type_struct\()e(3,)h(blocklen,)g (disp,)g(type,)g(&Particletype\);)147 1795 y(/*)g(If)h(compiler)f(does) g(padding)g(in)g(mysterious)g(ways,)147 1852 y(the)g(following)g(may)g (be)h(safer)f(*/)75 1965 y(MPI_Datatype)f(type1[4])h(=)h({MPI_INT,)e (MPI_DOUBLE,)h(MPI_CHAR,)f(MPI_UB};)75 2021 y(int)238 b(blocklen1[4])22 b(=)i({1,)g(6,)f(7,)h(1};)75 2077 y(MPI_Aint)118 b(disp1[4];)75 2190 y(/*)24 b(compute)e(displacements)h(of)g(structure) g(components)f(*/)75 2303 y(MPI_Address\()g(particle,)h(disp1\);)75 2360 y(MPI_Address\()f(particle[0].d,)g(disp1+1\);)75 2416 y(MPI_Address\()g(particle[0].b,)g(disp1+2\);)75 2473 y(MPI_Address\()g(particle+1,)h(disp1+3\);)75 2529 y(base)g(=)h(disp1[0];)75 2586 y(for)f(\(i=0;)h(i)f(<4;)h(i++\))f (disp1[i])g(-=)g(base;)75 2698 y(/*)h(build)f(datatype)g(describing)f (structure)h(*/)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 79 87 79 86 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(79)75 102 y Fu(MPI_Type_struct\()22 b(4,)h(blocklen1,)g (disp1,)g(type1,)g(&Particletype\);)409 271 y(/*)h(4.1:)266 327 y(send)f(the)h(entire)f(array)g(*/)75 440 y(MPI_Type_commit\()f (&Particletype\);)75 497 y(MPI_Send\()h(particle,)f(1000,)h (Particletype,)g(dest,)g(tag,)g(comm\);)409 666 y(/*)h(4.2:)266 723 y(send)f(only)h(the)f(entries)g(of)g(class)h(zero)f(particles,)266 779 y(preceded)g(by)g(the)h(number)f(of)g(such)h(entries)f(*/)75 892 y(MPI_Datatype)f(Zparticles;)70 b(/*)24 b(datatype)f(describing)f (all)i(particles)791 948 y(with)f(class)g(zero)h(\(needs)f(to)g(be)h (recomputed)791 1005 y(if)f(classes)g(change\))g(*/)75 1061 y(MPI_Datatype)f(Ztype;)75 1174 y(MPI_Aint)118 b(zdisp[1000];)75 1231 y(int)23 b(zblock[1000],)g(j,)g(k;)75 1287 y(int)g(zzblock[2])g(=) h({1,1};)75 1344 y(MPI_Aint)118 b(zzdisp[2];)75 1400 y(MPI_Datatype)22 b(zztype[2];)75 1513 y(/*)i(compute)e(displacements)h (of)g(class)g(zero)h(particles)e(*/)75 1569 y(j)i(=)f(0;)75 1626 y(for\(i=0;)g(i)g(<)h(1000;)f(i++\))123 1682 y(if)g (\(particle[i].class==0\))194 1739 y({)194 1795 y(zdisp[j])g(=)h(i;)194 1852 y(zblock[j])f(=)h(1;)194 1908 y(j++;)194 1965 y(})75 2077 y(/*)g(create)f(datatype)f(for)i(class)f(zero)g(particles)47 b(*/)75 2134 y(MPI_Type_indexed\()22 b(j,)h(zblock,)g(zdisp,)g (Particletype,)f(&Zparticles\);)75 2247 y(/*)i(prepend)e(particle)h (count)g(*/)75 2303 y(MPI_Address\(&j,)f(zzdisp\);)75 2360 y(MPI_Address\(particle,)f(zzdisp+1\);)75 2416 y(zztype[0])i(=)g (MPI_INT;)75 2473 y(zztype[1])g(=)g(Zparticles;)75 2529 y(MPI_Type_struct\(2,)e(zzblock,)i(zzdisp,)g(zztype,)g(&Ztype\);)75 2642 y(MPI_Type_commit\()f(&Ztype\);)75 2698 y(MPI_Send\()h (MPI_BOTTOM,)f(1,)i(Ztype,)f(dest,)g(tag,)g(comm\);)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 80 88 80 87 bop 75 -100 a Fx(80)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)242 158 y Fu(/*)24 b(A)f(probably)g(more)g(efficient)g(way)h(of)f(defining)g(Zparticles)f (*/)75 271 y(/*)i(consecutive)e(particles)h(with)g(index)g(zero)g(are)h (handled)f(as)g(one)h(block)f(*/)75 327 y(j=0;)75 384 y(for)g(\(i=0;)h(i)f(<)h(1000;)f(i++\))123 440 y(if)g (\(particle[i].index==0\))170 497 y({)170 553 y(for)h(\(k=i+1;)f(\(k)g (<)h(1000\)&&\(particle[k].index)d(==)i(0\))h(;)f(k++\);)170 610 y(zdisp[j])g(=)h(i;)170 666 y(zblock[j])f(=)h(k-i;)170 723 y(j++;)170 779 y(i)g(=)g(k;)170 835 y(})75 892 y (MPI_Type_indexed\()e(j,)h(zblock,)g(zdisp,)g(Particletype,)f (&Zparticles\);)457 1061 y(/*)h(4.3:)314 1118 y(send)g(the)g(first)h (two)f(coordinates)f(of)i(all)f(entries)g(*/)75 1231 y(MPI_Datatype)f(Allpairs;)142 b(/*)24 b(datatype)f(for)g(all)g(pairs)h (of)f(coordinates)g(*/)75 1344 y(MPI_Aint)g(sizeofentry;)75 1456 y(MPI_Type_extent\()f(Particletype,)g(&sizeofentry\);)194 1569 y(/*)i(sizeofentry)e(can)i(also)f(be)h(computed)e(by)i (subtracting)e(the)i(address)266 1626 y(of)f(particle[0])g(from)g(the)h (address)f(of)g(particle[1])f(*/)75 1739 y(MPI_Type_hvector\()g(1000,)h (2,)g(sizeofentry,)g(MPI_DOUBLE,)f(&Allpairs\);)75 1795 y(MPI_Type_commit\()g(&Allpairs\);)75 1852 y(MPI_Send\()h (particle[0].d,)f(1,)h(Allpairs,)g(dest,)g(tag,)g(comm\);)218 1965 y(/*)h(an)f(alternative)g(solution)f(to)i(4.3)g(*/)75 2077 y(MPI_Datatype)e(Onepair;)71 b(/*)23 b(datatype)g(for)h(one)f (pair)g(of)h(coordinates,)e(with)695 2134 y(the)i(extent)f(of)h(one)f (particle)g(entry)g(*/)75 2190 y(MPI_Aint)g(disp2[3];)75 2247 y(MPI_Datatype)f(type2[3])h(=)h({MPI_LB,)e(MPI_DOUBLE,)h(MPI_UB};) 75 2303 y(int)g(blocklen2[3])g(=)g({1,)h(2,)f(1};)75 2416 y(MPI_Address\()f(particle,)h(disp2\);)75 2473 y(MPI_Address\()f (particle[0].d,)g(disp2+1\);)75 2529 y(MPI_Address\()g(particle+1,)h (disp2+2\);)75 2586 y(base)g(=)h(disp2[0];)75 2642 y(for)f(\(i=0;)h (i<2;)f(i++\))g(disp2[i])g(-=)h(base;)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 81 89 81 88 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fx(81)75 45 y Fu(MPI_Type_struct\()22 b(3,)h(blocklen2,)g (disp2,)g(type2,)g(&Onepair\);)75 102 y(MPI_Type_commit\()f (&Onepair\);)75 158 y(MPI_Send\()h(particle[0].d,)f(1000,)h(Onepair,)g (dest,)g(tag,)g(comm\);)75 321 y Fq(Example)18 b(3.34)23 b Fx(The)15 b(same)g(manipulations)i(as)e(in)h(the)g(previous)g (example,)g(but)f(use)h(absolute)g(ad-)75 377 y(dresses)f(in)h(datat)o (yp)q(es.)75 483 y Fu(struct)23 b(Partstruct)147 540 y({)147 596 y(int)g(class;)147 653 y(double)g(d[6];)147 709 y(char)g(b[7];)147 766 y(};)75 879 y(struct)g(Partstruct)g (particle[1000];)337 992 y(/*)h(build)f(datatype)g(describing)g(first)g (array)g(entry)g(*/)75 1104 y(MPI_Datatype)f(Particletype;)75 1161 y(MPI_Datatype)g(type[3])h(=)h({MPI_INT,)e(MPI_DOUBLE,)h (MPI_CHAR};)75 1217 y(int)238 b(block[3])23 b(=)h({1,)f(6,)h(7};)75 1274 y(MPI_Aint)118 b(disp[3];)75 1387 y(MPI_Address\()22 b(particle,)h(disp\);)75 1443 y(MPI_Address\()f(particle[0].d,)g (disp+1\);)75 1500 y(MPI_Address\()g(particle[0].b,)g(disp+2\);)75 1556 y(MPI_Type_struct\()g(3,)h(block,)g(disp,)g(type,)h (&Particletype\);)75 1669 y(/*)g(Particletype)e(describes)g(first)i (array)f(entry)g(--)h(using)f(absolute)147 1725 y(addresses)f(*/)505 1838 y(/*)h(5.1:)361 1895 y(send)h(the)f(entire)g(array)g(*/)75 2008 y(MPI_Type_commit\()f(&Particletype\);)75 2064 y(MPI_Send\()h (MPI_BOTTOM,)f(1000,)h(Particletype,)f(dest,)h(tag,)h(comm\);)481 2234 y(/*)f(5.2:)290 2290 y(send)g(the)h(entries)e(of)i(class)f(zero,) 290 2346 y(preceded)g(by)g(the)h(number)f(of)g(such)h(entries)e(*/)75 2459 y(MPI_Datatype)g(Zparticles,)h(Ztype;)75 2572 y(MPI_Aint)g (zdisp[1000])75 2629 y(int)g(zblock[1000],)g(i,)g(j,)h(k;)75 2685 y(int)f(zzblock[2])g(=)h({1,1};)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 82 90 82 89 bop 75 -100 a Fx(82)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fu(MPI_Datatype)22 b(zztype[2];)75 102 y(MPI_Aint)118 b(zzdisp[2];)75 214 y(j=0;)75 271 y(for)23 b(\(i=0;)h(i)f(<)h(1000;)f(i++\))123 327 y(if)g(\(particle[i].index==0\))170 384 y({)170 440 y(for)h(\(k=i+1;)f(\(k)g(<)h(1000\)&&\(particle[k].index)d(=)i(0\))h(;) g(k++\);)170 497 y(zdisp[j])f(=)h(i;)170 553 y(zblock[j])f(=)h(k-i;)170 610 y(j++;)170 666 y(i)g(=)g(k;)170 723 y(})75 779 y (MPI_Type_indexed\()e(j,)h(zblock,)g(zdisp,)g(Particletype,)f (&Zparticles\);)75 835 y(/*)i(Zparticles)e(describe)h(particles)f(with) i(class)f(zero,)g(using)147 892 y(their)g(absolute)g(addresses*/)75 1005 y(/*)h(prepend)e(particle)h(count)g(*/)75 1061 y(MPI_Address\(&j,) f(zzdisp\);)75 1118 y(zzdisp[1])h(=)g(MPI_BOTTOM;)75 1174 y(zztype[0])g(=)g(MPI_INT;)75 1231 y(zztype[1])g(=)g(Zparticles;) 75 1287 y(MPI_Type_struct\(2,)e(zzblock,)i(zzdisp,)g(zztype,)g (&Ztype\);)75 1400 y(MPI_Type_commit\()f(&Ztype\);)75 1456 y(MPI_Send\()h(MPI_BOTTOM,)f(1,)i(Ztype,)f(dest,)g(tag,)g(comm\);) 75 1619 y Fq(Example)18 b(3.35)23 b Fx(Handling)16 b(of)f(unions.)75 1725 y Fu(union)23 b({)147 1782 y(int)118 b(ival;)147 1838 y(float)70 b(fval;)218 1895 y(})24 b(u[1000])75 2008 y(int)119 b(utype;)75 2121 y(/*)24 b(All)f(entries)g(of)h(u)f (have)h(identical)e(type;)h(variable)147 2177 y(utype)g(keeps)g(track)g (of)h(their)f(current)g(type)g(*/)75 2290 y(MPI_Datatype)70 b(type[2];)75 2346 y(int)286 b(blocklen[2])22 b(=)i({1,1};)75 2403 y(MPI_Aint)166 b(disp[2];)75 2459 y(MPI_Datatype)70 b(mpi_utype[2];)75 2516 y(MPI_Aint)166 b(i,j;)75 2629 y(/*)24 b(compute)e(an)i(MPI)g(datatype)e(for)i(each)f(possible)g (union)g(type;)147 2685 y(assume)g(values)g(are)g(left-aligned)f(in)i (union)f(storage.)g(*/)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 83 91 83 90 bop 75 -100 a Fm(3.13.)34 b(P)l(A)o(CK)15 b(AND)g(UNP)l(A)o(CK) 1169 b Fx(83)75 102 y Fu(MPI_Address\()22 b(u,)i(&i\);)75 158 y(MPI_Address\()e(u+1,)i(&j\);)75 214 y(disp[0])f(=)h(0;)f(disp[1]) g(=)h(j-i;)75 271 y(type[1])f(=)h(MPI_UB;)75 384 y(type[0])f(=)h (MPI_INT;)75 440 y(MPI_Type_struct\(2,)d(blocklen,)i(disp,)g(type,)g (&mpi_utype[0]\);)75 553 y(type[0])g(=)h(MPI_FLOAT;)75 610 y(MPI_Type_struct\(2,)d(blocklen,)i(disp,)g(type,)g (&mpi_utype[1]\);)75 723 y(for\(i=0;)g(i<2;)g(i++\))g (MPI_Type_commit\(&mpi_utype[i])o(\);)75 835 y(/*)h(actual)f (communication)f(*/)75 948 y(MPI_Send\(u,)g(1000,)i(mpi_utype[utype],)d (dest,)i(tag,)h(comm\);)75 1088 y Fn(3.13)59 b(P)n(ack)20 b(and)f(unpack)75 1189 y Fx(Some)c(existing)i(comm)o(unication)e (libraries)i(pro)o(vide)f(pac)o(k/unpac)o(k)f(functions)h(for)f (sending)h(noncon-)75 1246 y(tiguous)c(data.)19 b(In)13 b(these,)f(the)h(user)f(explicitly)j(pac)o(ks)d(data)g(in)o(to)g(a)g (con)o(tiguous)h(bu\013er)f(b)q(efore)h(sending)75 1302 y(it,)h(and)h(unpac)o(ks)g(it)f(from)g(a)g(con)o(tiguous)h(bu\013er)f (after)g(receiving)i(it.)k(Deriv)o(ed)14 b(datat)o(yp)q(es,)g(whic)o(h) h(are)75 1359 y(describ)q(ed)g(in)e(Section)h(3.12,)e(allo)o(w)h(one,)g (in)h(most)e(cases,)g(to)h(a)o(v)o(oid)f(explicit)j(pac)o(king)e(and)g (unpac)o(king.)75 1415 y(The)19 b(user)h(sp)q(eci\014es)h(the)e(la)o(y) o(out)g(of)g(the)g(data)f(to)h(b)q(e)h(sen)o(t)f(or)g(receiv)o(ed,)i (and)e(the)g(comm)o(unication)75 1472 y(library)h(directly)g(accesses)f (a)f(noncon)o(tiguous)h(bu\013er.)31 b(The)19 b(pac)o(k/unpac)o(k)g (routines)g(are)f(pro)o(vided)75 1528 y(for)d(compatibilit)o(y)i(with)f (previous)h(libraries.)22 b(Also,)16 b(they)g(pro)o(vide)g(some)f (functionalit)o(y)i(that)e(is)h(not)75 1585 y(otherwise)f(a)o(v)m (ailable)i(in)f Fv(MPI)p Fx(.)e(F)l(or)g(instance,)i(a)e(message)h(can) g(b)q(e)h(receiv)o(ed)g(in)g(sev)o(eral)f(parts,)f(where)75 1641 y(the)19 b(receiv)o(e)h(op)q(eration)g(done)f(on)h(a)f(later)g (part)f(ma)o(y)h(dep)q(end)i(on)e(the)g(con)o(ten)o(t)g(of)g(a)f (former)h(part.)75 1698 y(Another)f(use)h(is)f(that)g(outgoing)g (messages)f(ma)o(y)h(b)q(e)h(explicitly)i(bu\013ered)d(in)h(user)g (supplied)h(space,)75 1754 y(th)o(us)f(o)o(v)o(erriding)h(the)f(system) g(bu\013ering)g(p)q(olicy)l(.)34 b(Finally)l(,)22 b(the)d(a)o(v)m (ailabilit)o(y)i(of)e(pac)o(k)g(and)h(unpac)o(k)75 1810 y(op)q(erations)14 b(facilitates)h(the)f(dev)o(elopmen)o(t)h(of)e (additional)j(comm)o(unication)f(libraries)g(la)o(y)o(ered)f(on)g(top) 75 1867 y(of)h Fv(MPI)p Fx(.)75 2018 y Fv(MPI)p 160 2018 14 2 v 16 w(P)l(A)o(CK\(inbuf,)g(incount,)i(datat)o(yp)q(e,)f(outbuf,)g (outcount,)h(p)q(osition,)f(comm)m(\))117 2095 y Fk(IN)155 b Fv(inbuf)493 b Fk(input)14 b(bu\013er)h(start)f(\(c)o(hoice\))117 2166 y(IN)155 b Fv(incount)448 b Fk(n)o(um)o(b)q(er)13 b(of)h(input)f(data)h(items)f(\(in)o(teger\))117 2238 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(input)g(data)f(item)g(\(handle\))117 2309 y(OUT)108 b Fv(outbuf)465 b Fk(output)14 b(bu\013er)h(start)g(\(c) o(hoice\))117 2380 y(IN)155 b Fv(outcount)420 b Fk(output)14 b(bu\013er)h(size,)f(in)g(b)o(ytes)g(\(in)o(teger\))117 2451 y(INOUT)62 b Fv(p)q(osition)439 b Fk(curren)o(t)16 b(p)q(osition)d(in)g(bu\013er,)i(in)e(b)o(ytes)i(\(in)o(teger\))117 2523 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(for)j(pac)o(k)o (ed)g(message)g(\(handle\))75 2647 y Fu(int)23 b(MPI)p 245 2647 15 2 v 17 w(Pack\(void*)g(inbuf,)g(int)g(incount,)g(MPI)p 1074 2647 V 17 w(Datatype)g(datatype,)f(void)h(*outbuf,)393 2704 y(int)h(outcount,)e(int)i(*position,)e(MPI)p 1160 2704 V 17 w(Comm)h(comm\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 84 92 84 91 bop 75 -100 a Fx(84)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(PACK\(INBUF,)22 b(INCOUNT,)h(DATATYPE,)g(OUTBUF,)f (OUTCOUNT,)h(POSITION,)g(COMM,)393 102 y(IERROR\))170 158 y()g(INBUF\(*\),)g(OUTBUF\(*\))170 214 y(INTEGER)g(INCOUNT,)g (DATATYPE,)g(OUTCOUNT,)f(POSITION,)h(COMM,)g(IERROR)166 301 y Fx(P)o(ac)o(ks)11 b(the)i(message)e(in)i(the)g(send)f(bu\013er)h (sp)q(eci\014ed)h(b)o(y)e Fv(inbuf,)i(incount,)g(datat)o(yp)q(e)g Fx(in)o(to)e(the)g(bu\013er)75 357 y(space)j(sp)q(eci\014ed)i(b)o(y)e Fv(outbuf)h Fx(and)f Fv(outcount)p Fx(.)22 b(The)15 b(input)h(bu\013er) e(can)h(b)q(e)h(an)o(y)e(comm)o(unication)i(bu\013er)75 414 y(allo)o(w)o(ed)h(in)g Fv(MPI)p 377 414 14 2 v 16 w(SEND)p Fx(.)f(The)h(output)f(bu\013er)g(is)h(a)f(con)o(tiguous)h (storage)e(area)h(con)o(taining)h Fv(outcount)75 470 y Fx(b)o(ytes,)e(starting)f(at)h(the)g(address)h Fv(outbuf)g Fx(\(length)g(is)g(coun)o(ted)f(in)h Fv(b)o(ytes)p Fx(,)h(not)d(elemen) o(ts,)i(as)f(if)h(it)f(w)o(ere)75 527 y(a)g(comm)o(unication)h (bu\013er)f(for)g(a)f(message)h(of)g(t)o(yp)q(e)g Fv(MPI)p 1081 527 V 16 w(P)l(A)o(CKED)p Fx(\).)166 583 y(The)21 b(input)g(v)m(alue)h(of)e Fv(p)q(osition)h Fx(is)g(the)g(\014rst)f(lo)q (cation)h(in)g(the)g(output)f(bu\013er)h(to)e(b)q(e)j(used)f(for)75 640 y(pac)o(king.)27 b Fv(p)q(osition)19 b Fx(is)f(incremen)o(ted)g(b)o (y)g(the)f(size)h(of)f(the)h(pac)o(k)o(ed)f(message,)g(and)h(the)g (output)f(v)m(alue)75 696 y(of)e Fv(p)q(osition)i Fx(is)f(the)g (\014rst)f(lo)q(cation)i(in)f(the)g(output)f(bu\013er)h(follo)o(wing)h (the)e(lo)q(cations)i(o)q(ccupied)g(b)o(y)f(the)75 753 y(pac)o(k)o(ed)g(message.)k(The)c Fv(comm)10 b Fx(argumen)o(t)k(is)i (the)g(comm)o(unicator)f(that)g(will)i(b)q(e)g(subsequen)o(tly)f(used) 75 809 y(for)f(sending)h(the)f(pac)o(k)o(ed)h(message.)75 960 y Fv(MPI)p 160 960 V 16 w(UNP)l(A)o(CK\(inbuf,)g(insize,)f(p)q (osition,)h(outbuf,)g(outcount,)h(datat)o(yp)q(e,)g(comm)m(\))117 1037 y Fk(IN)155 b Fv(inbuf)493 b Fk(input)14 b(bu\013er)h(start)f(\(c) o(hoice\))117 1112 y(IN)155 b Fv(insize)485 b Fk(size)15 b(of)e(input)h(bu\013er,)g(in)g(b)o(ytes)g(\(in)o(teger\))117 1187 y(INOUT)62 b Fv(p)q(osition)439 b Fk(curren)o(t)16 b(p)q(osition)d(in)g(b)o(ytes)i(\(in)o(teger\))117 1263 y(OUT)108 b Fv(outbuf)465 b Fk(output)14 b(bu\013er)h(start)g(\(c)o (hoice\))117 1338 y(IN)155 b Fv(outcount)420 b Fk(n)o(um)o(b)q(er)13 b(of)h(items)f(to)g(b)q(e)i(unpac)o(k)o(ed)f(\(in)o(teger\))117 1413 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(output)g(data)g(item)e(\(handle\))117 1488 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(for)j(pac)o(k)o (ed)g(message)g(\(handle\))75 1612 y Fu(int)23 b(MPI)p 245 1612 15 2 v 17 w(Unpack\(void*)f(inbuf,)h(int)h(insize,)f(int)g (*position,)g(void)g(*outbuf,)393 1669 y(int)h(outcount,)e(MPI)p 802 1669 V 17 w(Datatype)h(datatype,)f(MPI)p 1344 1669 V 17 w(Comm)i(comm\))75 1755 y(MPI)p 150 1755 V 17 w(UNPACK\(INBUF,)e (INSIZE,)h(POSITION,)f(OUTBUF,)h(OUTCOUNT,)g(DATATYPE,)g(COMM,)393 1812 y(IERROR\))170 1868 y()g(INBUF\(*\),)g(OUTBUF\(*\))170 1925 y(INTEGER)g(INSIZE,)g(POSITION,)g(OUTCOUNT,)g(DATATYPE,)f(COMM,)h (IERROR)166 2011 y Fx(Unpac)o(ks)15 b(a)g(message)f(in)o(to)h(the)g (receiv)o(e)h(bu\013er)f(sp)q(eci\014ed)i(b)o(y)e Fv(outbuf,)h (outcount,)g(datat)o(yp)q(e)h Fx(from)75 2068 y(the)d(bu\013er)g(space) g(sp)q(eci\014ed)i(b)o(y)e Fv(inbuf)i Fx(and)e Fv(insize)p Fx(.)20 b(The)14 b(output)g(bu\013er)g(can)g(b)q(e)h(an)o(y)e(comm)o (unication)75 2124 y(bu\013er)h(allo)o(w)o(ed)g(in)g Fv(MPI)p 500 2124 14 2 v 16 w(RECV)p Fx(.)g(The)g(input)h(bu\013er)e (is)h(a)g(con)o(tiguous)g(storage)e(area)h(con)o(taining)h Fv(insize)75 2181 y Fx(b)o(ytes,)e(starting)f(at)g(address)h Fv(inbuf)p Fx(.)20 b(The)12 b(input)h(v)m(alue)g(of)e Fv(p)q(osition)i Fx(is)g(the)f(\014rst)f(lo)q(cation)i(in)f(the)g (output)75 2237 y(bu\013er)18 b(o)q(ccupied)h(b)o(y)f(the)f(pac)o(k)o (ed)h(message.)27 b Fv(p)q(osition)18 b Fx(is)g(incremen)o(ted)h(b)o(y) f(the)g(size)g(of)f(the)h(pac)o(k)o(ed)75 2293 y(message,)d(so)g(that)f (the)i(output)f(v)m(alue)i(of)e Fv(p)q(osition)i Fx(is)e(the)h(\014rst) f(lo)q(cation)h(in)h(the)e(output)g(bu\013er)h(after)75 2350 y(the)e(lo)q(cations)g(o)q(ccupied)i(b)o(y)e(the)g(message)f(that) g(w)o(as)g(unpac)o(k)o(ed.)20 b Fv(comm)8 b Fx(is)14 b(the)g(comm)o(unicator)f(used)75 2406 y(to)i(receiv)o(e)h(the)f(pac)o (k)o(ed)g(message.)189 2513 y Fl(A)n(dvic)n(e)21 b(to)i(users.)83 b Fx(Note)22 b(the)g(di\013erence)h(b)q(et)o(w)o(een)g Fv(MPI)p 1280 2513 V 16 w(RECV)g Fx(and)f Fv(MPI)p 1611 2513 V 16 w(UNP)l(A)o(CK)p Fx(:)g(in)189 2569 y Fv(MPI)p 274 2569 V 15 w(RECV)p Fx(,)g(the)g Fv(count)h Fx(argumen)o(t)e(sp)q (eci\014es)i(the)f(maxim)o(um)f(n)o(um)o(b)q(er)h(of)f(items)h(that)f (can)189 2626 y(b)q(e)g(receiv)o(ed.)39 b(The)22 b(actual)f(n)o(um)o(b) q(er)h(of)e(items)i(receiv)o(ed)g(is)g(determined)g(b)o(y)f(the)h (length)f(of)189 2682 y(the)h(incoming)h(message.)40 b(In)23 b Fv(MPI)p 837 2682 V 15 w(UNP)l(A)o(CK)p Fx(,)f(the)g Fv(count)i Fx(argumen)o(t)d(sp)q(eci\014es)j(the)e(actual)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 85 93 85 92 bop 75 -100 a Fm(3.13.)34 b(P)l(A)o(CK)15 b(AND)g(UNP)l(A)o(CK) 1169 b Fx(85)189 45 y(n)o(um)o(b)q(er)18 b(of)g(items)h(that)f(are)g (unpac)o(k)o(ed;)i(the)f(\\size")g(of)f(the)g(corresp)q(onding)i (message)e(is)h(the)189 102 y(incremen)o(t)c(in)g Fv(p)q(osition)p Fx(.)21 b(The)14 b(reason)g(for)g(this)g(c)o(hange)h(is)f(that)g(the)g (\\incoming)h(message)f(size")189 158 y(is)19 b(not)g(predetermined)i (since)g(the)e(user)g(decides)i(ho)o(w)e(m)o(uc)o(h)g(to)g(unpac)o(k;)i (nor)e(is)h(it)g(easy)f(to)189 214 y(determine)c(the)g(\\message)f (size")i(from)e(the)h(n)o(um)o(b)q(er)g(of)f(items)h(to)g(b)q(e)g (unpac)o(k)o(ed.)20 b(In)c(fact,)e(in)h(a)189 271 y(heterogeneous)i (system,)f(this)i(n)o(um)o(b)q(er)f(ma)o(y)f(not)h(b)q(e)g(determined)h Fl(a)g(priori)p Fx(.)26 b(\()p Fl(End)17 b(of)h(advic)n(e)189 327 y(to)e(users.)p Fx(\))166 434 y(T)l(o)g(understand)h(the)f(b)q(eha) o(vior)h(of)e(pac)o(k)h(and)h(unpac)o(k,)f(it)h(is)f(con)o(v)o(enien)o (t)h(to)f(think)h(of)e(the)i(data)75 490 y(part)11 b(of)g(a)g(message)g (as)g(b)q(eing)i(the)e(sequence)i(obtained)f(b)o(y)g(concatenating)f (the)h(successiv)o(e)h(v)m(alues)f(sen)o(t)75 547 y(in)k(that)f (message.)20 b(The)c(pac)o(k)f(op)q(eration)h(stores)f(this)g(sequence) i(in)f(the)g(bu\013er)f(space,)h(as)f(if)h(sending)75 603 y(the)d(message)g(to)g(that)f(bu\013er.)19 b(The)14 b(unpac)o(k)f(op)q(eration)h(retriev)o(es)f(this)h(sequence)g(from)f (bu\013er)g(space,)75 659 y(as)h(if)h(receiving)h(a)f(message)f(from)g (that)g(bu\013er.)19 b(\(It)c(is)g(helpful)h(to)e(think)i(of)e(in)o (ternal)h(F)l(ortran)f(\014les)h(or)75 716 y Fv(sscanf)h Fx(in)g(C,)f(for)g(a)g(similar)h(function.\))166 772 y(Sev)o(eral)h(messages)g(can)g(b)q(e)h(successiv)o(ely)g(pac)o(k)o(ed) f(in)o(to)g(one)h Fq(pac)o(king)i(unit)p Fx(.)26 b(This)17 b(is)h(e\013ected)75 829 y(b)o(y)c(sev)o(eral)h(successiv)o(e)h Fq(related)f Fx(calls)g(to)f Fv(MPI)p 909 829 14 2 v 16 w(P)l(A)o(CK)p Fx(,)g(where)g(the)h(\014rst)f(call)h(pro)o(vides)g Fv(p)q(osition)h(=)f(0)p Fx(,)75 885 y(and)i(eac)o(h)g(successiv)o(e)h (call)g(inputs)g(the)f(v)m(alue)i(of)d Fv(p)q(osition)i Fx(that)f(w)o(as)f(output)h(b)o(y)f(the)h(previous)h(call,)75 942 y(and)g(the)g(same)f(v)m(alues)i(for)e Fv(outbuf,)i(outcount)h Fx(and)e Fv(comm)m Fx(.)24 b(This)18 b(pac)o(king)g(unit)h(no)o(w)e (con)o(tains)h(the)75 998 y(equiv)m(alen)o(t)h(information)f(that)e(w)o (ould)i(ha)o(v)o(e)f(b)q(een)h(stored)f(in)h(a)f(message)g(b)o(y)h(one) f(send)h(call)h(with)e(a)75 1055 y(send)f(bu\013er)f(that)f(is)i(the)f (\\concatenation")g(of)g(the)g(individual)k(send)d(bu\013ers.)166 1111 y(A)g(pac)o(king)g(unit)g(can)g(b)q(e)h(sen)o(t)e(using)i(t)o(yp)q (e)f Fv(MPI)p 1029 1111 V 15 w(P)l(A)o(CKED)p Fx(.)g(An)o(y)f(p)q(oin)o (t)i(to)e(p)q(oin)o(t)h(or)f(collectiv)o(e)75 1168 y(comm)o(unication)g (function)h(can)f(b)q(e)g(used)g(to)f(mo)o(v)o(e)g(the)h(sequence)g(of) g(b)o(ytes)f(that)g(forms)g(the)g(pac)o(king)75 1224 y(unit)i(from)e(one)h(pro)q(cess)g(to)f(another.)19 b(This)d(pac)o (king)f(unit)h(can)f(no)o(w)f(b)q(e)i(receiv)o(ed)g(using)g(an)o(y)e (receiv)o(e)75 1280 y(op)q(eration,)j(with)g(an)o(y)f(datat)o(yp)q(e:) 22 b(the)16 b(t)o(yp)q(e)h(matc)o(hing)g(rules)g(are)f(relaxed)h(for)f (messages)g(sen)o(t)h(with)75 1337 y(t)o(yp)q(e)e Fv(MPI)p 262 1337 V 16 w(P)l(A)o(CKED)p Fx(.)166 1393 y(A)e(message)g(sen)o(t)g (with)h(an)o(y)e(t)o(yp)q(e)i(\(including)h Fv(MPI)p 1062 1393 V 16 w(P)l(A)o(CKED)p Fx(\))e(can)g(b)q(e)h(receiv)o(ed)h (using)f(the)f(t)o(yp)q(e)75 1450 y Fv(MPI)p 160 1450 V 16 w(P)l(A)o(CKED)p Fx(.)h(Suc)o(h)i(a)f(message)g(can)g(then)h(b)q (e)g(unpac)o(k)o(ed)f(b)o(y)h(calls)g(to)e Fv(MPI)p 1452 1450 V 16 w(UNP)l(A)o(CK)p Fx(.)166 1506 y(A)19 b(pac)o(king)h(unit)g (\(or)f(a)g(message)g(created)g(b)o(y)g(a)g(regular,)h(\\t)o(yp)q(ed")f (send\))h(can)f(b)q(e)h(unpac)o(k)o(ed)75 1563 y(in)o(to)j(sev)o(eral)g (successiv)o(e)h(messages.)42 b(This)23 b(is)h(e\013ected)f(b)o(y)f (sev)o(eral)h(successiv)o(e)h(related)g(calls)g(to)75 1619 y Fv(MPI)p 160 1619 V 16 w(UNP)l(A)o(CK)p Fx(,)19 b(where)h(the)f(\014rst)g(call)i(pro)o(vides)f Fv(p)q(osition)g(=)g(0)p Fx(,)g(and)f(eac)o(h)h(successiv)o(e)g(call)h(inputs)75 1676 y(the)d(v)m(alue)h(of)e Fv(p)q(osition)i Fx(that)e(w)o(as)g (output)h(b)o(y)f(the)h(previous)h(call,)g(and)f(the)g(same)f(v)m (alues)i(for)e Fv(inbuf,)75 1732 y(insize)f Fx(and)f Fv(comm)m Fx(.)166 1789 y(The)22 b(concatenation)f(of)g(t)o(w)o(o)g (pac)o(king)h(units)g(is)g(not)f(necessarily)i(a)e(pac)o(king)h(unit;)j (nor)c(is)h(a)75 1845 y(substring)17 b(of)g(a)f(pac)o(king)h(unit)h (necessarily)g(a)f(pac)o(king)g(unit.)25 b(Th)o(us,)17 b(one)g(cannot)g(concatenate)f(t)o(w)o(o)75 1901 y(pac)o(king)11 b(units)g(and)g(then)g(unpac)o(k)g(the)f(result)h(as)g(one)f(pac)o (king)h(unit;)i(nor)d(can)h(one)f(unpac)o(k)h(a)g(substring)75 1958 y(of)f(a)g(pac)o(king)h(unit)g(as)f(a)g(separate)g(pac)o(king)h (unit.)19 b(Eac)o(h)11 b(pac)o(king)f(unit,)i(that)e(w)o(as)g(created)g (b)o(y)g(a)h(related)75 2014 y(sequence)17 b(of)e(pac)o(k)h(calls,)g (or)f(b)o(y)h(a)f(regular)h(send,)g(m)o(ust)f(b)q(e)h(unpac)o(k)o(ed)h (as)e(a)g(unit,)i(b)o(y)e(a)h(sequence)g(of)75 2071 y(related)g(unpac)o (k)f(calls.)189 2177 y Fl(R)n(ationale.)74 b Fx(The)21 b(restriction)h(on)f(\\atomic")f(pac)o(king)h(and)g(unpac)o(king)h(of)f (pac)o(king)g(units)189 2234 y(allo)o(ws)13 b(the)f(implemen)o(tation)j (to)d(add)h(at)f(the)h(head)g(of)f(pac)o(king)h(units)h(additional)g (information,)189 2290 y(suc)o(h)i(as)g(a)g(description)h(of)f(the)g (sender)h(arc)o(hitecture)f(\(to)f(b)q(e)i(used)g(for)e(t)o(yp)q(e)h (con)o(v)o(ersion,)g(in)h(a)189 2346 y(heterogeneous)e(en)o(vironmen)o (t\))g(\()p Fl(End)h(of)g(r)n(ationale.)p Fx(\))166 2453 y(The)21 b(follo)o(wing)g(call)g(allo)o(ws)g(the)g(user)f(to)g(\014nd)h (out)f(ho)o(w)g(m)o(uc)o(h)h(space)f(is)h(needed)h(to)e(pac)o(k)g(a)75 2509 y(message)15 b(and,)g(th)o(us,)f(manage)h(space)h(allo)q(cation)g (for)e(bu\013ers.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 86 94 86 93 bop 75 -100 a Fx(86)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(P)l(A)o(CK)p 291 45 V 16 w(SIZE\(incount,)f(datat)o (yp)q(e,)g(comm)n(,)11 b(size\))117 122 y Fk(IN)155 b Fv(incount)448 b Fk(coun)o(t)14 b(argumen)o(t)f(to)h(pac)o(king)f(call) g(\(in)o(teger\))117 197 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(argumen)o(t)f(to)h(pac)o(king)f(call)g (\(handle\))117 271 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(argumen)o(t)i(to)h(pac)o(king)f(call)g(\(handle\))117 345 y(OUT)108 b Fv(size)519 b Fk(upp)q(er)16 b(b)q(ound)f(on)g(size)g (of)g(pac)o(k)o(ed)g(message,)f(in)h(b)o(ytes)g(\(in-)905 402 y(teger\))75 526 y Fu(int)23 b(MPI)p 245 526 15 2 v 17 w(Pack)p 358 526 V 17 w(size\(int)g(incount,)f(MPI)p 876 526 V 17 w(Datatype)h(datatype,)g(MPI)p 1419 526 V 16 w(Comm)h(comm,)393 583 y(int)g(*size\))75 669 y(MPI)p 150 669 V 17 w(PACK)p 263 669 V 16 w(SIZE\(INCOUNT,)e(DATATYPE,)h (COMM,)g(SIZE,)g(IERROR\))170 725 y(INTEGER)g(INCOUNT,)g(DATATYPE,)g (COMM,)g(SIZE,)g(IERROR)166 812 y Fx(A)12 b(call)h(to)f Fv(MPI)p 430 812 14 2 v 16 w(P)l(A)o(CK)p 561 812 V 16 w(SIZE\(incount,)h(datat)o(yp)q(e,)h(comm)m(,)c(size\))i Fx(returns)g(in)h Fv(size)f Fx(an)g(upp)q(er)h(b)q(ound)75 868 y(on)h(the)g(incremen)o(t)g(in)h Fv(p)q(osition)g Fx(that)e(is)i(e\013ected)f(b)o(y)f(a)h(call)h(to)e Fv(MPI)p 1263 868 V 16 w(P)l(A)o(CK\(inbuf,)h(incount,)i(datat)o(yp)q(e,)75 925 y(outbuf,)g(outcount,)h(p)q(osition,)f(comm)m(\))p Fx(.)189 1025 y Fl(R)n(ationale.)43 b Fx(The)16 b(call)h(returns)f(an)g (upp)q(er)h(b)q(ound,)f(rather)f(than)h(an)g(exact)f(b)q(ound,)i(since) g(the)189 1082 y(exact)e(amoun)o(t)g(of)g(space)g(needed)i(to)e(pac)o (k)g(the)h(message)f(ma)o(y)g(dep)q(end)i(on)e(the)h(con)o(text)f (\(e.g.,)189 1138 y(\014rst)f(message)h(pac)o(k)o(ed)g(in)h(a)f(pac)o (king)h(unit)g(ma)o(y)e(tak)o(e)h(more)g(space\).)20 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))75 1238 y Fq(Example)h(3.36)23 b Fx(An)15 b(example)h(using)g Fv(MPI)p 863 1238 V 16 w(P)l(A)o(CK)p Fx(.)75 1328 y Fu(int)23 b(position,)g(i,)h(j,)f(a[2];)75 1385 y(char)g(buff[1000];)75 1497 y(....)75 1610 y(MPI_Comm_rank\(MPI_COMM_WOR)o(LD,)e(&myrank\);)75 1667 y(if)j(\(myrank)e(==)i(0\))75 1723 y({)147 1780 y(/)f(*)h(SENDER)f(CODE)g(*/)123 1893 y(position)f(=)i(0;)123 1949 y(MPI_Pack\(&i,)e(1,)i(MPI_INT,)e(buff,)h(1000,)h(&position,)e (MPI_COMM_WORLD\);)123 2006 y(MPI_Pack\(&j,)g(1,)i(MPI_INT,)e(buff,)h (1000,)h(&position,)e(MPI_COMM_WORLD\);)123 2062 y(MPI_Send\()g(buff,)i (position,)e(MPI_PACKED,)h(1,)g(0,)h(MPI_COMM_WORLD\);)75 2118 y(})75 2175 y(else)47 b(/*)24 b(RECEIVER)f(CODE)g(*/)123 2231 y(MPI_Recv\()f(a,)i(2,)g(MPI_INT,)e(0,)i(0,)f(MPI_COMM_WORLD\))75 2344 y(})75 2445 y Fq(Example)18 b(3.37)23 b Fx(A)15 b(elab)q(orate)g(example.)75 2534 y Fu(int)23 b(position,)g(i;)75 2591 y(float)g(a[1000];)75 2647 y(char)g(buff[1000])1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 87 95 87 94 bop 75 -100 a Fm(3.13.)29 b(P)l(A)o(CK)15 b(AND)g(UNP)l(A)o(CK) 1174 b Fx(87)75 45 y Fu(....)75 158 y(MPI_Comm_rank\(MPI_Comm_wor)o (ld,)21 b(&myrank\);)75 214 y(if)j(\(myrank)e(==)i(0\))75 271 y({)123 327 y(/)f(*)h(SENDER)f(CODE)h(*/)123 440 y(int)f(len[2];)123 497 y(MPI_Aint)f(disp[2];)123 553 y(MPI_Datatype)g(type[2],)h(newtype;)123 666 y(/*)g(build)g(datatype)g (for)h(i)f(followed)g(by)h(a[0]...a[i-1])e(*/)123 779 y(len[0])h(=)g(1;)123 835 y(len[1])g(=)g(i;)123 892 y(MPI_Address\()f (&i,)h(disp\);)123 948 y(MPI_Address\()f(a,)i(disp+1\);)123 1005 y(type[0])f(=)g(MPI_INT;)123 1061 y(type[1])g(=)g(MPI_FLOAT;)123 1118 y(MPI_Type_struct\()e(2,)j(len,)f(disp,)g(type,)h(&newtype\);)123 1174 y(MPI_Type_commit\()d(&newtype\);)123 1287 y(/*)i(Pack)h(i)f (followed)g(by)h(a[0]...a[i-1]*/)123 1400 y(position)e(=)i(0;)123 1456 y(MPI_Pack\()e(MPI_BOTTOM,)h(1,)g(newtype,)g(buff,)g(1000,)g (&position,)g(MPI_COMM_WORLD\);)123 1569 y(/*)g(Send)h(*/)123 1682 y(MPI_Send\()e(buff,)i(position,)e(MPI_PACKED,)h(1,)g(0,)361 1739 y(MPI_COMM_WORLD\))75 1852 y(/*)h(*****)147 1908 y(One)f(can)g(replace)g(the)h(last)f(three)g(lines)h(with)147 1965 y(MPI_Send\()e(MPI_BOTTOM,)h(1,)g(newtype,)g(1,)h(0,)f (MPI_COMM_WORLD\);)147 2021 y(*****)g(*/)75 2077 y(})75 2134 y(else)g(/*)h(myrank)f(==)h(1)f(*/)75 2190 y({)147 2247 y(/*)g(RECEIVER)g(CODE)g(*/)123 2360 y(MPI_Status)f(status;)123 2473 y(/*)h(Receive)g(*/)123 2586 y(MPI_Recv\()f(buff,)i(1000,)f (MPI_PACKED,)f(0,)i(0,)f(&status\);)123 2698 y(/*)g(Unpack)g(i)h(*/)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 88 96 88 95 bop 75 -100 a Fx(88)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)99 102 y Fu(position)23 b(=)g(0;)99 158 y(MPI_Unpack\(buff,)f(1000,)h(&position,)f(&i,)i(1,)f (MPI_INT,)g(MPI_COMM_WORLD\);)99 271 y(/*)g(Unpack)g(a[0]...a[i-1])g (*/)99 327 y(MPI_Unpack\(buff,)f(1000,)h(&position,)f(a,)i(i,)f (MPI_FLOAT,)g(MPI_COMM_WORLD\);)75 384 y(})75 475 y Fq(Example)18 b(3.38)23 b Fx(Eac)o(h)16 b(pro)q(cess)g(sends)h(a)f(coun)o(t,)f(follo) o(w)o(ed)i(b)o(y)f(coun)o(t)g(c)o(haracters)g(to)f(the)h(ro)q(ot;)g (the)75 532 y(ro)q(ot)e(concatenate)h(all)h(c)o(haracters)f(in)o(to)g (one)g(string.)75 615 y Fu(int)23 b(count,)g(gsize,)h(counts[64],)e (totalcount,)g(k1,)i(k2,)f(k,)170 671 y(displs[64],)g(position,)f (concat_pos;)75 728 y(char)h(chr[100],)g(*lbuf,)g(*rbuf,)g(*cbuf;)75 784 y(...)75 841 y(MPI_Comm_size\(comm,)e(&gsize\);)75 897 y(MPI_Comm_rank\(comm,)g(&myrank\);)218 1010 y(/*)j(allocate)f (local)g(pack)g(buffer)g(*/)75 1066 y(MPI_Pack_size\(1,)f(MPI_INT,)g (comm,)i(&k1\);)75 1123 y(MPI_Pack_size\(count,)d(MPI_CHAR,)i(&k2\);)75 1179 y(k)h(=)f(k1+k2;)75 1236 y(lbuf)g(=)h(\(char)f(*\)malloc\(k\);)218 1349 y(/*)h(pack)f(count,)g(followed)g(by)g(count)h(characters)e(*/)75 1405 y(position)h(=)g(0;)75 1462 y(MPI_Pack\(&count,)f(1,)h(MPI_INT,)g (&lbuf,)g(k,)h(&position,)e(comm\);)75 1518 y(MPI_Pack\(chr,)g(count,)h (MPI_CHAR,)g(&lbuf,)g(k,)g(&position,)g(comm\);)75 1631 y(if)h(\(myrank)e(!=)i(root\))218 1687 y(/*)g(gather)f(at)g(root)h (sizes)f(of)g(all)h(packed)f(messages)g(*/)147 1744 y(MPI_Gather\()f (&position,)g(1,)i(MPI_INT,)f(NULL,)g(NULL,)385 1800 y(NULL,)g(root,)h(comm\);)218 1913 y(/*)g(gather)f(at)g(root)h(packed)f (messages)g(*/)123 1970 y(MPI_Gatherv\()f(&buf,)h(position,)g (MPI_PACKED,)f(NULL,)337 2026 y(NULL,)i(NULL,)f(NULL,)g(root,)g (comm\);)75 2139 y(else)g({)72 b(/*)23 b(root)h(code)f(*/)218 2195 y(/*)h(gather)f(sizes)g(of)h(all)f(packed)g(messages)g(*/)147 2252 y(MPI_Gather\()f(&position,)g(1,)i(MPI_INT,)f(counts,)g(1,)385 2308 y(MPI_INT,)g(root,)g(comm\);)218 2421 y(/*)h(gather)f(all)g (packed)g(messages)g(*/)123 2478 y(displs[0])f(=)i(0;)123 2534 y(for)f(\(i=1;)g(i)h(<)g(gsize;)f(i++\))170 2591 y(displs[i])g(=)h(displs[i-1])e(+)i(counts[i-1];)123 2647 y(totalcount)e(=)i(dipls[gsize-1])e(+)i(counts[gsize-1];)123 2704 y(rbuf)f(=)h(\(char)f(*\)malloc\(totalcount\);)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 89 97 89 96 bop 75 -100 a Fm(3.13.)34 b(P)l(A)o(CK)15 b(AND)g(UNP)l(A)o(CK) 1169 b Fx(89)123 45 y Fu(cbuf)23 b(=)h(\(char)f (*\)malloc\(totalcount\);)123 102 y(MPI_Gatherv\()f(lbuf,)h(position,)g (MPI_PACKED,)f(rbuf,)337 158 y(counts,)h(displs,)g(MPI_PACKED,)g(root,) g(comm\);)218 271 y(/*)h(unpack)f(all)g(messages)g(and)g(concatenate)g (strings)g(*/)123 327 y(concat_pos)f(=)i(0;)123 384 y(for)f(\(i=0;)g(i) h(<)g(gsize;)f(i++\))g({)170 440 y(position)g(=)h(0;)170 497 y(MPI_Unpack\()f(rbuf+displs[i],)f(totalcount-displs[i],)314 553 y(&position,)g(&count,)h(1,)h(MPI_INT,)e(comm\);)170 610 y(MPI_Unpack\()h(rbuf+displs[i],)f(totalcount-displs[i],)314 666 y(&position,)g(cbuf+concat_pos,)g(count,)h(MPI_CHAR,)f(comm\);)170 723 y(concat_pos)h(+=)h(count;)170 779 y(})123 835 y(cbuf[concat_pos])d (=)j(`\\0';)123 892 y(})-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 90 98 90 97 bop 75 356 a Fo(Chapter)34 b(4)75 564 y Fr(Collecti)q(v)m(e)42 b(Comm)m(unication)75 805 y Fn(4.1)59 b(Intro)r(duction)18 b(and)i(Overview)75 906 y Fx(Collectiv)o(e)e(comm)o(unication)g(is)f (de\014ned)h(as)e(comm)o(unication)i(that)e(in)o(v)o(olv)o(es)h(a)f (group)h(of)f(pro)q(cesses.)75 963 y(The)f(functions)h(of)f(this)h(t)o (yp)q(e)f(pro)o(vided)h(b)o(y)f Fv(MPI)g Fx(are)g(the)g(follo)o(wing:) 143 1056 y Ft(\017)23 b Fx(Barrier)15 b(sync)o(hronization)h(across)f (all)h(group)f(mem)o(b)q(ers)g(\(Sec.)20 b(4.3\).)143 1150 y Ft(\017)j Fx(Broadcast)15 b(from)h(one)h(mem)o(b)q(er)g(to)f (all)h(mem)o(b)q(ers)g(of)f(a)h(group)f(\(Sec.)25 b(4.4\).)d(This)c(is) f(sho)o(wn)f(in)189 1207 y(\014gure)f(4.1.)143 1301 y Ft(\017)23 b Fx(Gather)18 b(data)h(from)f(all)j(group)e(mem)o(b)q(ers)g (to)g(one)g(mem)o(b)q(er)g(\(Sec.)33 b(4.5\).)e(This)19 b(is)h(sho)o(wn)f(in)189 1357 y(\014gure)c(4.1.)143 1451 y Ft(\017)23 b Fx(Scatter)16 b(data)g(from)h(one)g(mem)o(b)q(er)g(to)f (all)i(mem)o(b)q(ers)g(of)e(a)h(group)g(\(Sec.)25 b(4.6\).)f(This)18 b(is)f(sho)o(wn)189 1507 y(in)f(\014gure)f(4.1.)143 1601 y Ft(\017)23 b Fx(A)16 b(v)m(ariation)i(on)e(Gather)g(where)h(all)h (mem)o(b)q(ers)f(of)f(the)h(group)f(receiv)o(e)i(the)f(result)g(\(Sec.) 25 b(4.7\).)189 1658 y(This)15 b(is)h(sho)o(wn)f(as)g(\\allgather")g (in)h(\014gure)f(4.1.)143 1751 y Ft(\017)23 b Fx(Scatter/Gather)10 b(data)h(from)h(all)g(mem)o(b)q(ers)g(to)g(all)g(mem)o(b)q(ers)g(of)g (a)g(group)f(\(also)h(called)h(complete)189 1808 y(exc)o(hange)i(or)g (all-to-all\))h(\(Sec.)k(4.8\).)f(This)d(is)g(sho)o(wn)e(as)h (\\alltoall")h(in)g(\014gure)g(4.1.)143 1902 y Ft(\017)23 b Fx(Global)15 b(reduction)h(op)q(erations)g(suc)o(h)f(as)g(sum,)g (max,)f(min,)i(or)e(user-de\014ned)j(functions,)f(where)189 1958 y(the)11 b(result)h(is)g(returned)g(to)e(all)j(group)e(mem)o(b)q (ers)g(and)h(a)f(v)m(ariation)h(where)f(the)h(result)g(is)g(returned) 189 2015 y(to)i(only)i(one)f(mem)o(b)q(er)h(\(Sec.)k(4.9\).)143 2108 y Ft(\017)j Fx(A)15 b(com)o(bined)h(reduction)g(and)g(scatter)e (op)q(eration)h(\(Sec.)21 b(4.10\).)143 2202 y Ft(\017)i Fx(Scan)15 b(across)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group)g(\(also)g (called)h(pre\014x\))g(\(Sec.)k(4.11\).)166 2296 y(A)e(collectiv)o(e)i (op)q(eration)e(is)g(executed)h(b)o(y)f(ha)o(ving)g(all)h(pro)q(cesses) f(in)h(the)f(group)g(call)h(the)f(com-)75 2352 y(m)o(unication)f (routine,)f(with)g(matc)o(hing)g(argumen)o(ts.)21 b(The)16 b(syn)o(tax)f(and)h(seman)o(tics)g(of)f(the)h(collectiv)o(e)75 2409 y(op)q(erations)e(are)g(de\014ned)i(to)e(b)q(e)g(consisten)o(t)h (with)f(the)h(syn)o(tax)e(and)i(seman)o(tics)f(of)g(the)g(p)q(oin)o (t-to-p)q(oin)o(t)75 2465 y(op)q(erations.)22 b(Th)o(us,)16 b(general)g(datat)o(yp)q(es)g(are)f(allo)o(w)o(ed)i(and)f(m)o(ust)f (matc)o(h)g(b)q(et)o(w)o(een)i(sending)g(and)f(re-)75 2522 y(ceiving)i(pro)q(cesses)e(as)g(sp)q(eci\014ed)i(in)f(Chapter)32 b(3.)23 b(One)17 b(of)e(the)i(k)o(ey)f(argumen)o(ts)f(is)h(a)g(comm)o (unicator)75 2578 y(that)f(de\014nes)i(the)f(group)g(of)g (participating)h(pro)q(cesses)f(and)g(pro)o(vides)h(a)e(con)o(text)h (for)f(the)h(op)q(eration.)75 2635 y(Sev)o(eral)f(collectiv)o(e)h (routines)f(suc)o(h)g(as)f(broadcast)g(and)h(gather)e(ha)o(v)o(e)i(a)f (single)i(originating)f(or)f(receiv-)75 2691 y(ing)i(pro)q(cess.)k(Suc) o(h)15 b(pro)q(cesses)h(are)f(called)h(the)f Fl(r)n(o)n(ot)p Fx(.)20 b(Some)15 b(argumen)o(ts)f(in)i(the)f(collectiv)o(e)i (functions)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 91 99 91 98 bop 75 -100 a Fm(4.1.)34 b(INTR)o(ODUCTION)16 b(AND)f(O)o(VER)-5 b(VIEW)890 b Fx(91)75 2444 y @beginspecial @setspecial %%BeginDocument: coll-fig1.ps /arrowdict 13 dict def % Local storage for the procedure % ``arrow.'' /arrow % The procedure ``arrow'' adds an { arrowdict begin % arrow shape to the current path. /headlength exch def % It takes seven arguments: the x /halfheadthickness exch 2 div def % and y coordinates of the tail /halfthickness exch 2 div def % (imagine that a line has been /tipy exch def /tipx exch def % drawn down the center of the /taily exch def /tailx exch def % arrow from the tip to the tail, % then x and y lie on this line), % the x and y coordinates of the % tip of the arrow, the thickness % of the arrow in the tail % portion, the thickness of the % arrow at the widest part of the % arrowhead and the length of the % arrowhead. /dx tipx tailx sub def % Compute the differences in x and /dy tipy taily sub def % y for the tip and tail. These /arrowlength dx dx mul dy dy mul add % will be used to compute the sqrt def % length of the arrow and to /angle dy dx atan def % compute the angle of direction % that the arrow is facing with % respect to the current user % coordinate system origin. /base arrowlength headlength sub def % Compute where the base of the % arrowhead will be. /savematrix matrix currentmatrix def % Save the current user coordinate % system. We are using the same % strategy to localize the effect % of transformations as was used % in the program to draw an % ellipse. tailx taily translate % Translate to the starting point % of the tail. angle rotate % Rotate the x-axis to correspond % with the center line of the % arrow. 0 halfthickness neg moveto % Add the arrow shape to the % current path. base halfthickness neg lineto base halfheadthickness neg lineto arrowlength 0 lineto base halfheadthickness lineto base halfthickness lineto 0 halfthickness lineto closepath savematrix setmatrix % Restore the current user % coordinate system. end } def /Box { /height exch def /length exch def length 0 rlineto 0 height rlineto length neg 0 rlineto closepath } def /Gdict 200 dict def /Grid { Gdict begin /ny exch def /nx exch def /dely exch def /delx exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def 0 1 ny { pop x y moveto lenx 0 rlineto stroke /y y dely add def} for /y ypos def /x xpos def 0 1 nx { pop x y moveto 0 leny rlineto stroke /x x delx add def} for end } def /GLdict 300 dict def /GridLabels { GLdict begin /shift exch def /raise exch def /yoff exch def /xoff exch def /p1 exch def /p2 exch def /ny exch def /nx exch def /dely exch def /delx exch def /Darray exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def /dx3 delx 3 div def /dy3 dely 3 div def /ix -1 def /iy ny 1 sub def Darray{ aload pop /Subc exch def /Text exch def /ix ix 1 add def ix nx ge { /ix 0 def /iy iy 1 sub def} if /x xpos delx ix 0.5 add mul add /Helvetica findfont p1 scalefont setfont Text stringwidth pop /Helvetica findfont p2 scalefont setfont Subc stringwidth pop add xoff add 2 div sub shift add def /y ypos dely iy 0.5 add mul add raise add def x y moveto /Helvetica findfont p1 scalefont Text show xoff yoff rmoveto /Helvetica findfont p2 scalefont Subc show } forall end clear } def 2 setlinecap 6.5 72 mul 320 sub 2 div 0 translate 0 300 moveto 20 20 6 6 Grid 0 300 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 380 200 380 12 24 18 arrow stroke 200 330 140 330 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (scatter) dup stringwidth pop 170 exch 2 div sub 400 moveto show (gather) dup stringwidth pop 170 exch 2 div sub 350 moveto show 220 300 moveto 20 20 6 6 Grid 220 300 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(1)] [()()] [()()] [()()] [()()] [()()] [(A)(2)] [()()] [()()] [()()] [()()] [()()] [(A)(3)] [()()] [()()] [()()] [()()] [()()] [(A)(4)] [()()] [()()] [()()] [()()] [()()] [(A)(5)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 0 moveto 20 20 6 6 Grid 0 0 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [(B)(0)] [(B)(1)] [(B)(2)] [(B)(3)] [(B)(4)] [(B)(5)] [(C)(0)] [(C)(1)] [(C)(2)] [(C)(3)] [(C)(4)] [(C)(5)] [(D)(0)] [(D)(1)] [(D)(2)] [(D)(3)] [(D)(4)] [(D)(5)] [(E)(0)] [(E)(1)] [(E)(2)] [(E)(3)] [(E)(4)] [(E)(5)] [(F)(0)] [(F)(1)] [(F)(2)] [(F)(3)] [(F)(4)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 220 0 moveto 20 20 6 6 Grid 220 0 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(1)] [(B)(1)] [(C)(1)] [(D)(1)] [(E)(1)] [(F)(1)] [(A)(2)] [(B)(2)] [(C)(2)] [(D)(2)] [(E)(2)] [(F)(2)] [(A)(3)] [(B)(3)] [(C)(3)] [(D)(3)] [(E)(3)] [(F)(3)] [(A)(4)] [(B)(4)] [(C)(4)] [(D)(4)] [(E)(4)] [(F)(4)] [(A)(5)] [(B)(5)] [(C)(5)] [(D)(5)] [(E)(5)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 60 200 60 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (alltoall) dup stringwidth pop 170 exch 2 div sub 77 moveto show 0 150 moveto 20 20 6 6 Grid 0 150 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(B)(0)] [()()] [()()] [()()] [()()] [()()] [(C)(0)] [()()] [()()] [()()] [()()] [()()] [(D)(0)] [()()] [()()] [()()] [()()] [()()] [(E)(0)] [()()] [()()] [()()] [()()] [()()] [(F)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 200 200 200 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (allgather) dup stringwidth pop 170 exch 2 div sub 220 moveto show 220 150 moveto 20 20 6 6 Grid 220 150 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 450 moveto 20 20 6 6 Grid 0 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 510 200 510 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont 0 575 moveto (data) show (data) stringwidth pop 4 add 575 4 add (data) stringwidth pop 33 add 575 4 add 1 4 5 arrow fill (broadcast) dup stringwidth pop 170 exch 2 div sub 527 moveto show gsave 0 570 (processes) stringwidth pop sub translate 90 rotate 0 5 moveto (processes) show -4 8 -33 8 1 4 5 arrow fill grestore 220 450 moveto 20 20 6 6 Grid 220 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels showpage %%EndDocument @endspecial 98 x(Figure)15 b(4.1:)k(Collectiv)o(e)e(mo)o(v)o(e)d (functions)i(illustrated)h(for)e(a)f(group)h(of)g(six)h(pro)q(cesses.)k (In)c(eac)o(h)f(case,)75 2598 y(eac)o(h)f(ro)o(w)e(of)i(b)q(o)o(xes)f (represen)o(ts)h(data)f(lo)q(cations)i(in)f(one)g(pro)q(cess.)20 b(Th)o(us,)13 b(in)i(the)e(broadcast,)g(initially)75 2655 y(just)i(the)g(\014rst)g(pro)q(cess)g(con)o(tains)g(the)g(data)g Fp(A)889 2662 y Fg(0)909 2655 y Fx(,)f(but)h(after)g(the)g(broadcast)f (all)i(pro)q(cesses)g(con)o(tain)f(it.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 92 100 92 99 bop 75 -100 a Fx(92)703 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fx(are)i(sp)q(eci\014ed)i (as)e(\\signi\014can)o(t)h(only)f(at)g(ro)q(ot,")f(and)i(are)e(ignored) i(for)f(all)h(participan)o(ts)f(except)h(the)75 102 y(ro)q(ot.)f(The)12 b(reader)g(is)g(referred)g(to)f(Chapter)h(3)g(for)f(information)h (concerning)h(comm)o(unication)f(bu\013ers,)75 158 y(general)19 b(datat)o(yp)q(es)f(and)h(t)o(yp)q(e)g(matc)o(hing)g(rules,)h(and)f(to) f(Chapter)h(5)f(for)h(information)g(on)f(ho)o(w)h(to)75 214 y(de\014ne)d(groups)f(and)h(create)f(comm)o(unicators.)166 271 y(The)d(t)o(yp)q(e-matc)o(hing)h(conditions)g(for)e(the)i (collectiv)o(e)g(op)q(erations)g(are)e(more)h(strict)g(than)g(the)g (cor-)75 327 y(resp)q(onding)j(conditions)g(b)q(et)o(w)o(een)f(sender)h (and)f(receiv)o(er)h(in)f(p)q(oin)o(t-to-p)q(oin)o(t.)20 b(Namely)l(,)15 b(for)e(collectiv)o(e)75 384 y(op)q(erations,)18 b(the)g(amoun)o(t)f(of)g(data)g(sen)o(t)g(m)o(ust)g(exactly)h(matc)o(h) f(the)h(amoun)o(t)f(of)g(data)g(sp)q(eci\014ed)j(b)o(y)75 440 y(the)15 b(receiv)o(er.)20 b(Distinct)c(t)o(yp)q(e)f(maps)f(\(the)h (la)o(y)o(out)f(in)i(memory)l(,)e(see)h(Sec.)20 b(3.12\))13 b(b)q(et)o(w)o(een)i(sender)h(and)75 497 y(receiv)o(er)g(are)f(still)h (allo)o(w)o(ed.)166 553 y(Collectiv)o(e)g(routine)f(calls)h(can)f (\(but)f(are)g(not)h(required)g(to\))f(return)h(as)f(so)q(on)h(as)f (their)h(participa-)75 610 y(tion)d(in)h(the)f(collectiv)o(e)i(comm)o (unication)e(is)h(complete.)19 b(The)12 b(completion)h(of)f(a)f(call)i (indicates)h(that)d(the)75 666 y(caller)k(is)g(no)o(w)f(free)g(to)g (access)g(lo)q(cations)h(in)g(the)g(comm)o(unication)g(bu\013er.)k(It) 14 b(do)q(es)h(not)f(indicate)i(that)75 723 y(other)c(pro)q(cesses)h (in)h(the)f(group)f(ha)o(v)o(e)h(completed)g(or)f(ev)o(en)i(started)d (the)i(op)q(eration)g(\(unless)h(otherwise)75 779 y(indicated)k(in)f (the)f(description)i(of)e(the)g(op)q(eration\).)23 b(Th)o(us,)16 b(a)g(collectiv)o(e)i(comm)o(unication)f(call)h(ma)o(y)l(,)75 835 y(or)c(ma)o(y)h(not,)f(ha)o(v)o(e)h(the)g(e\013ect)f(of)h(sync)o (hronizing)h(all)g(calling)h(pro)q(cesses.)j(This)c(statemen)o(t)e (excludes,)75 892 y(of)h(course,)g(the)g(barrier)g(function.)166 948 y(Collectiv)o(e)24 b(comm)o(unication)f(calls)g(ma)o(y)f(use)g(the) g(same)g(comm)o(unicators)g(as)g(p)q(oin)o(t-to-p)q(oin)o(t)75 1005 y(comm)o(unication;)16 b Fv(MPI)f Fx(guaran)o(tees)g(that)g (messages)g(generated)h(on)g(b)q(ehalf)h(of)e(collectiv)o(e)i(comm)o (uni-)75 1061 y(cation)c(calls)h(will)g(not)f(b)q(e)g(confused)h(with)f (messages)f(generated)h(b)o(y)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication.)75 1118 y(A)h(more)g(detailed)i(discussion)g(of)d(correct)h (use)h(of)e(collectiv)o(e)j(routines)f(is)g(found)f(in)h(Sec.)21 b(4.12.)189 1195 y Fl(R)n(ationale.)50 b Fx(The)18 b(equal-data)f (restriction)g(\(on)g(t)o(yp)q(e)g(matc)o(hing\))g(w)o(as)f(made)h(so)f (as)h(to)f(a)o(v)o(oid)189 1252 y(the)e(complexit)o(y)h(of)f(pro)o (viding)h(a)f(facilit)o(y)h(analogous)f(to)g(the)g(status)f(argumen)o (t)h(of)f Fv(MPI)p 1746 1252 14 2 v 16 w(RECV)189 1308 y Fx(for)f(disco)o(v)o(ering)j(the)e(amoun)o(t)g(of)g(data)f(sen)o(t.) 19 b(Some)14 b(of)f(the)g(collectiv)o(e)j(routines)d(w)o(ould)h (require)189 1365 y(an)h(arra)o(y)f(of)g(status)h(v)m(alues.)189 1436 y(The)e(statemen)o(ts)g(ab)q(out)g(sync)o(hronization)i(are)e (made)h(so)f(as)g(to)g(allo)o(w)h(a)f(v)m(ariet)o(y)h(of)f(implemen-) 189 1492 y(tations)h(of)h(the)g(collectiv)o(e)i(functions.)189 1563 y(The)c(collectiv)o(e)i(op)q(erations)e(do)g(not)g(accept)g(a)g (message)f(tag)g(argumen)o(t.)19 b(If)13 b(future)g(revisions)h(of)189 1619 y Fv(MPI)g Fx(de\014ne)j(non-blo)q(c)o(king)g(collectiv)o(e)g (functions,)f(then)f(tags)g(\(or)f(a)h(similar)i(mec)o(hanism\))e(will) 189 1676 y(need)i(to)g(b)q(e)g(added)h(so)f(as)f(to)h(allo)o(w)g(the)g (dis-am)o(biguation)h(of)e(m)o(ultiple,)j(p)q(ending,)g(collectiv)o(e) 189 1732 y(op)q(erations.)h(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 1810 y Fl(A)n(dvic)n(e)h(to)h(users.)59 b Fx(It)19 b(is)f(dangerous)h(to)e(rely)i(on)f(sync)o(hronization)i (side-e\013ects)f(of)f(the)g(col-)189 1866 y(lectiv)o(e)j(op)q (erations)f(for)f(program)g(correctness.)34 b(F)l(or)20 b(example,)h(ev)o(en)g(though)f(a)f(particular)189 1923 y(implemen)o(tation)i(ma)o(y)e(pro)o(vide)i(a)e(broadcast)g(routine)i (with)f(a)g(side-e\013ect)h(of)e(sync)o(hroniza-)189 1979 y(tion,)14 b(the)g(standard)g(do)q(es)h(not)f(require)h(this,)g (and)f(a)g(program)f(that)h(relies)h(on)g(this)f(will)i(not)e(b)q(e)189 2036 y(p)q(ortable.)189 2107 y(On)d(the)g(other)f(hand,)i(a)f(correct,) g(p)q(ortable)g(program)f(m)o(ust)g(allo)o(w)h(for)f(the)h(fact)g(that) f(a)g(collectiv)o(e)189 2163 y(call)j Fl(may)f Fx(b)q(e)g(sync)o (hronizing.)21 b(Though)11 b(one)h(cannot)g(rely)g(on)g(an)o(y)g(sync)o (hronization)g(side-e\013ect,)189 2220 y(one)17 b(m)o(ust)f(program)f (so)i(as)f(to)g(allo)o(w)h(it.)25 b(These)18 b(issues)f(are)g (discussed)h(further)f(in)g(Sec.)26 b(4.12.)189 2276 y(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fx(\))189 2353 y Fl(A)n(dvic)n(e)21 b(to)h(implementors.)77 b Fx(While)22 b(v)o(endors)g(ma)o(y)e(write)i(optimized)g(collectiv)o(e)i(routines) 189 2410 y(matc)o(hed)c(to)f(their)i(arc)o(hitectures,)g(a)f(complete)h (library)f(of)g(the)g(collectiv)o(e)i(comm)o(unication)189 2466 y(routines)d(can)h(b)q(e)f(written)h(en)o(tirely)g(using)g(the)f Fv(MPI)f Fx(p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)g(func-)189 2523 y(tions)11 b(and)g(a)f(few)h(auxiliary)i(functions.)19 b(If)11 b(implemen)o(ting)i(on)e(top)f(of)h(p)q(oin)o(t-to-p)q(oin)o (t,)h(a)f(hidden,)189 2579 y(sp)q(ecial)j(comm)o(unicator)e(m)o(ust)g (b)q(e)h(created)g(for)f(the)g(collectiv)o(e)i(op)q(eration)f(so)f(as)g (to)g(a)o(v)o(oid)h(in)o(ter-)189 2636 y(ference)j(with)f(an)o(y)g (on-going)h(p)q(oin)o(t-to-p)q(oin)o(t)f(comm)o(unication)h(at)f(the)g (time)h(of)f(the)g(collectiv)o(e)189 2692 y(call.)21 b(This)15 b(is)h(discussed)h(further)e(in)h(Sec.)k(4.12.)f(\()p Fl(End)d(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 93 101 93 100 bop 75 -100 a Fm(4.2.)34 b(COMMUNICA)l(TOR)16 b(AR)o(GUMENT)957 b Fx(93)75 45 y Fn(4.2)59 b(Communicato)n(r)19 b(a)n(rgument)75 147 y Fx(The)c(k)o(ey)h(concept)f(of)g(the)g (collectiv)o(e)i(functions)f(is)g(to)f(ha)o(v)o(e)f(a)h(\\group")g(of)g (participating)h(pro)q(cesses.)75 203 y(The)d(routines)f(do)h(not)e(ha) o(v)o(e)h(a)g(group)g(iden)o(ti\014er)i(as)e(an)g(explicit)j(argumen)o (t.)j(Instead,)13 b(there)f(is)h(a)f(com-)75 259 y(m)o(unicator)k (argumen)o(t.)21 b(F)l(or)15 b(the)h(purp)q(oses)h(of)e(this)i(c)o (hapter,)e(a)h(comm)o(unicator)g(can)g(b)q(e)g(though)o(t)f(of)75 316 y(as)d(a)g(group)g(iden)o(ti\014er)h(link)o(ed)h(with)f(a)f(con)o (text.)18 b(An)12 b(in)o(ter-comm)o(unicator,)h(that)e(is,)i(a)f(comm)o (unicator)75 372 y(that)i(spans)i(t)o(w)o(o)e(groups,)g(is)i Fl(not)f Fx(allo)o(w)o(ed)g(as)g(an)g(argumen)o(t)g(to)f(a)h(collectiv) o(e)i(function.)75 516 y Fn(4.3)59 b(Ba)n(rrier)21 b(synchronization)75 712 y Fv(MPI)p 160 712 14 2 v 16 w(BARRIER\()16 b(comm)9 b(\))117 789 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 913 y Fu(int)23 b(MPI)p 245 913 15 2 v 17 w(Barrier\(MPI)p 526 913 V 16 w(Comm)g(comm)h(\))75 1000 y(MPI)p 150 1000 V 17 w(BARRIER\(COMM,)e(IERROR\))170 1056 y(INTEGER)h(COMM,)h(IERROR)166 1143 y Fv(MPI)p 251 1143 14 2 v 16 w(BARRIER)13 b Fx(blo)q(c)o(ks)g(the)f(caller)i(un)o (til)f(all)g(group)f(mem)o(b)q(ers)h(ha)o(v)o(e)e(called)j(it.)19 b(The)13 b(call)g(returns)75 1199 y(at)i(an)o(y)f(pro)q(cess)i(only)g (after)e(all)i(group)f(mem)o(b)q(ers)g(ha)o(v)o(e)g(en)o(tered)h(the)f (call.)75 1342 y Fn(4.4)59 b(Broadcast)90 1538 y Fv(MPI)p 175 1538 V 16 w(BCAST\()16 b(bu\013er,)f(count,)h(datat)o(yp)q(e,)g(ro) q(ot,)f(comm)9 b(\))117 1615 y Fk(INOUT)78 b Fv(bu\013er)462 b Fk(starting)14 b(address)h(of)f(bu\013er)g(\(c)o(hoice\))117 1691 y(IN)171 b Fv(count)466 b Fk(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e(bu\013er)i(\(in)o(teger\))117 1766 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f (bu\013er)i(\(handle\))117 1841 y(IN)171 b Fv(ro)q(ot)492 b Fk(rank)14 b(of)f(broadcast)i(ro)q(ot)f(\(in)o(teger\))117 1916 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 2040 y Fu(int)23 b(MPI)p 245 2040 15 2 v 17 w(Bcast\(void*)g(buffer,)f (int)i(count,)f(MPI)p 1074 2040 V 17 w(Datatype)g(datatype,)f(int)i (root,)393 2097 y(MPI)p 468 2097 V 17 w(Comm)f(comm)h(\))75 2183 y(MPI)p 150 2183 V 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g (ROOT,)g(COMM,)g(IERROR\))170 2240 y()g(BUFFER\(*\))170 2296 y(INTEGER)g(COUNT,)g(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)166 2383 y Fv(MPI)p 251 2383 14 2 v 16 w(BCAST)17 b Fx(broadcasts)e(a)h (message)g(from)g(the)g(pro)q(cess)g(with)h(rank)f Fv(ro)q(ot)g Fx(to)g(all)h(pro)q(cesses)f(of)75 2439 y(the)h(group,)g(itself)h (included.)27 b(It)17 b(is)h(called)g(b)o(y)f(all)h(mem)o(b)q(ers)f(of) g(group)f(using)i(the)f(same)g(argumen)o(ts)75 2496 y(for)d Fv(comm)n(,)d(ro)q(ot)p Fx(.)20 b(On)15 b(return,)g(the)g(con)o(ten)o (ts)f(of)h Fv(ro)q(ot)p Fx('s)f(comm)o(unication)h(bu\013er)g(has)g(b)q (een)h(copied)h(to)75 2552 y(all)f(pro)q(cesses.)166 2609 y(General,)24 b(deriv)o(ed)f(datat)o(yp)q(es)e(are)h(allo)o(w)o (ed)g(for)g Fv(datat)o(yp)q(e)p Fx(.)42 b(The)22 b(t)o(yp)q(e)g (signature)g(of)g Fv(count,)75 2665 y(datat)o(yp)q(e)16 b Fx(on)f(an)o(y)f(pro)q(cess)h(m)o(ust)f(b)q(e)h(equal)g(to)f(the)h(t) o(yp)q(e)g(signature)g(of)f Fv(count,)h(datat)o(yp)q(e)i Fx(at)c(the)i(ro)q(ot.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 94 102 94 101 bop 75 -100 a Fx(94)703 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fx(This)g(implies)i(that)d (the)h(amoun)o(t)e(of)i(data)f(sen)o(t)g(m)o(ust)g(b)q(e)h(equal)h(to)e (the)h(amoun)o(t)e(receiv)o(ed,)j(pairwise)75 102 y(b)q(et)o(w)o(een)i (eac)o(h)h(pro)q(cess)f(and)g(the)g(ro)q(ot.)31 b Fv(MPI)p 909 102 14 2 v 16 w(BCAST)20 b Fx(and)f(all)h(other)f(data-mo)o(v)o (emen)o(t)e(collectiv)o(e)75 158 y(routines)j(mak)o(e)g(this)g (restriction.)34 b(Distinct)21 b(t)o(yp)q(e)f(maps)g(b)q(et)o(w)o(een)g (sender)g(and)g(receiv)o(er)h(are)f(still)75 214 y(allo)o(w)o(ed.)75 333 y Fj(4.4.1)49 b(Example)15 b(using)i(MPI)p 622 333 15 2 v 18 w(BCAST)75 419 y Fq(Example)h(4.1)k Fx(Broadcast)15 b(100)f(in)o(ts)h(from)g(pro)q(cess)g Fu(0)g Fx(to)g(ev)o(ery)g(pro)q (cess)g(in)i(the)e(group.)170 503 y Fu(MPI_Comm)23 b(comm;)170 559 y(int)h(array[100];)170 616 y(int)g(root=0;)170 672 y(...)170 728 y(MPI_Bcast\()f(array,)g(100,)g(MPI_INT,)g(root,)g (comm\);)75 812 y Fx(As)15 b(in)h(man)o(y)f(of)g(our)g(example)h(co)q (de)g(fragmen)o(ts,)e(w)o(e)h(assume)g(that)g(some)g(of)g(the)g(v)m (ariables)i(\(suc)o(h)e(as)75 869 y Fu(comm)g Fx(in)h(the)f(ab)q(o)o(v) o(e\))f(ha)o(v)o(e)h(b)q(een)i(assigned)e(appropriate)h(v)m(alues.)75 1009 y Fn(4.5)59 b(Gather)75 1205 y Fv(MPI)p 160 1205 14 2 v 16 w(GA)l(THER\()16 b(sendbuf,)h(sendcount,)h(sendt)o(yp)q(e,)f (recvbuf,)f(recvcount,)g(recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)m(\))117 1282 y Fk(IN)171 b Fv(sendbuf)428 b Fk(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 1354 y(IN)171 b Fv(sendcount)383 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h (send)h(bu\013er)f(\(in)o(teger\))117 1426 y(IN)171 b Fv(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er) g(elemen)o(ts)f(\(handle\))117 1498 y(OUT)124 b Fv(recvbuf)434 b Fk(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i (signi\014can)o(t)d(only)g(at)905 1554 y(ro)q(ot\))117 1626 y(IN)171 b Fv(recvcount)389 b Fk(n)o(um)o(b)q(er)11 b(of)g(elemen)o(ts)g(for)g(an)o(y)g(single)g(receiv)o(e)i(\(in)o (teger,)f(sig-)905 1682 y(ni\014can)o(t)i(only)f(at)h(ro)q(ot\))117 1754 y(IN)171 b Fv(recvt)o(yp)q(e)414 b Fk(data)16 b(t)o(yp)q(e)g(of)f (recv)i(bu\013er)g(elemen)o(ts)f(\(signi\014can)o(t)f(only)g(at)905 1811 y(ro)q(ot\))f(\(handle\))117 1883 y(IN)171 b Fv(ro)q(ot)492 b Fk(rank)14 b(of)f(receiving)i(pro)q(cess)g(\(in)o(teger\))117 1955 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 2079 y Fu(int)23 b(MPI)p 245 2079 15 2 v 17 w(Gather\(void*)f(sendbuf,) h(int)h(sendcount,)e(MPI)p 1217 2079 V 17 w(Datatype)h(sendtype,)393 2135 y(void*)g(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 2135 V 17 w(Datatype)h(recvtype,)f(int)i(root,)393 2192 y(MPI)p 468 2192 V 17 w(Comm)f(comm\))75 2278 y(MPI)p 150 2278 V 17 w(GATHER\(SENDBUF,)f(SENDCOUNT,)g(SENDTYPE,)h(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 2335 y(ROOT,)g(COMM,)h(IERROR\))170 2391 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 2448 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g (COMM,)g(IERROR)166 2534 y Fx(Eac)o(h)16 b(pro)q(cess)h(\(ro)q(ot)f (pro)q(cess)h(included\))h(sends)g(the)e(con)o(ten)o(ts)g(of)g(its)h (send)h(bu\013er)e(to)g(the)h(ro)q(ot)75 2591 y(pro)q(cess.)i(The)11 b(ro)q(ot)f(pro)q(cess)i(receiv)o(es)g(the)f(messages)f(and)i(stores)e (them)h(in)h(rank)f(order.)18 b(The)11 b(outcome)75 2647 y(is)j Fl(as)h(if)f Fx(eac)o(h)g(of)f(the)h Fu(n)g Fx(pro)q(cesses)g (in)h(the)f(group)f(\(including)k(the)d(ro)q(ot)f(pro)q(cess\))g(had)h (executed)h(a)f(call)75 2704 y(to)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 95 103 95 102 bop 75 -100 a Fm(4.5.)34 b(GA)l(THER)1452 b Fx(95)189 45 y Fu(MPI)p 264 45 15 2 v 16 w(Send)p Fx(\()p Fu(sendbuf)p Fp(;)8 b Fu(sendcount)p Fp(;)f Fu(se)o(ndtype)p Fp(;)g Fu(roo)o(t)p Fp(;)h(:)o(::)o Fx(\))p Fp(;)75 129 y Fx(and)15 b(the)h(ro)q(ot)e(had)h(executed)i Fu(n)e Fx(calls)h(to)189 213 y Fu(MPI)p 264 213 V 16 w(Recv)p Fx(\()p Fu(recvbuf)9 b Fx(+)h Fu(i)g Ft(\001)g Fu(recvcount)f Ft(\001)g Fu(extent)p Fx(\()p Fu(recvtype)p Fx(\))p Fp(;)f Fu(recvc)o(ount)p Fp(;)f Fu(recv)o(type)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))p Fp(;)75 298 y Fx(where)15 b Fu(extent\(recvtype\))e Fx(is)j(the)f(t)o(yp)q(e)h(exten)o(t)f(obtained)g(from)g(a)g(call)h(to) f Fu(MPI)p 1515 298 V 16 w(Type)p 1627 298 V 17 w(extent\(\))p Fx(.)166 354 y(An)j(alternativ)o(e)h(description)h(is)e(that)g(the)g Fu(n)g Fx(messages)g(sen)o(t)g(b)o(y)g(the)g(pro)q(cesses)h(in)g(the)f (group)75 410 y(are)c(concatenated)h(in)g(rank)f(order,)g(and)h(the)g (resulting)g(message)f(is)h(receiv)o(ed)h(b)o(y)e(the)h(ro)q(ot)f(as)g (if)h(b)o(y)f(a)75 467 y(call)i(to)f Fv(MPI)p 299 467 14 2 v 16 w(RECV\(recvbuf,)h(recvcount)p Ft(\001)p Fv(n,)g(recvt)o(yp)q (e,)g(...\))p Fx(.)166 523 y(The)f(receiv)o(e)h(bu\013er)g(is)f (ignored)h(for)f(all)h(non-ro)q(ot)f(pro)q(cesses.)166 580 y(General,)i(deriv)o(ed)h(datat)o(yp)q(es)e(are)g(allo)o(w)o(ed)h (for)f(b)q(oth)h Fv(sendt)o(yp)q(e)i Fx(and)e Fv(recvt)o(yp)q(e)p Fx(.)25 b(The)17 b(t)o(yp)q(e)g(sig-)75 636 y(nature)c(of)g Fv(sendcount,)j(sendt)o(yp)q(e)f Fx(on)e(pro)q(cess)h Fu(i)e Fx(m)o(ust)h(b)q(e)h(equal)g(to)e(the)h(t)o(yp)q(e)g(signature)h (of)e Fv(recvcount,)75 693 y(recvt)o(yp)q(e)h Fx(at)e(the)i(ro)q(ot.)18 b(This)12 b(implies)i(that)e(the)g(amoun)o(t)f(of)h(data)f(sen)o(t)h(m) o(ust)g(b)q(e)g(equal)h(to)f(the)g(amoun)o(t)75 749 y(of)j(data)g (receiv)o(ed,)i(pairwise)f(b)q(et)o(w)o(een)g(eac)o(h)g(pro)q(cess)g (and)g(the)g(ro)q(ot.)k(Distinct)c(t)o(yp)q(e)g(maps)g(b)q(et)o(w)o (een)75 806 y(sender)g(and)f(receiv)o(er)h(are)f(still)i(allo)o(w)o (ed.)166 862 y(All)e(argumen)o(ts)f(to)f(the)h(function)h(are)f (signi\014can)o(t)h(on)f(pro)q(cess)h Fv(ro)q(ot)p Fx(,)e(while)j(on)e (other)g(pro)q(cesses,)75 919 y(only)19 b(argumen)o(ts)f Fv(sendbuf,)k(sendcount,)g(sendt)o(yp)q(e,)g(ro)q(ot,)d(comm)13 b Fx(are)19 b(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 975 y Fv(ro)q(ot)c Fx(and)g Fv(comm)10 b Fx(m)o(ust)k(ha)o(v)o(e)h (iden)o(tical)i(v)m(alues)f(on)f(all)i(pro)q(cesses.)166 1031 y(The)f(sp)q(eci\014cation)h(of)f(coun)o(ts)f(and)h(t)o(yp)q(es)f (should)i(not)e(cause)h(an)o(y)g(lo)q(cation)g(on)g(the)f(ro)q(ot)g(to) g(b)q(e)75 1088 y(written)g(more)g(than)g(once.)20 b(Suc)o(h)c(a)f (call)h(is)g(erroneous.)166 1144 y(Note)e(that)f(the)h Fv(recvcount)i Fx(argumen)o(t)d(at)h(the)g(ro)q(ot)f(indicates)j(the)e (n)o(um)o(b)q(er)h(of)e(items)i(it)f(receiv)o(es)75 1201 y(from)g Fl(e)n(ach)i Fx(pro)q(cess,)f(not)f(the)i(total)e(n)o(um)o(b)q (er)i(of)f(items)g(it)h(receiv)o(es.)75 1352 y Fv(MPI)p 160 1352 V 16 w(GA)l(THERV\()24 b(sendbuf,)j(sendcount,)g(sendt)o(yp)q (e,)g(recvbuf,)e(recvcounts,)h(displs,)g(recvt)o(yp)q(e,)g(ro)q(ot,)75 1408 y(comm)m(\))117 1485 y Fk(IN)171 b Fv(sendbuf)428 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1559 y(IN)171 b Fv(sendcount)383 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1632 y(IN)171 b Fv(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 1705 y(OUT)124 b Fv(recvbuf)434 b Fk(address)21 b(of)e(receiv)o(e)j (bu\013er)e(\(c)o(hoice,)i(signi\014can)o(t)d(only)g(at)905 1761 y(ro)q(ot\))117 1835 y(IN)171 b Fv(recvcounts)372 b Fk(in)o(teger)10 b(arra)o(y)f(\(of)g(length)g(group)g(size\))h(con)o (taining)e(the)i(n)o(um-)905 1891 y(b)q(er)20 b(of)e(elemen)o(ts)h (that)f(are)i(receiv)o(ed)g(from)d(eac)o(h)i(pro)q(cess)905 1948 y(\(signi\014can)o(t)14 b(only)f(at)h(ro)q(ot\))117 2021 y(IN)171 b Fv(displs)469 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g (length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Fk(sp)q(eci\014es) 905 2077 y(the)h(displacemen)o(t)e(relativ)o(e)g(to)h Fi(recvbuf)i Fk(at)d(whic)o(h)h(to)g(place)905 2134 y(the)h(incoming)e (data)h(from)f(pro)q(cess)k Fb(i)d Fk(\(signi\014can)o(t)g(only)g(at) 905 2190 y(ro)q(ot\))117 2263 y(IN)171 b Fv(recvt)o(yp)q(e)414 b Fk(data)16 b(t)o(yp)q(e)g(of)f(recv)i(bu\013er)g(elemen)o(ts)f (\(signi\014can)o(t)f(only)g(at)905 2320 y(ro)q(ot\))f(\(handle\))117 2393 y(IN)171 b Fv(ro)q(ot)492 b Fk(rank)14 b(of)f(receiving)i(pro)q (cess)g(\(in)o(teger\))117 2466 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 2591 y Fu(int)23 b(MPI)p 245 2591 15 2 v 17 w(Gatherv\(void*)f(sendbuf,)h(int)h (sendcount,)e(MPI)p 1241 2591 V 17 w(Datatype)h(sendtype,)393 2647 y(void*)g(recvbuf,)g(int)h(*recvcounts,)e(int)h(*displs,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)g(recvtype,)f(int)i(root,)f(MPI)p 1249 2704 V 17 w(Comm)g(comm\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 96 104 96 103 bop 75 -100 a Fx(96)708 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(GATHERV\(SENDBUF,)21 b(SENDCOUNT,)i(SENDTYPE,)g (RECVBUF,)f(RECVCOUNTS,)h(DISPLS,)393 102 y(RECVTYPE,)g(ROOT,)g(COMM,)g (IERROR\))170 158 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 214 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g (DISPLS\(*\),)h(RECVTYPE,)g(ROOT,)170 271 y(COMM,)h(IERROR)166 357 y Fv(MPI)p 251 357 14 2 v 16 w(GA)l(THERV)15 b Fx(extends)g(the)f (functionalit)o(y)h(of)e Fv(MPI)p 1126 357 V 16 w(GA)l(THER)i Fx(b)o(y)f(allo)o(wing)h(a)f(v)m(arying)g(coun)o(t)75 414 y(of)i(data)g(from)g(eac)o(h)h(pro)q(cess,)g(since)h Fv(recvcounts)g Fx(is)f(no)o(w)f(an)h(arra)o(y)l(.)23 b(It)17 b(also)f(allo)o(ws)h(more)f(\015exibili)q(t)o(y)75 470 y(as)f(to)f(where)i(the)f(data)g(is)g(placed)i(on)e(the)g(ro)q(ot,) f(b)o(y)h(pro)o(viding)h(the)g(new)f(argumen)o(t,)f Fv(displs)p Fx(.)166 527 y(The)i(outcome)g(is)h Fl(as)g(if)f Fx(eac)o(h)h(pro)q (cess,)f(including)j(the)e(ro)q(ot)e(pro)q(cess,)i(sends)f(a)g(message) g(to)g(the)75 583 y(ro)q(ot,)189 677 y Fu(MPI)p 264 677 15 2 v 16 w(Send)p Fx(\()p Fu(sendbuf)p Fp(;)8 b Fu(sendcount)p Fp(;)f Fu(se)o(ndtype)p Fp(;)g Fu(roo)o(t)p Fp(;)h(:)o(::)o Fx(\))p Fp(;)75 771 y Fx(and)15 b(the)h(ro)q(ot)e(executes)i Fu(n)f Fx(receiv)o(es,)189 865 y Fu(MPI)p 264 865 V 16 w(Recv)p Fx(\()p Fu(recvbuf)9 b Fx(+)h Fu(disp)p Fx([)p Fu(i)p Fx(])f Ft(\001)h Fu(extent)p Fx(\()p Fu(recvtype)p Fx(\))p Fp(;)d Fu(recvc)o(ounts)p Fx([)o Fu(i)p Fx(])p Fp(;)g Fu(recvt)o(ype)p Fp(;)g Fu(i)p Fp(;)g(:::)o Fx(\))p Fp(:)166 958 y Fx(Messages)14 b(are)g(placed)i(in)f(the)g(receiv)o(e)g (bu\013er)g(of)f(the)g(ro)q(ot)g(pro)q(cess)h(in)g(rank)f(order,)g (that)g(is,)h(the)75 1015 y(data)e(sen)o(t)g(from)f(pro)q(cess)i Fu(j)f Fx(is)g(placed)i(in)f(the)f Fu(j)p Fx(th)g(p)q(ortion)h(of)f (the)g(receiv)o(e)h(bu\013er)g Fv(recvbuf)f Fx(on)h(pro)q(cess)75 1071 y Fv(ro)q(ot)p Fx(.)19 b(The)13 b Fu(j)p Fx(th)g(p)q(ortion)h(of)f Fv(recvbuf)h Fx(b)q(egins)g(at)f(o\013set)f Fv(displs[j])j Fx(elemen)o(ts)f(\(in)g(terms)e(of)h Fv(recvt)o(yp)q(e)p Fx(\))h(in)o(to)75 1128 y Fv(recvbuf)p Fx(.)166 1184 y(The)h(receiv)o(e)h(bu\013er)g(is)f(ignored)h(for)f(all)h(non-ro)q(ot) f(pro)q(cesses.)166 1241 y(The)i(t)o(yp)q(e)f(signature)h(implied)i(b)o (y)d Fv(sendcount,)j(sendt)o(yp)q(e)g Fx(on)e(pro)q(cess)g Fu(i)f Fx(m)o(ust)g(b)q(e)h(equal)g(to)f(the)75 1297 y(t)o(yp)q(e)f(signature)h(implied)i(b)o(y)d Fv(recvcounts[i],)i(recvt) o(yp)q(e)f Fx(at)f(the)g(ro)q(ot.)20 b(This)c(implies)i(that)c(the)i (amoun)o(t)75 1354 y(of)g(data)f(sen)o(t)h(m)o(ust)g(b)q(e)h(equal)g (to)e(the)i(amoun)o(t)e(of)h(data)f(receiv)o(ed,)j(pairwise)f(b)q(et)o (w)o(een)f(eac)o(h)h(pro)q(cess)75 1410 y(and)12 b(the)g(ro)q(ot.)17 b(Distinct)c(t)o(yp)q(e)f(maps)f(b)q(et)o(w)o(een)h(sender)g(and)g (receiv)o(er)g(are)g(still)h(allo)o(w)o(ed,)f(as)g(illustrated)75 1467 y(in)k(Example)g(4.6.)166 1523 y(All)f(argumen)o(ts)f(to)f(the)h (function)h(are)f(signi\014can)o(t)h(on)f(pro)q(cess)h Fv(ro)q(ot)p Fx(,)e(while)j(on)e(other)g(pro)q(cesses,)75 1579 y(only)19 b(argumen)o(ts)f Fv(sendbuf,)k(sendcount,)g(sendt)o(yp)q (e,)g(ro)q(ot,)d(comm)13 b Fx(are)19 b(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 1636 y Fv(ro)q(ot)c Fx(and)g Fv(comm)10 b Fx(m)o(ust)k(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)f(on)f (all)i(pro)q(cesses.)166 1692 y(The)d(sp)q(eci\014cation)i(of)d(coun)o (ts,)h(t)o(yp)q(es,)g(and)g(displacemen)o(ts)h(should)g(not)f(cause)g (an)o(y)f(lo)q(cation)i(on)75 1749 y(the)g(ro)q(ot)g(to)f(b)q(e)i (written)f(more)g(than)g(once.)20 b(Suc)o(h)c(a)f(call)h(is)g (erroneous.)75 1871 y Fj(4.5.1)49 b(Examples)14 b(using)j(MPI)p 640 1871 V 18 w(GA)l(THER,)g(MPI)p 963 1871 V 18 w(GA)l(THERV)75 1956 y Fq(Example)h(4.2)k Fx(Gather)15 b(100)f(in)o(ts)i(from)e(ev)o (ery)h(pro)q(cess)h(in)g(group)e(to)h(ro)q(ot.)k(See)d(\014gure)f(4.2.) 170 2063 y Fu(MPI_Comm)23 b(comm;)170 2119 y(int)h (gsize,sendarray[100];)170 2176 y(int)g(root,)f(*rbuf;)170 2232 y(...)170 2288 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 2345 y(rbuf)g(=)f(\(int)h(*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 2401 y(MPI_Gather\()f(sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(100,)f (MPI_INT,)g(root,)g(comm\);)75 2508 y Fq(Example)18 b(4.3)k Fx(Previous)15 b(example)g(mo)q(di\014ed)g({)f(only)h(the)f(ro)q(ot)f (allo)q(cates)i(memory)e(for)h(the)g(receiv)o(e)75 2564 y(bu\013er.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 97 105 97 104 bop 75 -100 a Fm(4.5.)29 b(GA)l(THER)1457 b Fx(97)458 4 y 16577003 6465030 0 0 33943388 13419479 startTexFig 458 4 a %%BeginDocument: mycoll-fig2.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 369.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 89 244 m 119 294 l gs 0.50 setgray fill gr gs col-1 s gr n 116.599 286.111 m 119.000 294.000 l 113.169 288.169 l gs 2 setlinejoin col-1 s gr n 189 244 m 189 294 l gs 0.50 setgray fill gr gs col-1 s gr n 191.000 286.000 m 189.000 294.000 l 187.000 286.000 l gs 2 setlinejoin col-1 s gr n 289 244 m 259 294 l gs 0.50 setgray fill gr gs col-1 s gr n 264.831 288.169 m 259.000 294.000 l 261.401 286.111 l gs 2 setlinejoin col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 409 m gs 1 -1 scale (rbuf) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr $F2psEnd %%EndDocument endTexFig 186 511 a Fx(Figure)16 b(4.2:)j(The)c(ro)q(ot)g(pro)q(cess)g(gathers)f (100)h Fu(int)p Fx(s)f(from)h(eac)o(h)g(pro)q(cess)h(in)g(the)f(group.) 170 644 y Fu(MPI_Comm)23 b(comm;)170 701 y(int)h(gsize,sendarray[100];) 170 757 y(int)g(root,)f(myrank,)g(*rbuf;)170 814 y(...)170 870 y(MPI_Comm_rank\()f(comm,)i(myrank\);)170 927 y(if)g(\()g(myrank)f (==)g(root\))h({)242 983 y(MPI_Comm_size\()e(comm,)h(&gsize\);)242 1040 y(rbuf)g(=)h(\(int)f(*\)malloc\(gsize*100*sizeof\(int)o(\)\);)242 1096 y(})170 1153 y(MPI_Gather\()g(sendarray,)f(100,)i(MPI_INT,)e (rbuf,)i(100,)f(MPI_INT,)g(root,)g(comm\);)75 1309 y Fq(Example)18 b(4.4)k Fx(Do)d(the)h(same)f(as)g(the)h(previous)g (example,)h(but)f(use)g(a)f(deriv)o(ed)i(datat)o(yp)q(e.)32 b(Note)75 1365 y(that)17 b(the)h(t)o(yp)q(e)g(cannot)g(b)q(e)h(the)f (en)o(tire)g(set)g(of)g Fu(gsize*100)k(int)p Fx(s)c(since)h(t)o(yp)q(e) f(matc)o(hing)g(is)h(de\014ned)75 1422 y(pairwise)d(b)q(et)o(w)o(een)g (the)f(ro)q(ot)f(and)i(eac)o(h)f(pro)q(cess)g(in)h(the)g(gather.)170 1528 y Fu(MPI_Comm)23 b(comm;)170 1584 y(int)h(gsize,sendarray[100];) 170 1641 y(int)g(root,)f(*rbuf;)170 1697 y(MPI_Datatype)g(rtype;)170 1754 y(...)170 1810 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 1867 y(MPI_Type_contiguous\()e(100,)h(MPI_INT,)g(&rtype)g(\);)170 1923 y(MPI_Type_commit\()f(&rtype)h(\);)170 1979 y(rbuf)h(=)f(\(int)h (*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 2036 y(MPI_Gather\()f (sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(1,)f(rtype,)g(root,)g(comm\);) 75 2142 y Fq(Example)18 b(4.5)k Fx(No)o(w)d(ha)o(v)o(e)g(eac)o(h)g(pro) q(cess)h(send)f(100)g(in)o(ts)g(to)g(ro)q(ot,)g(but)g(place)h(eac)o(h)g (set)f(\(of)f(100\))75 2199 y Fl(stride)f Fx(in)o(ts)f(apart)g(at)g (receiving)i(end.)25 b(Use)17 b Fv(MPI)p 945 2199 14 2 v 16 w(GA)l(THERV)h Fx(and)f(the)f Fv(displs)j Fx(argumen)o(t)c(to)h (ac)o(hiev)o(e)75 2255 y(this)g(e\013ect.)j(Assume)d Fp(str)q(ide)c Ft(\025)h Fx(100.)19 b(See)d(\014gure)f(4.3.)170 2361 y Fu(MPI_Comm)23 b(comm;)170 2418 y(int)h(gsize,sendarray[100];) 170 2474 y(int)g(root,)f(*rbuf,)g(stride;)170 2531 y(int)h (*displs,i,*rcounts;)170 2644 y(...)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 98 106 98 105 bop 75 -100 a Fx(98)703 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)458 4 y 16577003 6465030 0 0 33943388 13419479 startTexFig 458 4 a %%BeginDocument: mycoll-fig3.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 369.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 239 359 m 239 319 l 179 319 l 179 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 194 314 m gs 1 -1 scale (100) col-1 show gr n 319 359 m 319 319 l 259 319 l 259 359 l clp gs 0.50 setgray fill gr gs col-1 s gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 314 m gs 1 -1 scale (100) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 199 394 m gs 1 -1 scale (stride) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 89 244 m 119 294 l gs 0.50 setgray fill gr gs col-1 s gr n 116.599 286.111 m 119.000 294.000 l 113.169 288.169 l gs 2 setlinejoin col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 189 244 m 204 294 l gs col-1 s gr n 203.617 285.763 m 204.000 294.000 l 199.786 286.912 l gs 2 setlinejoin col-1 s gr n 289 244 m 289 294 l gs col-1 s gr n 291.000 286.000 m 289.000 294.000 l 287.000 286.000 l gs 2 setlinejoin col-1 s gr n 187.000 371.000 m 179.000 369.000 l 187.000 367.000 l gs 2 setlinejoin col-1 s gr n 179 369 m 259 369 l gs col-1 s gr n 251.000 367.000 m 259.000 369.000 l 251.000 371.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 409 m gs 1 -1 scale (rbuf) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 511 a Fx(Figure)h(4.3:)22 b(The)16 b(ro)q(ot)g(pro)q(cess)h (gathers)f(100)g Fu(int)p Fx(s)g(from)f(eac)o(h)i(pro)q(cess)g(in)g (the)g(group,)f(eac)o(h)h(set)f(is)75 568 y(placed)g Fu(stride)f Fx(in)o(ts)g(apart.)170 700 y Fu(MPI_Comm_size\()22 b(comm,)i(&gsize\);)170 757 y(rbuf)g(=)f(\(int)h (*\)malloc\(gsize*stride*siz)o(eof\(int)o(\)\);)170 813 y(displs)f(=)h(\(int)g(*\)malloc\(gsize*sizeof\(i)o(nt\)\);)170 870 y(rcounts)f(=)h(\(int)f(*\)malloc\(gsize*sizeof\(int\)\);)170 926 y(for)h(\(i=0;)f(i)f(SENDBUF\(*\),)g (RECVBUF\(*\))170 1450 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,) h(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR)166 1537 y Fv(MPI)p 251 1537 14 2 v 16 w(SCA)l(TTER)16 b Fx(is)g(the)f(in)o(v)o(erse)h(op)q (eration)f(to)g Fv(MPI)p 1097 1537 V 15 w(GA)l(THER)p Fx(.)166 1593 y(The)g(outcome)g(is)h Fl(as)g(if)f Fx(the)g(ro)q(ot)g (executed)h Fu(n)f Fx(send)h(op)q(erations,)189 1687 y Fu(MPI)p 264 1687 15 2 v 16 w(Send)p Fx(\()p Fu(sendbuf)9 b Fx(+)h Fu(i)g Ft(\001)g Fu(sendcount)f Ft(\001)g Fu(extent)p Fx(\()p Fu(sendtype)p Fx(\))p Fp(;)f Fu(sendc)o(ount)p Fp(;)f Fu(send)o(type)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))p Fp(;)75 1781 y Fx(and)15 b(eac)o(h)h(pro)q(cess)f(executed)h(a) f(receiv)o(e,)189 1875 y Fu(MPI)p 264 1875 V 16 w(Recv)p Fx(\()p Fu(recvbuf)p Fp(;)8 b Fu(recvcount)p Fp(;)f Fu(re)o(cvtype)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))o Fp(:)166 1968 y Fx(An)20 b(alternativ)o(e)g(description)h(is)g(that)e(the)h(ro)q(ot)f (sends)h(a)f(message)h(with)g Fv(MPI)p 1594 1968 14 2 v 16 w(Send\(sendbuf,)75 2025 y(sendcount)p Ft(\001)p Fv(n,)f(sendt)o(yp)q(e,)h(...\))p Fx(.)i(This)17 b(message)f(is)h (split)h(in)o(to)e Fu(n)g Fx(equal)i(segmen)o(ts,)e(the)g Fp(i)p Fx(th)g(segmen)o(t)g(is)75 2081 y(sen)o(t)f(to)g(the)g Fp(i)p Fx(th)g(pro)q(cess)g(in)h(the)f(group,)g(and)g(eac)o(h)g(pro)q (cess)h(receiv)o(es)g(this)g(message)e(as)h(ab)q(o)o(v)o(e.)166 2138 y(The)g(send)h(bu\013er)f(is)h(ignored)g(for)e(all)i(non-ro)q(ot)f (pro)q(cesses.)166 2194 y(The)j(t)o(yp)q(e)f(signature)h(asso)q(ciated) g(with)f Fv(sendcount,)k(sendt)o(yp)q(e)f Fx(at)d(the)h(ro)q(ot)e(m)o (ust)h(b)q(e)h(equal)h(to)75 2251 y(the)f(t)o(yp)q(e)g(signature)g (asso)q(ciated)g(with)g Fv(recvcount,)h(recvt)o(yp)q(e)g Fx(at)e(all)i(pro)q(cesses)f(\(ho)o(w)o(ev)o(er,)f(the)h(t)o(yp)q(e)75 2307 y(maps)f(ma)o(y)g(b)q(e)h(di\013eren)o(t\).)27 b(This)18 b(implies)i(that)d(the)g(amoun)o(t)g(of)g(data)g(sen)o(t)g(m)o(ust)g(b) q(e)i(equal)f(to)f(the)75 2364 y(amoun)o(t)f(of)f(data)h(receiv)o(ed,)i (pairwise)f(b)q(et)o(w)o(een)f(eac)o(h)h(pro)q(cess)f(and)h(the)f(ro)q (ot.)23 b(Distinct)17 b(t)o(yp)q(e)f(maps)75 2420 y(b)q(et)o(w)o(een)g (sender)f(and)h(receiv)o(er)g(are)e(still)j(allo)o(w)o(ed.)166 2477 y(All)e(argumen)o(ts)f(to)f(the)h(function)h(are)f(signi\014can)o (t)h(on)f(pro)q(cess)h Fv(ro)q(ot)p Fx(,)e(while)j(on)e(other)g(pro)q (cesses,)75 2533 y(only)f(argumen)o(ts)f Fv(recvbuf,)i(recvcount,)g (recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)7 b Fx(are)12 b(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fv(ro)q(ot)75 2589 y Fx(and)j Fv(comm)10 b Fx(m)o(ust)k(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)g(on)e (all)h(pro)q(cesses.)166 2646 y(The)g(sp)q(eci\014cation)h(of)f(coun)o (ts)f(and)h(t)o(yp)q(es)f(should)i(not)e(cause)h(an)o(y)g(lo)q(cation)g (on)g(the)f(ro)q(ot)g(to)g(b)q(e)75 2702 y(read)g(more)g(than)g(once.) -32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 104 112 104 111 bop 75 -100 a Fx(104)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)189 45 y Fl(R)n(ationale.)79 b Fx(Though)22 b(not)g(needed,)i(the)e(last)g(restriction)g(is)g(imp)q (osed)h(so)f(as)f(to)g(ac)o(hiev)o(e)189 102 y(symmetry)16 b(with)h Fv(MPI)p 593 102 14 2 v 16 w(GA)l(THER)p Fx(,)h(where)f(the)g (corresp)q(onding)h(restriction)g(\(a)e(m)o(ultiple-write)189 158 y(restriction\))f(is)h(necessary)l(.)k(\()p Fl(End)c(of)g(r)n (ationale.)p Fx(\))75 354 y Fv(MPI)p 160 354 V 16 w(SCA)l(TTERV\()21 b(sendbuf,)j(sendcounts)q(,)g(displs,)f(sendt)o(yp)q(e,)h(recvbuf,)f (recvcount,)f(recvt)o(yp)q(e,)h(ro)q(ot,)75 411 y(comm)m(\))117 488 y Fk(IN)171 b Fv(sendbuf)428 b Fk(address)13 b(of)d(send)i (bu\013er)h(\(c)o(hoice,)f(signi\014can)o(t)e(only)h(at)g(ro)q(ot\))117 562 y(IN)171 b Fv(sendcounts)366 b Fk(in)o(teger)10 b(arra)o(y)f(\(of)g (length)g(group)g(size\))h(sp)q(ecifying)g(the)g(n)o(um-)905 619 y(b)q(er)15 b(of)e(elemen)o(ts)h(to)g(send)h(to)e(eac)o(h)i(pro)q (cessor)117 693 y(IN)171 b Fv(displs)469 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Fk(sp)q(eci\014es)905 750 y(the)20 b(displacemen)o(t)e (\(relativ)o(e)h(to)f Fi(sendbuf)j Fk(from)c(whic)o(h)i(to)905 806 y(tak)o(e)14 b(the)h(outgoing)d(data)i(to)g(pro)q(cess)h Fb(i)117 881 y Fk(IN)171 b Fv(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 955 y(OUT)124 b Fv(recvbuf)434 b Fk(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 1030 y(IN)171 b Fv(recvcount)389 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 1104 y(IN)171 b Fv(recvt)o(yp)q(e)414 b Fk(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e (\(handle\))117 1179 y(IN)171 b Fv(ro)q(ot)492 b Fk(rank)14 b(of)f(sending)h(pro)q(cess)i(\(in)o(teger\))117 1253 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 1378 y Fu(int)23 b(MPI)p 245 1378 15 2 v 17 w(Scatterv\(void*)f (sendbuf,)h(int)g(*sendcounts,)g(int)g(*displs,)393 1434 y(MPI)p 468 1434 V 17 w(Datatype)g(sendtype,)f(void*)i(recvbuf,)e(int)i (recvcount,)393 1491 y(MPI)p 468 1491 V 17 w(Datatype)f(recvtype,)f (int)i(root,)f(MPI)p 1249 1491 V 17 w(Comm)g(comm\))75 1577 y(MPI)p 150 1577 V 17 w(SCATTERV\(SENDBUF,)e(SENDCOUNTS,)i (DISPLS,)g(SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)393 1634 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\))170 1690 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 1747 y(INTEGER)g(SENDCOUNTS\(*\),)f (DISPLS\(*\),)h(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)170 1803 y(COMM,)h(IERROR)166 1889 y Fv(MPI)p 251 1889 14 2 v 16 w(SCA)l(TTERV)16 b Fx(is)g(the)f(in)o(v)o(erse)h(op)q(eration)f (to)g Fv(MPI)p 1127 1889 V 16 w(GA)l(THERV)p Fx(.)166 1946 y Fv(MPI)p 251 1946 V 16 w(SCA)l(TTERV)24 b Fx(extends)f(the)g (functionalit)o(y)g(of)g Fv(MPI)p 1193 1946 V 15 w(SCA)l(TTER)h Fx(b)o(y)f(allo)o(wing)g(a)f(v)m(arying)75 2002 y(coun)o(t)17 b(of)h(data)f(to)g(b)q(e)h(sen)o(t)f(to)g(eac)o(h)h(pro)q(cess,)g (since)h Fv(sendcounts)i Fx(is)d(no)o(w)f(an)g(arra)o(y)l(.)27 b(It)17 b(also)h(allo)o(ws)75 2059 y(more)k(\015exibilit)o(y)i(as)e(to) f(where)h(the)g(data)g(is)g(tak)o(en)g(from)f(on)h(the)g(ro)q(ot,)h(b)o (y)f(pro)o(viding)h(the)f(new)75 2115 y(argumen)o(t,)14 b Fv(displs)p Fx(.)166 2172 y(The)h(outcome)g(is)h(as)f(if)g(the)h(ro)q (ot)e(executed)i Fu(n)f Fx(send)h(op)q(erations,)189 2262 y Fu(MPI)p 264 2262 15 2 v 16 w(Send)p Fx(\()p Fu(sendbuf)9 b Fx(+)h Fu(displs)p Fx([)p Fu(i)p Fx(])f Ft(\001)g Fu(extent)p Fx(\()p Fu(sendtype)p Fx(\))p Fp(;)f Fu(sen)o(dcounts)p Fx([)o Fu(i)p Fx(])p Fp(;)f Fu(sen)o(dtype)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))p Fp(;)75 2353 y Fx(and)15 b(eac)o(h)h(pro)q (cess)f(executed)h(a)f(receiv)o(e,)189 2444 y Fu(MPI)p 264 2444 V 16 w(Recv)p Fx(\()p Fu(recvbuf)p Fp(;)8 b Fu(recvcount)p Fp(;)f Fu(re)o(cvtype)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))o Fp(:)166 2534 y Fx(The)15 b(send)h(bu\013er)f (is)h(ignored)g(for)e(all)i(non-ro)q(ot)f(pro)q(cesses.)166 2591 y(The)h(t)o(yp)q(e)g(signature)g(implied)i(b)o(y)e Fv(sendcount[i],)j(sendt)o(yp)q(e)g Fx(at)c(the)h(ro)q(ot)f(m)o(ust)g (b)q(e)i(equal)g(to)e(the)75 2647 y(t)o(yp)q(e)f(signature)h(implied)i (b)o(y)d Fv(recvcount,)i(recvt)o(yp)q(e)f Fx(at)f(pro)q(cess)g Fu(i)g Fx(\(ho)o(w)o(ev)o(er,)f(the)i(t)o(yp)q(e)f(maps)h(ma)o(y)e(b)q (e)75 2704 y(di\013eren)o(t\).)20 b(This)c(implies)h(that)e(the)g (amoun)o(t)f(of)h(data)g(sen)o(t)g(m)o(ust)f(b)q(e)i(equal)g(to)f(the)g (amoun)o(t)g(of)g(data)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 105 113 105 112 bop 75 -100 a Fm(4.6.)29 b(SCA)l(TTER)1413 b Fx(105)458 4 y 16577003 6630800 0 0 33943388 13682606 startTexFig 458 4 a %%BeginDocument: mycoll-fig7.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 373.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 414 m gs 1 -1 scale (sendbuf) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 119 299 m 89 244 l gs col-1 s gr n 91.075 251.981 m 89.000 244.000 l 94.587 250.065 l gs 2 setlinejoin col-1 s gr n 189 299 m 189 244 l gs col-1 s gr n 187.000 252.000 m 189.000 244.000 l 191.000 252.000 l gs 2 setlinejoin col-1 s gr n 249 299 m 289 244 l gs col-1 s gr n 282.677 249.294 m 289.000 244.000 l 285.912 251.646 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr $F2psEnd %%EndDocument endTexFig 138 522 a Fx(Figure)16 b(4.7:)i(The)e(ro)q(ot)e(pro)q(cess)i(scatters) e(sets)h(of)g(100)f Fu(int)p Fx(s)h(to)f(eac)o(h)i(pro)q(cess)f(in)h (the)f(group.)75 655 y(receiv)o(ed,)j(pairwise)f(b)q(et)o(w)o(een)g (eac)o(h)g(pro)q(cess)g(and)g(the)g(ro)q(ot.)23 b(Distinct)17 b(t)o(yp)q(e)g(maps)f(b)q(et)o(w)o(een)h(sender)75 711 y(and)e(receiv)o(er)h(are)f(still)i(allo)o(w)o(ed.)166 768 y(All)e(argumen)o(ts)f(to)f(the)h(function)h(are)f(signi\014can)o (t)h(on)f(pro)q(cess)h Fv(ro)q(ot)p Fx(,)e(while)j(on)e(other)g(pro)q (cesses,)75 824 y(only)f(argumen)o(ts)f Fv(recvbuf,)i(recvcount,)g (recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)7 b Fx(are)12 b(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fv(ro)q(ot)75 881 y Fx(and)j Fv(comm)10 b Fx(m)o(ust)k(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)g(on)e (all)h(pro)q(cesses.)166 937 y(The)e(sp)q(eci\014cation)i(of)d(coun)o (ts,)h(t)o(yp)q(es,)g(and)g(displacemen)o(ts)h(should)g(not)f(cause)g (an)o(y)f(lo)q(cation)i(on)75 994 y(the)g(ro)q(ot)g(to)f(b)q(e)i(read)f (more)g(than)g(once.)75 1115 y Fj(4.6.1)49 b(Examples)14 b(using)j(MPI)p 640 1115 15 2 v 18 w(SCA)l(TTER,)h(MPI)p 988 1115 V 18 w(SCA)l(TTERV)75 1201 y Fq(Example)g(4.11)23 b Fx(The)15 b(rev)o(erse)g(of)f(Example)i(4.2.)j(Scatter)14 b(sets)h(of)g(100)f(in)o(ts)h(from)f(the)h(ro)q(ot)f(to)h(eac)o(h)75 1258 y(pro)q(cess)g(in)h(the)g(group.)j(See)d(\014gure)g(4.7.)170 1364 y Fu(MPI_Comm)23 b(comm;)170 1420 y(int)h(gsize,*sendbuf;)170 1477 y(int)g(root,)f(rbuf[100];)170 1533 y(...)170 1590 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 1646 y(sendbuf)f(=)h(\(int)f (*\)malloc\(gsize*100*sizeof\(int)o(\)\);)170 1703 y(...)170 1759 y(MPI_Scatter\()g(sendbuf,)f(100,)i(MPI_INT,)f(rbuf,)g(100,)g (MPI_INT,)g(root,)g(comm\);)75 1915 y Fq(Example)18 b(4.12)23 b Fx(The)16 b(rev)o(erse)h(of)f(Example)i(4.5.)23 b(The)17 b(ro)q(ot)f(pro)q(cess)h(scatters)f(sets)g(of)h(100)f(in)o(ts)h(to)75 1972 y(the)d(other)g(pro)q(cesses,)g(but)g(the)g(sets)g(of)g(100)f(are) h Fl(stride)f Fx(in)o(ts)i(apart)e(in)i(the)f(sending)h(bu\013er.)20 b(Requires)75 2028 y(use)c(of)e Fv(MPI)p 290 2028 14 2 v 16 w(SCA)l(TTERV)p Fx(.)i(Assume)g Fp(str)q(ide)c Ft(\025)h Fx(100.)19 b(See)d(\014gure)g(4.8.)170 2134 y Fu(MPI_Comm)23 b(comm;)170 2191 y(int)h(gsize,*sendbuf;)170 2247 y(int)g(root,)f(rbuf[100],)g(i,)g(*displs,)g(*scounts;)170 2360 y(...)170 2473 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 2530 y(sendbuf)f(=)h(\(int)f(*\)malloc\(gsize*stride*sizeof\()o (int\)\);)170 2586 y(...)170 2642 y(displs)g(=)h(\(int)g (*\)malloc\(gsize*sizeof\(i)o(nt\)\);)170 2699 y(scounts)f(=)h(\(int)f (*\)malloc\(gsize*sizeof\(int\)\);)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 106 114 106 113 bop 75 -100 a Fx(106)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)458 4 y 16577003 6796570 0 0 33943388 14011514 startTexFig 458 4 a %%BeginDocument: mycoll-fig8.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 378.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr n 239 359 m 239 319 l 179 319 l 179 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 194 314 m gs 1 -1 scale (100) col-1 show gr n 319 359 m 319 319 l 259 319 l 259 359 l clp gs 0.50 setgray fill gr gs col-1 s gr n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 314 m gs 1 -1 scale (100) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 99 419 m gs 1 -1 scale (sendbuf) col-1 show gr n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 187.000 371.000 m 179.000 369.000 l 187.000 367.000 l gs 2 setlinejoin col-1 s gr n 179 369 m 259 369 l gs col-1 s gr n 251.000 367.000 m 259.000 369.000 l 251.000 371.000 l gs 2 setlinejoin col-1 s gr n 119 299 m 84 244 l gs col-1 s gr n 86.608 251.823 m 84.000 244.000 l 89.982 249.676 l gs 2 setlinejoin col-1 s gr n 204 304 m 189 244 l gs col-1 s gr n 189.000 252.246 m 189.000 244.000 l 192.881 251.276 l gs 2 setlinejoin col-1 s gr n 284 304 m 289 244 l gs col-1 s gr n 286.343 251.806 m 289.000 244.000 l 290.329 252.138 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 199 394 m gs 1 -1 scale (stride) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 532 a Fx(Figure)f(4.8:)j(The)d(ro)q(ot)e(pro)q(cess)i(scatters)f (sets)g(of)g(100)f Fu(int)p Fx(s,)h(mo)o(ving)g(b)o(y)h Fu(stride)e Fx(in)o(ts)i(from)f(send)h(to)75 589 y(send)h(in)g(the)f (scatter.)170 710 y Fu(for)24 b(\(i=0;)f(i)g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2504 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)166 2591 y Fv(MPI)p 251 2591 14 2 v 16 w(ALLGA)l(THER)16 b Fx(can)g(b)q(e)h(though)o(t)e(of)g(as)h Fv(MPI)p 1047 2591 V 15 w(GA)l(THER)p Fx(,)h(but)f(where)g(all)h(pro)q(cesses)f (receiv)o(e)75 2647 y(the)e(result,)f(instead)i(of)e(just)g(the)g(ro)q (ot.)19 b(The)13 b Fu(j)p Fx(th)h(blo)q(c)o(k)g(of)f(data)g(sen)o(t)g (from)g(eac)o(h)g(pro)q(cess)h(is)g(receiv)o(ed)75 2704 y(b)o(y)h(ev)o(ery)g(pro)q(cess)h(and)f(placed)h(in)g(the)g Fu(j)p Fx(th)f(blo)q(c)o(k)h(of)e(the)i(bu\013er)f Fv(recvbuf)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 108 116 108 115 bop 75 -100 a Fx(108)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)166 45 y Fx(The)f(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)h Fv(sendcount,)h(sendt)o(yp)q(e)p Fx(,)h(at)c(a)h(pro)q(cess)g(m)o(ust)f(b)q(e)i(equal)g(to)75 102 y(the)f(t)o(yp)q(e)h(signature)f(asso)q(ciated)g(with)h Fv(recvcount,)g(recvt)o(yp)q(e)g Fx(at)f(an)o(y)f(other)h(pro)q(cess.) 166 158 y(The)f(outcome)g(of)g(a)g(call)h(to)f Fv(MPI)p 749 158 14 2 v 16 w(ALLGA)l(THER\(...\))19 b Fx(is)c(as)f(if)g(all)h (pro)q(cesses)g(executed)g Fu(n)f Fx(calls)i(to)147 250 y Fu(MPI_GATHER\(sendbuf,send)o(count,s)o(endtype,)o(recvbuf)o(,recvco) o(unt,)1244 306 y(recvtype,root,comm\),)75 398 y Fx(for)11 b Fu(root)23 b(=)h(0)g(,)f(...,)h(n-1)p Fx(.)18 b(The)12 b(rules)g(for)f(correct)g(usage)g(of)h Fv(MPI)p 1289 398 V 15 w(ALLGA)l(THER)h Fx(are)e(easily)i(found)75 455 y(from)h(the)i(corresp)q(onding)g(rules)g(for)e Fv(MPI)p 816 455 V 16 w(GA)l(THER)p Fx(.)75 606 y Fv(MPI)p 160 606 V 16 w(ALLGA)l(THERV\()d(sendbuf,)i(sendcount,)h(sendt)o(yp)q(e,)f (recvbuf,)f(recvcounts,)g(displs,)g(recvt)o(yp)q(e,)g(comm)m(\))117 739 y Fk(IN)171 b Fv(sendbuf)428 b Fk(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 814 y(IN)171 b Fv(sendcount)383 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 889 y(IN)171 b Fv(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 964 y(OUT)124 b Fv(recvbuf)434 b Fk(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1038 y(IN)171 b Fv(recvcounts)372 b Fk(in)o(teger)10 b(arra)o(y)f(\(of)g (length)g(group)g(size\))h(con)o(taining)e(the)i(n)o(um-)905 1095 y(b)q(er)15 b(of)e(elemen)o(ts)h(that)g(are)g(receiv)o(ed)h(from)e (eac)o(h)h(pro)q(cess)117 1170 y(IN)171 b Fv(displs)469 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Fk(sp)q(eci\014es)905 1226 y(the)d(displacemen)o (t)e(\(relativ)o(e)g(to)h Fi(recvbuf)p Fk(\))i(at)d(whic)o(h)h(to)g (place)905 1283 y(the)15 b(incoming)c(data)j(from)e(pro)q(cess)k Fb(i)117 1357 y Fk(IN)171 b Fv(recvt)o(yp)q(e)414 b Fk(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e(\(handle\)) 117 1432 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 1556 y Fu(int)23 b(MPI)p 245 1556 15 2 v 17 w(Allgatherv\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 1556 V 16 w(Datatype)g(sendtype,)393 1613 y(void*)g(recvbuf,)g (int)h(*recvcounts,)e(int)h(*displs,)393 1669 y(MPI)p 468 1669 V 17 w(Datatype)g(recvtype,)f(MPI)p 1010 1669 V 17 w(Comm)h(comm\))75 1756 y(MPI)p 150 1756 V 17 w (ALLGATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h (RECVCOUNTS,)f(DISPLS,)393 1812 y(RECVTYPE,)h(COMM,)g(IERROR\))170 1869 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1925 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g(DISPLS\(*\),)h (RECVTYPE,)g(COMM,)170 1982 y(IERROR)166 2068 y Fv(MPI)p 251 2068 14 2 v 16 w(ALLGA)l(THERV)11 b Fx(can)g(b)q(e)g(though)o(t)e (of)h(as)g Fv(MPI)p 1044 2068 V 16 w(GA)l(THERV)p Fx(,)h(but)g(where)f (all)h(pro)q(cesses)g(receiv)o(e)75 2125 y(the)j(result,)f(instead)i (of)e(just)g(the)g(ro)q(ot.)19 b(The)13 b Fu(j)p Fx(th)h(blo)q(c)o(k)g (of)f(data)g(sen)o(t)g(from)g(eac)o(h)g(pro)q(cess)h(is)g(receiv)o(ed) 75 2181 y(b)o(y)i(ev)o(ery)h(pro)q(cess)g(and)f(placed)i(in)f(the)g Fu(j)p Fx(th)f(blo)q(c)o(k)h(of)f(the)h(bu\013er)f Fv(recvbuf)p Fx(.)25 b(These)17 b(blo)q(c)o(ks)g(need)g(not)75 2237 y(all)f(b)q(e)g(the)f(same)g(size.)166 2294 y(The)g(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)g Fv(sendcount,)j(sendt)o(yp)q(e)p Fx(,)f(at)d(pro)q(cess)h Fu(j)g Fx(m)o(ust)f(b)q(e)i(equal)g(to)75 2350 y(the)f(t)o(yp)q(e)h(signature)f(asso)q(ciated)g(with)h Fv(recvcounts[j],)h(recvt)o(yp)q(e)f Fx(at)e(an)o(y)h(other)g(pro)q (cess.)166 2407 y(The)g(outcome)g(is)h(as)f(if)g(all)h(pro)q(cesses)g (executed)g(calls)g(to)170 2499 y Fu(MPI_GATHERV\(sendbuf,sendcount)o (,sendtyp)o(e,recvb)o(uf,recv)o(counts,d)o(ispls,)1292 2555 y(recvtype,root,comm\),)75 2647 y Fx(for)k Fu(root)j(=)h(0)f(,)h (...,)f(n-1)p Fx(.)35 b(The)20 b(rules)h(for)e(correct)h(usage)g(of)g Fv(MPI)p 1365 2647 V 16 w(ALLGA)l(THERV)h Fx(are)f(easily)75 2704 y(found)c(from)e(the)h(corresp)q(onding)h(rules)g(for)f Fv(MPI)p 944 2704 V 16 w(GA)l(THERV)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 109 117 109 116 bop 75 -100 a Fm(4.8.)34 b(ALL-TO-ALL)18 b(SCA)l(TTER/GA)l (THER)892 b Fx(109)75 45 y Fj(4.7.1)49 b(Examples)14 b(using)j(MPI)p 640 45 15 2 v 18 w(ALLGA)l(THER,)h(MPI)p 1048 45 V 18 w(ALLGA)l(THERV)75 131 y Fq(Example)g(4.14)23 b Fx(The)18 b(all-gather)h(v)o(ersion)g(of)f(Example)i(4.2.)29 b(Using)19 b Fv(MPI)p 1434 131 14 2 v 16 w(ALLGA)l(THER)p Fx(,)g(w)o(e)f(will)75 187 y(gather)d(100)f(in)o(ts)h(from)g(ev)o(ery)g (pro)q(cess)g(in)h(the)g(group)e(to)h(ev)o(ery)g(pro)q(cess.)170 294 y Fu(MPI_Comm)23 b(comm;)170 350 y(int)h(gsize,sendarray[100];)170 407 y(int)g(*rbuf;)170 463 y(...)170 519 y(MPI_Comm_size\()e(comm,)i (&gsize\);)170 576 y(rbuf)g(=)f(\(int)h(*\)malloc\(gsize*100*sizeof)o (\(int\)\);)170 632 y(MPI_Allgather\()e(sendarray,)h(100,)g(MPI_INT,)g (rbuf,)g(100,)g(MPI_INT,)g(comm\);)166 739 y Fx(After)15 b(the)g(call,)h(ev)o(ery)f(pro)q(cess)h(has)f(the)g(group-wide)h (concatenation)f(of)g(the)g(sets)g(of)g(data.)75 882 y Fn(4.8)59 b(All-to-All)20 b(Scatter/Gather)75 1078 y Fv(MPI)p 160 1078 V 16 w(ALL)l(TO)o(ALL\(sendbuf,)c(sendcount,)i (sendt)o(yp)q(e,)f(recvbuf,)f(recvcount,)g(recvt)o(yp)q(e,)g(comm)m(\)) 117 1155 y Fk(IN)171 b Fv(sendbuf)428 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1230 y(IN)171 b Fv(sendcount)383 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1305 y(IN)171 b Fv(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 1380 y(OUT)124 b Fv(recvbuf)434 b Fk(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 1456 y(IN)171 b Fv(recvcount)389 b Fk(n)o(um)o(b)q(er)17 b(of)f(elemen)o(ts)h(receiv)o(ed)i(from)c(an)o (y)i(pro)q(cess)i(\(in)o(te-)905 1512 y(ger\))117 1587 y(IN)171 b Fv(recvt)o(yp)q(e)414 b Fk(data)14 b(t)o(yp)q(e)g(of)f (receiv)o(e)j(bu\013er)f(elemen)o(ts)e(\(handle\))117 1662 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 1787 y Fu(int)23 b(MPI)p 245 1787 15 2 v 17 w(Alltoall\(void*)f (sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1787 V 17 w(Datatype)f (sendtype,)393 1843 y(void*)h(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 1843 V 17 w(Datatype)h(recvtype,)393 1900 y(MPI)p 468 1900 V 17 w(Comm)g(comm\))75 1986 y(MPI)p 150 1986 V 17 w(ALLTOALL\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h (RECVCOUNT,)g(RECVTYPE,)393 2043 y(COMM,)g(IERROR\))170 2099 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2155 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)166 2242 y Fv(MPI)p 251 2242 14 2 v 16 w(ALL)l(TO)o(ALL)18 b Fx(is)h(an)g(extension)h(of)e Fv(MPI)p 962 2242 V 16 w(ALLGA)l(THER)i Fx(to)e(the)h(case)g(where)g(eac)o(h)g(pro)q(cess)75 2298 y(sends)e(distinct)g(data)f(to)g(eac)o(h)g(of)g(the)g(receiv)o (ers.)24 b(The)17 b Fu(j)p Fx(th)f(blo)q(c)o(k)h(sen)o(t)f(from)f(pro)q (cess)i Fu(i)f Fx(is)h(receiv)o(ed)75 2355 y(b)o(y)e(pro)q(cess)h Fu(j)f Fx(and)g(is)h(placed)g(in)g(the)f Fu(i)p Fx(th)g(blo)q(c)o(k)h (of)f Fv(recvbuf)p Fx(.)166 2411 y(The)g(t)o(yp)q(e)g(signature)g(asso) q(ciated)g(with)h Fv(sendcount,)h(sendt)o(yp)q(e)p Fx(,)h(at)c(a)h(pro) q(cess)g(m)o(ust)f(b)q(e)i(equal)g(to)75 2468 y(the)i(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)g Fv(recvcount,)i(recvt)o(yp)q(e)f Fx(at)e(an)o(y)h(other)f(pro)q(cess.)28 b(This)19 b(implies)75 2524 y(that)10 b(the)h(amoun)o(t)g(of)f(data)h(sen)o(t)f(m)o(ust)h(b)q (e)h(equal)f(to)g(the)g(amoun)o(t)f(of)h(data)f(receiv)o(ed,)j (pairwise)f(b)q(et)o(w)o(een)75 2581 y(ev)o(ery)j(pair)h(of)e(pro)q (cesses.)21 b(As)15 b(usual,)g(ho)o(w)o(ev)o(er,)f(the)i(t)o(yp)q(e)f (maps)g(ma)o(y)f(b)q(e)i(di\013eren)o(t.)166 2637 y(The)d(outcome)g(is) h(as)e(if)i(eac)o(h)f(pro)q(cess)g(executed)h(a)f(send)h(to)e(eac)o(h)h (pro)q(cess)h(\(itself)f(included\))j(with)75 2694 y(a)f(call)h(to,)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 110 118 110 117 bop 75 -100 a Fx(110)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)189 45 y Fu(MPI)p 264 45 15 2 v 16 w(Send)p Fx(\()p Fu(sendbuf)9 b Fx(+)h Fu(i)g Ft(\001)g Fu(sendcount)f Ft(\001)g Fu(extent)p Fx(\()p Fu(sendtype)p Fx(\))p Fp(;)f Fu(sendc)o(ount)p Fp(;)f Fu(send)o(type)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))p Fp(;)75 139 y Fx(and)15 b(a)g(receiv)o(e)h(from)f(ev)o(ery)g (other)g(pro)q(cess)g(with)h(a)f(call)h(to,)189 233 y Fu(MPI)p 264 233 V 16 w(Recv)p Fx(\()p Fu(recvbuf)9 b Fx(+)h Fu(i)g Ft(\001)g Fu(recvcount)f Ft(\001)g Fu(extent)p Fx(\()p Fu(recvtype)p Fx(\))p Fp(;)f Fu(recvc)o(ount)p Fp(;)f Fu(i)p Fp(;)g(::)o(:)p Fx(\))p Fp(:)166 327 y Fx(All)14 b(argumen)o(ts)d(on)h(all)i(pro)q(cesses)f(are)f (signi\014can)o(t.)20 b(The)12 b(argumen)o(t)g Fv(comm)6 b Fx(m)o(ust)12 b(ha)o(v)o(e)g(iden)o(tical)75 383 y(v)m(alues)k(on)f (all)i(pro)q(cesses.)75 534 y Fv(MPI)p 160 534 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V\(sendbuf,)13 b(sendcounts)q(,)h(sdispls,)g (sendt)o(yp)q(e,)g(recvbuf,)e(recvcounts,)i(rdispls,)e(recvt)o(yp)q(e,) 75 590 y(comm)m(\))117 668 y Fk(IN)171 b Fv(sendbuf)428 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 743 y(IN)171 b Fv(sendcounts)366 b Fk(in)o(teger)20 b(arra)o(y)f(equal) h(to)f(the)h(group)g(size)g(sp)q(ecifying)f(the)905 799 y(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(to)h(send)h(to)f(eac)o(h)g (pro)q(cessor)117 874 y(IN)171 b Fv(sdispls)452 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(j)f Fk(sp)q(eci\014es)905 931 y(the)20 b(displacemen)o(t)e (\(relativ)o(e)h(to)f Fi(sendbuf)j Fk(from)c(whic)o(h)i(to)905 987 y(tak)o(e)14 b(the)h(outgoing)d(data)i(destined)h(for)e(pro)q(cess) j Fb(j)117 1062 y Fk(IN)171 b Fv(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 1138 y(OUT)124 b Fv(recvbuf)434 b Fk(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1213 y(IN)171 b Fv(recvcounts)372 b Fk(in)o(teger)20 b(arra)o(y)f(equal)h (to)f(the)h(group)g(size)g(sp)q(ecifying)f(the)905 1269 y(n)o(um)o(b)q(er)g(of)f(elemen)o(ts)h(that)g(can)h(b)q(e)g(receiv)o (ed)g(from)d(eac)o(h)905 1326 y(pro)q(cessor)117 1401 y(IN)171 b Fv(rdispls)453 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g (length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Fk(sp)q(eci\014es) 905 1457 y(the)f(displacemen)o(t)e(\(relativ)o(e)g(to)h Fi(recvbuf)i Fk(at)d(whic)o(h)h(to)g(place)905 1514 y(the)j(incoming)c (data)j(from)e(pro)q(cess)k Fb(i)117 1589 y Fk(IN)171 b Fv(recvt)o(yp)q(e)414 b Fk(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j (bu\013er)f(elemen)o(ts)e(\(handle\))117 1664 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 1788 y Fu(int)23 b(MPI)p 245 1788 15 2 v 17 w(Alltoallv\(void*)f (sendbuf,)h(int)g(*sendcounts,)f(int)i(*sdispls,)393 1845 y(MPI)p 468 1845 V 17 w(Datatype)f(sendtype,)f(void*)i(recvbuf,)e (int)i(*recvcounts,)393 1901 y(int)g(*rdispls,)e(MPI)p 802 1901 V 17 w(Datatype)h(recvtype,)f(MPI)p 1344 1901 V 17 w(Comm)i(comm\))75 1988 y(MPI)p 150 1988 V 17 w (ALLTOALLV\(SENDBUF,)d(SENDCOUNTS,)i(SDISPLS,)f(SENDTYPE,)h(RECVBUF,)g (RECVCOUNTS,)393 2044 y(RDISPLS,)g(RECVTYPE,)g(COMM,)g(IERROR\))170 2101 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2157 y(INTEGER)g(SENDCOUNTS\(*\),)f(SDISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNTS\(*\),)g(RDISPLS\(*\),)170 2214 y(RECVTYPE,)h(COMM,)g (IERROR)166 2300 y Fv(MPI)p 251 2300 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V)12 b Fx(adds)g(\015exibili)q(t)o(y)j(to)c Fv(MPI)p 959 2300 V 16 w(ALL)l(TO)o(ALL)h Fx(in)h(that)f(the)h(lo)q(cation)g(of) f(data)g(for)g(the)75 2356 y(send)j(is)f(sp)q(eci\014ed)j(b)o(y)d Fv(sdispls)i Fx(and)e(the)g(lo)q(cation)h(of)f(the)g(placemen)o(t)h(of) f(the)g(data)g(on)g(the)g(receiv)o(e)h(side)75 2413 y(is)h(sp)q (eci\014ed)h(b)o(y)e Fv(rdispls)p Fx(.)166 2469 y(The)k Fu(j)p Fx(th)f(blo)q(c)o(k)i(sen)o(t)e(from)g(pro)q(cess)h Fu(i)g Fx(is)g(receiv)o(ed)h(b)o(y)e(pro)q(cess)h Fu(j)g Fx(and)g(is)g(placed)h(in)f(the)g Fu(i)p Fx(th)75 2526 y(blo)q(c)o(k)d(of)f Fv(recvbuf)p Fx(.)20 b(These)c(blo)q(c)o(ks)g (need)g(not)f(all)h(ha)o(v)o(e)f(the)g(same)g(size.)166 2582 y(The)i(t)o(yp)q(e)h(signature)f(asso)q(ciated)h(with)f Fv(sendcount[)q(j],)j(sendt)o(yp)q(e)g Fx(at)d(pro)q(cess)g Fu(i)g Fx(m)o(ust)g(b)q(e)h(equal)75 2639 y(to)d(the)i(t)o(yp)q(e)f (signature)g(asso)q(ciated)g(with)h Fv(recvcount[i],)g(recvt)o(yp)q(e)g Fx(at)e(pro)q(cess)i Fu(j)p Fx(.)22 b(This)17 b(implies)h(that)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 111 119 111 118 bop 75 -100 a Fm(4.9.)34 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)814 b Fx(111)75 45 y(the)17 b(amoun)o(t)g(of)g(data)g(sen)o(t)g (m)o(ust)f(b)q(e)i(equal)g(to)f(the)h(amoun)o(t)e(of)h(data)g(receiv)o (ed,)h(pairwise)h(b)q(et)o(w)o(een)75 102 y(ev)o(ery)c(pair)h(of)e(pro) q(cesses.)21 b(Distinct)16 b(t)o(yp)q(e)f(maps)g(b)q(et)o(w)o(een)g (sender)h(and)f(receiv)o(er)h(are)f(still)i(allo)o(w)o(ed.)166 158 y(The)e(outcome)g(is)h(as)f(if)g(eac)o(h)h(pro)q(cess)f(sen)o(t)g (a)g(message)g(to)f(ev)o(ery)h(other)g(pro)q(cess)h(with,)189 235 y Fu(MPI)p 264 235 15 2 v 16 w(Send)p Fx(\()p Fu(sendbuf)9 b Fx(+)h Fu(displs)p Fx([)p Fu(i)p Fx(])f Ft(\001)g Fu(extent)p Fx(\()p Fu(sendtype)p Fx(\))p Fp(;)f Fu(sen)o(dcounts)p Fx([)o Fu(i)p Fx(])p Fp(;)f Fu(sen)o(dtype)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))p Fp(;)75 311 y Fx(and)15 b(receiv)o(ed)i(a)e (message)f(from)h(ev)o(ery)g(other)g(pro)q(cess)g(with)h(a)f(call)h(to) 189 388 y Fu(MPI)p 264 388 V 16 w(Recv)p Fx(\()p Fu(recvbuf)9 b Fx(+)h Fu(displs)p Fx([)p Fu(i)p Fx(])f Ft(\001)g Fu(extent)p Fx(\()p Fu(recvtype)p Fx(\))p Fp(;)f Fu(rec)o(vcounts)p Fx([)o Fu(i)p Fx(])p Fp(;)f Fu(rec)o(vtype)p Fp(;)g Fu(i)p Fp(;)g(::)o(:)p Fx(\))p Fp(:)166 464 y Fx(All)14 b(argumen)o(ts)d(on)h (all)i(pro)q(cesses)f(are)f(signi\014can)o(t.)20 b(The)12 b(argumen)o(t)g Fv(comm)6 b Fx(m)o(ust)12 b(ha)o(v)o(e)g(iden)o(tical) 75 521 y(v)m(alues)k(on)f(all)i(pro)q(cesses.)189 603 y Fl(R)n(ationale.)60 b Fx(The)19 b(de\014nitions)h(of)e Fv(MPI)p 911 603 14 2 v 16 w(ALL)l(TO)o(ALL)g Fx(and)h Fv(MPI)p 1338 603 V 16 w(ALL)l(TO)o(ALL)-5 b(V)18 b Fx(giv)o(e)h(as)f (m)o(uc)o(h)189 659 y(\015exibilit)o(y)e(as)d(one)h(w)o(ould)g(ac)o (hiev)o(e)g(b)o(y)f(sp)q(ecifying)j Fu(n)d Fx(indep)q(enden)o(t,)j(p)q (oin)o(t-to-p)q(oin)o(t)e(comm)o(uni-)189 716 y(cations,)i(with)h(t)o (w)o(o)f(exceptions:)23 b(all)18 b(messages)e(use)h(the)g(same)f(datat) o(yp)q(e,)g(and)h(messages)f(are)189 772 y(scattered)e(from)h(\(or)f (gathered)h(to\))f(sequen)o(tial)j(storage.)h(\()p Fl(End)e(of)g(r)n (ationale.)p Fx(\))189 854 y Fl(A)n(dvic)n(e)k(to)h(implementors.)70 b Fx(Although)21 b(the)f(discussion)i(of)e(collectiv)o(e)i(comm)o (unication)f(in)189 911 y(terms)f(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(op) q(eration)g(implies)i(that)d(eac)o(h)g(message)g(is)i(transferred)e (directly)189 967 y(from)i(sender)h(to)f(receiv)o(er,)j(implemen)o (tations)f(ma)o(y)e(use)h(a)g(tree)f(comm)o(unication)i(pattern.)189 1024 y(Messages)12 b(can)h(b)q(e)h(forw)o(arded)e(b)o(y)h(in)o (termediate)h(no)q(des)f(where)h(they)f(are)f(split)j(\(for)d (scatter\))f(or)189 1080 y(concatenated)k(\(for)f(gather\),)g(if)i (this)f(is)h(more)f(e\016cien)o(t.)20 b(\()p Fl(End)c(of)g(advic)n(e)g (to)h(implementors.)p Fx(\))75 1220 y Fn(4.9)59 b(Global)20 b(Reduction)e(Op)r(erations)75 1321 y Fx(The)e(functions)g(in)g(this)g (section)g(p)q(erform)f(a)g(global)h(reduce)g(op)q(eration)g(\(suc)o(h) f(as)g(sum,)g(max,)g(logical)75 1378 y(AND,)d(etc.\))19 b(across)12 b(all)i(the)e(mem)o(b)q(ers)h(of)f(a)h(group.)18 b(The)13 b(reduction)h(op)q(eration)f(can)g(b)q(e)g(either)h(one)e(of) 75 1434 y(a)j(prede\014ned)i(list)f(of)f(op)q(erations,)g(or)g(a)g (user-de\014ned)j(op)q(eration.)i(The)c(global)g(reduction)g(functions) 75 1491 y(come)h(in)h(sev)o(eral)g(\015a)o(v)o(ors:)23 b(a)17 b(reduce)h(that)f(returns)g(the)g(result)h(of)f(the)g(reduction) i(at)d(one)i(no)q(de,)g(an)75 1547 y(all-reduce)i(that)e(returns)g (this)g(result)h(at)e(all)i(no)q(des,)g(and)g(a)f(scan)g(\(parallel)h (pre\014x\))f(op)q(eration.)29 b(In)75 1604 y(addition,)15 b(a)f(reduce-scatter)g(op)q(eration)h(com)o(bines)g(the)f(functionalit) o(y)h(of)f(a)g(reduce)h(and)f(of)g(a)g(scatter)75 1660 y(op)q(eration.)75 1779 y Fj(4.9.1)49 b(Reduce)75 1959 y Fv(MPI)p 160 1959 V 16 w(REDUCE\()15 b(sendbuf,)j(recvbuf,)d(count,)h (datat)o(yp)q(e,)h(op,)e(ro)q(ot,)f(comm)m(\))117 2036 y Fk(IN)171 b Fv(sendbuf)428 b Fk(address)15 b(of)f(send)g(bu\013er)h (\(c)o(hoice\))117 2108 y(OUT)124 b Fv(recvbuf)434 b Fk(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i (signi\014can)o(t)d(only)g(at)905 2164 y(ro)q(ot\))117 2236 y(IN)171 b Fv(count)466 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 2308 y(IN)171 b Fv(datat)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(send)i(bu\013er)g(\(handle\))117 2379 y(IN)171 b Fv(op)525 b Fk(reduce)16 b(op)q(eration)e(\(handle\)) 117 2451 y(IN)171 b Fv(ro)q(ot)492 b Fk(rank)14 b(of)f(ro)q(ot)h(pro)q (cess)i(\(in)o(teger\))117 2523 y(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))75 2647 y Fu(int)23 b(MPI)p 245 2647 15 2 v 17 w(Reduce\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)h(count,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2704 V 17 w(Op)i(op,)f(int)h (root,)f(MPI)p 1505 2704 V 17 w(Comm)g(comm\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 112 120 112 119 bop 75 -100 a Fx(112)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(REDUCE\(SENDBUF,)22 b(RECVBUF,)g(COUNT,)h(DATATYPE,)g (OP,)g(ROOT,)h(COMM,)f(IERROR\))170 102 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 158 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(ROOT,)h (COMM,)f(IERROR)166 244 y Fv(MPI)p 251 244 14 2 v 16 w(REDUCE)c Fx(com)o(bines)g(the)g(elemen)o(ts)g(pro)o(vided)g(in)g(the) g(input)g(bu\013er)g(of)f(eac)o(h)g(pro)q(cess)h(in)75 301 y(the)d(group,)g(using)h(the)f(op)q(eration)g Fv(op)p Fx(,)g(and)h(returns)f(the)g(com)o(bined)h(v)m(alue)g(in)g(the)f (output)g(bu\013er)g(of)75 357 y(the)j(pro)q(cess)f(with)h(rank)f Fv(ro)q(ot)p Fx(.)30 b(The)19 b(input)g(bu\013er)g(is)g(de\014ned)h(b)o (y)e(the)h(argumen)o(ts)e Fv(sendbuf)p Fx(,)22 b Fv(count)75 414 y Fx(and)d Fv(datat)o(yp)q(e)p Fx(;)i(the)d(output)h(bu\013er)f(is) h(de\014ned)h(b)o(y)e(the)g(argumen)o(ts)g Fv(recvbuf)p Fx(,)h Fv(count)h Fx(and)f Fv(datat)o(yp)q(e)p Fx(;)75 470 y(b)q(oth)e(ha)o(v)o(e)f(the)h(same)g(n)o(um)o(b)q(er)g(of)f (elemen)o(ts,)h(with)h(the)e(same)h(t)o(yp)q(e.)24 b(The)17 b(routine)h(is)f(called)h(b)o(y)f(all)75 527 y(group)c(mem)o(b)q(ers)f (using)i(the)f(same)f(argumen)o(ts)g(for)h Fv(count,)h(datat)o(yp)q(e,) g(op,)f(ro)q(ot)g Fx(and)g Fv(comm)m Fx(.)j(Th)o(us,)d(all)75 583 y(pro)q(cesses)i(pro)o(vide)f(input)i(bu\013ers)e(and)g(output)g (bu\013ers)h(of)e(the)i(same)f(length,)h(with)f(elemen)o(ts)h(of)f(the) 75 640 y(same)g(t)o(yp)q(e.)20 b(Eac)o(h)14 b(pro)q(cess)h(can)f(pro)o (vide)h(one)g(elemen)o(t,)g(or)f(a)g(sequence)h(of)g(elemen)o(ts,)f(in) i(whic)o(h)f(case)75 696 y(the)d(com)o(bine)h(op)q(eration)g(is)f (executed)h(elemen)o(t-wise)h(on)e(eac)o(h)g(en)o(try)g(of)g(the)g (sequence.)20 b(F)l(or)11 b(example,)75 753 y(if)h(the)f(op)q(eration)h (is)g Fi(MPI)p 506 753 13 2 v 14 w(MAX)f Fx(and)g(the)h(send)g (bu\013er)f(con)o(tains)h(t)o(w)o(o)e(elemen)o(ts)i(that)e(are)h (\015oating)g(p)q(oin)o(t)75 809 y(n)o(um)o(b)q(ers)h(\()p Fv(count)h Fx(=)f(2)g(and)g Fv(datat)o(yp)q(e)i Fx(=)e Fv(MPI)p 867 809 14 2 v 16 w(FLO)o(A)l(T)p Fx(\),)f(then)h Fv(recvbuf)s Fx(\()p Fv(1)p Fx(\))h(=)g(global)8 b(max\()p Fv(sendbuf)s Fx(\()p Fv(1)p Fx(\)\))75 865 y(and)15 b Fv(recvbuf)s Fx(\()p Fv(2)p Fx(\))e(=)g(global)8 b(max\()p Fv(sendbuf)s Fx(\()p Fv(2)p Fx(\)\).)166 922 y(Sec.)31 b(4.9.2,)17 b(lists)j(the)e(set)h(of)f(prede\014ned)i(op)q(erations)f (pro)o(vided)g(b)o(y)g Fv(MPI)p Fx(.)f(That)g(section)h(also)75 978 y(en)o(umerates)d(the)g(datat)o(yp)q(es)f(eac)o(h)i(op)q(eration)f (can)g(b)q(e)h(applied)h(to.)k(In)16 b(addition,)h(users)f(ma)o(y)g (de\014ne)75 1035 y(their)i(o)o(wn)e(op)q(erations)i(that)e(can)h(b)q (e)h(o)o(v)o(erloaded)f(to)g(op)q(erate)f(on)i(sev)o(eral)f(datat)o(yp) q(es,)f(either)i(basic)75 1091 y(or)d(deriv)o(ed.)21 b(This)15 b(is)h(further)f(explained)i(in)f(Sec.)21 b(4.9.4.)166 1148 y(The)11 b(op)q(eration)g Fv(op)h Fx(is)f(alw)o(a)o(ys)f(assumed)h (to)g(b)q(e)g(asso)q(ciativ)o(e.)19 b(All)13 b(prede\014ned)f(op)q (erations)f(are)g(also)75 1204 y(assumed)f(to)g(b)q(e)h(comm)o(utativ)o (e.)17 b(Users)10 b(ma)o(y)g(de\014ne)h(op)q(erations)g(that)e(are)h (assumed)h(to)e(b)q(e)i(asso)q(ciativ)o(e,)75 1261 y(but)i(not)f(comm)o (utativ)o(e.)19 b(The)13 b(\\canonical")g(ev)m(aluation)h(order)e(of)h (a)f(reduction)i(is)f(determined)h(b)o(y)f(the)75 1317 y(ranks)18 b(of)f(the)h(pro)q(cesses)g(in)h(the)f(group.)28 b(Ho)o(w)o(ev)o(er,)17 b(the)h(implemen)o(tation)h(can)f(tak)o(e)f(adv) m(an)o(tage)g(of)75 1374 y(asso)q(ciativit)o(y)l(,)d(or)f(asso)q (ciativit)o(y)g(and)h(comm)o(utativit)o(y)f(in)h(order)f(to)f(c)o (hange)i(the)f(order)g(of)g(ev)m(aluation.)75 1430 y(This)j(ma)o(y)e(c) o(hange)h(the)g(result)g(of)g(the)g(reduction)h(for)e(op)q(erations)i (that)e(are)h(not)f(strictly)i(asso)q(ciativ)o(e)75 1486 y(and)f(comm)o(utativ)o(e,)f(suc)o(h)i(as)f(\015oating)g(p)q(oin)o(t)h (addition.)189 1593 y Fl(A)n(dvic)n(e)h(to)i(implementors.)53 b Fx(It)17 b(is)h(strongly)f(recommended)i(that)d Fv(MPI)p 1480 1593 V 16 w(REDUCE)i Fx(b)q(e)g(imple-)189 1649 y(men)o(ted)e(so)g(that)g(the)g(same)h(result)f(b)q(e)i(obtained)f (whenev)o(er)f(the)h(function)g(is)g(applied)i(on)d(the)189 1706 y(same)f(argumen)o(ts,)f(app)q(earing)i(in)g(the)f(same)g(order.) 20 b(Note)14 b(that)h(this)h(ma)o(y)e(prev)o(en)o(t)h(optimiza-)189 1762 y(tions)k(that)f(tak)o(e)h(adv)m(an)o(tage)f(of)h(the)g(ph)o (ysical)h(lo)q(cation)g(of)f(pro)q(cessors.)31 b(\()p Fl(End)19 b(of)h(advic)n(e)g(to)189 1819 y(implementors.)p Fx(\))166 1925 y(The)c Fv(datat)o(yp)q(e)i Fx(argumen)o(t)e(of)f Fv(MPI)p 783 1925 V 16 w(REDUCE)i Fx(m)o(ust)f(b)q(e)g(compatible)i (with)e Fv(op)p Fx(.)23 b(Prede\014ned)18 b(op-)75 1981 y(erators)h(w)o(ork)g(only)i(with)f(the)g Fv(MPI)g Fx(t)o(yp)q(es)g (listed)h(in)g(Sec.)35 b(4.9.2)18 b(and)j(Sec.)35 b(4.9.3.)d (User-de\014ned)75 2038 y(op)q(erators)14 b(ma)o(y)g(op)q(erate)h(on)g (general,)g(deriv)o(ed)h(datat)o(yp)q(es.)k(In)15 b(this)h(case,)e(eac) o(h)h(argumen)o(t)g(that)f(the)75 2094 y(reduce)21 b(op)q(eration)g(is) g(applied)h(to)e(is)h(one)f(elemen)o(t)h(describ)q(ed)i(b)o(y)d(suc)o (h)h(a)f(datat)o(yp)q(e,)g(whic)o(h)i(ma)o(y)75 2151 y(con)o(tain)15 b(sev)o(eral)h(basic)g(v)m(alues.)21 b(This)15 b(is)h(further)f(explained)i(in)f(Section)h(4.9.4.)75 2272 y Fj(4.9.2)49 b(Prede\014ned)15 b(reduce)g(op)q(erations)75 2358 y Fx(The)k(follo)o(wing)h(prede\014ned)h(op)q(erations)e(are)f (supplied)k(for)c Fv(MPI)p 1238 2358 V 16 w(REDUCE)i Fx(and)f(related)g(functions)75 2415 y Fv(MPI)p 160 2415 V 16 w(ALLREDUCE)p Fx(,)14 b Fv(MPI)p 541 2415 V 15 w(REDUCE)p 732 2415 V 17 w(SCA)l(TTER)p Fx(,)h(and)f Fv(MPI)p 1144 2415 V 16 w(SCAN)p Fx(.)g(These)g(op)q(erations)g(are)g(in)o(v)o(ok)o (ed)75 2471 y(b)o(y)h(placing)i(the)e(follo)o(wing)h(in)g Fv(op)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 113 121 113 120 bop 75 -100 a Fm(4.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)819 b Fx(113)117 102 y Fi(Name)727 b Fx(Meaning)117 210 y Fi(MPI)p 194 210 13 2 v 14 w(MAX)649 b Fx(maxim)o(um)117 265 y Fi(MPI)p 194 265 V 14 w(MIN)664 b Fx(minim)o(um)117 319 y Fi(MPI)p 194 319 V 14 w(SUM)653 b Fx(sum)117 373 y Fi(MPI)p 194 373 V 14 w(PROD)626 b Fx(pro)q(duct)117 428 y Fi(MPI)p 194 428 V 14 w(LAND)632 b Fx(logical)16 b(and)117 482 y Fi(MPI)p 194 482 V 14 w(BAND)626 b Fx(bit-wise)16 b(and)117 537 y Fi(MPI)p 194 537 V 14 w(LOR)661 b Fx(logical)16 b(or)117 591 y Fi(MPI)p 194 591 V 14 w(BOR)655 b Fx(bit-wise)16 b(or)117 645 y Fi(MPI)p 194 645 V 14 w(LX)o(OR)634 b Fx(logical)16 b(xor)117 700 y Fi(MPI)p 194 700 V 14 w(BX)o(OR)628 b Fx(bit-wise)16 b(xor)117 754 y Fi(MPI)p 194 754 V 14 w(MAXLOC)569 b Fx(max)14 b(v)m(alue)j(and)e(lo)q(cation)117 808 y Fi(MPI)p 194 808 V 14 w(MINLOC)584 b Fx(min)16 b(v)m(alue)g(and)f(lo)q(cation)166 891 y(The)22 b(t)o(w)o(o)e(op)q (erations)h Fi(MPI)p 662 891 V 14 w(MINLOC)h Fx(and)f Fi(MPI)p 1024 891 V 14 w(MAXLOC)g Fx(are)g(discussed)i(separately)e(in) i(Sec.)75 948 y(4.9.3.)g(F)l(or)17 b(the)g(other)f(prede\014ned)j(op)q (erations,)e(w)o(e)g(en)o(umerate)f(b)q(elo)o(w)i(the)f(allo)o(w)o(ed)g (com)o(binations)75 1004 y(of)e Fv(op)g Fx(and)g Fv(datat)o(yp)q(e)i Fx(argumen)o(ts.)i(First,)14 b(de\014ne)j(groups)d(of)h Fv(MPI)g Fx(basic)h(datat)o(yp)q(es)e(in)i(the)f(follo)o(wing)75 1061 y(w)o(a)o(y)l(.)117 1221 y Fi(C)e(integer:)661 b Fv(MPI)p 1032 1221 14 2 v 16 w(INT,)124 b(MPI)p 1343 1221 V 15 w(LONG,)h(MPI)p 1698 1221 V 16 w(SHORT,)905 1277 y(MPI)p 990 1277 V 16 w(UNSIGNED)p 1229 1277 V 17 w(SHORT,)135 b(MPI)p 1626 1277 V 16 w(UNSIGNED,)905 1334 y(MPI)p 990 1334 V 16 w(UNSIGNED)p 1229 1334 V 17 w(LONG)117 1388 y Fi(F)o(o)o(rtran)13 b(integer:)561 b Fv(MPI)p 1032 1388 V 16 w(INTEGER)117 1443 y Fi(Floating)13 b(p)q(oint:)574 b Fv(MPI)p 1032 1443 V 16 w(FLO)o(A)l(T,)82 b(MPI)p 1365 1443 V 16 w(DOUBLE,)h(MPI)p 1738 1443 V 16 w(REAL,)905 1499 y(MPI)p 990 1499 V 16 w(DOUBLE)p 1185 1499 V 17 w(PRECISION,)16 b(MPI)p 1543 1499 V 15 w(LONG)p 1678 1499 V 17 w(DOUBLE)117 1553 y Fi(Logical:)696 b Fv(MPI)p 1032 1553 V 16 w(LOGICAL)117 1608 y Fi(Com)o(plex:)667 b Fv(MPI)p 1032 1608 V 16 w(COMPLEX)117 1662 y Fi(Byte:)738 b Fv(MPI)p 1032 1662 V 16 w(BYTE)166 1745 y Fx(No)o(w,)14 b(the)h(v)m(alid)i(datat)o(yp)q(es)e(for)f(eac)o(h)i(option)f(is)h(sp)q (eci\014ed)h(b)q(elo)o(w.)117 1905 y Fi(Op)778 b Fx(Allo)o(w)o(ed)16 b(T)o(yp)q(es)117 2014 y Fi(MPI)p 194 2014 13 2 v 14 w(MAX,)d(MPI)p 400 2014 V 14 w(MIN)458 b(C)13 b(integer,)i(F)o(o)o (rtran)e(integer,)i(Floating)e(p)q(oint,)h(Com)o(plex)117 2068 y(MPI)p 194 2068 V 14 w(SUM,)f(MPI)p 396 2068 V 14 w(PROD)424 b(C)13 b(integer,)i(F)o(o)o(rtran)e(integer,)i(Floating)e (p)q(oint,)h(Com)o(plex)117 2123 y(MPI)p 194 2123 V 14 w(LAND,)g(MPI)p 418 2123 V 14 w(LOR,)f(MPI)p 612 2123 V 15 w(LX)o(OR)215 b(C)13 b(integer,)i(Logical)117 2177 y(MPI)p 194 2177 V 14 w(BAND,)e(MPI)p 423 2177 V 14 w(BOR,)g(MPI)p 623 2177 V 14 w(BX)o(OR)199 b(C)13 b(integer,)i(F)o(o)o(rtran)e (integer,)i(Byte)75 2310 y Fq(Example)j(4.15)23 b Fx(A)10 b(routine)i(that)e(computes)h(the)g(dot)f(pro)q(duct)i(of)e(t)o(w)o(o)g (v)o(ectors)g(that)g(are)g(distributed)75 2366 y(across)k(a)h(group)g (of)g(pro)q(cesses)h(and)f(returns)g(the)g(answ)o(er)g(at)g(no)q(de)g (zero.)75 2473 y Fu(SUBROUTINE)22 b(PAR_BLAS1\(m,)h(a,)g(b,)h(c,)f (comm\))75 2529 y(REAL)g(a\(m\),)g(b\(m\))167 b(!)24 b(local)f(slice)g(of)h(array)75 2586 y(REAL)f(c)382 b(!)24 b(result)f(\(at)g(node)h(zero\))75 2642 y(REAL)f(sum)75 2699 y(INTEGER)g(m,)g(comm,)h(i,)f(ierr)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 114 122 114 121 bop 75 -100 a Fx(114)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 102 y Fu(!)24 b(local)f(sum)75 158 y(sum)g(=)h(0.0)75 214 y(DO)g(i)f(=)h(1,)g(m)147 271 y(sum)f(=)h(sum)f(+)h(a\(i\)*b\(i\))75 327 y(END)f(DO)75 440 y(!)h(global)f(sum)75 497 y(CALL)g(MPI_REDUCE\(sum,)f(c,)i(1,)f (MPI_REAL,)g(MPI_SUM,)g(0,)g(comm,)g(ierr\))75 553 y(RETURN)75 654 y Fq(Example)18 b(4.16)23 b Fx(A)c(routine)h(that)f(computes)h(the) f(pro)q(duct)h(of)g(a)f(v)o(ector)g(and)h(an)f(arra)o(y)f(that)h(are)75 711 y(distributed)e(across)d(a)h(group)g(of)g(pro)q(cesses)g(and)h (returns)f(the)g(answ)o(er)g(at)f(no)q(de)i(zero.)75 812 y Fu(SUBROUTINE)22 b(PAR_BLAS2\(m,)h(n,)g(a,)h(b,)f(c,)h(comm\))75 868 y(REAL)f(a\(m\),)g(b\(m,n\))95 b(!)24 b(local)f(slice)g(of)h(array) 75 925 y(REAL)f(c\(n\))286 b(!)24 b(result)75 981 y(REAL)f(sum\(n\))75 1038 y(INTEGER)g(n,)g(comm,)h(i,)f(j,)h(ierr)75 1151 y(!)g(local)f(sum)75 1207 y(DO)h(j=)f(1,)h(n)123 1264 y(sum\(j\))f(=)g(0.0)123 1320 y(DO)g(i)h(=)g(1,)f(m)170 1376 y(sum\(j\))g(=)h(sum\(j\))f(+)h(a\(i\)*b\(i,j\))123 1433 y(END)f(DO)75 1489 y(END)g(DO)75 1602 y(!)h(global)f(sum)75 1659 y(CALL)g(MPI_REDUCE\(sum,)f(c,)i(n,)f(MPI_REAL,)g(MPI_SUM,)g(0,)g (comm,)g(ierr\))75 1772 y(!)h(return)f(result)g(at)g(node)h(zero)f (\(and)g(garbage)g(at)h(the)f(other)g(nodes\))75 1828 y(RETURN)75 1949 y Fj(4.9.3)49 b(MINLOC)18 b(and)e(MAXLOC)75 2035 y Fx(The)d(op)q(erator)e Fi(MPI)p 423 2035 13 2 v 15 w(MINLOC)h Fx(is)h(used)g(to)f(compute)g(a)h(global)g(minim)o(um)g (and)g(also)f(an)h(index)h(attac)o(hed)75 2091 y(to)e(the)g(minim)o(um) i(v)m(alue.)20 b Fi(MPI)p 615 2091 V 14 w(MAXLOC)12 b Fx(similarly)i(computes)f(a)f(global)h(maxim)o(um)f(and)h(index.)20 b(One)75 2148 y(application)13 b(of)e(these)h(is)g(to)f(compute)g(a)g (global)h(minim)o(um)h(\(maxim)o(um\))e(and)g(the)h(rank)f(of)g(the)h (pro)q(cess)75 2204 y(con)o(taining)k(this)g(v)m(alue.)166 2261 y(The)f(op)q(eration)h(that)e(de\014nes)i Fi(MPI)p 787 2261 V 15 w(MAXLOC)e Fx(is:)189 2320 y Fe( )242 2364 y Fp(u)247 2420 y(i)289 2320 y Fe(!)332 2392 y Ft(\016)365 2320 y Fe( )418 2364 y Fp(v)420 2420 y(j)463 2320 y Fe(!)508 2392 y Fx(=)556 2320 y Fe( )610 2364 y Fp(w)614 2420 y(k)664 2320 y Fe(!)75 2523 y Fx(where)189 2613 y Fp(w)f Fx(=)g(max\()p Fp(u;)8 b(v)r Fx(\))75 2704 y(and)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 115 123 115 122 bop 75 -100 a Fm(4.9.)34 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)814 b Fx(115)189 102 y Fp(k)13 b Fx(=)274 4 y Fe(8)274 41 y(>)274 53 y(<)274 128 y(>)274 141 y(:)332 46 y Fp(i)194 b Fx(if)15 b Fp(u)e(>)g(v)332 102 y Fx(min)q(\()p Fp(i;)8 b(j)s Fx(\))39 b(if)15 b Fp(u)e Fx(=)g Fp(v)332 159 y(j)191 b Fx(if)15 b Fp(u)e(<)g(v)166 251 y Fi(MPI)p 243 251 13 2 v 14 w(MINLOC)i Fx(is)h(de\014ned)g(similarly:)189 315 y Fe( )242 359 y Fp(u)247 416 y(i)289 315 y Fe(!)332 387 y Ft(\016)365 315 y Fe( )418 359 y Fp(v)420 416 y(j)463 315 y Fe(!)508 387 y Fx(=)556 315 y Fe( )610 359 y Fp(w)614 416 y(k)664 315 y Fe(!)75 524 y Fx(where)189 618 y Fp(w)d Fx(=)g(min)q(\()p Fp(u;)8 b(v)r Fx(\))75 712 y(and)189 852 y Fp(k)13 b Fx(=)274 753 y Fe(8)274 790 y(>)274 803 y(<)274 878 y(>)274 890 y(:)332 795 y Fp(i)194 b Fx(if)15 b Fp(u)e(<)g(v)332 851 y Fx(min)q(\()p Fp(i;)8 b(j)s Fx(\))39 b(if)15 b Fp(u)e Fx(=)g Fp(v)332 908 y(j)191 b Fx(if)15 b Fp(u)e(>)g(v)166 1000 y Fx(Both)g(op)q(erations)g(are)g (asso)q(ciativ)o(e)h(and)f(comm)o(utativ)o(e.)19 b(Note)12 b(that)h(if)h Fi(MPI)p 1491 1000 V 14 w(MAXLOC)e Fx(is)i(applied)75 1057 y(to)k(reduce)i(a)e(sequence)i(of)e(pairs)h(\()p Fp(u)729 1064 y Fg(0)748 1057 y Fp(;)8 b Fx(0\))p Fp(;)g Fx(\()p Fp(u)875 1064 y Fg(1)893 1057 y Fp(;)g Fx(1\))p Fp(;)g(:)g(:)f(:)t(;)h Fx(\()p Fp(u)1099 1064 y Ff(n)p Fh(\000)p Fg(1)1167 1057 y Fp(;)g(n)j Ft(\000)i Fx(1\),)19 b(then)g(the)f(v)m(alue)i(returned)f(is)75 1113 y(\()p Fp(u;)8 b(r)q Fx(\),)15 b(where)i Fp(u)f Fx(=)f(max)518 1120 y Ff(i)540 1113 y Fp(u)566 1120 y Ff(i)597 1113 y Fx(and)i Fp(r)g Fx(is)g(the)g(index)i(of)d(the)h(\014rst)f(global)i (maxim)o(um)f(in)g(the)g(sequence.)75 1170 y(Th)o(us,)12 b(if)h(eac)o(h)f(pro)q(cess)g(supplies)i(a)e(v)m(alue)h(and)f(its)h (rank)e(within)j(the)e(group,)g(then)g(a)g(reduce)h(op)q(eration)75 1226 y(with)18 b Fv(op)g Fx(=)g Fi(MPI)p 375 1226 V 14 w(MAXLOC)f Fx(will)i(return)f(the)g(maxim)o(um)f(v)m(alue)i(and)f(the)g (rank)f(of)h(the)f(\014rst)h(pro)q(cess)75 1283 y(with)g(that)f(v)m (alue.)29 b(Similarly)l(,)21 b Fi(MPI)p 708 1283 V 14 w(MINLOC)d Fx(can)g(b)q(e)g(used)h(to)e(return)h(a)f(minim)o(um)i(and)f (its)g(index.)75 1339 y(More)10 b(generally)l(,)i Fi(MPI)p 463 1339 V 14 w(MINLOC)e Fx(computes)h(a)f Fl(lexic)n(o)n(gr)n(aphic)h (minimum)p Fx(,)g(where)g(elemen)o(ts)g(are)f(ordered)75 1396 y(according)k(to)g(the)g(\014rst)g(comp)q(onen)o(t)g(of)f(eac)o(h) h(pair,)h(and)f(ties)g(are)g(resolv)o(ed)g(according)h(to)e(the)h (second)75 1452 y(comp)q(onen)o(t.)166 1509 y(The)i(reduce)h(op)q (eration)f(is)h(de\014ned)g(to)e(op)q(erate)h(on)g(argumen)o(ts)f(that) g(consist)i(of)e(a)h(pair:)22 b(v)m(alue)75 1565 y(and)12 b(index.)20 b(F)l(or)12 b(b)q(oth)g(F)l(ortran)f(and)h(C,)g(t)o(yp)q (es)g(are)g(pro)o(vided)h(to)e(describ)q(e)j(the)e(pair.)19 b(The)13 b(p)q(oten)o(tially)75 1621 y(mixed-t)o(yp)q(e)g(nature)e(of)g (suc)o(h)h(argumen)o(ts)f(is)h(a)f(problem)i(in)f(F)l(ortran.)17 b(The)12 b(problem)h(is)f(circum)o(v)o(en)o(ted,)75 1678 y(for)k(F)l(ortran,)g(b)o(y)h(ha)o(ving)h(the)f Fv(MPI)p Fx(-pro)o(vided)g(t)o(yp)q(e)g(consist)g(of)g(a)g(pair)g(of)g(the)g (same)f(t)o(yp)q(e)h(as)g(v)m(alue,)75 1734 y(and)j(co)q(ercing)g(the)g (index)g(to)f(this)h(t)o(yp)q(e)f(also.)33 b(In)20 b(C,)f(the)g Fv(MPI)p Fx(-pro)o(vided)h(pair)g(t)o(yp)q(e)f(has)h(distinct)75 1791 y(t)o(yp)q(es)15 b(and)h(the)f(index)h(is)g(an)f Fv(int)p Fx(.)166 1847 y(In)f(order)g(to)f(use)h Fi(MPI)p 547 1847 V 14 w(MINLOC)f Fx(and)h Fi(MPI)p 893 1847 V 15 w(MAXLOC)e Fx(in)j(a)e(reduce)i(op)q(eration,)f(one)f(m)o(ust)g(pro) o(vide)75 1904 y(a)19 b Fv(datat)o(yp)q(e)j Fx(argumen)o(t)d(that)g (represen)o(ts)h(a)f(pair)h(\(v)m(alue)h(and)f(index\).)34 b Fv(MPI)19 b Fx(pro)o(vides)i(sev)o(en)f(suc)o(h)75 1960 y(prede\014ned)14 b(datat)o(yp)q(es.)19 b(The)13 b(op)q(erations)g Fi(MPI)p 905 1960 V 14 w(MAXLOC)f Fx(and)h Fi(MPI)p 1264 1960 V 14 w(MINLOC)f Fx(can)h(b)q(e)h(used)f(with)g(eac)o (h)75 2017 y(of)i(the)g(follo)o(wing)h(datat)o(yp)q(es.)117 2120 y Fi(F)o(o)o(rtran:)117 2175 y(Name)727 b Fx(Description)117 2229 y Fi(MPI)p 194 2229 V 14 w(2REAL)618 b Fx(pair)15 b(of)g Fu(REAL)p Fx(s)117 2283 y Fi(MPI)p 194 2283 V 14 w(2DOUBLE)p 394 2283 V 14 w(PRECISION)328 b Fx(pair)15 b(of)g Fu(DOUBLE)23 b(PRECISION)14 b Fx(v)m(ariables)117 2338 y Fi(MPI)p 194 2338 V 14 w(2INTEGER)546 b Fx(pair)15 b(of)g Fu(INTEGER)p Fx(s)117 2392 y Fi(MPI)p 194 2392 V 14 w(2COMPLEX)524 b Fx(pair)15 b(of)g Fu(COMPLEX)p Fx(es)117 2579 y Fi(C:)117 2633 y(Name)727 b Fx(Description)117 2688 y Fi(MPI)p 194 2688 V 14 w(FLO)o(A)m(T)p 337 2688 V 14 w(INT)529 b Fv(\015oat)15 b Fx(and)h Fv(int)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 116 124 116 123 bop 75 -100 a Fx(116)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)117 45 y Fi(MPI)p 194 45 13 2 v 14 w(DOUBLE)p 373 45 V 14 w(INT)493 b Fv(double)16 b Fx(and)g Fv(int)117 99 y Fi(MPI)p 194 99 V 14 w(LONG)p 318 99 V 15 w(INT)547 b Fv(long)15 b Fx(and)g Fv(int)117 154 y Fi(MPI)p 194 154 V 14 w(2INT)651 b Fx(pair)15 b(of)g Fv(int)117 208 y Fi(MPI)p 194 208 V 14 w(SHORT)p 346 208 V 15 w(INT)519 b Fv(sho)o(rt)15 b Fx(and)h Fv(int)117 262 y Fi(MPI)p 194 262 V 14 w(LONG)p 318 262 V 15 w(DOUBLE)p 498 262 V 14 w(INT)368 b Fv(long)15 b(double)h Fx(and)f Fv(int)166 345 y Fx(The)g(datat)o(yp)q(e)g Fi(MPI)p 527 345 V 14 w(2REAL)g Fx(is)h Fl(as)g(if)f Fx(de\014ned)h(b)o(y)g(the)f (follo)o(wing)h(\(see)f(Section)h(3.12\).)75 451 y Fu (MPI_TYPE_CONTIGUOUS\(2,)21 b(MPI_REAL,)h(MPI_2REAL\))166 556 y Fx(Similar)17 b(statemen)o(ts)d(apply)i(for)e Fi(MPI)p 823 556 V 15 w(2INTEGER)p Fx(,)f Fi(MPI)p 1134 556 V 15 w(2DOUBLE)p 1335 556 V 13 w(PRECISION)p Fx(,)g(and)i Fi(MPI)p 1750 556 V 15 w(2INT)p Fx(.)166 613 y(The)f(datat)o(yp)q(e)g Fi(MPI)p 525 613 V 14 w(FLO)o(A)m(T)p 668 613 V 14 w(INT)g Fx(is)h Fl(as)g(if)e Fx(de\014ned)j(b)o(y)e(the)g(follo)o(wing)h (sequence)g(of)e(instructions.)75 706 y Fu(type[0])23 b(=)h(MPI_FLOAT)75 762 y(type[1])f(=)h(MPI_INT)75 819 y(disp[0])f(=)h(0)75 875 y(disp[1])f(=)h(sizeof\(float\))75 932 y(block[0])f(=)g(1)75 988 y(block[1])g(=)g(1)75 1044 y(MPI_TYPE_STRUCT\(2,)e(block,)i(disp,)h(type,)f(MPI_FLOAT_INT\))75 1138 y Fx(Similar)17 b(statemen)o(ts)d(apply)i(for)e Fi(MPI)p 732 1138 V 15 w(LONG)p 857 1138 V 15 w(INT)h Fx(and)g Fi(MPI)p 1119 1138 V 15 w(DOUBLE)p 1299 1138 V 14 w(INT)p Fx(.)75 1243 y Fq(Example)j(4.17)23 b Fx(Eac)o(h)12 b(pro)q(cess)g(has)g(an)g(arra)o(y)f(of)h(30)g Fu(double)p Fx(s,)f(in)i(C.)f(F)l(or)g(eac)o(h)g(of)g(the)g(30)g(lo)q(cations,)75 1300 y(compute)j(the)h(v)m(alue)g(and)f(rank)g(of)g(the)g(pro)q(cess)h (con)o(taining)g(the)f(largest)g(v)m(alue.)170 1405 y Fu(...)170 1462 y(/*)24 b(each)f(process)g(has)h(an)f(array)g(of)h(30)g (double:)f(ain[30])194 1518 y(*/)170 1574 y(double)g(ain[30],)g (aout[30];)170 1631 y(int)48 b(ind[30];)170 1687 y(struct)23 b({)266 1744 y(double)g(val;)266 1800 y(int)71 b(rank;)170 1857 y(})24 b(in[30],)f(out[30];)170 1913 y(int)h(i,)f(myrank,)g(root;) 170 2026 y(MPI_Comm_rank\(MPI_COMM_WORLD,)d(&myrank\);)170 2083 y(for)k(\(i=0;)f(i<30;)g(++i\))h({)266 2139 y(in[i].val)f(=)g (ain[i];)266 2195 y(in[i].rank)f(=)i(myrank;)170 2252 y(})170 2308 y(MPI_Reduce\()f(in,)g(out,)h(30,)f(MPI_DOUBLE_INT,)f (MPI_MAXLOC,)g(root,)h(comm)h(\);)170 2365 y(/*)g(At)g(this)f(point,)g (the)g(answer)g(resides)g(on)h(process)f(root)194 2421 y(*/)170 2478 y(if)h(\(myrank)f(==)h(root\))f({)266 2534 y(/*)g(read)h(ranks)f(out)290 2591 y(*/)266 2647 y(for)g(\(i=0;)h (i<30;)f(++i\))g({)361 2704 y(aout[i])g(=)h(out[i].val;)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 117 125 117 124 bop 75 -100 a Fm(4.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)819 b Fx(117)361 45 y Fu(ind[i])23 b(=)h(out[i].rank;)266 102 y(})170 158 y(})75 258 y Fq(Example)18 b(4.18)23 b Fx(Same)15 b(example,)g(in)h(F)l(ortran.)170 358 y Fu(...)170 415 y(!)24 b(each)f(process)g(has)h(an)f(array)h(of)f(30)h (double:)f(ain\(30\))170 528 y(DOUBLE)g(PRECISION)g(ain\(30\),)g (aout\(30\))170 584 y(INTEGER)g(ind\(30\);)170 640 y(DOUBLE)g (PRECISION)g(in\(2,30\),)g(out\(2,30\))170 697 y(INTEGER)g(i,)h (myrank,)f(root,)g(ierr;)170 810 y(MPI_COMM_RANK\(MPI_COMM_WORLD,)d (myrank\);)170 866 y(DO)k(I=1,)f(30)266 923 y(in\(1,i\))g(=)h(ain\(i\)) 266 979 y(in\(2,i\))f(=)h(myrank)94 b(!)24 b(myrank)f(is)h(coerced)e (to)i(a)g(double)170 1036 y(END)g(DO)170 1149 y(MPI_REDUCE\()f(in,)g (out,)h(30,)f(MPI_2DOUBLE_PRECISION,)e(MPI_MAXLOC,)h(root,)1459 1205 y(comm,)h(ierr)h(\);)170 1261 y(!)g(At)g(this)f(point,)g(the)h (answer)f(resides)g(on)g(process)g(root)170 1374 y(IF)h(\(myrank)f (.EQ.)g(root\))g(THEN)266 1431 y(!)h(read)f(ranks)g(out)266 1487 y(DO)g(I=)h(1,)g(30)361 1544 y(aout\(i\))f(=)h(out\(1,i\))361 1600 y(ind\(i\))f(=)h(out\(2,i\))47 b(!)23 b(rank)h(is)f(coerced)g (back)h(to)f(an)h(integer)266 1657 y(END)f(DO)170 1713 y(END)h(IF)75 1813 y Fq(Example)18 b(4.19)23 b Fx(Eac)o(h)16 b(pro)q(cess)h(has)f(a)h(non-empt)o(y)f(arra)o(y)g(of)g(v)m(alues.)25 b(Find)18 b(the)f(minim)o(um)g(global)75 1870 y(v)m(alue,)f(the)f(rank) g(of)g(the)g(pro)q(cess)h(that)e(holds)i(it)g(and)f(its)g(index)i(on)e (this)h(pro)q(cess.)75 1970 y Fu(#define)47 b(LEN)71 b(1000)75 2083 y(float)23 b(val[LEN];)190 b(/*)23 b(local)h(array)f(of) g(values)g(*/)75 2139 y(int)g(count;)310 b(/*)23 b(local)h(number)f(of) g(values)g(*/)75 2195 y(int)g(myrank,)g(minrank,)g(minindex;)75 2252 y(float)g(minval;)75 2365 y(struct)g({)170 2421 y(float)h(value;)170 2478 y(int)72 b(index;)75 2534 y(})24 b(in,)f(out;)170 2647 y(/*)h(local)f(minloc)g(*/)75 2704 y(in.value)g(=)g(val[0];)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 118 126 118 125 bop 75 -100 a Fx(118)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fu(in.index)23 b(=)g(0;)75 102 y(for)g(\(i=1;)h(i)f(<)h(count;)f(i++\))170 158 y(if)h(\(in.value)f(>)g(val[i]\))g({)266 214 y(in.value)g(=)g (val[i];)266 271 y(in.index)g(=)g(i;)170 327 y(})170 440 y(/*)h(global)f(minloc)g(*/)75 497 y(MPI_Comm_rank\(MPI_COMM_WOR)o (LD,)e(&myrank\);)75 553 y(in.index)i(=)g(myrank*LEN)g(+)h(in.index;)75 610 y(MPI_Reduce\()e(in,)i(out,)f(1,)h(MPI_FLOAT_INT,)e(MPI_MINLOC,)g (root,)h(comm)h(\);)170 666 y(/*)g(At)g(this)f(point,)g(the)g(answer)g (resides)g(on)h(process)f(root)194 723 y(*/)75 779 y(if)h(\(myrank)e (==)i(root\))f({)170 835 y(/*)h(read)f(answer)g(out)194 892 y(*/)170 948 y(minval)g(=)h(out.value;)170 1005 y(minrank)f(=)h (out.index)f(/)g(LEN;)170 1061 y(minindex)g(=)h(out.index)f(\045)g (LEN;)75 1118 y(})189 1224 y Fl(R)n(ationale.)84 b Fx(The)23 b(de\014nition)h(of)e Fi(MPI)p 921 1224 13 2 v 14 w(MINLOC)h Fx(and)f Fi(MPI)p 1285 1224 V 15 w(MAXLOC)f Fx(giv)o(en)i(here)g(has)g (the)189 1280 y(adv)m(an)o(tage)14 b(that)h(it)h(do)q(es)f(not)g (require)h(an)o(y)f(sp)q(ecial-case)j(handling)f(of)e(these)g(t)o(w)o (o)f(op)q(erations:)189 1337 y(they)i(are)g(handled)i(lik)o(e)g(an)o(y) e(other)g(reduce)i(op)q(eration.)24 b(A)16 b(programmer)g(can)g(pro)o (vide)h(his)g(or)189 1393 y(her)e(o)o(wn)g(de\014nition)i(of)f Fi(MPI)p 694 1393 V 14 w(MAXLOC)e Fx(and)i Fi(MPI)p 1058 1393 V 14 w(MINLOC)p Fx(,)f(if)h(so)f(desired.)22 b(The)16 b(disadv)m(an)o(tage)189 1450 y(is)d(that)f(v)m(alues)h(and)g(indices)i (ha)o(v)o(e)d(to)g(b)q(e)h(\014rst)f(in)o(terlea)o(v)o(ed,)i(and)e (that)g(indices)j(and)e(v)m(alues)g(ha)o(v)o(e)189 1506 y(to)h(b)q(e)i(co)q(erced)g(to)f(the)g(same)g(t)o(yp)q(e,)g(in)h(F)l (ortran.)i(\()p Fl(End)e(of)g(r)n(ationale.)p Fx(\))75 1628 y Fj(4.9.4)49 b(User-De\014ned)15 b(Op)q(erations)75 1808 y Fv(MPI)p 160 1808 14 2 v 16 w(OP)p 238 1808 V 17 w(CREA)l(TE\()h(function,)g(comm)m(ute,)d(op\))117 1886 y Fk(IN)155 b Fv(function)434 b Fk(user)15 b(de\014ned)g(function) f(\(function\))117 1961 y(IN)155 b Fv(comm)n(ute)409 b Fb(true)13 b Fk(if)g(comm)o(utativ)o(e;)e Fb(false)i Fk(otherwise.)117 2036 y(OUT)108 b Fv(op)541 b Fk(op)q(eration)14 b(\(handle\))75 2160 y Fu(int)23 b(MPI)p 245 2160 15 2 v 17 w(Op)p 310 2160 V 17 w(create\(MPI)p 567 2160 V 16 w(User)p 679 2160 V 17 w(function)f(*function,)h(int)g(commute,)g (MPI)p 1555 2160 V 17 w(Op)g(*op\))75 2247 y(MPI)p 150 2247 V 17 w(OP)p 215 2247 V 17 w(CREATE\()g(FUNCTION,)f(COMMUTE,)h(OP,) g(IERROR\))170 2303 y(EXTERNAL)g(FUNCTION)170 2360 y(LOGICAL)g(COMMUTE) 170 2416 y(INTEGER)g(OP,)h(IERROR)166 2503 y Fv(MPI)p 251 2503 14 2 v 16 w(OP)p 329 2503 V 17 w(CREA)l(TE)h Fx(binds)h(a)f(user-de\014ned)h(global)f(op)q(eration)g(to)f(an)h Fv(op)g Fx(handle)h(that)e(can)75 2559 y(subsequen)o(tly)e(b)q(e)f (used)g(in)g Fv(MPI)p 669 2559 V 16 w(REDUCE)p Fx(,)g Fv(MPI)p 977 2559 V 15 w(ALLREDUCE)p Fx(,)g Fv(MPI)p 1364 2559 V 15 w(REDUCE)p 1555 2559 V 18 w(SCA)l(TTER)p Fx(,)g(and)75 2615 y Fv(MPI)p 160 2615 V 16 w(SCAN)p Fx(.)c(The)g(user-de\014ned)i(op)q(eration)e(is)g(assumed)g(to)g(b)q(e) g(asso)q(ciativ)o(e.)25 b(If)18 b Fv(comm)m(ute)d Fx(=)i Fu(true)p Fx(,)75 2672 y(then)k(the)g(op)q(eration)g(should)h(b)q(e)g (b)q(oth)e(comm)o(utativ)o(e)h(and)g(asso)q(ciativ)o(e.)37 b(If)21 b Fv(comm)m(ute)e Fx(=)i Fu(false)p Fx(,)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 119 127 119 126 bop 75 -100 a Fm(4.9.)34 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)814 b Fx(119)75 45 y(then)16 b(the)g(order)g(of)f(op)q(erations) h(is)g(\014xed)h(and)f(is)g(de\014ned)h(to)e(b)q(e)i(in)f(ascending,)h (pro)q(cess)f(rank)g(order,)75 102 y(b)q(eginning)h(with)f(pro)q(cess)f (zero.)166 158 y Fv(function)20 b Fx(is)g(the)e(user-de\014ned)j (function,)f(whic)o(h)g(m)o(ust)e(ha)o(v)o(e)g(the)h(follo)o(wing)h (four)e(argumen)o(ts:)75 214 y Fv(invec,)d(inoutvec,)i(len)e Fx(and)h Fv(datat)o(yp)q(e)p Fx(.)166 271 y(The)f(ANSI-C)h(protot)o(yp) q(e)f(for)f(the)i(function)g(is)f(the)h(follo)o(wing.)75 365 y Fu(typedef)23 b(void)g(MPI_User_function\()f(void)h(*invec,)g (void)g(*inoutvec,)g(int)g(*len,)1220 421 y(MPI_Datatype)g (*datatype\);)166 515 y Fx(The)15 b(F)l(ortran)f(declaration)i(of)f (the)g(user-de\014ned)i(function)f(app)q(ears)g(b)q(elo)o(w.)75 609 y Fu(FUNCTION)23 b(USER_FUNCTION\()f(INVEC\(*\),)g(INOUTVEC\(*\),)h (LEN,)g(TYPE\))75 665 y()g(INVEC\(LEN\),)f(INOUTVEC\(LEN\))99 722 y(INTEGER)h(LEN,)g(TYPE)166 816 y Fx(The)18 b Fv(datat)o(yp)q(e)h Fx(argumen)o(t)e(is)h(a)f(handle)i(to)e(the)h(data)e(t)o(yp)q(e)i(that) f(w)o(as)g(passed)h(in)o(to)f(the)h(call)h(to)75 872 y Fv(MPI)p 160 872 14 2 v 16 w(REDUCE)p Fx(.)f(The)g(user)g(reduce)h (function)g(should)g(b)q(e)f(written)g(suc)o(h)h(that)e(the)h(follo)o (wing)h(holds:)75 928 y(Let)h Fv(u[0],)g(...)32 b(,)20 b(u[len-1])g Fx(b)q(e)g(the)g Fv(len)g Fx(elemen)o(ts)g(in)g(the)g (comm)o(unication)g(bu\013er)f(describ)q(ed)j(b)o(y)d(the)75 985 y(argumen)o(ts)14 b Fv(invec,)i(len)f Fx(and)g Fv(datat)o(yp)q(e)i Fx(when)f(the)f(function)h(is)f(in)o(v)o(ok)o(ed;)g(let)h Fv(v[0],)e(...)19 b(,)c(v[len-1])h Fx(b)q(e)f Fv(len)75 1041 y Fx(elemen)o(ts)e(in)g(the)f(comm)o(unication)h(bu\013er)f (describ)q(ed)i(b)o(y)e(the)g(argumen)o(ts)f Fv(inoutvec,)j(len)e Fx(and)g Fv(datat)o(yp)q(e)75 1098 y Fx(when)17 b(the)g(function)g(is)h (in)o(v)o(ok)o(ed;)f(let)g Fv(w[0],)g(...)23 b(,)17 b(w[len-1])g Fx(b)q(e)g Fv(len)h Fx(elemen)o(ts)f(in)g(the)g(comm)o(unication)75 1154 y(bu\013er)i(describ)q(ed)j(b)o(y)d(the)h(argumen)o(ts)e Fv(inoutvec,)k(len)e Fx(and)g Fv(datat)o(yp)q(e)h Fx(when)f(the)f (function)h(returns;)75 1211 y(then)d Fv(w[i])f(=)h(u[i])p Ft(\016)p Fv(v[i])p Fx(,)g(for)f Fv(i=0)g(,)g(...)23 b(,)16 b(len-1)p Fx(,)h(where)g Ft(\016)f Fx(is)h(the)f(reduce)h(op)q (eration)g(that)f(the)g(function)75 1267 y(computes.)166 1324 y(Informally)l(,)j(w)o(e)e(can)g(think)h(of)f Fv(invec)h Fx(and)g Fv(inoutvec)h Fx(as)e(arra)o(ys)f(of)h Fv(len)h Fx(elemen)o(ts)g(that)e Fv(function)75 1380 y Fx(is)i(com)o(bining.)28 b(The)18 b(result)g(of)f(the)h(reduction)h(o)o(v)o(er-writes)e(v)m (alues)h(in)h Fv(inoutvec)p Fx(,)g(hence)g(the)e(name.)75 1437 y(Eac)o(h)g(in)o(v)o(o)q(cation)g(of)f(the)h(function)h(results)f (in)h(the)f(p)q(oin)o(t)o(wise)h(ev)m(aluation)g(of)e(the)h(reduce)h (op)q(erator)75 1493 y(on)h Fv(len)h Fx(elemen)o(ts:)29 b(I.e,)21 b(the)e(function)h(returns)g(in)g Fv(inoutvec[i])h Fx(the)e(v)m(alue)i Fv(invec)p Fx([)p Fv(i)p Fx(])12 b Ft(\016)h Fv(inoutvec)p Fx([)p Fv(i)p Fx(],)21 b(for)75 1549 y Fv(i)12 b Fx(=)h Fv(0)p Fp(;)8 b(:)g(:)g(:)d(;)j Fv(count)j Ft(\000)f Fv(1)p Fx(,)15 b(where)g Ft(\016)g Fx(is)h(the)f(com)o(bining)h(op)q(eration)g(computed)f(b)o(y)g(the)h (function.)189 1656 y Fl(R)n(ationale.)47 b Fx(The)16 b(len)i(argumen)o(t)d(allo)o(ws)i Fv(MPI)p 1033 1656 V 16 w(REDUCE)g Fx(to)f(a)o(v)o(oid)g(calling)i(the)e(function)i(for) 189 1712 y(eac)o(h)13 b(elemen)o(t)h(in)g(the)f(input)h(bu\013er.)19 b(Rather,)13 b(the)h(system)e(can)h(c)o(ho)q(ose)h(to)e(apply)i(the)f (function)189 1769 y(to)k(c)o(h)o(unks)i(of)f(input.)30 b(In)19 b(C,)e(it)i(is)g(passed)f(in)h(as)f(a)g(reference)h(for)f (reasons)g(of)f(compatibilit)o(y)189 1825 y(with)e(F)l(ortran.)189 1900 y(By)e(in)o(ternally)h(comparing)f(the)g(v)m(alue)h(of)e(the)h Fv(datat)o(yp)q(e)h Fx(argumen)o(t)e(to)g(kno)o(wn,)h(global)g (handles,)189 1957 y(it)h(is)h(p)q(ossible)h(to)e(o)o(v)o(erload)g(the) g(use)h(of)f(a)g(single)h(user-de\014ned)i(function)e(for)e(sev)o (eral,)i(di\013eren)o(t)189 2013 y(data)f(t)o(yp)q(es.)20 b(\()p Fl(End)c(of)g(r)n(ationale.)p Fx(\))166 2119 y(General)e(datat)o (yp)q(es)f(ma)o(y)g(b)q(e)i(passed)f(to)f(the)g(user)h(function.)21 b(Ho)o(w)o(ev)o(er,)12 b(use)i(of)g(datat)o(yp)q(es)f(that)75 2176 y(are)i(not)g(con)o(tiguous)g(is)h(lik)o(ely)h(to)d(lead)i(to)f (ine\016ciencies.)166 2232 y(No)g Fv(MPI)g Fx(comm)o(unication)i (function)f(ma)o(y)f(b)q(e)h(called)h(inside)h(the)d(user)h(function.) 22 b Fv(MPI)p 1708 2232 V 16 w(ABORT)75 2289 y Fx(ma)o(y)14 b(b)q(e)i(called)h(inside)g(the)e(function)h(in)h(case)e(of)g(an)g (error.)189 2395 y Fl(A)n(dvic)n(e)g(to)i(users.)42 b Fx(Supp)q(ose)17 b(one)e(de\014nes)i(a)e(library)i(of)e(user-de\014ned) i(reduce)f(functions)h(that)189 2451 y(are)10 b(o)o(v)o(erloaded:)17 b(the)11 b Fv(datat)o(yp)q(e)h Fx(argumen)o(t)d(is)i(used)g(to)f (select)h(the)g(righ)o(t)f(execution)i(path)e(at)g(eac)o(h)189 2508 y(in)o(v)o(o)q(cation,)15 b(according)h(to)e(the)i(t)o(yp)q(es)f (of)g(the)g(op)q(erands.)21 b(The)15 b(user-de\014ned)i(reduce)f (function)189 2564 y(cannot)11 b(\\deco)q(de")h(the)f Fv(datat)o(yp)q(e)i Fx(argumen)o(t)d(that)h(it)h(is)f(passed,)h(and)g (cannot)f(iden)o(tify)l(,)i(b)o(y)e(itself,)189 2621 y(the)18 b(corresp)q(ondence)h(b)q(et)o(w)o(een)f(the)h(datat)o(yp)q(e) e(handles)i(and)g(the)f(datat)o(yp)q(e)f(they)h(represen)o(t.)189 2677 y(This)i(corresp)q(ondence)h(w)o(as)d(established)j(when)g(the)e (datat)o(yp)q(es)g(w)o(ere)g(created.)33 b(Before)20 b(the)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 120 128 120 127 bop 75 -100 a Fx(120)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)189 45 y Fx(library)k(is)h(used,)h (a)d(library)i(initialization)i(pream)o(ble)e(m)o(ust)e(b)q(e)i (executed.)36 b(This)20 b(pream)o(ble)189 102 y(co)q(de)14 b(will)i(de\014ne)f(the)e(datat)o(yp)q(es)h(that)f(are)g(used)i(b)o(y)f (the)g(library)l(,)g(and)g(store)f(handles)j(to)d(these)189 158 y(datat)o(yp)q(es)i(in)h(global,)g(static)f(v)m(ariables)i(that)e (are)g(shared)h(b)o(y)f(the)h(user)g(co)q(de)g(and)g(the)f(library)189 214 y(co)q(de.)189 290 y(The)f(F)l(ortran)f(v)o(ersion)i(of)e Fv(MPI)p 734 290 14 2 v 16 w(REDUCE)i Fx(will)h(in)o(v)o(ok)o(e)e(a)g (user-de\014ned)i(reduce)f(function)g(using)189 346 y(the)g(F)l(ortran) g(calling)i(con)o(v)o(en)o(tions)f(and)g(will)h(pass)f(a)f(F)l (ortran-t)o(yp)q(e)g(datat)o(yp)q(e)g(argumen)o(t;)g(the)189 402 y(C)d(v)o(ersion)h(will)h(use)f(C)g(calling)h(con)o(v)o(en)o(tion)f (and)g(the)f(C)h(represen)o(tation)f(of)h(a)f(datat)o(yp)q(e)g(handle.) 189 459 y(Users)i(who)h(plan)h(to)e(mix)h(languages)g(should)h (de\014ne)g(their)g(reduction)f(functions)h(accordingly)l(.)189 515 y(\()p Fl(End)f(of)i(advic)n(e)f(to)g(users.)p Fx(\))189 622 y Fl(A)n(dvic)n(e)f(to)h(implementors.)39 b Fx(W)l(e)15 b(outline)h(b)q(elo)o(w)f(a)g(naiv)o(e)g(and)g(ine\016cien)o(t)h (implemen)o(tation)g(of)189 678 y Fv(MPI)p 274 678 V 15 w(REDUCE)p Fx(.)189 803 y Fu(if)23 b(\(rank)g(>)h(0\))g({)284 859 y(RECV\(tempbuf,)e(count,)h(datatype,)g(rank-1,...\))284 916 y(User_reduce\()f(tempbuf,)h(sendbuf,)g(count,)g(datatype\))189 972 y(})189 1029 y(if)g(\(rank)g(<)h(groupsize-1\))e({)284 1085 y(SEND\()h(sendbuf,)g(count,)g(datatype,)g(rank+1,)g(...\))189 1142 y(})189 1198 y(/*)g(answer)g(now)h(resides)f(in)g(process)g (groupsize-1)f(...)i(now)f(send)h(to)f(root)212 1254 y(*/)189 1311 y(if)g(\(rank)g(==)h(groupsize-1\))e({)284 1367 y(SEND\()h(sendbuf,)g(count,)g(datatype,)g(root,)g(...\))189 1424 y(})189 1480 y(if)g(\(rank)g(==)h(root\))f({)284 1537 y(RECV\(recvbuf,)f(count,)h(datatype,)g(groupsize-1,...\))189 1593 y(})189 1718 y Fx(The)16 b(reduction)h(computation)e(pro)q(ceeds,) i(sequen)o(tially)l(,)h(from)d(pro)q(cess)h Fu(0)f Fx(to)h(pro)q(cess)g Fu(group-)189 1775 y(size-1)p Fx(.)i(This)13 b(order)g(is)g(c)o(hosen)g (so)f(as)h(to)f(resp)q(ect)h(the)g(order)f(of)h(a)f(p)q(ossibly)i (non-comm)o(utativ)o(e)189 1831 y(op)q(erator)g(de\014ned)j(b)o(y)f (the)g(function)g Fu(User)p 949 1831 15 2 v 17 w(reduce\(\))p Fx(.)j(A)d(more)f(e\016cien)o(t)i(implemen)o(tation)f(is)189 1887 y(ac)o(hiev)o(ed)f(b)o(y)g(taking)g(adv)m(an)o(tage)g(of)f(asso)q (ciativit)o(y)h(and)h(using)f(a)g(logarithmic)h(tree)e(reduction.)189 1944 y(Comm)o(utativit)o(y)e(can)i(b)q(e)g(used)g(to)f(adv)m(an)o (tage,)g(for)g(those)g(cases)g(in)i(whic)o(h)f(the)g Fv(comm)m(ute)d Fx(argu-)189 2000 y(men)o(t)i(to)g Fv(MPI)p 441 2000 14 2 v 16 w(OP)p 519 2000 V 17 w(CREA)l(TE)i Fx(is)f(true.)20 b(Also,)14 b(the)g(amoun)o(t)f(of)g(temp)q(orary)g (bu\013er)h(required)h(can)189 2057 y(b)q(e)i(reduced,)i(and)e(comm)o (unication)h(can)f(b)q(e)h(pip)q(elined)j(with)c(computation,)g(b)o(y)g (transferring)189 2113 y(and)e(reducing)h(the)g(elemen)o(ts)g(in)g(c)o (h)o(unks)f(of)g(size)h Fu(len)f Fp(<)p Fu(count)p Fx(.)189 2188 y(The)21 b(prede\014ned)h(reduce)g(op)q(erations)f(can)g(b)q(e)h (implemen)o(ted)g(as)f(a)f(library)i(of)e(user-de\014ned)189 2245 y(op)q(erations.)g(Ho)o(w)o(ev)o(er,)14 b(b)q(etter)h(p)q (erformance)g(migh)o(t)g(b)q(e)h(ac)o(hiev)o(ed)g(if)g Fv(MPI)p 1521 2245 V 16 w(REDUCE)g Fx(handles)189 2301 y(these)f(functions)h(as)f(a)g(sp)q(ecial)i(case.)j(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 2502 y Fv(MPI)p 160 2502 V 16 w(OP)p 238 2502 V 17 w(FREE\()f(op\))117 2579 y Fk(IN)155 b Fv(op)541 b Fk(op)q(eration)14 b(\(handle\))75 2704 y Fu(int)23 b(MPI)p 245 2704 15 2 v 17 w(op)p 310 2704 V 17 w(free\()g(MPI)p 542 2704 V 17 w(Op)h(*op\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 121 129 121 128 bop 75 -100 a Fm(4.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)819 b Fx(121)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(OP)p 215 45 V 17 w(FREE\()23 b(OP,)g(IERROR\))170 102 y(INTEGER)g(OP,)h(IERROR)166 188 y Fx(Marks)12 b(a)h(user-de\014ned)i (reduction)f(op)q(eration)g(for)f(deallo)q(cation)i(and)e(sets)g Fv(op)g Fx(to)g Fi(MPI)p 1675 188 13 2 v 14 w(OP)p 1747 188 V 14 w(NULL)p Fx(.)75 308 y Fv(Example)g(of)i(User-de\014ned)i (Reduce)75 394 y Fx(It)e(is)h(time)f(for)g(an)g(example)h(of)f (user-de\014ned)i(reduction.)75 500 y Fq(Example)h(4.20)23 b Fx(Compute)14 b(the)i(pro)q(duct)f(of)g(an)g(arra)o(y)f(of)h(complex) h(n)o(um)o(b)q(ers,)f(in)h(C.)75 606 y Fu(typedef)23 b(struct)g({)170 663 y(double)g(real,imag;)75 719 y(})h(Complex;)75 832 y(/*)g(the)f(user-defined)f(function)99 889 y(*/)75 945 y(void)h(myProd\()g(Complex)g(*in,)g(Complex)g(*inout,)g(int)h (*len,)f(MPI_Datatype)f(*dptr)h(\))75 1002 y({)170 1058 y(int)h(i;)170 1115 y(Complex)f(c;)170 1227 y(for)h(\(i=0;)f(i<)h (*len;)f(++i\))g({)266 1284 y(c.real)g(=)h(inout->real*in->real)d(-)528 1340 y(inout->imag*in->imag;)266 1397 y(c.imag)i(=)h (inout->real*in->imag)d(+)528 1453 y(inout->imag*in->real;)266 1510 y(*inout)i(=)h(c;)266 1566 y(in++;)f(inout++;)170 1623 y(})75 1679 y(})75 1792 y(/*)h(and,)f(to)g(call)h(it...)99 1848 y(*/)75 1905 y(...)170 2018 y(/*)g(each)f(process)g(has)h(an)f (array)g(of)h(100)g(Complexes)194 2074 y(*/)170 2131 y(Complex)f(a[100],)g(answer[100];)170 2187 y(MPI_Op)g(myOp;)170 2244 y(MPI_Datatype)g(ctype;)170 2357 y(/*)h(explain)f(to)h(MPI)f(how)g (type)h(Complex)f(is)g(defined)194 2413 y(*/)170 2469 y(MPI_Type_contiguous\()f(2,)h(MPI_DOUBLE,)g(&ctype)g(\);)170 2526 y(MPI_Type_commit\()f(&ctype)h(\);)170 2582 y(/*)h(create)f(the)h (complex-product)d(user-op)194 2639 y(*/)170 2695 y(MPI_Op_create\()h (myProd,)h(True,)g(&myOp)h(\);)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 122 130 122 129 bop 75 -100 a Fx(122)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)170 102 y Fu(MPI_Reduce\()23 b(a,)g(answer,)g(100,)h(ctype,)f(myOp,)g(root,)g(comm)g(\);)170 214 y(/*)h(At)g(this)f(point,)g(the)g(answer,)g(which)h(consists)e(of)i (100)f(Complexes,)194 271 y(*)h(resides)f(on)h(process)e(root)194 327 y(*/)75 449 y Fj(4.9.5)49 b(All-Reduce)75 535 y Fv(MPI)17 b Fx(includes)j(v)m(arian)o(ts)e(of)f(eac)o(h)h(of)f(the)h(reduce)g(op) q(erations)g(where)g(the)g(result)g(is)g(returned)g(to)f(all)75 591 y(pro)q(cesses)i(in)h(the)f(group.)32 b Fv(MPI)18 b Fx(requires)i(that)e(all)i(pro)q(cesses)g(participating)g(in)g(these) f(op)q(erations)75 647 y(receiv)o(e)d(iden)o(tical)h(results.)75 798 y Fv(MPI)p 160 798 14 2 v 16 w(ALLREDUCE\()e(sendbuf,)i(recvbuf,)f (count,)g(datat)o(yp)q(e,)g(op,)f(comm)m(\))117 876 y Fk(IN)155 b Fv(sendbuf)444 b Fk(starting)14 b(address)h(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 950 y(OUT)108 b Fv(recvbuf)450 b Fk(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er)g(\(c)o (hoice\))117 1025 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1100 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(send)i(bu\013er)g(\(handle\))117 1174 y(IN)155 b Fv(op)541 b Fk(op)q(eration)14 b(\(handle\))117 1249 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 1374 y Fu(int)23 b(MPI)p 245 1374 15 2 v 17 w(Allreduce\(void*)f (sendbuf,)h(void*)g(recvbuf,)g(int)g(count,)393 1430 y(MPI)p 468 1430 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 1430 V 17 w(Op)i(op,)f(MPI)p 1266 1430 V 17 w(Comm)g(comm\))75 1517 y(MPI)p 150 1517 V 17 w(ALLREDUCE\(SENDBUF,)e(RECVBUF,)i(COUNT,)g (DATATYPE,)g(OP,)g(COMM,)g(IERROR\))170 1573 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1629 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 1716 y Fx(Same)18 b(as)g Fv(MPI)p 434 1716 14 2 v 15 w(REDUCE)h Fx(except)g(that)e(the)h(result)g(app)q(ears)h(in) f(the)g(receiv)o(e)h(bu\013er)f(of)g(all)h(the)75 1772 y(group)c(mem)o(b)q(ers.)189 1876 y Fl(A)n(dvic)n(e)20 b(to)h(implementors.)71 b Fx(The)21 b(all-reduce)h(op)q(erations)f(can) f(b)q(e)i(implemen)o(ted)g(as)e(a)g(re-)189 1932 y(duce,)d(follo)o(w)o (ed)h(b)o(y)e(a)h(broadcast.)24 b(Ho)o(w)o(ev)o(er,)16 b(a)g(direct)i(implemen)o(tation)g(can)f(lead)h(to)e(b)q(etter)189 1989 y(p)q(erformance.)k(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g (implementors.)p Fx(\))75 2092 y Fq(Example)i(4.21)23 b Fx(A)c(routine)h(that)f(computes)h(the)f(pro)q(duct)h(of)g(a)f(v)o (ector)g(and)h(an)f(arra)o(y)f(that)h(are)75 2149 y(distributed)13 b(across)d(a)h(group)g(of)g(pro)q(cesses)h(and)f(returns)g(the)h(answ)o (er)e(at)h(all)h(no)q(des)g(\(see)f(also)g(Example)75 2205 y(4.16\).)75 2308 y Fu(SUBROUTINE)22 b(PAR_BLAS2\(m,)h(n,)g(a,)h (b,)f(c,)h(comm\))75 2365 y(REAL)f(a\(m\),)g(b\(m,n\))95 b(!)24 b(local)f(slice)g(of)h(array)75 2421 y(REAL)f(c\(n\))286 b(!)24 b(result)75 2478 y(REAL)f(sum\(n\))75 2534 y(INTEGER)g(n,)g (comm,)h(i,)f(j,)h(ierr)75 2647 y(!)g(local)f(sum)75 2704 y(DO)h(j=)f(1,)h(n)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 123 131 123 130 bop 75 -100 a Fm(4.10.)34 b(REDUCE-SCA)l(TTER)1173 b Fx(123)123 45 y Fu(sum\(j\))23 b(=)g(0.0)123 102 y(DO)g(i)h(=)g(1,)f (m)170 158 y(sum\(j\))g(=)h(sum\(j\))f(+)h(a\(i\)*b\(i,j\))123 214 y(END)f(DO)75 271 y(END)g(DO)75 384 y(!)h(global)f(sum)75 440 y(CALL)g(MPI_ALLREDUCE\(sum,)f(c,)h(n,)h(MPI_REAL,)f(MPI_SUM,)f(0,) i(comm,)f(ierr\))75 553 y(!)h(return)f(result)g(at)g(all)h(nodes)75 610 y(RETURN)75 753 y Fn(4.10)59 b(Reduce-Scatte)o(r)75 854 y Fv(MPI)16 b Fx(includes)k(v)m(arian)o(ts)d(of)f(eac)o(h)h(of)g (the)g(reduce)h(op)q(erations)f(where)g(the)g(result)h(is)f(scattered)g (to)f(all)75 911 y(pro)q(cesses)g(in)g(the)f(group)g(on)g(return.)75 1062 y Fv(MPI)p 160 1062 14 2 v 16 w(REDUCE)p 352 1062 V 17 w(SCA)l(TTER\()h(sendbuf,)h(recvbuf,)f(recvcounts,)g(datat)o(yp)q (e,)h(op,)e(comm)m(\))117 1139 y Fk(IN)155 b Fv(sendbuf)444 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1214 y(OUT)108 b Fv(recvbuf)450 b Fk(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 1289 y(IN)155 b Fv(recvcounts)388 b Fk(in)o(teger)14 b(arra)o(y)f(sp)q(ecifying)g (the)h(n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g(in)f(re-)905 1346 y(sult)i(distributed)h(to)f(eac)o(h)h(pro)q(cess.)21 b(Arra)o(y)14 b(m)o(ust)f(b)q(e)i(iden-)905 1402 y(tical)e(on)h(all)f (calling)f(pro)q(cesses.)117 1477 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 1552 y(IN)155 b Fv(op)541 b Fk(op)q(eration)14 b(\(handle\))117 1628 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 1752 y Fu(int)23 b(MPI)p 245 1752 15 2 v 17 w(Reduce)p 406 1752 V 17 w(scatter\(void*)f(sendbuf,)g(void*)i (recvbuf,)e(int)i(*recvcounts,)393 1808 y(MPI)p 468 1808 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 1808 V 17 w(Op)i(op,)f(MPI)p 1266 1808 V 17 w(Comm)g(comm\))75 1895 y(MPI)p 150 1895 V 17 w(REDUCE)p 311 1895 V 16 w(SCATTER\(SENDBUF,)f(RECVBUF,)h (RECVCOUNTS,)f(DATATYPE,)h(OP,)g(COMM,)393 1951 y(IERROR\))170 2008 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2064 y(INTEGER)g(RECVCOUNTS\(*\),)f(DATATYPE,)h(OP,)g(COMM,)h(IERROR)166 2151 y Fv(MPI)p 251 2151 14 2 v 16 w(REDUCE)p 443 2151 V 17 w(SCA)l(TTER)f Fx(\014rst)e(do)q(es)h(an)g(elemen)o(t-wise)h (reduction)g(on)e(v)o(ector)g(of)h Fv(count)j Fx(=)75 2175 y Fe(P)119 2220 y Fi(i)138 2207 y Fv(recvcounts)p Fx([)p Fv(i)p Fx(])18 b(elemen)o(ts)g(in)f(the)g(send)h(bu\013er)f (de\014ned)h(b)o(y)f Fv(sendbuf,)i(count)g Fx(and)e Fv(datat)o(yp)q(e)p Fx(.)26 b(Next,)75 2264 y(the)19 b(resulting)h(v)o(ector)e(of)g (results)h(is)h(split)g(in)o(to)e Fu(n)h Fx(disjoin)o(t)h(segmen)o(ts,) e(where)h Fu(n)g Fx(is)g(the)g(n)o(um)o(b)q(er)g(of)75 2320 y(mem)o(b)q(ers)c(in)h(the)f(group.)k(Segmen)o(t)c Fu(i)g Fx(con)o(tains)g Fv(recvcounts[i])h Fx(elemen)o(ts.)21 b(The)15 b Fu(i)p Fx(th)f(segmen)o(t)h(is)g(sen)o(t)75 2377 y(to)g(pro)q(cess)g Fu(i)g Fx(and)g(stored)g(in)h(the)f(receiv)o (e)i(bu\013er)e(de\014ned)h(b)o(y)f Fv(recvbuf,)h(recvcounts[i])h Fx(and)e Fv(datat)o(yp)q(e)p Fx(.)189 2483 y Fl(A)n(dvic)n(e)d(to)h (implementors.)37 b Fx(The)12 b Fv(MPI)p 875 2483 V 16 w(REDUCE)p 1067 2483 V 17 w(SCA)l(TTER)h Fx(routine)f(is)g (functionally)i(equiv)m(a-)189 2539 y(len)o(t)c(to:)17 b(A)10 b Fv(MPI)p 473 2539 V 16 w(REDUCE)h Fx(op)q(eration)g(function)g (with)g Fv(count)g Fx(equal)g(to)f(the)g(sum)g(of)g Fv(recvcounts[i]) 189 2596 y Fx(follo)o(w)o(ed)17 b(b)o(y)g Fv(MPI)p 516 2596 V 16 w(SCA)l(TTERV)h Fx(with)f Fv(sendcounts)k Fx(equal)c(to)g Fv(recvcounts)p Fx(.)27 b(Ho)o(w)o(ev)o(er,)16 b(a)g(direct)189 2652 y(implemen)o(tation)g(ma)o(y)f(run)g(faster.)k(\()p Fl(End)d(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 124 132 124 131 bop 75 -100 a Fx(124)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fn(4.11)59 b(Scan)75 241 y Fv(MPI)p 160 241 14 2 v 16 w(SCAN\()15 b(sendbuf,)j(recvbuf,)d(count,)h(datat)o(yp)q(e,)h(op,)e(comm)9 b(\))117 318 y Fk(IN)155 b Fv(sendbuf)444 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 393 y(OUT)108 b Fv(recvbuf)450 b Fk(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 469 y(IN)155 b Fv(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input) g(bu\013er)g(\(in)o(teger\))117 544 y(IN)155 b Fv(datat)o(yp)q(e)424 b Fk(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 619 y(IN)155 b Fv(op)541 b Fk(op)q(eration)14 b(\(handle\))117 694 y(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))75 818 y Fu(int)23 b(MPI)p 245 818 15 2 v 17 w(Scan\(void*)g(sendbuf,)f(void*)i(recvbuf,)e(int)i(count,)393 875 y(MPI)p 468 875 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 875 V 17 w(Op)i(op,)f(MPI)p 1266 875 V 17 w(Comm)g(comm)h(\))75 961 y(MPI)p 150 961 V 17 w(SCAN\(SENDBUF,)e(RECVBUF,)h(COUNT,)g (DATATYPE,)f(OP,)i(COMM,)f(IERROR\))170 1018 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1074 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 1161 y Fv(MPI)p 251 1161 14 2 v 16 w(SCAN)15 b Fx(is)h(used)f(to)g(p)q(erform)f(a)h(pre\014x)g(reduction)h(on)f (data)f(distributed)i(across)f(the)g(group.)75 1217 y(The)k(op)q (eration)g(returns,)h(in)f(the)g(receiv)o(e)h(bu\013er)f(of)f(the)h (pro)q(cess)g(with)h(rank)e Fu(i)p Fx(,)i(the)f(reduction)g(of)75 1274 y(the)h(v)m(alues)h(in)g(the)f(send)h(bu\013ers)e(of)h(pro)q (cesses)g(with)h(ranks)e Fu(0,...,i)g Fx(\(inclusiv)o(e\).)36 b(The)20 b(t)o(yp)q(e)g(of)75 1330 y(op)q(erations)c(supp)q(orted,)h (their)g(seman)o(tics,)f(and)g(the)h(constrain)o(ts)f(on)g(send)h(and)f (receiv)o(e)h(bu\013ers)f(are)75 1386 y(as)f(for)f Fv(MPI)p 285 1386 V 16 w(REDUCE)p Fx(.)189 1493 y Fl(R)n(ationale.)38 b Fx(W)l(e)13 b(ha)o(v)o(e)g(de\014ned)h(an)f(inclusiv)o(e)i(scan,)e (that)f(is,)i(the)f(pre\014x)g(reduction)h(on)f(pro)q(cess)189 1549 y Fu(i)k Fx(includes)j(the)e(data)f(from)f(pro)q(cess)i Fu(i)p Fx(.)27 b(An)18 b(alternativ)o(e)g(is)g(to)f(de\014ne)i(scan)e (in)i(an)e(exclusiv)o(e)189 1606 y(manner,)11 b(where)f(the)h(result)f (on)h Fu(i)f Fx(only)h(includes)h(data)e(up)h(to)e Fu(i-1)p Fx(.)18 b(Both)10 b(de\014nitions)i(are)e(useful.)189 1662 y(The)k(latter)g(has)g(some)g(adv)m(an)o(tages:)19 b(the)c(inclusiv)o(e)h(scan)f(can)f(alw)o(a)o(ys)g(b)q(e)h(computed)f (from)g(the)189 1719 y(exclusiv)o(e)k(scan)f(with)g(no)g(additional)h (comm)o(unication;)g(for)f(non-in)o(v)o(ertible)i(op)q(erations)e(suc)o (h)189 1775 y(as)f(max)g(and)g(min,)i(comm)o(unication)f(is)g(required) g(to)f(compute)h(the)f(exclusiv)o(e)j(scan)d(from)g(the)189 1831 y(inclusiv)o(e)i(scan.)23 b(There)16 b(is,)g(ho)o(w)o(ev)o(er,)f (a)h(complication)i(with)e(exclusiv)o(e)i(scan)e(since)h(one)g(m)o(ust) 189 1888 y(de\014ne)g(the)f(\\unit")g(elemen)o(t)h(for)e(the)h (reduction)h(in)g(this)f(case.)22 b(That)15 b(is,)i(one)f(m)o(ust)f (explicitly)189 1944 y(sa)o(y)21 b(what)h(o)q(ccurs)g(for)g(pro)q(cess) g Fu(0)p Fx(.)41 b(This)23 b(w)o(as)e(though)o(t)g(to)h(b)q(e)h (complex)g(for)e(user-de\014ned)189 2001 y(op)q(erations)15 b(and)g(hence,)h(the)g(exclusiv)o(e)g(scan)g(w)o(as)e(dropp)q(ed.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))75 2123 y Fj(4.11.1)49 b(Example)15 b(using)i(MPI)p 646 2123 15 2 v 18 w(SCAN)75 2208 y Fq(Example)h(4.22)23 b Fx(This)17 b(example)h(uses)f(a)f(user-de\014ned)j(op)q(eration)e(to)f(pro)q(duce) i(a)e Fl(se)n(gmente)n(d)g(sc)n(an)p Fx(.)75 2265 y(A)k(segmen)o(ted)g (scan)g(tak)o(es,)f(as)h(input,)h(a)f(set)f(of)h(v)m(alues)h(and)f(a)f (set)h(of)f(logicals,)j(and)e(the)g(logicals)75 2321 y(delineate)d(the)e(v)m(arious)h(segmen)o(ts)f(of)f(the)i(scan.)k(F)l (or)14 b(example:)321 2419 y Fp(v)r(al)q(ues)66 b(v)540 2426 y Fg(1)650 2419 y Fp(v)672 2426 y Fg(2)781 2419 y Fp(v)803 2426 y Fg(3)913 2419 y Fp(v)935 2426 y Fg(4)1143 2419 y Fp(v)1165 2426 y Fg(5)1323 2419 y Fp(v)1345 2426 y Fg(6)1455 2419 y Fp(v)1477 2426 y Fg(7)1587 2419 y Fp(v)1609 2426 y Fg(8)321 2476 y Fp(l)q(og)r(ical)q(s)49 b Fx(0)109 b(0)g(1)g(1)206 b(1)158 b(0)109 b(0)g(1)321 2532 y Fp(r)q(esul)q(t)76 b(v)540 2539 y Fg(1)601 2532 y Fp(v)623 2539 y Fg(1)653 2532 y Fx(+)10 b Fp(v)720 2539 y Fg(2)781 2532 y Fp(v)803 2539 y Fg(3)865 2532 y Fp(v)887 2539 y Fg(3)917 2532 y Fx(+)g Fp(v)984 2539 y Fg(4)1045 2532 y Fp(v)1067 2539 y Fg(3)1097 2532 y Fx(+)h Fp(v)1165 2539 y Fg(4)1194 2532 y Fx(+)g Fp(v)1262 2539 y Fg(5)1323 2532 y Fp(v)1345 2539 y Fg(6)1406 2532 y Fp(v)1428 2539 y Fg(6)1458 2532 y Fx(+)g Fp(v)1526 2539 y Fg(7)1587 2532 y Fp(v)1609 2539 y Fg(8)166 2631 y Fx(The)k(op)q(erator)g(that)f(pro)q(duces)i(this)g(e\013ect)f(is,) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 125 133 125 132 bop 75 -100 a Fm(4.11.)29 b(SCAN)1482 b Fx(125)711 51 y Fe( )764 95 y Fp(u)769 151 y(i)811 51 y Fe(!)854 123 y Ft(\016)887 51 y Fe( )940 95 y Fp(v)942 151 y(j)985 51 y Fe(!)1030 123 y Fx(=)1078 51 y Fe( )1132 95 y Fp(w)1138 151 y(j)1186 51 y Fe(!)1227 123 y Fp(;)166 234 y Fx(where,)735 369 y Fp(w)13 b Fx(=)829 297 y Fe(\()883 340 y Fp(u)d Fx(+)h Fp(v)43 b Fx(if)16 b Fp(i)c Fx(=)h Fp(j)883 397 y(v)125 b Fx(if)16 b Fp(i)c Ft(6)p Fx(=)h Fp(j)1203 369 y(:)166 480 y Fx(Note)20 b(that)g(this)h(is)g(a)f(non-comm)o(utativ)o (e)g(op)q(erator.)36 b(C)20 b(co)q(de)h(that)f(implemen)o(ts)i(it)e(is) h(giv)o(en)75 537 y(b)q(elo)o(w.)75 638 y Fu(typedef)i(struct)g({)170 694 y(double)g(val;)170 751 y(int)h(log;)75 807 y(})g(SegScanPair;)75 920 y(/*)g(the)f(user-defined)f(function)99 977 y(*/)75 1033 y(void)h(segScan\()g(SegScanPair)f(*in,)i(SegScanPair)e(*inout,)h (int)g(*len,)75 1090 y(MPI_Datatype)f(*dptr)h(\))75 1146 y({)170 1202 y(int)h(i;)170 1259 y(SegScanPair)f(c;)170 1372 y(for)h(\(i=0;)f(i<)h(*len;)f(++i\))g({)266 1428 y(if)g(\()h(in->log)f(==)h(inout->log)e(\))361 1485 y(c.val)i(=)f (in->val)g(+)h(inout->val;)266 1541 y(else)361 1598 y(c.val)g(=)f (inout->val;)266 1654 y(c.log)g(=)h(inout->log;)266 1711 y(*inout)f(=)h(c;)266 1767 y(in++;)f(inout++;)170 1823 y(})75 1880 y(})166 1981 y Fx(Note)c(that)f(the)i Fv(inout)g Fx(argumen)o(t)e(to)h(the)g(user-de\014ned)i(function)f(corresp)q(onds) g(to)f(the)g(righ)o(t-)75 2038 y(hand)13 b(op)q(erand)g(of)g(the)f(op)q (erator.)19 b(When)13 b(using)g(this)g(op)q(erator,)f(w)o(e)h(m)o(ust)f (b)q(e)h(careful)h(to)e(sp)q(ecify)i(that)75 2094 y(it)h(is)h(non-comm) o(utativ)o(e,)f(as)f(in)i(the)g(follo)o(wing.)170 2195 y Fu(int)24 b(i,base;)170 2252 y(SeqScanPair)47 b(a,)23 b(answer;)170 2308 y(MPI_Op)167 b(myOp;)170 2365 y(MPI_Datatype)23 b(type[2])g(=)g({MPI_DOUBLE,)g(MPI_INT};)170 2421 y(MPI_Aint)119 b(disp[2];)170 2478 y(int)239 b(blocklen[2])22 b(=)i({)f(1,)h(1};)170 2534 y(MPI_Datatype)f(sspair;)170 2647 y(/*)h(explain)f(to)h(MPI)f(how) g(type)h(SegScanPair)e(is)i(defined)194 2704 y(*/)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 126 134 126 133 bop 75 -100 a Fx(126)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)170 45 y Fu(MPI_Address\()23 b(a,)g(disp\);)170 102 y(MPI_Address\()g(a.log,)g(disp+1\);)170 158 y(base)h(=)f(disp[0];)170 214 y(for)h(\(i=0;)f(i<2;)g(++i\))h (disp[i])f(-=)g(base;)170 271 y(MPI_Type_struct\()f(2,)i(blocklen,)e (disp,)i(type,)f(&sspair)g(\);)170 327 y(MPI_Type_commit\()f(&sspair)h (\);)170 384 y(/*)h(create)f(the)h(segmented-scan)d(user-op)194 440 y(*/)170 497 y(MPI_Op_create\()h(segScan,)h(False,)g(&myOp)g(\);) 170 553 y(...)170 610 y(MPI_Scan\()g(a,)h(answer,)f(1,)g(sspair,)g (myOp,)g(root,)g(comm)h(\);)75 753 y Fn(4.12)59 b(Co)n(rrectness)75 854 y Fx(A)11 b(correct,)f(p)q(ortable)h(program)f(m)o(ust)g(in)o(v)o (ok)o(e)g(collectiv)o(e)i(comm)o(unications)g(so)e(that)g(deadlo)q(c)o (k)h(will)h(not)75 911 y(o)q(ccur,)h(whether)g(collectiv)o(e)h(comm)o (unications)g(are)e(sync)o(hronizing)i(or)e(not.)19 b(The)13 b(follo)o(wing)g(examples)75 967 y(illustrate)j(dangerous)f(use)h(of)f (collectiv)o(e)i(routines.)75 1074 y Fq(Example)h(4.23)23 b Fx(The)15 b(follo)o(wing)h(is)g(erroneous.)75 1180 y Fu(switch\(rank\))22 b({)170 1236 y(case)i(0:)266 1293 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h(comm\);)266 1349 y(MPI_Bcast\(buf2,)e(count,)h(type,)g(1,)h(comm\);)266 1406 y(break;)170 1462 y(case)g(1:)266 1519 y(MPI_Bcast\(buf2,)e (count,)h(type,)g(1,)h(comm\);)266 1575 y(MPI_Bcast\(buf1,)e(count,)h (type,)g(0,)h(comm\);)266 1631 y(break;)75 1688 y(})166 1794 y Fx(W)l(e)19 b(assume)h(that)f(the)g(group)g(of)g Fv(comm)14 b Fx(is)20 b Ft(f)p Fx(0,1)p Ft(g)p Fx(.)31 b(Tw)o(o)19 b(pro)q(cesses)g(execute)i(t)o(w)o(o)d(broadcast)75 1851 y(op)q(erations)d(in)h(rev)o(erse)f(order.)20 b(If)15 b(the)h(op)q(eration)f(is)h(sync)o(hronizing)h(then)e(a)g(deadlo)q(c)o (k)h(will)h(o)q(ccur.)166 1907 y(Collectiv)o(e)h(op)q(erations)e(m)o (ust)g(b)q(e)g(executed)i(in)f(the)f(same)g(order)g(at)f(all)j(mem)o(b) q(ers)e(of)g(the)g(com-)75 1964 y(m)o(unication)g(group.)75 2070 y Fq(Example)i(4.24)23 b Fx(The)15 b(follo)o(wing)h(is)g (erroneous.)75 2176 y Fu(switch\(rank\))22 b({)170 2233 y(case)i(0:)266 2289 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h (comm0\);)266 2345 y(MPI_Bcast\(buf2,)e(count,)h(type,)g(2,)h(comm2\);) 266 2402 y(break;)170 2458 y(case)g(1:)266 2515 y(MPI_Bcast\(buf1,)e (count,)h(type,)g(1,)h(comm1\);)266 2571 y(MPI_Bcast\(buf2,)e(count,)h (type,)g(0,)h(comm0\);)266 2628 y(break;)170 2684 y(case)g(2:)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 127 135 127 134 bop 75 -100 a Fm(4.12.)29 b(CORRECTNESS)1262 b Fx(127)266 45 y Fu(MPI_Bcast\(buf1,)22 b(count,)h(type,)g(2,)h (comm2\);)266 102 y(MPI_Bcast\(buf2,)e(count,)h(type,)g(1,)h(comm1\);) 266 158 y(break;)75 214 y(})166 319 y Fx(Assume)13 b(that)f(the)i (group)e(of)h Fv(comm)m(0)d Fx(is)j Ft(f)p Fx(0,1)p Ft(g)p Fx(,)f(of)h Fv(comm)m(1)c Fx(is)14 b Ft(f)p Fx(1,)e(2)p Ft(g)h Fx(and)g(of)g Fv(comm)m(2)d Fx(is)j Ft(f)p Fx(2,0)p Ft(g)p Fx(.)18 b(If)75 375 y(the)12 b(broadcast)g(is)h(a)f(sync)o (hronizing)i(op)q(eration,)e(then)h(there)g(is)f(a)g(cyclic)j(dep)q (endency:)20 b(the)13 b(broadcast)75 432 y(in)i Fv(comm)m(2)c Fx(completes)k(only)f(after)g(the)g(broadcast)f(in)i Fv(comm)m(0)p Fx(;)c(the)j(broadcast)f(in)i Fv(comm)n(0)10 b Fx(completes)75 488 y(only)18 b(after)e(the)h(broadcast)g(in)h Fv(comm)m(1)p Fx(;)c(and)j(the)h(broadcast)e(in)i Fv(comm)m(1)c Fx(completes)k(only)f(after)g(the)75 545 y(broadcast)d(in)i Fv(comm)n(2)o Fx(.)h(Th)o(us,)e(the)g(co)q(de)h(will)h(deadlo)q(c)o(k.) 166 601 y(Collectiv)o(e)c(op)q(erations)f(m)o(ust)f(b)q(e)i(executed)g (in)f(an)g(order)g(so)f(that)g(no)h(cyclic)i(dep)q(endences)g(o)q (ccur.)75 705 y Fq(Example)k(4.25)23 b Fx(The)15 b(follo)o(wing)h(is)g (erroneous.)75 810 y Fu(switch\(rank\))22 b({)170 866 y(case)i(0:)266 923 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h(comm\);) 266 979 y(MPI_Send\(buf2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)266 1036 y(break;)170 1092 y(case)h(1:)266 1149 y(MPI_Recv\(buf2,)e(count,) h(type,)g(0,)h(tag,)f(comm\);)266 1205 y(MPI_Bcast\(buf1,)f(count,)h (type,)g(0,)h(comm\);)266 1261 y(break;)75 1318 y(})166 1422 y Fx(Pro)q(cess)15 b(zero)f(executes)i(a)e(broadcast,)g(follo)o(w) o(ed)h(b)o(y)g(a)f(blo)q(c)o(king)j(send)e(op)q(eration.)20 b(Pro)q(cess)15 b(one)75 1479 y(\014rst)21 b(executes)g(a)g(blo)q(c)o (king)h(receiv)o(e)g(that)e(matc)o(hes)h(the)g(send,)h(follo)o(w)o(ed)g (b)o(y)e(broadcast)h(call)h(that)75 1535 y(matc)o(hes)14 b(the)h(broadcast)e(of)i(pro)q(cess)f(zero.)20 b(This)15 b(program)e(ma)o(y)h(deadlo)q(c)o(k.)21 b(The)15 b(broadcast)e(call)j (on)75 1592 y(pro)q(cess)g(zero)f Fl(may)g Fx(blo)q(c)o(k)h(un)o(til)h (pro)q(cess)e(one)h(executes)f(the)h(matc)o(hing)f(broadcast)f(call,)i (so)f(that)g(the)75 1648 y(send)i(is)h(not)e(executed.)26 b(Pro)q(cess)17 b(one)g(will)h(de\014nitely)h(blo)q(c)o(k)f(on)e(the)h (receiv)o(e)h(and)f(so,)f(in)i(this)f(case,)75 1705 y(nev)o(er)e (executes)h(the)f(broadcast.)166 1761 y(The)j(relativ)o(e)g(order)f(of) g(execution)h(of)f(collectiv)o(e)i(op)q(erations)f(and)g(p)q(oin)o (t-to-p)q(oin)o(t)g(op)q(erations)75 1817 y(should)f(b)q(e)h(suc)o(h,)e (so)g(that)g(ev)o(en)h(if)g(the)g(collectiv)o(e)h(op)q(erations)e(and)h (the)g(p)q(oin)o(t-to-p)q(oin)o(t)g(op)q(erations)75 1874 y(are)e(sync)o(hronizing,)h(no)f(deadlo)q(c)o(k)i(will)f(o)q (ccur.)75 1978 y Fq(Example)i(4.26)23 b Fx(A)15 b(correct,)f(but)h (non-deterministic)j(program.)75 2083 y Fu(switch\(rank\))k({)170 2139 y(case)i(0:)266 2195 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h (comm\);)266 2252 y(MPI_Send\(buf2,)e(count,)h(type,)g(1,)h(tag,)f (comm\);)266 2308 y(break;)170 2365 y(case)h(1:)266 2421 y(MPI_Recv\(buf2,)e(count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h(comm\);) 266 2478 y(MPI_Bcast\(buf1,)f(count,)h(type,)g(0,)h(comm\);)266 2534 y(MPI_Recv\(buf2,)e(count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h (comm\);)266 2591 y(break;)170 2647 y(case)h(2:)266 2704 y(MPI_Send\(buf2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 128 136 128 135 bop 75 -100 a Fx(128)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)383 4 y 18945146 16482269 0 0 29733355 25918013 startTexFig 383 4 a %%BeginDocument: coll-matchings.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -50 0 translate -40.0 436.0 translate 0.900 -0.900 scale 0.500 setlinewidth n 469 134 m 334 134 l gs col-1 s gr n 342.000 136.000 m 334.000 134.000 l 342.000 132.000 l gs 2 setlinejoin col-1 s gr n 154 399 m 284 399 l gs col-1 s gr n 276.000 397.000 m 284.000 399.000 l 276.000 401.000 l gs 2 setlinejoin col-1 s gr n 469 454 m 329 454 l gs col-1 s gr n 337.000 456.000 m 329.000 454.000 l 337.000 452.000 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 279 59 m gs 1 -1 scale (First Execution) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 274 299 m gs 1 -1 scale (Second Execution) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 139 94 m gs 1 -1 scale (0) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 319 94 m gs 1 -1 scale (1) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 509 94 m gs 1 -1 scale (2) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 194 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 484 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 194 m gs 1 -1 scale (send) col-1 show gr n 159 189 m 279 189 l gs col-1 s gr n 271.000 187.000 m 279.000 189.000 l 271.000 191.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 16.00 scalefont setfont 294 139 m gs 1 -1 scale (recv) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 44 94 m gs 1 -1 scale (process:) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 484 139 m gs 1 -1 scale (send) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 199 184 m gs 1 -1 scale (match) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 384 129 m gs 1 -1 scale (match) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 429 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 404 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 459 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 479 459 m gs 1 -1 scale (send) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 479 484 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 404 m gs 1 -1 scale (send) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 379 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 194 394 m gs 1 -1 scale (match) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 379 449 m gs 1 -1 scale (match) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 1146 a Fx(Figure)d(4.10:)k(A)c(race)f(condition)i(causes)f (non-deterministic)i(matc)o(hing)d(of)g(sends)i(and)e(receiv)o(es.)20 b(One)75 1202 y(cannot)15 b(rely)h(on)f(sync)o(hronization)h(from)e(a)h (broadcast)g(to)f(mak)o(e)h(the)g(program)f(deterministic.)266 1335 y Fu(MPI_Bcast\(buf1,)22 b(count,)h(type,)g(0,)h(comm\);)266 1392 y(break;)75 1448 y(})166 1555 y Fx(All)19 b(three)f(pro)q(cesses)g (participate)h(in)g(a)e(broadcast.)28 b(Pro)q(cess)17 b(0)h(sends)g(a)g(message)f(to)h(pro)q(cess)75 1611 y(1)h(after)g(the)h (broadcast,)f(and)h(pro)q(cess)f(2)h(sends)g(a)f(message)g(to)g(pro)q (cess)h(1)f(b)q(efore)g(the)h(broadcast.)75 1667 y(Pro)q(cess)15 b(1)g(receiv)o(es)h(b)q(efore)g(and)f(after)f(the)i(broadcast,)e(with)h (a)g(wildcard)i(source)e(argumen)o(t.)166 1724 y(Tw)o(o)10 b(p)q(ossible)j(executions)g(of)e(this)g(program,)g(with)h(di\013eren)o (t)f(matc)o(hings)g(of)g(sends)h(and)f(receiv)o(es,)75 1780 y(are)k(illustrated)i(in)f(\014gure)g(4.10.)j(Note)c(that)g(the)g (second)h(execution)g(has)g(the)f(p)q(eculiar)i(e\013ect)f(that)e(a)75 1837 y(send)k(executed)g(after)f(the)h(broadcast)e(is)i(receiv)o(ed)h (at)e(another)g(no)q(de)h(b)q(efore)g(the)f(broadcast.)26 b(This)75 1893 y(example)15 b(illustrates)g(the)g(fact)f(that)f(one)i (should)g(not)f(rely)h(on)f(collectiv)o(e)i(comm)o(unication)f (functions)75 1950 y(to)h(ha)o(v)o(e)g(particular)h(sync)o(hronization) h(e\013ects.)23 b(A)17 b(program)f(that)f(w)o(orks)h(correctly)h(only)g (when)g(the)75 2006 y(\014rst)e(execution)h(o)q(ccurs)g(\(only)f(when)h (broadcast)e(is)i(sync)o(hronizing\))g(is)g(erroneous.)166 2112 y(Finally)l(,)h(in)g(m)o(ultithreaded)g(implemen)o(tations,)g(one) f(can)g(ha)o(v)o(e)g(more)g(than)f(one,)h(concurren)o(tly)75 2169 y(executing,)e(collectiv)o(e)h(comm)o(unication)f(call)h(at)d(a)h (pro)q(cess.)20 b(In)13 b(these)h(situations,)f(it)h(is)g(the)f(user's) g(re-)75 2225 y(sp)q(onsibilit)o(y)i(to)d(ensure)h(that)f(the)h(same)f (comm)o(unicator)h(is)g(not)f(used)h(concurren)o(tly)h(b)o(y)e(t)o(w)o (o)g(di\013eren)o(t)75 2282 y(collectiv)o(e)17 b(comm)o(unication)f (calls)g(at)f(the)g(same)g(pro)q(cess.)189 2388 y Fl(A)n(dvic)n(e)e(to) h(implementors.)39 b Fx(Assume)13 b(that)f(broadcast)g(is)h(implemen)o (ted)i(using)e(p)q(oin)o(t-to-p)q(oin)o(t)189 2445 y Fv(MPI)h Fx(comm)o(unication.)21 b(Supp)q(ose)16 b(the)f(follo)o(wing)h (t)o(w)o(o)e(rules)i(are)f(follo)o(w)o(ed.)231 2538 y(1.)22 b(All)16 b(receiv)o(es)g(sp)q(ecify)h(their)e(source)h(explicitly)i (\(no)c(wildcards\).)231 2611 y(2.)22 b(Eac)o(h)13 b(pro)q(cess)g (sends)h(all)g(messages)f(that)f(p)q(ertain)i(to)f(one)g(collectiv)o(e) i(call)f(b)q(efore)g(sending)289 2668 y(an)o(y)g(message)h(that)g(p)q (ertain)h(to)e(a)h(subsequen)o(t)h(collectiv)o(e)h(call.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 129 137 129 136 bop 75 -100 a Fm(4.12.)34 b(CORRECTNESS)1257 b Fx(129)189 45 y(Then,)17 b(messages)f(b)q(elonging)j(to)d(successiv)o (e)i(broadcasts)e(cannot)g(b)q(e)i(confused,)f(as)f(the)h(order)189 102 y(of)d(p)q(oin)o(t-to-p)q(oin)o(t)i(messages)f(is)h(preserv)o(ed.) 189 177 y(It)g(is)h(the)g(implemen)o(tor's)g(resp)q(onsibilit)o(y)i(to) d(ensure)i(that)e(p)q(oin)o(t-to-p)q(oin)o(t)h(messages)f(are)g(not)189 233 y(confused)d(with)g(collectiv)o(e)h(messages.)19 b(One)13 b(w)o(a)o(y)f(to)g(accomplish)i(this)f(is,)g(whenev)o(er)g(a)f (comm)o(u-)189 290 y(nicator)j(is)g(created,)g(to)g(also)g(create)g(a)g (\\hidden)h(comm)o(unicator")f(for)f(collectiv)o(e)j(comm)o(unica-)189 346 y(tion.)j(One)15 b(could)h(ac)o(hiev)o(e)f(a)f(similar)i(e\013ect)e (more)g(c)o(heaply)l(,)i(for)e(example,)h(b)o(y)g(using)g(a)f(hidden) 189 403 y(tag)f(or)h(con)o(text)g(bit)h(to)f(indicate)i(whether)f(the)f (comm)o(unicator)g(is)h(used)g(for)f(p)q(oin)o(t-to-p)q(oin)o(t)h(or) 189 459 y(collectiv)o(e)i(comm)o(unication.)j(\()p Fl(End)c(of)g(advic) n(e)g(to)h(implementors.)p Fx(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 130 138 130 137 bop 75 356 a Fo(Chapter)34 b(5)75 564 y Fr(Groups,)40 b(Con)m(texts,)g(and)75 689 y(Comm)m(unicators)75 929 y Fn(5.1)59 b(Intro)r(duction)75 1031 y Fx(This)20 b(c)o(hapter)g(in)o (tro)q(duces)h Fv(MPI)e Fx(features)g(that)h(supp)q(ort)g(the)f(dev)o (elopmen)o(t)i(of)e(parallel)j(libraries.)75 1087 y(P)o(arallel)15 b(libraries)h(are)f(needed)g(to)f(encapsulate)i(the)e(distracting)h (complications)h(inheren)o(t)f(in)h(paral-)75 1144 y(lel)i(implemen)o (tations)g(of)f(k)o(ey)g(algorithms.)24 b(They)18 b(help)g(to)e(ensure) h(consisten)o(t)g(correctness)g(of)g(suc)o(h)75 1200 y(pro)q(cedures,)22 b(and)e(pro)o(vide)h(a)f(\\higher)h(lev)o(el")g(of) f(p)q(ortabilit)o(y)h(than)f Fv(MPI)g Fx(itself)h(can)f(pro)o(vide.)36 b(As)75 1257 y(suc)o(h,)23 b(libraries)g(prev)o(en)o(t)e(eac)o(h)g (programmer)f(from)h(rep)q(eating)h(the)g(w)o(ork)e(of)h(de\014ning)i (consisten)o(t)75 1313 y(data)14 b(structures,)f(data)h(la)o(y)o(outs,) g(and)g(metho)q(ds)g(that)g(implemen)o(t)i(k)o(ey)e(algorithms)g(\(suc) o(h)g(as)g(matrix)75 1369 y(op)q(erations\).)24 b(Since)18 b(the)e(b)q(est)h(libraries)h(come)e(with)h(sev)o(eral)g(v)m(ariations) g(on)f(parallel)i(systems)e(\(dif-)75 1426 y(feren)o(t)f(data)g(la)o(y) o(outs,)g(di\013eren)o(t)g(strategies)g(dep)q(ending)j(on)e(the)f(size) i(of)e(the)g(system)h(or)f(problem,)g(or)75 1482 y(t)o(yp)q(e)g(of)g (\015oating)g(p)q(oin)o(t\),)g(this)h(to)q(o)e(needs)i(to)f(b)q(e)h (hidden)h(from)d(the)h(user.)166 1539 y(W)l(e)h(refer)f(the)g(reader)h (to)f([26)o(])g(and)h([3)o(])f(for)g(further)g(information)h(on)g (writing)g(libraries)h(in)f Fv(MPI)p Fx(,)75 1595 y(using)g(the)f (features)g(describ)q(ed)i(in)f(this)g(c)o(hapter.)75 1717 y Fj(5.1.1)49 b(F)o(eatures)14 b(Needed)i(to)g(Supp)q(o)o(rt)g (Lib)o(ra)o(ries)75 1803 y Fx(The)f(k)o(ey)h(features)e(needed)j(to)d (supp)q(ort)i(the)f(creation)g(of)g(robust)g(parallel)i(libraries)f (are)f(as)g(follo)o(ws:)143 1897 y Ft(\017)23 b Fx(Safe)18 b(comm)o(unication)i(space,)g(that)e(guaran)o(tees)g(that)g(libraries)i (can)f(comm)o(unicate)g(as)g(they)189 1953 y(need)d(to,)e(without)h (con\015icting)i(with)e(comm)o(unication)h(extraneous)f(to)g(the)g (library)l(,)143 2047 y Ft(\017)23 b Fx(Group)14 b(scop)q(e)h(for)g (collectiv)o(e)h(op)q(erations,)f(that)f(allo)o(w)h(libraries)h(to)e(a) o(v)o(oid)h(unnecessarily)i(syn-)189 2103 y(c)o(hronizing)f(unin)o(v)o (olv)o(ed)h(pro)q(cesses)e(\(p)q(oten)o(tially)i(running)f(unrelated)g (co)q(de\),)143 2197 y Ft(\017)23 b Fx(Abstract)15 b(pro)q(cess)h (naming)h(to)e(allo)o(w)h(libraries)i(to)d(describ)q(e)j(their)e(comm)o (unication)h(in)g(terms)189 2254 y(suitable)f(to)f(their)g(o)o(wn)g (data)g(structures)f(and)i(algorithms,)143 2347 y Ft(\017)23 b Fx(The)14 b(abilit)o(y)i(to)e(\\adorn")f(a)h(set)h(of)f(comm)o (unicating)h(pro)q(cesses)g(with)f(additional)i(user-de\014ned)189 2404 y(attributes,)23 b(suc)o(h)f(as)g(extra)g(collectiv)o(e)i(op)q (erations.)41 b(This)23 b(mec)o(hanism)f(should)i(pro)o(vide)e(a)189 2460 y(means)15 b(for)f(the)h(user)g(or)g(library)h(writer)f (e\013ectiv)o(ely)h(to)e(extend)i(a)f(message-passing)g(notation.)75 2554 y(In)f(addition,)h(a)f(uni\014ed)h(mec)o(hanism)g(or)e(ob)s(ject)g (is)i(needed)g(for)e(con)o(v)o(enien)o(tly)i(denoting)f(comm)o(unica-) 75 2611 y(tion)g(con)o(text,)f(the)h(group)f(of)g(comm)o(unicating)h (pro)q(cesses,)g(to)f(house)h(abstract)f(pro)q(cess)h(naming,)g(and)75 2667 y(to)h(store)f(adornmen)o(ts.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 131 139 131 138 bop 75 -100 a Fm(5.1.)34 b(INTR)o(ODUCTION)1256 b Fx(131)75 45 y Fj(5.1.2)49 b(MPI's)16 b(Supp)q(o)o(rt)g(fo)o(r)g(Lib) o(ra)o(ries)75 131 y Fx(The)f(corresp)q(onding)h(concepts)g(that)e Fv(MPI)h Fx(pro)o(vides,)g(sp)q(eci\014cally)j(to)c(supp)q(ort)h (robust)g(libraries,)h(are)75 187 y(as)f(follo)o(ws:)143 281 y Ft(\017)23 b Fq(Con)o(texts)14 b Fx(of)h(comm)o(unication,)143 375 y Ft(\017)23 b Fq(Groups)14 b Fx(of)h(pro)q(cesses,)143 469 y Ft(\017)23 b Fq(Virtual)18 b(top)q(ologies)p Fx(,)143 563 y Ft(\017)23 b Fq(A)o(ttribute)17 b(cac)o(hing)p Fx(,)143 656 y Ft(\017)23 b Fq(Comm)o(unicators)p Fx(.)75 750 y Fq(Comm)o(unicators)18 b Fx(\(see)g([16)o(,)f(24,)h(27)o(]\))g (encapsulate)h(all)g(of)f(these)g(ideas)h(in)g(order)f(to)f(pro)o(vide) i(the)75 807 y(appropriate)h(scop)q(e)h(for)f(all)h(comm)o(unication)g (op)q(erations)f(in)h Fv(MPI)p Fx(.)e(Comm)o(unicators)h(are)g(divided) 75 863 y(in)o(to)15 b(t)o(w)o(o)f(kinds:)21 b(in)o(tra-comm)o (unicators)15 b(for)f(op)q(erations)h(within)i(a)e(single)h(group)f(of) g(pro)q(cesses,)g(and)75 920 y(in)o(ter-comm)o(unicators,)g(for)f(p)q (oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)g(b)q(et)o(w)o(een)g(t)o(w)o (o)e(groups)g(of)h(pro)q(cesses.)75 1040 y Fv(Caching.)46 b Fx(Comm)o(unicators)17 b(\(see)g(b)q(elo)o(w\))i(pro)o(vide)f(a)f (\\cac)o(hing")h(mec)o(hanism)h(that)e(allo)o(ws)h(one)g(to)75 1096 y(asso)q(ciate)h(new)g(attributes)g(with)g(comm)o(unicators,)g(on) g(a)g(par)g(with)g Fv(MPI)f Fx(built-in)k(features.)30 b(This)75 1153 y(can)15 b(b)q(e)h(used)g(b)o(y)f(adv)m(anced)h(users)f (to)f(adorn)h(comm)o(unicators)g(further,)f(and)i(b)o(y)f Fv(MPI)f Fx(to)h(implemen)o(t)75 1209 y(some)20 b(comm)o(unicator)g (functions.)35 b(F)l(or)19 b(example,)j(the)f(virtual-top)q(ology)f (functions)h(describ)q(ed)h(in)75 1266 y(Chapter)15 b(6)g(are)g(lik)o (ely)i(to)d(b)q(e)i(supp)q(orted)g(this)f(w)o(a)o(y)l(.)75 1386 y Fv(Groups.)46 b Fx(Groups)16 b(de\014ne)h(an)f(ordered)g (collection)i(of)e(pro)q(cesses,)g(eac)o(h)g(with)h(a)f(rank,)f(and)i (it)f(is)h(this)75 1442 y(group)d(that)g(de\014nes)i(the)f(lo)o(w-lev)o (el)h(names)e(for)g(in)o(ter-pro)q(cess)i(comm)o(unication)f(\(ranks)f (are)g(used)i(for)75 1498 y(sending)k(and)g(receiving\).)33 b(Th)o(us,)20 b(groups)e(de\014ne)j(a)d(scop)q(e)i(for)f(pro)q(cess)g (names)g(in)h(p)q(oin)o(t-to-p)q(oin)o(t)75 1555 y(comm)o(unication.)i (In)16 b(addition,)h(groups)e(de\014ne)i(the)e(scop)q(e)i(of)e (collectiv)o(e)i(op)q(erations.)k(Groups)16 b(ma)o(y)75 1611 y(b)q(e)k(manipulated)h(separately)f(from)e(comm)o(unicators)h(in) i Fv(MPI)p Fx(,)d(but)i(only)g(comm)o(unicators)f(can)g(b)q(e)75 1668 y(used)d(in)g(comm)o(unication)g(op)q(erations.)75 1788 y Fv(Intra-comm)m(unicato)o(rs.)43 b Fx(The)18 b(most)f(commonly)g (used)i(means)e(for)g(message)g(passing)h(in)h Fv(MPI)e Fx(is)h(via)75 1844 y(in)o(tra-comm)o(unicators.)41 b(In)o(tra-comm)o (unicators)22 b(con)o(tain)g(an)h(instance)g(of)f(a)g(group,)i(con)o (texts)d(of)75 1901 y(comm)o(unication)g(for)e(b)q(oth)h(p)q(oin)o (t-to-p)q(oin)o(t)h(and)f(collectiv)o(e)i(comm)o(unication,)g(and)e (the)g(abilit)o(y)i(to)75 1957 y(include)c(virtual)d(top)q(ology)g(and) h(other)e(attributes.)20 b(These)c(features)f(w)o(ork)f(as)h(follo)o (ws:)143 2051 y Ft(\017)23 b Fq(Con)o(texts)13 b Fx(pro)o(vide)i(the)g (abilit)o(y)h(to)d(ha)o(v)o(e)h(separate)g(safe)g(\\univ)o(erses")h(of) f(message)g(passing)h(in)189 2108 y Fv(MPI)p Fx(.)f(A)h(con)o(text)g (is)h(akin)f(to)g(an)g(additional)i(tag)d(that)h(di\013eren)o(tiates)g (messages.)20 b(The)15 b(system)189 2164 y(manages)20 b(this)h(di\013eren)o(tiation)g(pro)q(cess.)36 b(The)21 b(use)g(of)f(separate)g(comm)o(unication)i(con)o(texts)189 2220 y(b)o(y)13 b(distinct)i(libraries)g(\(or)e(distinct)i(library)f (in)o(v)o(o)q(cations\))g(insulates)h(comm)o(unication)f(in)o(ternal) 189 2277 y(to)h(the)h(library)h(execution)g(from)e(external)h(comm)o (unication.)23 b(This)17 b(allo)o(ws)f(the)g(in)o(v)o(o)q(cation)g(of) 189 2333 y(the)e(library)g(ev)o(en)h(if)f(there)g(are)g(p)q(ending)i (comm)o(unications)e(on)g(\\other")f(comm)o(unicators,)h(and)189 2390 y(a)o(v)o(oids)f(the)h(need)g(to)f(sync)o(hronize)i(en)o(try)e(or) g(exit)i(in)o(to)e(library)i(co)q(de.)20 b(P)o(ending)14 b(p)q(oin)o(t-to-p)q(oin)o(t)189 2446 y(comm)o(unications)j(are)g(also) f(guaran)o(teed)h(not)f(to)g(in)o(terfere)i(with)f(collectiv)o(e)h (comm)o(unications)189 2503 y(within)e(a)f(single)h(comm)o(unicator.) 143 2597 y Ft(\017)23 b Fq(Groups)12 b Fx(de\014ne)i(the)e(participan)o (ts)i(in)f(the)g(comm)o(unication)g(\(see)g(ab)q(o)o(v)o(e\))f(of)g(a)h (comm)o(unicator.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 132 140 132 139 bop 75 -100 a Fx(132)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)143 45 y Ft(\017)23 b Fx(A)14 b Fq(virtual)j(top)q(ology)f Fx(de\014nes)f(a)f(sp)q(ecial)i(mapping)f(of)f(the)h(ranks)f(in)h(a)f (group)g(to)f(and)i(from)e(a)189 102 y(top)q(ology)l(.)19 b(Sp)q(ecial)f(constructors)c(for)g(comm)o(unicators)h(are)g(de\014ned) h(in)g(c)o(hapter)f(6)g(to)f(pro)o(vide)189 158 y(this)f(feature.)19 b(In)o(tra-comm)o(unicators)12 b(as)h(describ)q(ed)i(in)f(this)f(c)o (hapter)g(do)g(not)f(ha)o(v)o(e)h(top)q(ologies.)143 252 y Ft(\017)23 b Fq(A)o(ttributes)14 b Fx(de\014ne)h(the)g(lo)q(cal)g (information)g(that)f(the)g(user)h(or)f(library)h(has)f(added)h(to)f(a) g(com-)189 308 y(m)o(unicator)h(for)f(later)h(reference.)189 415 y Fl(A)n(dvic)n(e)f(to)i(users.)40 b Fx(The)15 b(curren)o(t)f (practice)i(in)f(man)o(y)f(comm)o(unication)i(libraries)g(is)f(that)f (there)189 471 y(is)k(a)f(unique,)i(prede\014ned)g(comm)o(unication)g (univ)o(erse)f(that)f(includes)j(all)e(pro)q(cesses)g(a)o(v)m(ailable) 189 527 y(when)f(the)f(parallel)i(program)d(is)i(initiated;)i(the)d (pro)q(cesses)h(are)f(assigned)h(consecutiv)o(e)h(ranks.)189 584 y(P)o(articipan)o(ts)d(in)h(a)g(p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o (unication)g(are)g(iden)o(ti\014ed)h(b)o(y)f(their)g(rank;)f(a)g (collec-)189 640 y(tiv)o(e)i(comm)o(unication)h(\(suc)o(h)g(as)f (broadcast\))f(alw)o(a)o(ys)g(in)o(v)o(olv)o(es)i(all)h(pro)q(cesses.) 26 b(This)18 b(practice)189 697 y(can)e(b)q(e)g(follo)o(w)o(ed)h(in)g Fv(MPI)e Fx(b)o(y)h(using)h(the)f(prede\014ned)i(comm)o(unicator)d Fv(MPI)p 1529 697 14 2 v 16 w(COMM)p 1687 697 V 16 w(W)o(ORLD)p Fx(.)189 753 y Fl(Users)i(who)h(ar)n(e)g(satis\014e)n(d)f(with)h(this)g (pr)n(actic)n(e)f(c)n(an)h(plug)g(in)f Fv(MPI)p 1351 753 V 16 w(COMM)p 1509 753 V 16 w(W)o(ORLD)h Fl(wher)n(ever)189 810 y(a)f(c)n(ommunic)n(ator)g(ar)n(gument)g(is)f(r)n(e)n(quir)n(e)n (d,)g(and)h(c)n(an)f(c)n(onse)n(quently)f(disr)n(e)n(gar)n(d)h(the)h(r) n(est)f(of)h(this)189 866 y(chapter.)k Fx(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fx(\))75 986 y Fv(Inter-comm)n (unicato)o(rs.)42 b Fx(The)18 b(discussion)h(has)f(dealt)g(so)f(far)g (with)g Fq(in)o(tra-comm)o(unication)p Fx(:)27 b(com-)75 1043 y(m)o(unication)21 b(within)h(a)e(group.)36 b Fv(MPI)19 b Fx(also)i(supp)q(orts)f Fq(in)o(ter-comm)o(unication)p Fx(:)32 b(comm)o(unication)75 1099 y(b)q(et)o(w)o(een)18 b(t)o(w)o(o)f(non-o)o(v)o(erlapping)i(groups.)28 b(When)18 b(an)g(application)i(is)e(built)h(b)o(y)f(comp)q(osing)h(sev)o(eral)75 1156 y(parallel)e(mo)q(dules,)g(it)f(is)h(con)o(v)o(enien)o(t)f(to)f (allo)o(w)h(one)g(mo)q(dule)h(to)e(comm)o(unicate)i(with)f(another)f (using)75 1212 y(lo)q(cal)22 b(ranks)e(for)f(addressing)i(within)h(the) f(second)f(mo)q(dule.)37 b(This)21 b(is)g(esp)q(ecially)i(con)o(v)o (enien)o(t)e(in)g(a)75 1269 y(clien)o(t-serv)o(er)e(computing)f (paradigm,)g(where)g(either)h(clien)o(t)g(or)e(serv)o(er)g(are)h (parallel.)29 b(The)18 b(supp)q(ort)75 1325 y(of)f(in)o(ter-comm)o (unication)h(also)f(pro)o(vides)g(a)g(mec)o(hanism)g(for)f(the)i (extension)f(of)g Fv(MPI)f Fx(to)g(a)h(dynamic)75 1381 y(mo)q(del)h(where)g(not)f(all)i(pro)q(cesses)f(are)f(preallo)q(cated)h (at)f(initialization)k(time.)27 b(In)18 b(suc)o(h)g(a)f(situation,)75 1438 y(it)h(b)q(ecomes)g(necessary)g(to)g(supp)q(ort)f(comm)o (unication)i(across)e(\\univ)o(erses.")28 b(In)o(ter-comm)o(unication) 75 1494 y(is)17 b(supp)q(orted)f(b)o(y)g(ob)s(jects)g(called)h Fq(in)o(ter-comm)o(unicators)p Fx(.)23 b(These)16 b(ob)s(jects)g(bind)h (t)o(w)o(o)e(groups)g(to-)75 1551 y(gether)c(with)h(comm)o(unication)g (con)o(texts)e(shared)i(b)o(y)f(b)q(oth)g(groups.)18 b(F)l(or)11 b(in)o(ter-comm)o(unicators,)h(these)75 1607 y(features)j(w)o(ork)f(as)h(follo)o(ws:)143 1701 y Ft(\017)23 b Fq(Con)o(texts)16 b Fx(pro)o(vide)i(the)f(abilit)o(y)i(to)d(ha)o(v)o (e)h(a)g(separate)g(safe)f(\\univ)o(erse")i(of)f(message)g(passing)189 1758 y(b)q(et)o(w)o(een)k(the)f(t)o(w)o(o)g(groups.)36 b(A)21 b(send)g(in)g(the)g(lo)q(cal)h(group)e(is)i(alw)o(a)o(ys)e(a)g (receiv)o(e)i(in)f(the)g(re-)189 1814 y(mote)d(group,)g(and)h(vice)h(v) o(ersa.)29 b(The)19 b(system)f(manages)g(this)h(di\013eren)o(tiation)g (pro)q(cess.)31 b(The)189 1870 y(use)19 b(of)f(separate)g(comm)o (unication)i(con)o(texts)e(b)o(y)g(distinct)i(libraries)h(\(or)d (distinct)h(library)h(in-)189 1927 y(v)o(o)q(cations\))f(insulates)i (comm)o(unication)g(in)o(ternal)f(to)g(the)g(library)g(execution)h (from)f(external)189 1983 y(comm)o(unication.)29 b(This)19 b(allo)o(ws)f(the)g(in)o(v)o(o)q(cation)h(of)f(the)g(library)h(ev)o(en) f(if)h(there)f(are)g(p)q(ending)189 2040 y(comm)o(unications)c(on)f (\\other")f(comm)o(unicators,)h(and)h(a)o(v)o(oids)f(the)g(need)h(to)f (sync)o(hronize)h(en)o(try)189 2096 y(or)k(exit)h(in)o(to)g(library)h (co)q(de.)32 b(There)19 b(is)g(no)g(general-purp)q(ose)i(collectiv)o(e) f(comm)o(unication)g(on)189 2153 y(in)o(ter-comm)o(unicators,)d(so)g (con)o(texts)f(are)h(used)h(just)e(to)h(isolate)h(p)q(oin)o(t-to-p)q (oin)o(t)f(comm)o(unica-)189 2209 y(tion.)143 2303 y Ft(\017)23 b Fx(A)c(lo)q(cal)i(and)e(remote)g(group)g(sp)q(ecify)i(the) f(recipien)o(ts)h(and)e(destinations)h(for)f(an)h(in)o(ter-com-)189 2359 y(m)o(unicator.)143 2453 y Ft(\017)j Fx(Virtual)16 b(top)q(ology)e(is)i(unde\014ned)h(for)e(an)g(in)o(ter-comm)o (unicator.)143 2547 y Ft(\017)23 b Fx(As)16 b(b)q(efore,)g(attributes)g (cac)o(he)h(de\014nes)g(the)f(lo)q(cal)i(information)e(that)g(the)g (user)g(or)g(library)h(has)189 2604 y(added)e(to)g(a)g(comm)o(unicator) g(for)f(later)i(reference.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 133 141 133 140 bop 75 -100 a Fm(5.2.)34 b(BASIC)16 b(CONCEPTS)1215 b Fx(133)166 45 y Fv(MPI)17 b Fx(pro)o(vides)i(mec)o(hanisms)g(for)e (creating)h(and)h(manipulating)g(in)o(ter-comm)o(unicators.)29 b(They)75 102 y(are)18 b(used)h(for)f(p)q(oin)o(t-to-p)q(oin)o(t)h (comm)o(unication)g(in)g(an)f(related)h(manner)f(to)g(in)o(tra-comm)o (unicators.)75 158 y(Users)j(who)g(do)g(not)g(need)h(in)o(ter-comm)o (unication)h(in)f(their)f(applications)i(can)e(safely)h(ignore)g(this) 75 214 y(extension.)k(Users)18 b(who)e(need)i(collectiv)o(e)h(op)q (erations)f(via)f(in)o(ter-comm)o(unicators)g(m)o(ust)g(la)o(y)o(er)g (it)g(on)75 271 y(top)e(of)h Fv(MPI)p Fx(.)f(Users)g(who)h(require)h (in)o(ter-comm)o(unication)g(b)q(et)o(w)o(een)f(o)o(v)o(erlapping)g (groups)g(m)o(ust)f(also)75 327 y(la)o(y)o(er)g(this)h(capabilit)o(y)g (on)f(top)g(of)g Fv(MPI)p Fx(.)75 471 y Fn(5.2)59 b(Basic)19 b(Concepts)75 572 y Fx(In)d(this)f(section,)h(w)o(e)f(turn)g(to)f(a)h (more)g(formal)g(de\014nition)i(of)e(the)g(concepts)h(in)o(tro)q(duced) g(ab)q(o)o(v)o(e.)75 694 y Fj(5.2.1)49 b(Groups)75 780 y Fx(A)24 b Fq(group)h Fx(is)g(an)f(ordered)h(set)f(of)g(pro)q(cess)g (iden)o(ti\014ers)i(\(henceforth)f(pro)q(cesses\);)j(pro)q(cesses)d (are)75 836 y(implemen)o(tation-dep)q(endent)e(ob)s(jects.)36 b(Eac)o(h)20 b(pro)q(cess)i(in)f(a)g(group)f(is)h(asso)q(ciated)g(with) h(an)e(in)o(te-)75 893 y(ger)e Fq(rank)p Fx(.)28 b(Ranks)18 b(are)g(con)o(tiguous)g(and)g(start)f(from)g(zero.)28 b(Groups)18 b(are)f(represen)o(ted)i(b)o(y)f(opaque)75 949 y Fq(group)g(ob)s(jects)p Fx(,)f(and)f(hence)h(cannot)f(b)q(e)h (directly)g(transferred)f(from)f(one)i(pro)q(cess)f(to)g(another.)22 b(A)75 1005 y(group)13 b(is)h(used)g(within)g(a)f(comm)o(unicator)g(to) g(describ)q(e)i(the)e(participan)o(ts)h(in)g(a)f(comm)o(unication)h (\\uni-)75 1062 y(v)o(erse")d(and)i(to)e(rank)g(suc)o(h)i(participan)o (ts)f(\(th)o(us)f(giving)i(them)f(unique)h(names)f(within)h(that)f (\\univ)o(erse")75 1118 y(of)j(comm)o(unication\).)166 1175 y(There)20 b(is)f(a)h(sp)q(ecial)h(pre-de\014ned)g(group:)28 b Fi(MPI)p 1017 1175 13 2 v 14 w(GROUP)p 1173 1175 V 13 w(EMPTY)p Fx(,)19 b(whic)o(h)h(is)g(a)f(group)g(with)h(no)75 1231 y(mem)o(b)q(ers.)33 b(The)20 b(prede\014ned)h(constan)o(t)d Fi(MPI)p 884 1231 V 15 w(GROUP)p 1041 1231 V 13 w(NULL)i Fx(is)g(the)g(v)m(alue)g(used)h(for)d(in)o(v)m(alid)k(group)75 1288 y(handles.)189 1394 y Fl(A)n(dvic)n(e)c(to)i(users.)62 b Fi(MPI)p 650 1394 V 14 w(GROUP)p 806 1394 V 13 w(EMPTY)p Fx(,)18 b(whic)o(h)i(is)f(a)g(v)m(alid)h(handle)g(to)e(an)h(empt)o(y)g (group,)189 1450 y(should)h(not)f(b)q(e)h(confused)h(with)e Fi(MPI)p 863 1450 V 15 w(GROUP)p 1020 1450 V 13 w(NULL)p Fx(,)h(whic)o(h)g(in)g(turn)g(is)g(an)f(in)o(v)m(alid)j(handle.)189 1507 y(The)e(former)f(ma)o(y)g(b)q(e)h(used)g(as)g(an)f(argumen)o(t)g (to)h(group)f(op)q(erations;)j(the)e(latter,)g(whic)o(h)g(is)189 1563 y(returned)15 b(when)h(a)f(group)g(is)g(freed,)h(in)g(not)e(a)h(v) m(alid)i(argumen)o(t.)i(\()p Fl(End)d(of)g(advic)n(e)g(to)h(users.)p Fx(\))189 1670 y Fl(A)n(dvic)n(e)g(to)h(implementors.)53 b Fx(A)18 b(group)f(ma)o(y)f(b)q(e)i(represen)o(ted)g(b)o(y)f(a)g (virtual-to-real)i(pro)q(cess-)189 1726 y(address-translation)11 b(table.)19 b(Eac)o(h)11 b(comm)o(unicator)f(ob)s(ject)h(\(see)g(b)q (elo)o(w\))g(w)o(ould)g(ha)o(v)o(e)g(a)f(p)q(oin)o(ter)189 1782 y(to)k(suc)o(h)i(a)f(table.)189 1858 y(Simple)j(implemen)o (tations)g(of)e Fv(MPI)g Fx(will)i(en)o(umerate)e(groups,)g(suc)o(h)h (as)f(in)i(a)e(table.)24 b(Ho)o(w)o(ev)o(er,)189 1914 y(more)11 b(adv)m(anced)i(data)e(structures)g(mak)o(e)g(sense)i(in)f (order)g(to)f(impro)o(v)o(e)g(scalabilit)o(y)j(and)e(memory)189 1971 y(usage)j(with)g(large)h(n)o(um)o(b)q(ers)f(of)g(pro)q(cesses.)21 b(Suc)o(h)16 b(implemen)o(tations)g(are)f(p)q(ossible)i(with)f Fv(MPI)p Fx(.)189 2027 y(\()p Fl(End)f(of)i(advic)n(e)f(to)g (implementors.)p Fx(\))75 2149 y Fj(5.2.2)49 b(Contexts)75 2235 y Fx(A)18 b Fq(con)o(text)g Fx(is)h(a)f(prop)q(ert)o(y)f(of)h (comm)o(unicators)g(\(de\014ned)h(next\))f(that)f(allo)o(ws)h (partitioning)h(of)f(the)75 2291 y(comm)o(unication)13 b(space.)19 b(A)12 b(message)g(sen)o(t)g(in)h(one)f(con)o(text)g (cannot)g(b)q(e)h(receiv)o(ed)g(in)g(another)f(con)o(text.)75 2347 y(F)l(urthermore,)17 b(where)g(p)q(ermitted,)h(collectiv)o(e)h(op) q(erations)e(are)g(indep)q(enden)o(t)i(of)e(p)q(ending)i(p)q(oin)o (t-to-)75 2404 y(p)q(oin)o(t)g(op)q(erations.)28 b(Con)o(texts)17 b(are)h(not)f(explicit)k Fv(MPI)c Fx(ob)s(jects;)h(they)g(app)q(ear)h (only)f(as)g(part)f(of)h(the)75 2460 y(realization)e(of)f(comm)o (unicators)g(\(b)q(elo)o(w\).)189 2567 y Fl(A)n(dvic)n(e)h(to)i (implementors.)48 b Fx(Distinct)17 b(comm)o(unicators)f(in)h(the)g (same)f(pro)q(cess)h(ha)o(v)o(e)f(distinct)189 2623 y(con)o(texts.)31 b(A)20 b(con)o(text)f(is)g(essen)o(tially)i(a)e(system-managed)g(tag)g (\(or)f(tags\))g(needed)j(to)d(mak)o(e)189 2680 y(a)i(comm)o(unicator)h (safe)g(for)f(p)q(oin)o(t-to-p)q(oin)o(t)i(and)f Fv(MPI)p Fx(-de\014ned)h(collectiv)o(e)h(comm)o(unication.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 134 142 134 141 bop 75 -100 a Fx(134)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)189 45 y Fx(Safet)o(y)j(means)h(that)f(collectiv)o(e)j(and)e(p)q(oin)o (t-to-p)q(oin)o(t)g(comm)o(unication)h(within)g(one)f(comm)o(u-)189 102 y(nicator)d(do)g(not)g(in)o(terfere,)h(and)g(that)e(comm)o (unication)i(o)o(v)o(er)f(distinct)h(comm)o(unicators)f(don't)189 158 y(in)o(terfere.)189 230 y(A)c(p)q(ossible)i(implemen)o(tation)f (for)e(a)h(con)o(text)g(is)g(as)g(a)g(supplemen)o(tal)h(tag)f(attac)o (hed)f(to)g(messages)189 287 y(on)j(send)g(and)h(matc)o(hed)f(on)g (receiv)o(e.)20 b(Eac)o(h)14 b(in)o(tra-comm)o(unicator)g(stores)g(the) g(v)m(alue)h(of)f(its)g(t)o(w)o(o)189 343 y(tags)i(\(one)i(for)f(p)q (oin)o(t-to-p)q(oin)o(t)h(and)g(one)g(for)f(collectiv)o(e)i(comm)o (unication\).)28 b(Comm)o(unicator-)189 400 y(generating)19 b(functions)i(use)f(a)f(collectiv)o(e)j(comm)o(unication)e(to)f(agree)g (on)h(a)f(new)h(group-wide)189 456 y(unique)c(con)o(text.)189 528 y(Analogously)l(,)c(in)f(in)o(ter-comm)o(unication)h(\(whic)o(h)f (is)g(strictly)g(p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o(unication\),)189 585 y(t)o(w)o(o)f(con)o(text)h(tags)g(are)h(stored)f(p)q(er)i(comm)o (unicator,)f(one)g(used)g(b)o(y)g(group)g(A)f(to)h(send)g(and)g(group) 189 641 y(B)j(to)g(receiv)o(e,)g(and)h(a)f(second)g(used)h(b)o(y)f (group)g(B)g(to)g(send)h(and)f(for)g(group)g(A)g(to)f(receiv)o(e.)189 713 y(Since)h(con)o(texts)f(are)f(not)h(explicit)i(ob)s(jects,)e(other) f(implemen)o(tations)j(are)d(also)h(p)q(ossible.)22 b(\()p Fl(End)189 770 y(of)16 b(advic)n(e)g(to)h(implementors.)p Fx(\))75 889 y Fj(5.2.3)49 b(Intra-Comm)o(unicato)o(rs)75 975 y Fx(In)o(tra-comm)o(unicators)26 b(bring)i(together)e(the)h (concepts)h(of)e(group)h(and)g(con)o(text.)55 b(T)l(o)27 b(supp)q(ort)75 1031 y(implemen)o(tation-sp)q(eci\014c)15 b(optimizations,)e(and)f(application)h(top)q(ologies)g(\(de\014ned)f (in)h(the)f(next)g(c)o(hap-)75 1087 y(ter,)17 b(c)o(hapter)g(6\),)f (comm)o(unicators)h(ma)o(y)f(also)h(\\cac)o(he")f(additional)j (information)e(\(see)g(section)h(5.7\).)75 1144 y Fv(MPI)h Fx(comm)o(unication)g(op)q(erations)h(reference)f(comm)o(unicators)g (to)g(determine)h(the)f(scop)q(e)h(and)f(the)75 1200 y(\\comm)o(unication)d(univ)o(erse")g(in)g(whic)o(h)g(a)e(p)q(oin)o (t-to-p)q(oin)o(t)i(or)f(collectiv)o(e)i(op)q(eration)e(is)h(to)f(op)q (erate.)166 1257 y(Eac)o(h)g(comm)o(unicator)h(con)o(tains)g(a)f(group) g(of)h(v)m(alid)h(participan)o(ts;)f(this)g(group)g(alw)o(a)o(ys)f (includes)75 1313 y(the)k(lo)q(cal)h(pro)q(cess.)32 b(The)20 b(source)f(and)g(destination)h(of)f(a)f(message)h(is)h(iden)o(ti\014ed) h(b)o(y)e(pro)q(cess)g(rank)75 1370 y(within)d(that)f(group.)166 1426 y(F)l(or)10 b(collectiv)o(e)j(comm)o(unication,)f(the)f(in)o (tra-comm)o(unicator)g(sp)q(eci\014es)i(the)e(set)g(of)f(pro)q(cesses)i (that)75 1483 y(participate)g(in)g(the)g(collectiv)o(e)h(op)q(eration)e (\(and)h(their)f(order,)h(when)g(signi\014can)o(t\).)19 b(Th)o(us,)12 b(the)f(comm)o(u-)75 1539 y(nicator)16 b(restricts)g(the)g(\\spatial")g(scop)q(e)h(of)e(comm)o(unication,)i (and)f(pro)o(vides)h(mac)o(hine-indep)q(endent)75 1596 y(pro)q(cess)e(addressing)h(through)f(ranks.)166 1652 y(In)o(tra-comm)o(unicators)h(are)h(represen)o(ted)g(b)o(y)g(opaque)g Fq(in)o(tra-comm)o(unicator)k(ob)s(jects)p Fx(,)c(and)75 1708 y(hence)f(cannot)f(b)q(e)h(directly)h(transferred)d(from)h(one)g (pro)q(cess)h(to)e(another.)75 1827 y Fj(5.2.4)49 b(Prede\014ned)15 b(Intra-Comm)o(unicato)o(rs)75 1913 y Fx(An)22 b(initial)i(in)o (tra-comm)o(unicator)e Fi(MPI)p 785 1913 13 2 v 14 w(COMM)p 929 1913 V 15 w(W)o(ORLD)f Fx(of)h(all)h(pro)q(cesses)f(the)g(lo)q(cal) h(pro)q(cess)f(can)75 1970 y(comm)o(unicate)d(with)g(after)f (initialization)k(\(itself)d(included\))j(is)d(de\014ned)h(once)f Fv(MPI)p 1578 1970 14 2 v 16 w(INIT)f Fx(has)h(b)q(een)75 2026 y(called.)i(In)13 b(addition,)h(the)e(comm)o(unicator)g Fi(MPI)p 911 2026 13 2 v 15 w(COMM)p 1056 2026 V 14 w(SELF)h Fx(is)g(pro)o(vided,)g(whic)o(h)h(includes)h(only)e(the)75 2083 y(pro)q(cess)i(itself.)166 2139 y(The)i(prede\014ned)i(constan)o (t)d Fi(MPI)p 745 2139 V 14 w(COMM)p 889 2139 V 15 w(NULL)i Fx(is)f(the)g(v)m(alue)h(used)g(for)e(in)o(v)m(alid)j(comm)o(unicator) 75 2195 y(handles.)166 2252 y(In)d(a)g(static-pro)q(cess-mo)q(del)h (implemen)o(tation)g(of)f Fv(MPI)p Fx(,)e(all)j(pro)q(cesses)g(that)e (participate)h(in)h(the)75 2308 y(computation)i(are)f(a)o(v)m(ailable)j (after)d Fv(MPI)g Fx(is)h(initialized.)34 b(F)l(or)18 b(this)h(case,)g Fi(MPI)p 1479 2308 V 15 w(COMM)p 1624 2308 V 14 w(W)o(ORLD)g Fx(is)g(a)75 2365 y(comm)o(unicator)g(of)g(all)i (pro)q(cesses)f(a)o(v)m(ailable)h(for)e(the)g(computation;)i(this)f (comm)o(unicator)g(has)f(the)75 2421 y(same)h(v)m(alue)h(in)g(all)f (pro)q(cesses.)35 b(In)21 b(an)e(implemen)o(tation)j(of)d Fv(MPI)h Fx(where)g(pro)q(cesses)g(can)g(dynami-)75 2478 y(cally)d(join)g(an)f Fv(MPI)g Fx(execution,)h(it)f(ma)o(y)g(b)q(e)g (the)h(case)f(that)f(a)h(pro)q(cess)g(starts)f(an)h Fv(MPI)g Fx(computation)75 2534 y(without)j(ha)o(ving)g(access)g(to)f(all)i (other)f(pro)q(cesses.)31 b(In)20 b(suc)o(h)f(situations,)h Fi(MPI)p 1479 2534 V 14 w(COMM)p 1623 2534 V 15 w(W)o(ORLD)f Fx(is)g(a)75 2591 y(comm)o(unicator)d(incorp)q(orating)h(all)h(pro)q (cesses)f(with)g(whic)o(h)g(the)g(joining)h(pro)q(cess)f(can)f (immediately)75 2647 y(comm)o(unicate.)28 b(Therefore,)17 b Fi(MPI)p 675 2647 V 15 w(COMM)p 820 2647 V 14 w(W)o(ORLD)h Fx(ma)o(y)f(sim)o(ultaneously)i(ha)o(v)o(e)e(di\013eren)o(t)h(v)m (alues)h(in)75 2704 y(di\013eren)o(t)c(pro)q(cesses.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 135 143 135 142 bop 75 -100 a Fm(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b Fx(135)166 45 y(All)18 b Fv(MPI)d Fx(implemen)o(tations)j(are)e (required)h(to)f(pro)o(vide)h(the)f Fi(MPI)p 1315 45 13 2 v 15 w(COMM)p 1460 45 V 14 w(W)o(ORLD)g Fx(comm)o(unica-)75 102 y(tor.)34 b(It)21 b(cannot)f(b)q(e)h(deallo)q(cated)h(during)f(the) f(life)i(of)e(a)g(pro)q(cess.)36 b(The)20 b(group)g(corresp)q(onding)i (to)75 158 y(this)14 b(comm)o(unicator)g(do)q(es)g(not)g(app)q(ear)g (as)g(a)f(pre-de\014ned)j(constan)o(t,)d(but)h(it)h(ma)o(y)e(b)q(e)i (accessed)f(using)75 214 y Fv(MPI)p 160 214 14 2 v 16 w(COMM)p 318 214 V 16 w(GROUP)21 b Fx(\(see)f(b)q(elo)o(w\).)34 b Fv(MPI)19 b Fx(do)q(es)h(not)g(sp)q(ecify)h(the)f(corresp)q(ondence)h (b)q(et)o(w)o(een)f(the)75 271 y(pro)q(cess)15 b(rank)f(in)h Fi(MPI)p 466 271 13 2 v 15 w(COMM)p 611 271 V 14 w(W)o(ORLD)g Fx(and)f(its)h(\(mac)o(hine-dep)q(enden)o(t\))i(absolute)d(address.)20 b(Neither)75 327 y(do)q(es)14 b Fv(MPI)e Fx(sp)q(ecify)j(the)f (function)g(of)f(the)g(host)g(pro)q(cess,)h(if)g(an)o(y)l(.)19 b(Other)13 b(implemen)o(tation-dep)q(end)q(en)o(t,)75 384 y(prede\014ned)k(comm)o(unicators)e(ma)o(y)f(also)h(b)q(e)h(pro)o (vided.)75 527 y Fn(5.3)59 b(Group)20 b(Management)75 629 y Fx(This)h(section)g(describ)q(es)h(the)e(manipulation)i(of)e(pro) q(cess)h(groups)f(in)h Fv(MPI)p Fx(.)e(These)i(op)q(erations)f(are)75 685 y(lo)q(cal)c(and)g(their)f(execution)i(do)e(not)g(require)h(in)o (terpro)q(cess)f(comm)o(unication.)75 807 y Fj(5.3.1)49 b(Group)16 b(Accesso)o(rs)75 987 y Fv(MPI)p 160 987 14 2 v 16 w(GROUP)p 328 987 V 18 w(SIZE\(group,)e(size\))117 1064 y Fk(IN)155 b Fv(group)479 b Fk(group)14 b(\(handle\))117 1139 y(OUT)108 b Fv(size)519 b Fk(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g(group)g(\(in)o(teger\))75 1264 y Fu(int)23 b(MPI)p 245 1264 15 2 v 17 w(Group)p 382 1264 V 17 w(size\(MPI)p 591 1264 V 16 w(Group)g(group,)g(int)g (*size\))75 1350 y(MPI)p 150 1350 V 17 w(GROUP)p 287 1350 V 16 w(SIZE\(GROUP,)g(SIZE,)g(IERROR\))170 1407 y(INTEGER)g(GROUP,)g(SIZE,)h(IERROR)75 1588 y Fv(MPI)p 160 1588 14 2 v 16 w(GROUP)p 328 1588 V 18 w(RANK\(group,)14 b(rank\))117 1665 y Fk(IN)155 b Fv(group)479 b Fk(group)14 b(\(handle\))117 1740 y(OUT)108 b Fv(rank)504 b Fk(rank)50 b(of)g(the)h(calling)e(pro)q(cess)j(in)e(group,)59 b(or)905 1796 y Fi(MPI)p 982 1796 13 2 v 15 w(UNDEFINED)13 b Fk(if)h(the)h(pro)q (cess)i(is)d(not)h(a)f(mem)o(b)q(er)f(\(in-)905 1853 y(teger\))75 1977 y Fu(int)23 b(MPI)p 245 1977 15 2 v 17 w(Group)p 382 1977 V 17 w(rank\(MPI)p 591 1977 V 16 w(Group)g(group,)g(int)g(*rank\))75 2064 y(MPI)p 150 2064 V 17 w(GROUP)p 287 2064 V 16 w(RANK\(GROUP,)g(RANK,)g(IERROR\))170 2120 y(INTEGER)g(GROUP,)g(RANK,)h(IERROR)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 136 144 136 143 bop 75 -100 a Fx(136)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(TRANSLA)l(TE)p 602 45 V 17 w(RANKS)h(\(group1,)e(n,)h(ranks1,)g (group2,)g(ranks2\))117 122 y Fk(IN)155 b Fv(group1)456 b Fk(group1)14 b(\(handle\))117 197 y(IN)155 b Fv(n)564 b Fk(n)o(um)o(b)q(er)11 b(of)h(ranks)g(in)23 b Fi(ranks1)13 b Fk(and)f Fi(ranks2)g Fk(arra)o(ys)g(\(in)o(teger\))117 273 y(IN)155 b Fv(ranks1)464 b Fk(arra)o(y)14 b(of)f(zero)i(or)f(more)f (v)n(alid)f(ranks)i(in)g(group1)117 348 y(IN)155 b Fv(group2)456 b Fk(group2)14 b(\(handle\))117 423 y(OUT)108 b Fv(ranks2)464 b Fk(arra)o(y)17 b(of)g(corresp)q(onding)h(ranks)f(in)g(group2,)g Fi(MPI)p 1736 423 13 2 v 14 w(UNDE-)905 479 y(FINED)c Fk(when)i(no)e(corresp)q(ondence)k(exists.)75 604 y Fu(int)23 b(MPI)p 245 604 15 2 v 17 w(Group)p 382 604 V 17 w(translate)p 615 604 V 16 w(ranks)g(\(MPI)p 870 604 V 16 w(Group)h(group1,)f(int)g (n,)h(int)f(*ranks1,)393 660 y(MPI)p 468 660 V 17 w(Group)g(group2,)g (int)g(*ranks2\))75 747 y(MPI)p 150 747 V 17 w(GROUP)p 287 747 V 16 w(TRANSLATE)p 519 747 V 16 w(RANKS\(GROUP1,)f(N,)i (RANKS1,)f(GROUP2,)g(RANKS2,)g(IERROR\))170 803 y(INTEGER)g(GROUP1,)g (N,)h(RANKS1\(*\),)e(GROUP2,)h(RANKS2\(*\),)g(IERROR)166 890 y Fx(This)12 b(function)g(is)g(imp)q(ortan)o(t)f(for)f(determining) j(the)e(relativ)o(e)h(n)o(um)o(b)q(ering)g(of)f(the)h(same)f(pro)q (cesses)75 946 y(in)h(t)o(w)o(o)e(di\013eren)o(t)h(groups.)18 b(F)l(or)11 b(instance,)h(if)g(one)f(kno)o(ws)f(the)i(ranks)e(of)h (certain)h(pro)q(cesses)f(in)h(the)f(group)75 1002 y(of)k Fi(MPI)p 204 1002 13 2 v 14 w(COMM)p 348 1002 V 15 w(W)o(ORLD)p Fx(,)f(one)i(migh)o(t)f(w)o(an)o(t)f(to)g(kno)o(w)h(their)h(ranks)f(in) h(a)f(subset)g(of)g(that)f(group.)75 1153 y Fv(MPI)p 160 1153 14 2 v 16 w(GROUP)p 328 1153 V 18 w(COMP)l(ARE\(group1,)h (group2,)g(result\))117 1231 y Fk(IN)155 b Fv(group1)456 b Fk(\014rst)15 b(group)f(\(handle\))117 1306 y(IN)155 b Fv(group2)456 b Fk(second)15 b(group)f(\(handle\))117 1381 y(OUT)108 b Fv(result)484 b Fk(result)15 b(\(in)o(teger\))75 1505 y Fu(int)23 b(MPI)p 245 1505 15 2 v 17 w(Group)p 382 1505 V 17 w(compare\(MPI)p 663 1505 V 15 w(Group)h(group1,MPI)p 1062 1505 V 15 w(Group)g(group2,)e(int)i(*result\))75 1592 y(MPI)p 150 1592 V 17 w(GROUP)p 287 1592 V 16 w(COMPARE\(GROUP1,)e (GROUP2,)h(RESULT,)g(IERROR\))170 1648 y(INTEGER)g(GROUP1,)g(GROUP2,)g (RESULT,)g(IERROR)75 1735 y Fi(MPI)p 152 1735 13 2 v 14 w(IDENT)12 b Fx(results)g(if)h(the)f(group)f(mem)o(b)q(ers)h(and)g (group)g(order)g(is)g(exactly)g(the)g(same)g(in)h(b)q(oth)f(groups.)75 1791 y(This)i(happ)q(ens)h(for)f(instance)g(if)h Fv(group1)e Fx(and)h Fv(group2)g Fx(are)f(the)h(same)g(handle.)21 b Fi(MPI)p 1520 1791 V 14 w(SIMILAR)13 b Fx(results)h(if)75 1848 y(the)f(group)f(mem)o(b)q(ers)h(are)f(the)h(same)f(but)h(the)g (order)f(is)h(di\013eren)o(t.)19 b Fi(MPI)p 1316 1848 V 15 w(UNEQUAL)11 b Fx(results)i(otherwise.)75 1969 y Fj(5.3.2)49 b(Group)16 b(Constructo)o(rs)75 2055 y Fx(Group)j (constructors)f(are)h(used)g(to)g(subset)g(and)g(sup)q(erset)h (existing)g(groups.)31 b(These)20 b(constructors)75 2112 y(construct)14 b(new)g(groups)g(from)f(existing)i(groups.)k(These)14 b(are)g(lo)q(cal)h(op)q(erations,)f(and)g(distinct)h(groups)75 2168 y(ma)o(y)20 b(b)q(e)h(de\014ned)h(on)e(di\013eren)o(t)g(pro)q (cesses;)j(a)e(pro)q(cess)f(ma)o(y)g(also)g(de\014ne)i(a)e(group)g (that)g(do)q(es)g(not)75 2225 y(include)i(itself.)33 b(Consisten)o(t)19 b(de\014nitions)i(are)e(required)h(when)g(groups)f (are)g(used)h(as)f(argumen)o(ts)f(in)75 2281 y(comm)o (unicator-building)23 b(functions.)39 b Fv(MPI)20 b Fx(do)q(es)h(not)g (pro)o(vide)h(a)e(mec)o(hanism)i(to)f(build)i(a)d(group)75 2337 y(from)j(scratc)o(h,)i(but)f(only)h(from)e(other,)i(previously)g (de\014ned)g(groups.)46 b(The)24 b(base)g(group,)h(up)q(on)75 2394 y(whic)o(h)d(all)g(other)e(groups)g(are)h(de\014ned,)i(is)f(the)f (group)f(asso)q(ciated)h(with)g(the)g(initial)i(comm)o(unica-)75 2450 y(tor)14 b Fv(MPI)p 233 2450 14 2 v 16 w(COMM)p 391 2450 V 17 w(W)o(ORLD)h Fx(\(accessible)h(through)f(the)h(function)g Fv(MPI)p 1318 2450 V 15 w(COMM)p 1475 2450 V 17 w(GROUP)p Fx(\).)189 2557 y Fl(R)n(ationale.)74 b Fx(In)21 b(what)g(follo)o(ws,)h (there)e(is)i(no)f(group)f(duplication)j(function)f(analogous)f(to)189 2613 y Fv(MPI)p 274 2613 V 15 w(COMM)p 431 2613 V 17 w(DUP)p Fx(,)16 b(de\014ned)j(later)e(in)g(this)h(c)o(hapter.)24 b(There)17 b(is)h(no)f(need)g(for)g(a)f(group)h(dupli-)189 2670 y(cator.)27 b(A)18 b(group,)g(once)g(created,)g(can)g(ha)o(v)o(e)g (sev)o(eral)g(references)g(to)g(it)g(b)o(y)g(making)g(copies)g(of)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 137 145 137 144 bop 75 -100 a Fm(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b Fx(137)189 45 y(the)15 b(handle.)23 b(The)15 b(follo)o(wing)i (constructors)d(address)i(the)g(need)g(for)f(subsets)h(and)f(sup)q (ersets)h(of)189 102 y(existing)g(groups.)j(\()p Fl(End)d(of)g(r)n (ationale.)p Fx(\))189 208 y Fl(A)n(dvic)n(e)j(to)h(implementors.)65 b Fx(Eac)o(h)19 b(group)g(constructor)g(b)q(eha)o(v)o(es)h(as)e(if)i (it)g(returned)g(a)f(new)189 264 y(group)h(ob)s(ject.)35 b(When)20 b(this)h(new)g(group)f(is)h(a)f(cop)o(y)g(of)g(an)g(existing) i(group,)f(then)f(one)h(can)189 321 y(a)o(v)o(oid)16 b(creating)g(suc)o(h)h(new)g(ob)s(jects,)e(using)j(a)e(reference-coun)o (t)h(mec)o(hanism.)24 b(\()p Fl(End)17 b(of)g(advic)n(e)189 377 y(to)f(implementors.)p Fx(\))75 578 y Fv(MPI)p 160 578 14 2 v 16 w(COMM)p 318 578 V 16 w(GROUP\(comm)n(,)c(group\))117 655 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 730 y(OUT)108 b Fv(group)479 b Fk(group)14 b(corresp)q(onding)h(to)f Fi(comm)8 b Fk(\(handle\))75 855 y Fu(int)23 b(MPI)p 245 855 15 2 v 17 w(Comm)p 358 855 V 17 w(group\(MPI)p 591 855 V 16 w(Comm)g(comm,)g(MPI)p 941 855 V 17 w(Group)g(*group\))75 941 y(MPI)p 150 941 V 17 w(COMM)p 263 941 V 16 w(GROUP\(COMM,)g(GROUP,)g(IERROR\))170 998 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)166 1084 y Fv(MPI)p 251 1084 14 2 v 16 w(COMM)p 409 1084 V 16 w(GROUP)17 b Fx(returns)e(in)h Fv(group)f Fx(a)g(handle)h(to)f(the)g(group)g(of)g Fv(comm)m Fx(.)75 1235 y Fv(MPI)p 160 1235 V 16 w(GROUP)p 328 1235 V 18 w(UNION\(group1,)f(group2,)h(newgroup\))117 1312 y Fk(IN)155 b Fv(group1)456 b Fk(\014rst)15 b(group)f(\(handle\)) 117 1387 y(IN)155 b Fv(group2)456 b Fk(second)15 b(group)f(\(handle\)) 117 1463 y(OUT)108 b Fv(newgroup)405 b Fk(union)14 b(group)f (\(handle\))75 1587 y Fu(int)23 b(MPI)p 245 1587 15 2 v 17 w(Group)p 382 1587 V 17 w(union\(MPI)p 615 1587 V 16 w(Group)g(group1,)g(MPI)p 1037 1587 V 17 w(Group)g(group2,)g(MPI)p 1460 1587 V 16 w(Group)h(*newgroup\))75 1673 y(MPI)p 150 1673 V 17 w(GROUP)p 287 1673 V 16 w(UNION\(GROUP1,)e(GROUP2,)h (NEWGROUP,)g(IERROR\))170 1730 y(INTEGER)g(GROUP1,)g(GROUP2,)g (NEWGROUP,)g(IERROR)75 1911 y Fv(MPI)p 160 1911 14 2 v 16 w(GROUP)p 328 1911 V 18 w(INTERSECTION\(group1,)15 b(group2,)f(newgroup\))117 1988 y Fk(IN)155 b Fv(group1)456 b Fk(\014rst)15 b(group)f(\(handle\))117 2063 y(IN)155 b Fv(group2)456 b Fk(second)15 b(group)f(\(handle\))117 2138 y(OUT)108 b Fv(newgroup)405 b Fk(in)o(tersection)15 b(group)f(\(handle\))75 2263 y Fu(int)23 b(MPI)p 245 2263 15 2 v 17 w(Group)p 382 2263 V 17 w(intersection\(MPI)p 783 2263 V 15 w(Group)g(group1,)g(MPI)p 1204 2263 V 17 w(Group)g(group2,)393 2319 y(MPI)p 468 2319 V 17 w(Group)g(*newgroup\)) 75 2406 y(MPI)p 150 2406 V 17 w(GROUP)p 287 2406 V 16 w(INTERSECTION\(GROUP1,)e(GROUP2,)i(NEWGROUP,)g(IERROR\))170 2462 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 138 146 138 145 bop 75 -100 a Fx(138)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(DIFFERENCE\(group1,)f(group2,)g(newgroup\))117 122 y Fk(IN)155 b Fv(group1)456 b Fk(\014rst)15 b(group)f(\(handle\))117 197 y(IN)155 b Fv(group2)456 b Fk(second)15 b(group)f(\(handle\))117 273 y(OUT)108 b Fv(newgroup)405 b Fk(di\013erence)16 b(group)e(\(handle\))75 397 y Fu(int)23 b(MPI)p 245 397 15 2 v 17 w(Group)p 382 397 V 17 w(difference\(MPI)p 735 397 V 15 w(Group)g(group1,)g(MPI)p 1156 397 V 17 w(Group)g(group2,)393 453 y(MPI)p 468 453 V 17 w(Group)g(*newgroup\))75 540 y(MPI)p 150 540 V 17 w(GROUP)p 287 540 V 16 w(DIFFERENCE\(GROUP1,)f (GROUP2,)h(NEWGROUP,)f(IERROR\))170 596 y(INTEGER)h(GROUP1,)g(GROUP2,)g (NEWGROUP,)g(IERROR)75 683 y Fx(The)15 b(set-lik)o(e)i(op)q(erations)e (are)g(de\014ned)i(as)d(follo)o(ws:)75 777 y Fq(union)24 b Fx(All)19 b(elemen)o(ts)g(of)f(the)h(\014rst)f(group)g(\()p Fv(group1)p Fx(\),)g(follo)o(w)o(ed)h(b)o(y)f(all)h(elemen)o(ts)g(of)f (second)h(group)189 833 y(\()p Fv(group2)p Fx(\))14 b(not)h(in)h (\014rst.)75 927 y Fq(in)o(tersect)23 b Fx(all)18 b(elemen)o(ts)h(of)e (the)g(\014rst)h(group)f(that)g(are)g(also)g(in)i(the)f(second)g (group,)f(ordered)h(as)f(in)189 983 y(\014rst)d(group.)75 1077 y Fq(di\013erence)22 b Fx(all)c(elemen)o(ts)f(of)g(the)g(\014rst)f (group)g(that)g(are)h(not)f(in)h(the)g(second)g(group,)g(ordered)g(as)f (in)189 1134 y(the)f(\014rst)g(group.)75 1227 y(Note)k(that)g(for)g (these)h(op)q(erations)g(the)f(order)h(of)f(pro)q(cesses)h(in)g(the)g (output)f(group)h(is)g(determined)75 1284 y(primarily)g(b)o(y)f(order)g (in)h(the)f(\014rst)g(group)f(\(if)h(p)q(ossible\))i(and)e(then,)h(if)f (necessary)l(,)h(b)o(y)f(order)g(in)h(the)75 1340 y(second)c(group.)j (Neither)d(union)h(nor)e(in)o(tersection)h(are)e(comm)o(utativ)o(e,)h (but)g(b)q(oth)g(are)g(asso)q(ciativ)o(e.)166 1397 y(The)g(new)h(group) f(can)g(b)q(e)h(empt)o(y)l(,)f(that)f(is,)h(equal)h(to)f Fi(MPI)p 1184 1397 13 2 v 14 w(GROUP)p 1340 1397 V 13 w(EMPTY)p Fx(.)75 1548 y Fv(MPI)p 160 1548 14 2 v 16 w(GROUP)p 328 1548 V 18 w(INCL\(group,)e(n,)j(ranks,)f(newgroup\))117 1625 y Fk(IN)155 b Fv(group)479 b Fk(group)14 b(\(handle\))117 1700 y(IN)155 b Fv(n)564 b Fk(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g (in)f(arra)o(y)h(ranks)g(\(and)g(size)h(of)e Fi(new-)905 1757 y(group)p Fk(\))f(\(in)o(teger\))117 1832 y(IN)155 b Fv(ranks)487 b Fk(ranks)13 b(of)f(pro)q(cesses)i(in)e Fi(group)h Fk(to)f(app)q(ear)h(in)f Fi(newgroup)i Fk(\(ar-)905 1888 y(ra)o(y)g(of)f(in)o(tegers\))117 1963 y(OUT)108 b Fv(newgroup)405 b Fk(new)12 b(group)f(deriv)o(ed)h(from)e(ab)q(o)o(v) o(e,)h(in)g(the)h(order)h(de\014ned)f(b)o(y)905 2020 y Fi(ranks)j Fk(\(handle\))75 2144 y Fu(int)23 b(MPI)p 245 2144 15 2 v 17 w(Group)p 382 2144 V 17 w(incl\(MPI)p 591 2144 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 2144 V 17 w(Group)h(*newgroup\))75 2231 y(MPI)p 150 2231 V 17 w(GROUP)p 287 2231 V 16 w(INCL\(GROUP,)g(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 2287 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)166 2374 y Fx(The)17 b(function)h Fv(MPI)p 526 2374 14 2 v 16 w(GROUP)p 694 2374 V 18 w(INCL)f Fx(creates)g(a)g(group)f Fv(newgroup)i Fx(that)f(consists)g(of)g(the)g Fv(n)h Fx(pro-)75 2430 y(cesses)e(in)h Fv(group)f Fx(with)h(ranks)e Fv(rank[0],)p Fp(:)8 b(:)g(:)n Fv(,)16 b(rank[n-1])p Fx(;)g(the)h(pro)q(cess)f(with)g(rank)g Fv(i)g Fx(in)h Fv(newgroup)f Fx(is)h(the)75 2487 y(pro)q(cess)h(with)f(rank)g Fv(ranks[i])h Fx(in)g Fv(group)p Fx(.)26 b(Eac)o(h)17 b(of)g(the)g Fv(n)h Fx(elemen)o(ts)g(of)f Fv(ranks)h Fx(m)o(ust)e(b)q(e)i(a)f(v)m(alid)i(rank)75 2543 y(in)h Fv(group)f Fx(and)g(all)h(elemen)o(ts)g(m)o(ust)f(b)q(e)g(distinct,)i (or)e(else)h(the)f(program)f(is)i(erroneous.)31 b(If)20 b Fv(n)f Fx(=)39 b(0,)75 2599 y(then)19 b Fv(newgroup)f Fx(is)h Fi(MPI)p 508 2599 13 2 v 14 w(GROUP)p 664 2599 V 13 w(EMPTY)p Fx(.)f(This)g(function)h(can,)g(for)f(instance,)h(b)q(e) g(used)g(to)e(reorder)75 2656 y(the)e(elemen)o(ts)h(of)f(a)g(group.)k (See)d(also)f Fv(MPI)p 832 2656 14 2 v 16 w(GROUP)p 1000 2656 V 18 w(COMP)l(ARE)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 139 147 139 146 bop 75 -100 a Fm(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b Fx(139)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(EX)o(CL\(group,)14 b(n,)h(ranks,)g(newgroup\))117 122 y Fk(IN)155 b Fv(group)479 b Fk(group)14 b(\(handle\))117 196 y(IN)155 b Fv(n)564 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f (in)h(arra)o(y)f(ranks)i(\(in)o(teger\))117 269 y(IN)155 b Fv(ranks)487 b Fk(arra)o(y)15 b(of)f(in)o(teger)i(ranks)f(in)g Fi(group)g Fk(not)g(to)g(app)q(ear)g(in)g Fi(new-)905 326 y(group)117 399 y Fk(OUT)108 b Fv(newgroup)405 b Fk(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i(preserving)h (the)f(order)905 455 y(de\014ned)d(b)o(y)28 b Fi(group)14 b Fk(\(handle\))75 580 y Fu(int)23 b(MPI)p 245 580 15 2 v 17 w(Group)p 382 580 V 17 w(excl\(MPI)p 591 580 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 580 V 17 w(Group)h(*newgroup\))75 666 y(MPI)p 150 666 V 17 w(GROUP)p 287 666 V 16 w(EXCL\(GROUP,)g(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 723 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)166 809 y Fx(The)12 b(function)h Fv(MPI)p 516 809 14 2 v 16 w(GROUP)p 684 809 V 17 w(EX)o(CL)f Fx(creates)g(a)f(group)h(of)g(pro)q(cesses)g Fv(newgroup)h Fx(that)e(is)h(obtained)75 866 y(b)o(y)k(deleting)i(from)e Fv(group)g Fx(those)g(pro)q(cesses)h(with)f(ranks)g Fv(ranks[0])h(,)p Fp(:)8 b(:)g(:)14 b Fv(ranks[n-1])p Fx(.)24 b(The)17 b(ordering)f(of)75 922 y(pro)q(cesses)f(in)h Fv(newgroup)g Fx(is)f(iden)o(tical)i(to)e(the)g(ordering)g(in)h Fv(group)p Fx(.)j(Eac)o(h)c(of)g(the)g Fv(n)g Fx(elemen)o(ts)h(of)e Fv(ranks)75 979 y Fx(m)o(ust)h(b)q(e)h(a)f(v)m(alid)i(rank)e(in)h Fv(group)f Fx(and)h(all)g(elemen)o(ts)g(m)o(ust)f(b)q(e)h(distinct;)g (otherwise,)f(the)g(program)f(is)75 1035 y(erroneous.)20 b(If)15 b Fv(n)h Fx(=)28 b(0,)14 b(then)i Fv(newgroup)g Fx(is)g(iden)o(tical)h(to)d Fv(group)p Fx(.)75 1186 y Fv(MPI)p 160 1186 V 16 w(GROUP)p 328 1186 V 18 w(RANGE)p 494 1186 V 17 w(INCL\(group,)g(n,)h(ranges,)g(newgroup\))117 1263 y Fk(IN)155 b Fv(group)479 b Fk(group)14 b(\(handle\))117 1336 y(IN)155 b Fv(n)564 b Fk(n)o(um)o(b)q(er)13 b(of)h(triplets)g(in)f (arra)o(y)28 b Fi(ranges)14 b Fk(\(in)o(teger\))117 1410 y(IN)155 b Fv(ranges)466 b Fk(an)11 b(arra)o(y)g(of)f(in)o(teger)i (triplets,)g(of)e(the)i(form)d(\(\014rst)j(rank,)f(last)905 1466 y(rank,)j(stride\))g(indicating)f(ranks)h(in)g Fi(group)g Fk(of)g(pro)q(cesses)i(to)905 1523 y(b)q(e)f(included)f(in)f Fi(newgroup)117 1596 y Fk(OUT)108 b Fv(newgroup)405 b Fk(new)12 b(group)f(deriv)o(ed)h(from)e(ab)q(o)o(v)o(e,)h(in)g(the)h (order)h(de\014ned)f(b)o(y)905 1653 y Fi(ranges)j Fk(\(handle\))75 1777 y Fu(int)23 b(MPI)p 245 1777 15 2 v 17 w(Group)p 382 1777 V 17 w(range)p 519 1777 V 16 w(incl\(MPI)p 727 1777 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(ranges[][3],)393 1834 y(MPI)p 468 1834 V 17 w(Group)g(*newgroup\))75 1920 y(MPI)p 150 1920 V 17 w(GROUP)p 287 1920 V 16 w(RANGE)p 423 1920 V 17 w(INCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1976 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(3,*\),)e(NEWGROUP,)h(IERROR)75 2063 y Fx(If)31 b Fv(ranges)15 b Fx(consist)g(of)g(the)h(triplets)189 2148 y(\()p Fp(f)5 b(ir)q(st)309 2155 y Fg(1)328 2148 y Fp(;)j(l)q(ast)425 2155 y Fg(1)444 2148 y Fp(;)g(str)q(ide)585 2155 y Fg(1)604 2148 y Fx(\))p Fp(;)g(:::;)g Fx(\()p Fp(f)d(irst)822 2155 y Ff(n)844 2148 y Fp(;)j(l)q(ast)941 2155 y Ff(n)964 2148 y Fp(;)g(str)q(ide)1105 2155 y Ff(n)1127 2148 y Fx(\))75 2233 y(then)16 b Fv(newgroup)f Fx(consists)h(of)f(the)g (sequence)h(of)f(pro)q(cesses)h(in)g Fv(group)f Fx(with)g(ranks)189 2337 y Fp(f)5 b(ir)q(st)291 2344 y Fg(1)311 2337 y Fp(;)j(f)d(ir)q(st) 434 2344 y Fg(1)463 2337 y Fx(+)10 b Fp(str)q(ide)628 2344 y Fg(1)648 2337 y Fp(;)e(:::;)g(f)d(ir)q(st)831 2344 y Fg(1)859 2337 y Fx(+)904 2277 y Fe(\026)933 2306 y Fp(l)q(ast)1009 2313 y Fg(1)1039 2306 y Ft(\000)11 b Fp(f)5 b(ir)q(st)1187 2313 y Fg(1)p 933 2326 274 2 v 1000 2368 a Fp(str)q(ide)1120 2375 y Fg(1)1212 2277 y Fe(\027)1243 2337 y Fp(str)q(ide)1363 2344 y Fg(1)1383 2337 y Fp(;)j(:::)189 2479 y(f)d(ir)q(st)291 2486 y Ff(n)314 2479 y Fp(;)j(f)d(ir)q(st)437 2486 y Ff(n)471 2479 y Fx(+)10 b Fp(str)q(ide)636 2486 y Ff(n)659 2479 y Fp(;)e(:::;)g(f)d(ir) q(st)842 2486 y Ff(n)874 2479 y Fx(+)919 2420 y Fe(\026)948 2448 y Fp(l)q(ast)1024 2455 y Ff(n)1058 2448 y Ft(\000)10 b Fp(f)5 b(ir)q(st)1205 2455 y Ff(n)p 948 2469 282 2 v 1017 2510 a Fp(str)q(ide)1137 2517 y Ff(n)1234 2420 y Fe(\027)1266 2479 y Fp(str)q(ide)1386 2486 y Ff(n)1409 2479 y Fp(:)166 2591 y Fx(Eac)o(h)17 b(computed)h(rank)f(m)o(ust)g(b)q (e)i(a)e(v)m(alid)i(rank)e(in)i Fv(group)e Fx(and)h(all)g(computed)g (ranks)f(m)o(ust)g(b)q(e)75 2647 y(distinct,)c(or)f(else)g(the)g (program)f(is)h(erroneous.)19 b(Note)11 b(that)g(w)o(e)h(ma)o(y)f(ha)o (v)o(e)h Fp(f)5 b(ir)q(st)1466 2654 y Ff(i)1493 2647 y Fp(>)13 b(l)q(ast)1617 2654 y Ff(i)1631 2647 y Fx(,)f(and)g Fp(str)q(ide)1861 2654 y Ff(i)75 2704 y Fx(ma)o(y)i(b)q(e)i(negativ)o (e,)f(but)h(cannot)f(b)q(e)g(zero.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 140 148 140 147 bop 75 -100 a Fx(140)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)166 45 y Fx(The)j(functionalit)o(y)h(of)e(this)i(routine)f(is)g(sp)q (eci\014ed)i(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)e(expanding)i(the)f (arra)o(y)75 102 y(of)g(ranges)g(to)g(an)h(arra)o(y)e(of)h(the)h (included)i(ranks)d(and)h(passing)g(the)g(resulting)g(arra)o(y)e(of)i (ranks)f(and)75 158 y(other)e(argumen)o(ts)g(to)g Fv(MPI)p 560 158 14 2 v 16 w(GROUP)p 728 158 V 18 w(INCL)p Fx(.)g(A)g(call)i(to) e Fv(MPI)p 1148 158 V 16 w(GROUP)p 1316 158 V 18 w(INCL)g Fx(is)h(equiv)m(alen)o(t)h(to)e(a)h(call)75 214 y(to)e Fv(MPI)p 216 214 V 16 w(GROUP)p 384 214 V 18 w(RANGE)p 550 214 V 18 w(INCL)g Fx(with)i(eac)o(h)f(rank)g Fv(i)g Fx(in)h Fv(ranks)f Fx(replaced)i(b)o(y)e(the)g(triplet)h Fu(\(i,i,1\))e Fx(in)75 271 y(the)g(argumen)o(t)g Fv(ranges)p Fx(.)75 422 y Fv(MPI)p 160 422 V 16 w(GROUP)p 328 422 V 18 w(RANGE)p 494 422 V 17 w(EX)o(CL\(group,)g(n,)g(ranges,)g (newgroup\))117 499 y Fk(IN)155 b Fv(group)479 b Fk(group)14 b(\(handle\))117 574 y(IN)155 b Fv(n)564 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(arra)o(y)f(ranks)i(\(in)o(teger\))117 649 y(IN)155 b Fv(ranges)466 b Fk(a)13 b(one-dimensional)d(arra)o(y)j (of)f(in)o(teger)h(triplets)g(of)f(the)i(form)905 706 y(\(\014rst)j(rank,)f(last)g(rank,)g(stride\),)h(indicating)e(the)i (ranks)f(in)905 762 y Fi(group)22 b Fk(of)f(pro)q(cesses)j(to)d(b)q(e)h (excluded)g(from)e(the)i(output)905 819 y(group)14 b Fi(newgroup)p Fk(.)117 894 y(OUT)108 b Fv(newgroup)405 b Fk(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i(preserving) h(the)f(order)905 950 y(in)c Fi(group)g Fk(\(handle\))75 1075 y Fu(int)23 b(MPI)p 245 1075 15 2 v 17 w(Group)p 382 1075 V 17 w(range)p 519 1075 V 16 w(excl\(MPI)p 727 1075 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(ranges[][3],)393 1131 y(MPI)p 468 1131 V 17 w(Group)g(*newgroup\))75 1218 y(MPI)p 150 1218 V 17 w(GROUP)p 287 1218 V 16 w(RANGE)p 423 1218 V 17 w(EXCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1274 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(3,*\),)e(NEWGROUP,)h(IERROR)75 1361 y Fx(Eac)o(h)12 b(computed)g(rank)g(m)o(ust)f(b)q(e)i(a)f(v)m (alid)h(rank)f(in)h Fv(group)f Fx(and)g(all)h(computed)f(ranks)g(m)o (ust)f(b)q(e)i(distinct,)75 1417 y(or)i(else)h(the)f(program)f(is)i (erroneous.)166 1473 y(The)i(functionalit)o(y)h(of)e(this)i(routine)f (is)g(sp)q(eci\014ed)i(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)e(expanding)i (the)f(arra)o(y)75 1530 y(of)g(ranges)f(to)h(an)g(arra)o(y)f(of)h(the)g (excluded)i(ranks)e(and)g(passing)h(the)f(resulting)h(arra)o(y)e(of)h (ranks)g(and)75 1586 y(other)c(argumen)o(ts)g(to)g Fv(MPI)p 554 1586 14 2 v 16 w(GROUP)p 722 1586 V 18 w(EX)o(CL)p Fx(.)g(A)h(call)h(to)e Fv(MPI)p 1146 1586 V 16 w(GROUP)p 1314 1586 V 18 w(EX)o(CL)g Fx(is)i(equiv)m(alen)o(t)g(to)e(a)h(call)75 1643 y(to)g Fv(MPI)p 216 1643 V 15 w(GROUP)p 383 1643 V 18 w(RANGE)p 549 1643 V 18 w(EX)o(CL)g Fx(with)h(eac)o(h)f(rank)g Fv(i)g Fx(in)h Fv(ranks)g Fx(replaced)g(b)o(y)f(the)h(triplet)g Fu(\(i,i,1\))e Fx(in)75 1699 y(the)h(argumen)o(t)g Fv(ranges)p Fx(.)189 1806 y Fl(A)n(dvic)n(e)22 b(to)h(users.)86 b Fx(The)23 b(range)f(op)q(erations)h(do)f(not)h(explicitly)i(en)o (umerate)e(ranks,)h(and)189 1862 y(therefore)c(are)h(more)f(scalable)i (if)f(implemen)o(ted)i(e\016cien)o(tly)l(.)38 b(Hence,)23 b(w)o(e)d(recommend)i Fv(MPI)189 1918 y Fx(programmers)14 b(to)i(use)g(them)h(whenenev)o(er)g(p)q(ossible,)g(as)f(high-qualit)o (y)i(implemen)o(tations)f(will)189 1975 y(tak)o(e)d(adv)m(an)o(tage)h (of)g(this)g(fact.)k(\()p Fl(End)d(of)g(advic)n(e)h(to)f(users.)p Fx(\))189 2081 y Fl(A)n(dvic)n(e)h(to)h(implementors.)52 b Fx(The)17 b(range)g(op)q(erations)g(should)h(b)q(e)g(implemen)o(ted,) h(if)f(p)q(ossible,)189 2138 y(without)e(en)o(umerating)i(the)f(group)f (mem)o(b)q(ers,)h(in)h(order)e(to)g(obtain)i(b)q(etter)e(scalabilit)o (y)j(\(time)189 2194 y(and)c(space\).)20 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 2316 y Fj(5.3.3)49 b(Group)16 b(Destructo)o(rs)75 2496 y Fv(MPI)p 160 2496 V 16 w(GROUP)p 328 2496 V 18 w(FREE\(group\))117 2573 y Fk(INOUT)62 b Fv(group)479 b Fk(group)14 b(\(handle\))75 2698 y Fu(int)23 b(MPI)p 245 2698 15 2 v 17 w(Group)p 382 2698 V 17 w(free\(MPI)p 591 2698 V 16 w(Group)g(*group\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 141 149 141 148 bop 75 -100 a Fm(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b Fx(141)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(GROUP)p 287 45 V 16 w(FREE\(GROUP,)23 b(IERROR\))170 102 y(INTEGER)g(GROUP,)g(IERROR)166 188 y Fx(This)f(op)q(eration)h (marks)e(a)g(group)h(ob)s(ject)f(for)h(deallo)q(cation.)41 b(The)22 b(handle)i Fv(group)d Fx(is)i(set)f(to)75 244 y Fi(MPI)p 152 244 13 2 v 14 w(GROUP)p 308 244 V 13 w(NULL)17 b Fx(b)o(y)g(the)f(call.)24 b(An)o(y)17 b(on-going)f(op)q(eration)g (using)h(this)g(group)f(will)i(complete)f(nor-)75 301 y(mally)l(.)189 395 y Fl(A)n(dvic)n(e)c(to)h(implementors.)39 b Fx(One)13 b(can)h(k)o(eep)f(a)f(reference)i(coun)o(t)f(that)f(is)h (incremen)o(ted)i(for)d(eac)o(h)189 452 y(call)j(to)f Fv(MPI)p 411 452 14 2 v 16 w(COMM)p 569 452 V 16 w(CREA)l(TE)i Fx(and)e Fv(MPI)p 939 452 V 16 w(COMM)p 1097 452 V 17 w(DUP)p Fx(,)g(and)g(decremen)o(ted)i(for)d(eac)o(h)i(call)h(to)189 508 y Fv(MPI)p 274 508 V 15 w(GROUP)p 441 508 V 18 w(FREE)h Fx(or)g Fv(MPI)p 725 508 V 15 w(COMM)p 882 508 V 17 w(FREE)p Fx(;)g(the)g(group)f(ob)s(ject)g(is)i(ultimately)g(deallo)q(cated)189 565 y(when)d(the)h(reference)g(coun)o(t)f(drops)g(to)f(zero.)20 b(\()p Fl(End)c(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))75 706 y Fn(5.4)59 b(Communicato)n(r)19 b(Management)75 808 y Fx(This)e(section)g(describ)q(es)h(the)f(manipulation)h(of)e (comm)o(unicators)g(in)h Fv(MPI)p Fx(.)f(Op)q(erations)h(that)f(access) 75 864 y(comm)o(unicators)f(are)g(lo)q(cal)h(and)f(their)h(execution)g (do)q(es)g(not)f(require)h(in)o(terpro)q(cess)g(comm)o(unication.)75 921 y(Op)q(erations)g(that)e(create)h(comm)o(unicators)f(are)h (collectiv)o(e)i(and)e(ma)o(y)f(require)i(in)o(terpro)q(cess)f(comm)o (u-)75 977 y(nication.)189 1072 y Fl(A)n(dvic)n(e)k(to)h(implementors.) 65 b Fx(High-qualit)o(y)21 b(implemen)o(tations)g(should)f(amortize)f (the)h(o)o(v)o(er-)189 1128 y(heads)15 b(asso)q(ciated)h(with)g(the)f (creation)h(of)e(comm)o(unicators)h(\(for)g(the)g(same)g(group,)g(or)g (subsets)189 1185 y(thereof)t(\))d(o)o(v)o(er)h(sev)o(eral)g(calls,)i (b)o(y)e(allo)q(cating)i(m)o(ultiple)g(con)o(texts)e(with)g(one)h (collectiv)o(e)h(comm)o(u-)189 1241 y(nication.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 1361 y Fj(5.4.1)49 b(Comm)o(unicato)o(r)13 b(Accesso)o(rs)75 1447 y Fx(The)i(follo)o(wing)h(are)f(all)h(lo)q(cal)h(op)q(erations.)75 1598 y Fv(MPI)p 160 1598 V 16 w(COMM)p 318 1598 V 16 w(SIZE\(comm)m(,)12 b(size\))117 1675 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 1749 y(OUT)108 b Fv(size)519 b Fk(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g (group)g(of)27 b Fi(comm)8 b Fk(\(in)o(teger\))75 1873 y Fu(int)23 b(MPI)p 245 1873 15 2 v 17 w(Comm)p 358 1873 V 17 w(size\(MPI)p 567 1873 V 16 w(Comm)g(comm,)g(int)h(*size\))75 1960 y(MPI)p 150 1960 V 17 w(COMM)p 263 1960 V 16 w(SIZE\(COMM,)f (SIZE,)g(IERROR\))170 2016 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)189 2140 y Fl(R)n(ationale.)51 b Fx(This)17 b(function)h(is)g(equiv)m(alen) o(t)g(to)f(accessing)h(the)f(comm)o(unicator's)f(group)h(with)189 2197 y Fv(MPI)p 274 2197 14 2 v 15 w(COMM)p 431 2197 V 17 w(GROUP)k Fx(\(see)g(b)q(elo)o(w\),)g(computing)g(the)f(size)i (using)f Fv(MPI)p 1494 2197 V 15 w(GROUP)p 1661 2197 V 18 w(SIZE)p Fx(,)f(and)189 2253 y(then)13 b(freeing)h(the)f(group)g (temp)q(orary)f(via)h Fv(MPI)p 1016 2253 V 16 w(GROUP)p 1184 2253 V 18 w(FREE)p Fx(.)g(Ho)o(w)o(ev)o(er,)f(this)i(function)g (is)f(so)189 2310 y(commonly)i(used,)g(that)g(this)h(shortcut)e(w)o(as) h(in)o(tro)q(duced.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 2404 y Fl(A)n(dvic)n(e)k(to)i(users.)80 b Fx(This)23 b(function)f(indicates)i(the)e(n)o(um)o(b)q(er)g(of)f(pro)q(cesses)i (in)o(v)o(olv)o(ed)g(in)f(a)189 2461 y(comm)o(unicator.)35 b(F)l(or)21 b Fi(MPI)p 684 2461 13 2 v 14 w(COMM)p 828 2461 V 14 w(W)o(ORLD)p Fx(,)g(it)g(indicates)h(the)e(total)g(n)o(um)o (b)q(er)h(of)g(pro)q(cesses)189 2517 y(a)o(v)m(ailable)c(\(for)f(this)g (v)o(ersion)h(of)f Fv(MPI)p Fx(,)f(there)h(is)h(no)f(standard)g(w)o(a)o (y)f(to)g(c)o(hange)i(the)f(n)o(um)o(b)q(er)g(of)189 2574 y(pro)q(cesses)f(once)h(initialization)i(has)d(tak)o(en)g (place\).)189 2647 y(This)20 b(call)h(is)f(often)g(used)g(with)h(the)f (next)g(call)h(to)e(determine)i(the)f(amoun)o(t)f(of)g(concurrency)189 2704 y(a)o(v)m(ailable)j(for)e(a)h(sp)q(eci\014c)i(library)e(or)g (program.)36 b(The)21 b(follo)o(wing)g(call,)i Fv(MPI)p 1580 2704 14 2 v 16 w(COMM)p 1738 2704 V 17 w(RANK)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 142 150 142 149 bop 75 -100 a Fx(142)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)189 45 y Fx(indicates)g(the)f(rank)f(of)h(the)g(pro)q(cess)g(that)f(calls)i (it)f(in)h(the)f(range)f(from)g(0)8 b Fp(:)g(:)g(:)n Fv(size)p Ft(\000)p Fx(1,)14 b(where)g Fv(size)189 102 y Fx(is)h(the)h(return)f(v)m(alue)h(of)f Fv(MPI)p 706 102 14 2 v 16 w(COMM)p 864 102 V 16 w(SIZE)p Fx(.\()p Fl(End)g(of)i(advic)n(e)f(to)g(users.)p Fx(\))75 290 y Fv(MPI)p 160 290 V 16 w(COMM)p 318 290 V 16 w(RANK\(comm)m(,)c (rank\))117 367 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 440 y(OUT)108 b Fv(rank)504 b Fk(rank)12 b(of)f(the)h(calling)e(pro)q(cess)j(in)f(group)f(of)23 b Fi(comm)5 b Fk(\(in)o(teger\))75 565 y Fu(int)23 b(MPI)p 245 565 15 2 v 17 w(Comm)p 358 565 V 17 w(rank\(MPI)p 567 565 V 16 w(Comm)g(comm,)g(int)h(*rank\))75 651 y(MPI)p 150 651 V 17 w(COMM)p 263 651 V 16 w(RANK\(COMM,)f(RANK,)g(IERROR\))170 708 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)189 831 y Fl(R)n(ationale.)51 b Fx(This)17 b(function)h(is)g(equiv)m(alen)o(t)g(to)f(accessing)h(the) f(comm)o(unicator's)f(group)h(with)189 888 y Fv(MPI)p 274 888 14 2 v 15 w(COMM)p 431 888 V 17 w(GROUP)h Fx(\(see)e(b)q(elo)o (w\),)h(computing)g(the)g(size)g(using)h Fv(MPI)p 1467 888 V 15 w(GROUP)p 1634 888 V 18 w(RANK)p Fx(,)f(and)189 944 y(then)c(freeing)h(the)f(group)g(temp)q(orary)f(via)h Fv(MPI)p 1016 944 V 16 w(GROUP)p 1184 944 V 18 w(FREE)p Fx(.)g(Ho)o(w)o(ev)o(er,)f(this)i(function)g(is)f(so)189 1001 y(commonly)i(used,)g(that)g(this)h(shortcut)e(w)o(as)h(in)o(tro)q (duced.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 1094 y Fl(A)n(dvic)n(e)c(to)h(users.)38 b Fx(This)14 b(function)f(giv)o(es)g(the)g(rank)f(of)h(the)f(pro)q(cess)h(in)h(the)f (particular)g(comm)o(u-)189 1151 y(nicator's)h(group.)20 b(It)15 b(is)h(useful,)g(as)f(noted)g(ab)q(o)o(v)o(e,)f(in)i (conjunction)h(with)e Fv(MPI)p 1569 1151 V 16 w(COMM)p 1727 1151 V 16 w(SIZE)p Fx(.)189 1224 y(Man)o(y)e(programs)g(will)i(b)q (e)g(written)f(with)g(the)g(master-sla)o(v)o(e)f(mo)q(del,)i(where)f (one)g(pro)q(cess)g(\(suc)o(h)189 1280 y(as)19 b(the)g(rank-zero)h(pro) q(cess\))f(will)i(pla)o(y)f(a)f(sup)q(ervisory)h(role,)h(and)f(the)f (other)g(pro)q(cesses)h(will)189 1337 y(serv)o(e)g(as)f(compute)h(no)q (des.)35 b(In)21 b(this)g(framew)o(ork,)e(the)h(t)o(w)o(o)f(preceding)j (calls)f(are)e(useful)j(for)189 1393 y(determining)c(the)e(roles)h(of)f (the)h(v)m(arious)g(pro)q(cesses)g(of)f(a)g(comm)o(unicator.)24 b(\()p Fl(End)17 b(of)g(advic)n(e)h(to)189 1450 y(users.)p Fx(\))75 1638 y Fv(MPI)p 160 1638 V 16 w(COMM)p 318 1638 V 16 w(COMP)l(ARE\(comm)n(1,)12 b(comm)m(2,)g(result\))117 1715 y Fk(IN)155 b Fv(comm)n(1)445 b Fk(\014rst)15 b(comm)o(unicator)c (\(handle\))117 1788 y(IN)155 b Fv(comm)n(2)445 b Fk(second)15 b(comm)o(unicator)c(\(handle\))117 1862 y(OUT)108 b Fv(result)484 b Fk(result)15 b(\(in)o(teger\))75 1986 y Fu(int)23 b(MPI)p 245 1986 15 2 v 17 w(Comm)p 358 1986 V 17 w(compare\(MPI)p 639 1986 V 16 w(Comm)g(comm1,MPI)p 990 1986 V 16 w(Comm)g(comm2,)g(int) h(*result\))75 2073 y(MPI)p 150 2073 V 17 w(COMM)p 263 2073 V 16 w(COMPARE\(COMM1,)e(COMM2,)h(RESULT,)g(IERROR\))170 2129 y(INTEGER)g(COMM1,)g(COMM2,)g(RESULT,)g(IERROR)75 2215 y Fi(MPI)p 152 2215 13 2 v 14 w(IDENT)11 b Fx(results)g(if)h(and)f (only)g(if)h Fv(comm)m(1)c Fx(and)j Fv(comm)m(2)c Fx(are)k(handles)h (for)e(the)h(same)g(ob)s(ject)f(\(iden)o(tical)75 2272 y(groups)15 b(and)g(same)g(con)o(texts\).)j Fi(MPI)p 715 2272 V 15 w(CONGRUENT)c Fx(results)h(if)h(the)f(underlying)i (groups)d(are)h(iden)o(tical)75 2328 y(in)j(constituen)o(ts)f(and)h (rank)f(order;)h(these)f(comm)o(unicators)g(di\013er)h(only)f(b)o(y)h (con)o(text.)25 b Fi(MPI)p 1703 2328 V 15 w(SIMILAR)75 2385 y Fx(results)12 b(if)h(the)f(group)g(mem)o(b)q(ers)g(of)g(b)q(oth) g(comm)o(unicators)g(are)g(the)g(same)g(but)g(the)g(rank)g(order)g (di\013ers.)75 2441 y Fi(MPI)p 152 2441 V 14 w(UNEQUAL)i Fx(results)i(otherwise.)75 2561 y Fj(5.4.2)49 b(Comm)o(unicato)o(r)13 b(Constructo)o(rs)75 2647 y Fx(The)i(follo)o(wing)g(are)g(collectiv)o (e)h(functions)f(that)f(are)h(in)o(v)o(ok)o(ed)g(b)o(y)f(all)i(pro)q (cesses)f(in)g(the)g(group)f(asso)q(ci-)75 2704 y(ated)h(with)h Fv(comm)m Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 143 151 143 150 bop 75 -100 a Fm(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b Fx(143)189 45 y Fl(R)n(ationale.)37 b Fx(Note)10 b(that)g(there)h(is)g(a)g(c)o(hic)o(k)o(en-and-egg)h(asp)q (ect)f(to)f Fv(MPI)g Fx(in)i(that)e(a)g(comm)o(unicator)189 102 y(is)18 b(needed)h(to)e(create)g(a)h(new)f(comm)o(unicator.)27 b(The)18 b(base)g(comm)o(unicator)f(for)g(all)i Fv(MPI)e Fx(com-)189 158 y(m)o(unicators)f(is)h(prede\014ned)h(outside)f(of)f Fv(MPI)p Fx(,)g(and)g(is)h Fi(MPI)p 1227 158 13 2 v 15 w(COMM)p 1372 158 V 14 w(W)o(ORLD)p Fx(.)f(This)h(mo)q(del)h(w)o(as)189 214 y(arriv)o(ed)c(at)g(after)g(considerable)i(debate,)e(and)h(w)o(as)e (c)o(hosen)i(to)f(increase)h(\\safet)o(y")e(of)h(programs)189 271 y(written)h(in)h Fv(MPI)p Fx(.)e(\()p Fl(End)i(of)g(r)n(ationale.)p Fx(\))75 472 y Fv(MPI)p 160 472 14 2 v 16 w(COMM)p 318 472 V 16 w(DUP\(comm)m(,)c(new)o(comm)n(\))117 549 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 624 y(OUT)108 b Fv(new)o(comm)393 b Fk(cop)o(y)14 b(of)f Fi(comm)8 b Fk(\(handle\))75 748 y Fu(int)23 b(MPI)p 245 748 15 2 v 17 w(Comm)p 358 748 V 17 w(dup\(MPI)p 543 748 V 16 w(Comm)g(comm,)h(MPI)p 894 748 V 16 w(Comm)g(*newcomm\))75 835 y(MPI)p 150 835 V 17 w(COMM)p 263 835 V 16 w(DUP\(COMM,)f(NEWCOMM,) g(IERROR\))170 891 y(INTEGER)g(COMM,)h(NEWCOMM,)e(IERROR)166 978 y Fv(MPI)p 251 978 14 2 v 16 w(COMM)p 409 978 V 16 w(DUP)12 b Fx(Duplicates)i(the)e(existing)h(comm)o(unicator)f Fv(comm)6 b Fx(with)13 b(asso)q(ciated)f(k)o(ey)g(v)m(al-)75 1034 y(ues.)19 b(F)l(or)11 b(eac)o(h)h(k)o(ey)f(v)m(alue,)j(the)d(resp) q(ectiv)o(e)i(cop)o(y)f(callbac)o(k)g(function)h(determines)f(the)g (attribute)g(v)m(alue)75 1091 y(asso)q(ciated)f(with)h(this)f(k)o(ey)g (in)h(the)f(new)h(comm)o(unicator;)f(one)h(particular)f(action)h(that)e (a)h(cop)o(y)g(callbac)o(k)75 1147 y(ma)o(y)18 b(tak)o(e)g(is)h(to)f (delete)h(the)g(attribute)g(from)e(the)i(new)g(comm)o(unicator.)30 b(Returns)19 b(in)g Fv(new)o(comm)13 b Fx(a)75 1204 y(new)h(comm)o (unicator)f(with)g(the)h(same)f(group,)g(an)o(y)g(copied)i(cac)o(hed)f (information,)f(but)h(a)f(new)h(con)o(text)75 1260 y(\(see)h(section)h (5.7.1\).)189 1366 y Fl(A)n(dvic)n(e)d(to)h(users.)39 b Fx(This)14 b(op)q(eration)f(is)g(used)h(to)e(pro)o(vide)i(a)e (parallel)j(library)f(call)g(with)f(a)g(dupli-)189 1423 y(cate)f(comm)o(unication)i(space)f(that)f(has)h(the)f(same)h(prop)q (erties)g(as)g(the)g(original)h(comm)o(unicator.)189 1479 y(This)i(includes)j(an)o(y)d(attributes)g(\(see)g(b)q(elo)o(w\),)g (and)g(top)q(ologies)h(\(see)f(c)o(hapter)g(6\).)22 b(This)17 b(call)g(is)189 1536 y(v)m(alid)g(ev)o(en)g(if)f(there)g(are)g(p)q (ending)i(p)q(oin)o(t-to-p)q(oin)o(t)e(comm)o(unications)h(in)o(v)o (olving)g(the)f(comm)o(u-)189 1592 y(nicator)i Fv(comm)m Fx(.)25 b(A)18 b(t)o(ypical)h(call)g(migh)o(t)e(in)o(v)o(olv)o(e)i(a)f Fv(MPI)p 1206 1592 V 15 w(COMM)p 1363 1592 V 17 w(DUP)g Fx(at)f(the)h(b)q(eginning)i(of)189 1649 y(the)e(parallel)h(call,)g (and)f(an)g Fv(MPI)p 775 1649 V 16 w(COMM)p 933 1649 V 16 w(FREE)h Fx(of)e(that)g(duplicated)j(comm)o(unicator)e(at)f(the) 189 1705 y(end)e(of)g(the)h(call.)21 b(Other)15 b(mo)q(dels)h(of)f (comm)o(unicator)g(managemen)o(t)f(are)h(also)g(p)q(ossible.)189 1780 y(This)g(call)i(applies)g(to)d(b)q(oth)h(in)o(tra-)h(and)f(in)o (ter-comm)o(unicators.)20 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g (users.)p Fx(\))189 1887 y Fl(A)n(dvic)n(e)c(to)j(implementors.)38 b Fx(One)13 b(need)g(not)f(actually)i(cop)o(y)e(the)h(group)f (information,)h(but)f(only)189 1943 y(add)17 b(a)g(new)g(reference)h (and)g(incremen)o(t)g(the)f(reference)h(coun)o(t.)26 b(Cop)o(y)17 b(on)g(write)g(can)g(b)q(e)h(used)189 1999 y(for)c(the)h(cac)o(hed)h(information.\()p Fl(End)g(of)g(advic)n(e)g (to)h(implementors.)p Fx(\))75 2200 y Fv(MPI)p 160 2200 V 16 w(COMM)p 318 2200 V 16 w(CREA)l(TE\(comm)n(,)12 b(group,)j(new)o(comm)m(\))117 2277 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 2353 y(IN)155 b Fv(group)479 b Fk(Group,)14 b(whic)o(h)h(is)f(a)g(subset)i(of)e(the)h (group)g(of)e Fi(comm)c Fk(\(han-)905 2409 y(dle\))117 2484 y(OUT)108 b Fv(new)o(comm)393 b Fk(new)15 b(comm)o(uni)o(cator)d (\(handle\))75 2609 y Fu(int)23 b(MPI)p 245 2609 15 2 v 17 w(Comm)p 358 2609 V 17 w(create\(MPI)p 615 2609 V 16 w(Comm)g(comm,)g(MPI)p 965 2609 V 17 w(Group)g(group,)g(MPI)p 1364 2609 V 17 w(Comm)g(*newcomm\))75 2695 y(MPI)p 150 2695 V 17 w(COMM)p 263 2695 V 16 w(CREATE\(COMM,)g(GROUP,)g(NEWCOMM,)f (IERROR\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 144 152 144 151 bop 75 -100 a Fx(144)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)170 45 y Fu(INTEGER)23 b(COMM,)h(GROUP,)f(NEWCOMM,)f(IERROR)75 132 y Fx(This)14 b(function)g(creates)f(a)g(new)h(comm)o(unicator)f Fv(new)o(comm)8 b Fx(with)14 b(comm)o(unication)g(group)f(de\014ned)i (b)o(y)75 188 y Fv(group)f Fx(and)h(a)e(new)i(con)o(text.)k(No)14 b(cac)o(hed)h(information)f(propagates)f(from)h Fv(comm)8 b Fx(to)14 b Fv(new)o(comm)m Fx(.)j(The)75 244 y(function)f(returns)f Fi(MPI)p 487 244 13 2 v 14 w(COMM)p 631 244 V 15 w(NULL)h Fx(to)e(pro)q(cesses)i(that)e(are)h(not)g(in)h Fv(group)p Fx(.)k(The)15 b(call)h(is)g(erroneous)75 301 y(if)k(not)f(all)h Fv(group)f Fx(argumen)o(ts)g(ha)o(v)o(e)f(the)i(same)f(v)m(alue,)i(or)e (if)h Fv(group)f Fx(is)h(not)f(a)f(subset)i(of)f(the)g(group)75 357 y(asso)q(ciated)d(with)f Fv(comm)n Fx(.)i(Note)e(that)g(the)g(call) i(is)f(to)e(b)q(e)i(executed)h(b)o(y)e(all)h(pro)q(cesses)g(in)g Fv(comm)n Fx(,)c(ev)o(en)75 414 y(if)k(they)f(do)g(not)g(b)q(elong)h (to)f(the)g(new)g(group.)20 b(This)c(call)g(applies)h(only)f(to)e(in)o (tra-comm)o(unicators.)189 516 y Fl(R)n(ationale.)58 b Fx(The)19 b(requiremen)o(t)g(that)f(the)g(en)o(tire)h(group)f(of)g Fv(comm)12 b Fx(participate)19 b(in)g(the)g(call)189 572 y(stems)14 b(from)h(the)g(follo)o(wing)h(considerations:)243 665 y Ft(\017)23 b Fx(It)e(allo)o(ws)g(the)g(implemen)o(tation)h(to)e (la)o(y)o(er)h Fv(MPI)p 1161 665 14 2 v 16 w(COMM)p 1319 665 V 16 w(CREA)l(TE)h Fx(on)f(top)f(of)h(regular)289 721 y(collectiv)o(e)c(comm)o(unications.)243 793 y Ft(\017)23 b Fx(It)12 b(pro)o(vides)h(additional)h(safet)o(y)l(,)e(in)h (particular)g(in)g(the)g(case)f(where)h(partially)g(o)o(v)o(erlapping) 289 849 y(groups)h(are)h(used)h(to)f(create)g(new)g(comm)o(unicators.) 243 921 y Ft(\017)23 b Fx(It)11 b(p)q(ermits)g(implemen)o(tations)i (sometimes)e(to)f(a)o(v)o(oid)h(comm)o(unication)h(related)f(to)g(con)o (text)289 978 y(creation.)189 1070 y(\()p Fl(End)k(of)i(r)n(ationale.)p Fx(\))189 1172 y Fl(A)n(dvic)n(e)g(to)i(users.)53 b Fv(MPI)p 647 1172 V 16 w(COMM)p 805 1172 V 16 w(CREA)l(TE)19 b Fx(pro)o(vides)f(a)f(means)g(to)g(subset)h(a)f(group)g(of)g(pro-)189 1228 y(cesses)e(for)f(the)g(purp)q(ose)i(of)e(separate)g(MIMD)g (computation,)g(with)h(separate)f(comm)o(unication)189 1285 y(space.)33 b Fv(new)o(comm)n Fx(,)18 b(whic)o(h)i(emerges)g(from) f Fv(MPI)p 1072 1285 V 16 w(COMM)p 1230 1285 V 16 w(CREA)l(TE)i Fx(can)f(b)q(e)g(used)h(in)f(subse-)189 1341 y(quen)o(t)15 b(calls)i(to)e Fv(MPI)p 558 1341 V 16 w(COMM)p 716 1341 V 16 w(CREA)l(TE)i Fx(\(or)d(other)i(comm)o(unicator)f(constructors\))f (further)i(to)189 1398 y(sub)q(divide)21 b(a)d(computation)h(in)o(to)g (parallel)h(sub-computations.)31 b(A)18 b(more)h(general)g(service)g (is)189 1454 y(pro)o(vided)d(b)o(y)f Fv(MPI)p 524 1454 V 16 w(COMM)p 682 1454 V 16 w(SPLIT)p Fx(,)f(b)q(elo)o(w.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fx(\))189 1556 y Fl(A)n(dvic)n(e)32 b(to)i(implementors.)151 b Fx(Since)35 b(all)g(pro)q(cesses)f(calling)h Fv(MPI)p 1536 1556 V 16 w(COMM)p 1694 1556 V 17 w(DUP)e Fx(or)189 1612 y Fv(MPI)p 274 1612 V 15 w(COMM)p 431 1612 V 17 w(CREA)l(TE)21 b Fx(pro)o(vide)f(the)g(same)g Fv(group)g Fx(argumen)o(t,)f(it)i(is)f(theoretically)h(p)q(ossible)189 1669 y(to)13 b(agree)g(on)g(a)h(group-wide)g(unique)h(con)o(text)e (with)h(no)g(comm)o(unication.)20 b(Ho)o(w)o(ev)o(er,)13 b(lo)q(cal)h(exe-)189 1725 y(cution)f(of)f(these)h(functions)g (requires)h(use)f(of)f(a)g(larger)h(con)o(text)f(name)h(space)f(and)h (reduces)h(error)189 1782 y(c)o(hec)o(king.)20 b(Implemen)o(tations)c (ma)o(y)e(strik)o(e)h(v)m(arious)g(compromises)g(b)q(et)o(w)o(een)g (these)f(con\015icting)189 1838 y(goals,)g(suc)o(h)i(as)e(bulk)j(allo)q (cation)f(of)f(m)o(ultiple)i(con)o(texts)d(in)i(one)g(collectiv)o(e)h (op)q(eration.)189 1913 y(Imp)q(ortan)o(t:)28 b(If)20 b(new)g(comm)o(unicators)g(are)f(created)h(without)g(sync)o(hronizing)i (the)e(pro)q(cesses)189 1969 y(in)o(v)o(olv)o(ed)11 b(then)g(the)g (comm)o(unication)g(system)f(should)i(b)q(e)f(able)g(to)f(cop)q(e)h (with)g(messages)f(arriving)189 2026 y(in)16 b(a)g(con)o(text)f(that)g (has)g(not)h(y)o(et)f(b)q(een)i(allo)q(cated)f(at)f(the)h(receiving)i (pro)q(cess.)j(\()p Fl(End)16 b(of)h(advic)n(e)189 2082 y(to)f(implementors.)p Fx(\))75 2278 y Fv(MPI)p 160 2278 V 16 w(COMM)p 318 2278 V 16 w(SPLIT\(com)o(m)n(,)11 b(colo)o(r,)j(k)o (ey)l(,)h(new)o(comm)n(\))117 2356 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(\(handle\))117 2430 y(IN)155 b Fv(colo)o(r)495 b Fk(con)o(trol)14 b(of)f(subset)i(assignmen)o(t)e (\(in)o(teger\))117 2505 y(IN)155 b Fv(k)o(ey)525 b Fk(con)o(trol)14 b(of)f(rank)h(assigmen)o(t)f(\(in)o(teger\))117 2579 y(OUT)108 b Fv(new)o(comm)393 b Fk(new)15 b(comm)o(uni)o(cator)d (\(handle\))75 2704 y Fu(int)23 b(MPI)p 245 2704 15 2 v 17 w(Comm)p 358 2704 V 17 w(split\(MPI)p 591 2704 V 16 w(Comm)g(comm,)g(int)h(color,)f(int)g(key,)h(MPI)p 1419 2704 V 16 w(Comm)g(*newcomm\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 145 153 145 152 bop 75 -100 a Fm(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b Fx(145)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(COMM)p 263 45 V 16 w(SPLIT\(COMM,)23 b(COLOR,)g(KEY,)g (NEWCOMM,)g(IERROR\))170 102 y(INTEGER)g(COMM,)h(COLOR,)f(KEY,)g (NEWCOMM,)g(IERROR)75 188 y Fx(This)c(function)f(partitions)g(the)h (group)e(asso)q(ciated)h(with)h Fv(comm)12 b Fx(in)o(to)18 b(disjoin)o(t)g(subgroups,)h(one)f(for)75 244 y(eac)o(h)g(v)m(alue)g (of)f Fv(colo)o(r)p Fx(.)26 b(Eac)o(h)17 b(subgroup)h(con)o(tains)f (all)i(pro)q(cesses)f(of)f(the)g(same)g(color.)27 b(Within)19 b(eac)o(h)75 301 y(subgroup,)24 b(the)f(pro)q(cesses)g(are)f(rank)o(ed) h(in)g(the)g(order)g(de\014ned)h(b)o(y)e(the)h(v)m(alue)h(of)e(the)h (argumen)o(t)75 357 y Fv(k)o(ey)p Fx(,)d(with)g(ties)g(brok)o(en)g (according)g(to)f(their)h(rank)f(in)i(the)f(old)g(group.)33 b(A)19 b(new)h(comm)o(unicator)f(is)75 414 y(created)14 b(for)f(eac)o(h)h(subgroup)g(and)g(returned)g(in)g Fv(new)o(comm)n Fx(.)i(A)e(pro)q(cess)g(ma)o(y)f(supply)i(the)f(color)g(v)m(alue)75 470 y Fv(MPI)p 160 470 14 2 v 16 w(UNDEFINED)p Fx(,)19 b(in)i(whic)o(h)g(case)e Fv(new)o(comm)c Fx(returns)20 b Fi(MPI)p 1200 470 13 2 v 14 w(COMM)p 1344 470 V 15 w(NULL)p Fx(.)g(This)g(is)h(a)e(collectiv)o(e)75 527 y(call,)d(but)f(eac)o(h)h(pro)q(cess)f(is)h(p)q(ermitted)g(to)e(pro)o (vide)i(di\013eren)o(t)f(v)m(alues)i(for)d Fv(colo)o(r)g Fx(and)i Fv(k)o(ey)p Fx(.)166 583 y(A)47 b(call)i(to)e Fv(MPI)p 536 583 14 2 v 16 w(COMM)p 694 583 V 16 w(CREA)l(TE\(comm)n(,) 52 b(group,)k(new)o(comm)m(\))44 b Fx(is)k(equiv)m(alen)o(t)i(to)75 640 y(a)15 b(call)i(to)d Fv(MPI)p 337 640 V 16 w(COMM)p 495 640 V 17 w(SPLIT\(com)o(m)m(,)e(colo)o(r,)i(k)o(ey)l(,)h(new)o (comm)n(\))o Fx(,)d(where)k(all)g(mem)o(b)q(ers)g(of)f Fv(group)g Fx(pro-)75 696 y(vide)i Fv(colo)o(r)e Fx(=)30 b(0)15 b(and)h Fv(k)o(ey)h Fx(=)32 b(rank)15 b(in)i Fv(group)p Fx(,)f(and)g(all)h(pro)q(cesses)f(that)f(are)h(not)f(mem)o(b)q(ers)i (of)e Fv(group)75 753 y Fx(pro)o(vide)k Fv(colo)o(r)e Fx(=)37 b Fv(MPI)p 506 753 V 16 w(UNDEFINED)p Fx(.)18 b(The)g(function)h Fv(MPI)p 1166 753 V 16 w(COMM)p 1324 753 V 16 w(SPLIT)f Fx(allo)o(ws)h(more)e(general)75 809 y(partitioning)22 b(of)e(a)g(group)h(in)o(to)g(one)g(or)f(more)g (subgroups)h(with)g(optional)h(reordering.)36 b(This)22 b(call)75 865 y(applies)17 b(only)f(in)o(tra-comm)o(unicators.)189 951 y Fl(A)n(dvic)n(e)e(to)h(users.)39 b Fx(This)15 b(is)f(an)g (extremely)h(p)q(o)o(w)o(erful)f(mec)o(hanism)h(for)e(dividing)j(a)e (single)h(com-)189 1007 y(m)o(unicating)h(group)e(of)h(pro)q(cesses)g (in)o(to)g Fp(k)h Fx(subgroups,)f(with)g Fp(k)h Fx(c)o(hosen)g (implicitly)i(b)o(y)d(the)g(user)189 1064 y(\(b)o(y)f(the)h(n)o(um)o(b) q(er)g(of)g(colors)g(asserted)f(o)o(v)o(er)g(all)i(the)f(pro)q (cesses\).)20 b(Eac)o(h)14 b(resulting)i(comm)o(unica-)189 1120 y(tor)g(will)i(b)q(e)f(non-o)o(v)o(erlapping.)26 b(Suc)o(h)17 b(a)g(division)i(could)e(b)q(e)h(useful)g(for)e (de\014ning)i(a)f(hierarc)o(h)o(y)189 1177 y(of)d(computations,)h(suc)o (h)h(as)e(for)h(m)o(ultigrid,)h(or)f(linear)h(algebra.)189 1249 y(Multiple)k(calls)f(to)f Fv(MPI)p 623 1249 V 16 w(COMM)p 781 1249 V 16 w(SPLIT)g Fx(can)h(b)q(e)g(used)g(to)f(o)o(v)o (ercome)f(the)i(requiremen)o(t)g(that)189 1305 y(an)o(y)d(call)h(ha)o (v)o(e)f(no)g(o)o(v)o(erlap)g(of)g(the)g(resulting)i(comm)o(unicators)e (\(eac)o(h)g(pro)q(cess)g(is)h(of)f(only)h(one)189 1362 y(color)i(p)q(er)g(call\).)32 b(In)19 b(this)g(w)o(a)o(y)l(,)g(m)o (ultiple)i(o)o(v)o(erlapping)e(comm)o(unication)h(structures)e(can)h(b) q(e)189 1418 y(created.)h(Creativ)o(e)14 b(use)i(of)f(the)g Fv(colo)o(r)f Fx(and)h Fv(k)o(ey)h Fx(in)g(suc)o(h)f(splitting)i(op)q (erations)e(is)h(encouraged.)189 1490 y(Note)g(that,)g(for)h(a)f (\014xed)i(color,)f(the)g(k)o(eys)f(need)i(not)e(b)q(e)i(unique.)26 b(It)17 b(is)h Fv(MPI)p 1551 1490 V 16 w(COMM)p 1709 1490 V 16 w(SPLIT)p Fx('s)189 1547 y(resp)q(onsibilit)o(y)f(to)c(sort)h (pro)q(cesses)h(in)g(ascending)h(order)e(according)h(to)e(this)i(k)o (ey)l(,)g(and)f(to)g(break)189 1603 y(ties)19 b(in)h(a)e(consisten)o(t) i(w)o(a)o(y)l(.)30 b(If)19 b(all)h(the)f(k)o(eys)g(are)f(sp)q (eci\014ed)k(in)d(the)g(same)g(w)o(a)o(y)l(,)g(then)g(all)h(the)189 1660 y(pro)q(cesses)15 b(in)g(a)f(giv)o(en)h(color)g(will)h(ha)o(v)o(e) e(the)g(relativ)o(e)h(rank)g(order)f(as)g(they)g(did)i(in)f(their)g (paren)o(t)189 1716 y(group.)k(\(In)d(general,)f(they)g(will)i(ha)o(v)o (e)e(di\013eren)o(t)g(ranks.\))189 1788 y(Essen)o(tially)l(,)f(making)e (the)h(k)o(ey)f(v)m(alue)i(zero)f(for)e(all)j(pro)q(cesses)f(of)f(a)g (giv)o(en)h(color)g(means)f(that)g(one)189 1845 y(do)q(esn't)j(really)h (care)f(ab)q(out)g(the)g(rank-order)g(of)g(the)g(pro)q(cesses)h(in)g (the)f(new)h(comm)o(unicator.)189 1917 y(\()p Fl(End)f(of)i(advic)n(e)f (to)g(users.)p Fx(\))75 2036 y Fj(5.4.3)49 b(Comm)o(unicato)o(r)13 b(Destructo)o(rs)75 2216 y Fv(MPI)p 160 2216 V 16 w(COMM)p 318 2216 V 16 w(FREE\(comm)n(\))117 2293 y Fk(INOUT)62 b Fv(comm)466 b Fk(comm)o(unicator)11 b(to)j(b)q(e)h(destro)o(y)o(ed)g (\(handle\))75 2418 y Fu(int)23 b(MPI)p 245 2418 15 2 v 17 w(Comm)p 358 2418 V 17 w(free\(MPI)p 567 2418 V 16 w(Comm)g(*comm\))75 2504 y(MPI)p 150 2504 V 17 w(COMM)p 263 2504 V 16 w(FREE\(COMM,)g(IERROR\))170 2561 y(INTEGER)g(COMM,)h (IERROR)166 2647 y Fx(This)13 b(collectiv)o(e)g(op)q(eration)g(marks)e (the)h(comm)o(unication)h(ob)s(ject)f(for)f(deallo)q(cation.)20 b(The)13 b(handle)75 2704 y(is)e(set)g(to)f Fi(MPI)p 311 2704 13 2 v 14 w(COMM)p 455 2704 V 15 w(NULL)p Fx(.)h(An)o(y)g(p)q (ending)h(op)q(erations)f(that)f(use)h(this)g(comm)o(unicator)g(will)h (complete)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 146 154 146 153 bop 75 -100 a Fx(146)370 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fx(normally;)j(the)f(ob)s(ject)g(is)g(actually)h(deallo)q(cated)h (only)e(if)h(there)f(are)g(no)g(other)g(activ)o(e)g(references)h(to)75 102 y(it.)28 b(This)18 b(call)h(applies)g(to)e(in)o(tra-)h(and)f(in)o (ter-comm)o(unicators.)28 b(The)18 b(delete)g(callbac)o(k)h(functions)g (for)75 158 y(all)d(cac)o(hed)g(attributes)f(\(see)g(section)h(5.7\))e (are)h(called)h(in)g(arbitrary)f(order.)189 264 y Fl(A)n(dvic)n(e)i(to) h(implementors.)52 b Fx(A)17 b(reference-coun)o(t)h(mec)o(hanism)f(ma)o (y)g(b)q(e)h(used:)24 b(the)17 b(reference)189 321 y(coun)o(t)d(is)h (incremen)o(ted)h(b)o(y)e(eac)o(h)h(call)g(to)f Fv(MPI)p 1000 321 14 2 v 16 w(COMM)p 1158 321 V 17 w(DUP)p Fx(,)g(and)g (decremen)o(ted)i(b)o(y)e(eac)o(h)h(call)189 377 y(to)j Fv(MPI)p 333 377 V 16 w(COMM)p 491 377 V 16 w(FREE)p Fx(.)h(The)h(ob)s(ject)e(is)i(ultimately)g(deallo)q(cated)g(when)g(the) f(coun)o(t)g(reac)o(hes)189 434 y(zero.)189 509 y(Though)10 b(collectiv)o(e,)j(it)d(is)h(an)o(ticipated)g(that)f(this)g(op)q (eration)h(will)h(normally)f(b)q(e)g(implemen)o(ted)h(to)189 565 y(b)q(e)h(lo)q(cal,)h(though)f(the)f(debugging)i(v)o(ersion)f(of)f (an)h Fv(MPI)f Fx(library)i(migh)o(t)e(c)o(ho)q(ose)h(to)f(sync)o (hronize.)189 622 y(\()p Fl(End)j(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 765 y Fn(5.5)59 b(Motivating)19 b(Examples)75 868 y Fj(5.5.1)49 b(Current)15 b(Practice)g(#1)75 954 y Fx(Example)h(#1a:)147 1048 y Fu(main\(int)22 b(argc,)i(char)f (**argv\))147 1104 y({)194 1161 y(int)h(me,)f(size;)194 1217 y(...)194 1273 y(MPI_Init)g(\()h(&argc,)f(&argv)g(\);)194 1330 y(MPI_Comm_rank)f(\(MPI_COMM_WORLD,)g(&me\);)194 1386 y(MPI_Comm_size)g(\(MPI_COMM_WORLD,)g(&size\);)194 1499 y(\(void\)printf)h(\("Process)f(\045d)i(size)f(\045d\\n",)g(me,)h (size\);)194 1556 y(...)194 1612 y(MPI_Finalize\(\);)147 1669 y(})75 1762 y Fx(Example)19 b(#1a)f(is)h(a)f(do-nothing)h(program) e(that)h(initializes)k(itself)d(legally)l(,)i(and)d(refers)g(to)g(the)h (the)75 1819 y(\\all")f(comm)o(unicator,)g(and)g(prin)o(ts)g(a)f (message.)27 b(It)18 b(terminates)g(itself)h(legally)g(to)q(o.)27 b(This)18 b(example)75 1875 y(do)q(es)e(not)e(imply)j(that)d Fv(MPI)h Fx(supp)q(orts)g Fu(printf)p Fx(-lik)o(e)h(comm)o(unication)g (itself.)75 1932 y(Example)g(#1b)f(\(supp)q(osing)h(that)e Fu(size)h Fx(is)h(ev)o(en\):)170 2026 y Fu(main\(int)23 b(argc,)g(char)h(**argv\))170 2082 y({)242 2139 y(int)f(me,)h(size;)242 2195 y(int)f(SOME_TAG)g(=)h(0;)242 2251 y(...)242 2308 y(MPI_Init\(&argc,)e(&argv\);)242 2421 y(MPI_Comm_rank\(MPI_COMM_WOR)o (LD,)f(&me\);)71 b(/*)23 b(local)h(*/)242 2477 y (MPI_Comm_size\(MPI_COMM_WOR)o(LD,)d(&size\);)i(/*)g(local)h(*/)242 2590 y(if\(\(me)f(\045)h(2\))f(==)h(0\))242 2647 y({)314 2703 y(/*)f(send)h(unless)f(highest-numbered)e(process)i(*/)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 147 155 147 154 bop 75 -100 a Fm(5.5.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b Fx(147)314 45 y Fu(if\(\(me)23 b(+)g(1\))h(<)g (size\))385 102 y(MPI_Send\(...,)e(me)i(+)g(1,)f(SOME_TAG,)g (MPI_COMM_WORLD\);)242 158 y(})242 214 y(else)314 271 y(MPI_Recv\(...,)f(me)h(-)h(1,)g(SOME_TAG,)e(MPI_COMM_WORLD\);)242 384 y(...)242 440 y(MPI_Finalize\(\);)170 497 y(})75 591 y Fx(Example)16 b(#1b)f(sc)o(hematically)i(illustrates)g(message)e (exc)o(hanges)h(b)q(et)o(w)o(een)g(\\ev)o(en")f(and)h(\\o)q(dd")f(pro-) 75 647 y(cesses)h(in)g(the)f(\\all")g(comm)o(unicator.)75 769 y Fj(5.5.2)49 b(Current)15 b(Practice)g(#2)147 855 y Fu(main\(int)22 b(argc,)i(char)f(**argv\))147 911 y({)194 967 y(int)h(me,)f(count;)194 1024 y(void)h(*data;)194 1080 y(...)194 1193 y(MPI_Init\(&argc,)e(&argv\);)194 1250 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&me\);)194 1363 y(if\(me)i(==)h(0\))194 1419 y({)290 1476 y(/*)f(get)h(input,)f (create)g(buffer)g(``data'')g(*/)290 1532 y(...)194 1588 y(})194 1701 y(MPI_Bcast\(data,)f(count,)h(MPI_BYTE,)g(0,)g (MPI_COMM_WORLD\);)194 1814 y(...)194 1927 y(MPI_Finalize\(\);)147 1984 y(})75 2090 y Fx(This)16 b(example)g(illustrates)g(the)f(use)h(of) f(a)g(collectiv)o(e)h(comm)o(unication.)75 2212 y Fj(5.5.3)49 b(\(App)o(ro)o(ximate\))14 b(Current)h(Practice)g(#3)123 2297 y Fu(main\(int)22 b(argc,)i(char)f(**argv\))123 2354 y({)170 2410 y(int)h(me,)f(count,)g(count2;)170 2467 y(void)h(*send_buf,)e(*recv_buf,)h(*send_buf2,)f(*recv_buf2;)170 2523 y(MPI_Group)h(MPI_GROUP_WORLD,)f(grprem;)170 2580 y(MPI_Comm)h(commslave;)170 2636 y(static)g(int)h(ranks[])f(=)h({0};) 170 2693 y(...)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 148 156 148 155 bop 75 -100 a Fx(148)370 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)170 45 y Fu(MPI_Init\(&argc,)22 b(&argv\);)170 102 y (MPI_Comm_group\(MPI_COMM_WORLD)o(,)f(&MPI_GROUP_WORLD\);)170 158 y(MPI_Comm_rank\(MPI_COMM_WORLD,)f(&me\);)47 b(/*)24 b(local)f(*/)170 271 y(MPI_Group_excl\(MPI_GROUP_WORL)o(D,)e(1,)j (ranks,)f(&grprem\);)46 b(/*)24 b(local)f(*/)170 327 y(MPI_Comm_create\(MPI_COMM_WORL)o(D,)e(grprem,)i(&commslave\);)170 440 y(if\(me)h(!=)f(0\))170 497 y({)218 553 y(/*)h(compute)f(on)g (slave)g(*/)218 610 y(...)218 666 y(MPI_Reduce\(send_buf,recv_bu)o (ff,count)o(,)e(MPI_INT,)i(MPI_SUM,)g(1,)g(commslave\);)218 723 y(...)170 779 y(})170 835 y(/*)h(zero)f(falls)h(through)e (immediately)h(to)g(this)h(reduce,)f(others)g(do)g(later...)g(*/)170 892 y(MPI_Reduce\(send_buf2,)e(recv_buff2,)i(count2,)433 948 y(MPI_INT,)g(MPI_SUM,)f(0,)i(MPI_COMM_WORLD\);)170 1061 y(MPI_Comm_free\(&commslave\);)170 1118 y (MPI_Group_free\(&MPI_GROUP_WOR)o(LD\);)170 1174 y (MPI_Group_free\(&grprem\);)170 1231 y(MPI_Finalize\(\);)123 1287 y(})75 1389 y Fx(This)18 b(example)h(illustrates)g(ho)o(w)e(a)h (group)f(consisting)i(of)e(all)i(but)f(the)g(zeroth)f(pro)q(cess)h(of)g (the)g(\\all")75 1446 y(group)13 b(is)h(created,)f(and)h(then)g(ho)o(w) f(a)g(comm)o(unicator)g(is)h(formed)f(\()g Fv(comm)m(slave)p Fx(\))d(for)j(that)g(new)h(group.)75 1502 y(The)f(new)h(comm)o (unicator)e(is)i(used)g(in)g(a)e(collectiv)o(e)j(call,)g(and)e(all)h (pro)q(cesses)f(execute)h(a)f(collectiv)o(e)i(call)75 1559 y(in)j(the)33 b Fi(MPI)p 303 1559 13 2 v 15 w(COMM)p 448 1559 V 14 w(W)o(ORLD)18 b Fx(con)o(text.)25 b(This)18 b(example)h(illustrates)f(ho)o(w)f(the)g(t)o(w)o(o)f(comm)o(unicators) 75 1615 y(\(that)c(inheren)o(tly)k(p)q(ossess)d(distinct)i(con)o (texts\))d(protect)h(comm)o(unication.)20 b(That)13 b(is,)h(comm)o (unication)75 1672 y(in)30 b Fi(MPI)p 219 1672 V 14 w(COMM)p 363 1672 V 15 w(W)o(ORLD)15 b Fx(is)h(insulated)g(from)f(comm)o (unication)h(in)31 b Fv(comm)m(slave)p Fx(,)12 b(and)j(vice)i(v)o (ersa.)166 1728 y(In)h(summary)l(,)g(\\group)f(safet)o(y")f(is)i(ac)o (hiev)o(ed)h(via)f(comm)o(unicators)f(b)q(ecause)i(distinct)g(con)o (texts)75 1785 y(within)d(comm)o(unicators)f(are)g(enforced)h(to)e(b)q (e)i(unique)h(on)e(an)o(y)g(pro)q(cess.)75 1906 y Fj(5.5.4)49 b(Example)15 b(#4)75 1992 y Fx(The)g(follo)o(wing)g(example)g(is)g (mean)o(t)f(to)f(illustrate)j(\\safet)o(y")d(b)q(et)o(w)o(een)h(p)q (oin)o(t-to-p)q(oin)o(t)i(and)e(collectiv)o(e)75 2048 y(comm)o(unication.)20 b Fv(MPI)13 b Fx(guaran)o(tees)f(that)h(a)g (single)h(comm)o(unicator)f(can)h(do)f(safe)g(p)q(oin)o(t-to-p)q(oin)o (t)h(and)75 2105 y(collectiv)o(e)j(comm)o(unication.)147 2195 y Fu(#define)22 b(TAG_ARBITRARY)h(12345)147 2252 y(#define)f(SOME_COUNT)166 b(50)147 2365 y(main\(int)22 b(argc,)i(char)f(**argv\))147 2421 y({)194 2478 y(int)h(me;)194 2534 y(MPI_Request)f(request[2];)194 2591 y(MPI_Status)g(status[2];)194 2647 y(MPI_Group)g(MPI_GROUP_WORLD,)f(subgroup;)194 2704 y(int)i(ranks[])f(=)g({2,)h(4,)f(6,)h(8};)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 149 157 149 156 bop 75 -100 a Fm(5.5.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b Fx(149)194 45 y Fu(MPI_Comm)23 b(the_comm;)194 102 y(...)194 158 y(MPI_Init\(&argc,)f(&argv\);)194 214 y(MPI_Comm_group\(MPI_COMM_WORL)o(D,)f(&MPI_GROUP_WORLD\);)194 327 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)g(4,)j(ranks,)f(&subgroup\);) f(/*)i(local)f(*/)194 384 y(MPI_Group_rank\(subgroup,)e(&me\);)119 b(/*)23 b(local)g(*/)194 497 y(MPI_Comm_create\(MPI_COMM_WOR)o(LD,)e (subgroup,)i(&the_comm\);)194 610 y(if\(me)g(!=)h(MPI_UNDEFINED\))194 666 y({)290 723 y(MPI_Irecv\(buff1,)d(count,)i(MPI_DOUBLE,)g (MPI_ANY_SOURCE,)f(TAG_ARBITRARY,)719 779 y(the_comm,)h(request\);)290 835 y(MPI_Isend\(buff2,)e(count,)i(MPI_DOUBLE,)g(\(me+1\)\0454,)f (TAG_ARBITRARY,)719 892 y(the_comm,)h(request+1\);)194 948 y(})194 1061 y(for\(i)g(=)h(0;)g(i)g(<)f(SOME_COUNT,)g(i++\))242 1118 y(MPI_Reduce\(...,)f(the_comm\);)194 1174 y(MPI_Waitall\(2,)g (request,)h(status\);)194 1287 y(MPI_Comm_free\(t&he_comm\);)194 1344 y(MPI_Group_free\(&MPI_GROUP_WO)o(RLD\);)194 1400 y(MPI_Group_free\(&subgroup\);)194 1456 y(MPI_Finalize\(\);)147 1513 y(})75 1632 y Fj(5.5.5)49 b(Lib)o(ra)o(ry)16 b(Example)f(#1)75 1718 y Fx(The)g(main)h(program:)147 1800 y Fu(main\(int)22 b(argc,)i(char)f(**argv\))147 1857 y({)194 1913 y(int)h(done)f(=)h(0;) 194 1970 y(user_lib_t)f(*libh_a,)g(*libh_b;)194 2026 y(void)h(*dataset1,)e(*dataset2;)194 2083 y(...)194 2139 y(MPI_Init\(&argc,)g(&argv\);)194 2195 y(...)194 2252 y(init_user_lib\(MPI_COMM_WORLD)o(,)f(&libh_a\);)194 2308 y(init_user_lib\(MPI_COMM_WORLD)o(,)g(&libh_b\);)194 2365 y(...)194 2421 y(user_start_op\(libh_a,)g(dataset1\);)194 2478 y(user_start_op\(libh_b,)g(dataset2\);)194 2534 y(...)194 2591 y(while\(!done\))194 2647 y({)266 2704 y(/*)i(work)h(*/)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 150 158 150 157 bop 75 -100 a Fx(150)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)266 45 y Fu(...)266 102 y(MPI_Reduce\(...,)22 b(MPI_COMM_WORLD\);)266 158 y(...)266 214 y(/*)h(see)h(if)g(done)f(*/)266 271 y(...)194 327 y(})194 384 y(user_end_op\(libh_a\);)194 440 y(user_end_op\(libh_b\);)194 553 y(uninit_user_lib\(libh_a\);)194 610 y(uninit_user_lib\(libh_b\);)194 666 y(MPI_Finalize\(\);)147 723 y(})75 800 y Fx(The)15 b(user)h(library)g(initialization)i(co)q (de:)147 878 y Fu(void)23 b(init_user_lib\(MPI_Comm)e(*comm,)i (user_lib_t)f(**handle\))147 935 y({)194 991 y(user_lib_t)h(*save;)194 1104 y(user_lib_initsave\(&save\);)e(/*)i(local)h(*/)194 1160 y(MPI_Comm_dup\(comm,)e(&\(save)h(->)g(comm\)\);)194 1273 y(/*)h(other)f(inits)g(*/)194 1330 y(...)194 1443 y(*handle)g(=)h(save;)147 1499 y(})75 1577 y Fx(User)15 b(start-up)g(co)q(de:)147 1655 y Fu(void)23 b (user_start_op\(user_lib_t)d(*handle,)j(void)h(*data\))147 1711 y({)194 1768 y(MPI_Irecv\()f(...,)g(handle->comm,)f(&\(handle)h (->)h(irecv_handle\))e(\);)194 1824 y(MPI_Isend\()h(...,)g (handle->comm,)f(&\(handle)h(->)h(isend_handle\))e(\);)147 1881 y(})75 1958 y Fx(User)15 b(comm)o(unication)h(clean-up)h(co)q(de:) 147 2036 y Fu(void)23 b(user_end_op\(user_lib_t)e(*handle\))147 2093 y({)194 2149 y(MPI_Status)i(*status;)194 2205 y(MPI_Wait\(handle)f (->)i(isend_handle,)e(status\);)194 2262 y(MPI_Wait\(handle)g(->)i (irecv_handle,)e(status\);)147 2318 y(})75 2396 y Fx(User)15 b(ob)s(ject)g(clean-up)i(co)q(de:)147 2474 y Fu(void)23 b(uninit_user_lib\(user_lib_)o(t)e(*handle\))147 2530 y({)194 2587 y(MPI_Comm_free\(&\(handle)g(->)j(comm\)\);)194 2643 y(free\(handle\);)147 2700 y(})1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 151 159 151 158 bop 75 -100 a Fm(5.5.)34 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1050 b Fx(151)75 45 y Fj(5.5.6)49 b(Lib)o(ra)o(ry)16 b(Example)f(#2)75 131 y Fx(The)g(main)h(program:)147 225 y Fu(main\(int)22 b(argc,)i(char)f(**argv\))147 281 y({)194 338 y(int)h(ma,)f(mb;)194 394 y(MPI_Group)g(MPI_GROUP_WORLD,)f (group_a,)g(group_b;)194 451 y(MPI_Comm)h(comm_a,)g(comm_b;)194 563 y(static)g(int)h(list_a[])f(=)g({0,)h(1};)75 620 y(#if)47 b(defined\(EXAMPLE_2B\))22 b(|)h(defined\(EXAMPLE_2C\))194 676 y(static)g(int)h(list_b[])f(=)g({0,)h(2)f(,3};)75 733 y(#else/*)g(EXAMPLE_2A)f(*/)194 789 y(static)h(int)h(list_b[])f(=)g ({0,)h(2};)75 846 y(#endif)194 902 y(int)g(size_list_a)e(=)i (sizeof\(list_a\)/sizeof\(int)o(\);)194 959 y(int)g(size_list_b)e(=)i (sizeof\(list_b\)/sizeof\(int)o(\);)194 1072 y(...)194 1128 y(MPI_Init\(&argc,)e(&argv\);)194 1184 y (MPI_Comm_group\(MPI_COMM_WORL)o(D,)f(&MPI_GROUP_WORLD\);)194 1297 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)g(size_list_a,)h(list_a,)h (&group_a\);)194 1354 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)e (size_list_b,)h(list_b,)h(&group_b\);)194 1467 y (MPI_Comm_create\(MPI_COMM_WOR)o(LD,)e(group_a,)i(&comm_a\);)194 1523 y(MPI_Comm_create\(MPI_COMM_WOR)o(LD,)e(group_b,)i(&comm_b\);)194 1636 y(MPI_Comm_rank\(comm_a,)e(&ma\);)194 1693 y (MPI_Comm_rank\(comm_b,)g(&mb\);)194 1805 y(if\(ma)i(!=)h (MPI_UNDEFINED\))266 1862 y(lib_call\(comm_a\);)194 1975 y(if\(mb)f(!=)h(MPI_UNDEFINED\))194 2031 y({)242 2088 y(lib_call\(comm_b\);)242 2144 y(lib_call\(comm_b\);)194 2201 y(})194 2314 y(MPI_Comm_free\(&comm_a\);)194 2370 y(MPI_Comm_free\(&comm_b\);)194 2426 y(MPI_Group_free\(&group_a\);)194 2483 y(MPI_Group_free\(&group_b\);)194 2539 y (MPI_Group_free\(&MPI_GROUP_WO)o(RLD\);)194 2596 y(MPI_Finalize\(\);) 147 2652 y(})-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 152 160 152 159 bop 75 -100 a Fx(152)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fx(The)g(library:)147 139 y Fu(void)23 b(lib_call\(MPI_Comm)e (comm\))147 195 y({)194 252 y(int)j(me,)f(done)h(=)f(0;)194 308 y(MPI_Comm_rank\(comm,)f(&me\);)194 365 y(if\(me)h(==)h(0\))266 421 y(while\(!done\))266 478 y({)337 534 y(MPI_Recv\(...,)f (MPI_ANY_SOURCE,)e(MPI_ANY_TAG,)i(comm\);)337 591 y(...)266 647 y(})194 703 y(else)194 760 y({)242 816 y(/*)h(work)f(*/)242 873 y(MPI_Send\(...,)f(0,)i(ARBITRARY_TAG,)e(comm\);)242 929 y(....)194 986 y(})75 1042 y(#ifdef)h(EXAMPLE_2C)194 1099 y(/*)h(include)f(\(resp,)g(exclude\))g(for)g(safety)g(\(resp,)g (no)h(safety\):)f(*/)194 1155 y(MPI_Barrier\(comm\);)75 1212 y(#endif)147 1268 y(})75 1362 y Fx(The)13 b(ab)q(o)o(v)o(e)f (example)h(is)h(really)f(three)g(examples,)g(dep)q(ending)i(on)e (whether)f(or)g(not)h(one)f(includes)j(rank)75 1418 y(3)f(in)h Fv(list)p 222 1418 14 2 v 17 w(b)p Fx(,)g(and)f(whether)h(or)f(not)f(a) i(sync)o(hronize)g(is)g(included)i(in)e Fv(lib)p 1290 1418 V 17 w(call)p Fx(.)k(This)c(example)g(illustrates)75 1475 y(that,)i(despite)h(con)o(texts,)f(subsequen)o(t)h(calls)h(to)d Fv(lib)p 984 1475 V 17 w(call)h Fx(with)h(the)f(same)g(con)o(text)g (need)h(not)f(b)q(e)h(safe)75 1531 y(from)g(one)h(another)g(\(collo)q (quially)l(,)j(\\bac)o(k-masking"\).)30 b(Safet)o(y)18 b(is)i(realized)g(if)g(the)37 b Fv(MPI)p 1684 1531 V 16 w(Ba)o(rrier)17 b Fx(is)75 1588 y(added.)34 b(What)19 b(this)h(demonstrates)f(is)h(that)f(libraries)i(ha)o(v)o(e)f(to)f(b)q (e)h(written)g(carefully)l(,)i(ev)o(en)e(with)75 1644 y(con)o(texts.)j(When)17 b(rank)f(3)g(is)h(excluded,)i(then)d(the)h (sync)o(hronize)g(is)g(not)g(needed)g(to)f(get)g(safet)o(y)g(from)75 1701 y(bac)o(k)f(masking.)166 1757 y(Algorithms)i(lik)o(e)h(\\reduce")e (and)h(\\allreduce")h(ha)o(v)o(e)e(strong)g(enough)h(source)g (selectivit)o(y)h(prop-)75 1813 y(erties)c(so)g(that)g(they)g(are)f (inheren)o(tly)j(ok)m(a)o(y)e(\(no)f(bac)o(kmasking\),)h(pro)o(vided)h (that)e Fv(MPI)h Fx(pro)o(vides)g(basic)75 1870 y(guaran)o(tees.)21 b(So)16 b(are)g(m)o(ultiple)i(calls)f(to)e(a)h(t)o(ypical)h (tree-broadcast)e(algorithm)h(with)g(the)g(same)g(ro)q(ot)75 1926 y(or)h(di\013eren)o(t)h(ro)q(ots)f(\(see)h([28)o(]\).)27 b(Here)18 b(w)o(e)f(rely)h(on)g(t)o(w)o(o)f(guaran)o(tees)f(of)i Fv(MPI)p Fx(:)f(pairwise)h(ordering)g(of)75 1983 y(messages)f(b)q(et)o (w)o(een)h(pro)q(cesses)g(in)g(the)g(same)f(con)o(text,)g(and)h(source) f(selectivit)o(y)i(|)f(deleting)h(either)75 2039 y(feature)c(remo)o(v)o (es)f(the)i(guaran)o(tee)e(that)g(bac)o(kmasking)i(cannot)f(b)q(e)h (required.)166 2096 y(Algorithms)c(that)g(try)g(to)f(do)h (non-deterministic)j(broadcasts)c(or)h(other)g(calls)h(that)f(include)j (wild-)75 2152 y(card)g(op)q(erations)g(will)i(not)e(generally)h(ha)o (v)o(e)f(the)g(go)q(o)q(d)g(prop)q(erties)h(of)f(the)g(deterministic)i (implemen-)75 2209 y(tations)g(of)h(\\reduce,")g(\\allreduce,")i(and)e (\\broadcast.")27 b(Suc)o(h)18 b(algorithms)g(w)o(ould)g(ha)o(v)o(e)g (to)f(utilize)75 2265 y(the)e(monotonically)h(increasing)h(tags)d (\(within)i(a)f(comm)o(unicator)g(scop)q(e\))g(to)g(k)o(eep)g(things)h (straigh)o(t.)166 2322 y(All)k(of)e(the)h(foregoing)f(is)h(a)g(supp)q (osition)h(of)e(\\collectiv)o(e)i(calls")f(implemen)o(ted)i(with)e(p)q (oin)o(t-to-)75 2378 y(p)q(oin)o(t)g(op)q(erations.)30 b Fv(MPI)18 b Fx(implemen)o(tations)i(ma)o(y)e(or)g(ma)o(y)g(not)g (implemen)o(t)i(collectiv)o(e)h(calls)e(using)75 2434 y(p)q(oin)o(t-to-p)q(oin)o(t)e(op)q(erations.)22 b(These)16 b(algorithms)g(are)f(used)i(to)e(illustrate)i(the)f(issues)h(of)e (correctness)75 2491 y(and)g(safet)o(y)l(,)g(indep)q(enden)o(t)i(of)e (ho)o(w)g Fv(MPI)f Fx(implemen)o(ts)j(its)e(collectiv)o(e)i(calls.)k (See)16 b(also)f(section)h(5.8.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 153 161 153 160 bop 75 -100 a Fm(5.6.)34 b(INTER-COMMUNICA)l(TION)1047 b Fx(153)75 45 y Fn(5.6)59 b(Inter-Communication)75 147 y Fx(This)18 b(section)h(in)o(tro)q(duces)f(the)g(concept)g(of)f(in)o (ter-comm)o(unication)i(and)f(describ)q(es)i(the)d(p)q(ortions)h(of)75 203 y Fv(MPI)j Fx(that)g(supp)q(ort)h(it.)40 b(It)22 b(describ)q(es)h(supp)q(ort)f(for)f(writing)i(programs)d(that)h(con)o (tain)h(user-lev)o(el)75 259 y(serv)o(ers.)166 316 y(All)c(p)q(oin)o (t-to-p)q(oin)o(t)g(comm)o(unication)g(describ)q(ed)h(th)o(us)d(far)h (has)g(in)o(v)o(olv)o(ed)g(comm)o(unication)h(b)q(e-)75 372 y(t)o(w)o(een)c(pro)q(cesses)g(that)f(are)h(mem)o(b)q(ers)g(of)f (the)h(same)g(group.)19 b(This)c(t)o(yp)q(e)f(of)f(comm)o(unication)i (is)f(called)75 429 y(\\in)o(tra-comm)o(unication")j(and)g(the)h(comm)o (unicator)e(used)i(is)f(called)i(an)e(\\in)o(tra-comm)o(unicator,")f (as)75 485 y(w)o(e)f(ha)o(v)o(e)g(noted)g(earlier)h(in)g(the)f(c)o (hapter.)166 542 y(In)c(mo)q(dular)g(and)f(m)o(ulti-discipl)q(inary)j (applications,)g(di\013eren)o(t)d(pro)q(cess)h(groups)f(execute)h (distinct)75 598 y(mo)q(dules)i(and)f(pro)q(cesses)h(within)g (di\013eren)o(t)f(mo)q(dules)h(comm)o(unicate)f(with)h(one)f(another)f (in)i(a)f(pip)q(eline)75 655 y(or)j(a)f(more)h(general)h(mo)q(dule)g (graph.)j(In)d(these)f(applications,)i(the)e(most)f(natural)h(w)o(a)o (y)f(for)h(a)g(pro)q(cess)75 711 y(to)g(sp)q(ecify)i(a)e(target)f(pro)q (cess)i(is)g(b)o(y)g(the)g(rank)f(of)g(the)h(target)e(pro)q(cess)i (within)h(the)e(target)g(group.)20 b(In)75 768 y(applications)13 b(that)e(con)o(tain)h(in)o(ternal)h(user-lev)o(el)g(serv)o(ers,)f(eac)o (h)f(serv)o(er)h(ma)o(y)f(b)q(e)h(a)f(pro)q(cess)h(group)g(that)75 824 y(pro)o(vides)17 b(services)f(to)g(one)g(or)g(more)f(clien)o(ts,)i (and)g(eac)o(h)f(clien)o(t)h(ma)o(y)f(b)q(e)g(a)g(pro)q(cess)h(group)e (that)h(uses)75 880 y(the)h(services)h(of)f(one)g(or)g(more)g(serv)o (ers.)25 b(It)17 b(is)h(again)f(most)f(natural)i(to)e(sp)q(ecify)j(the) e(target)f(pro)q(cess)75 937 y(b)o(y)d(rank)g(within)i(the)e(target)f (group)i(in)g(these)f(applications.)21 b(This)14 b(t)o(yp)q(e)g(of)e (comm)o(unication)j(is)e(called)75 993 y(\\in)o(ter-comm)o(unication") 18 b(and)g(the)g(comm)o(unicator)f(used)h(is)g(called)h(an)f(\\in)o (ter-comm)o(unicator,")f(as)75 1050 y(in)o(tro)q(duced)f(earlier.)166 1106 y(An)d(in)o(ter-comm)o(unication)g(is)g(a)f(p)q(oin)o(t-to-p)q (oin)o(t)h(comm)o(unication)g(b)q(et)o(w)o(een)g(pro)q(cesses)f(in)h (di\013er-)75 1163 y(en)o(t)f(groups.)19 b(The)12 b(group)g(con)o (taining)h(a)f(pro)q(cess)h(that)e(initiates)j(an)e(in)o(ter-comm)o (unication)i(op)q(eration)75 1219 y(is)i(called)i(the)d(\\lo)q(cal)i (group,")e(that)g(is,)h(the)g(sender)g(in)h(a)e(send)i(and)f(the)g (receiv)o(er)g(in)h(a)e(receiv)o(e.)23 b(The)75 1276 y(group)16 b(con)o(taining)g(the)g(target)f(pro)q(cess)h(is)g(called)i (the)e(\\remote)f(group,")g(that)g(is,)h(the)g(receiv)o(er)h(in)f(a)75 1332 y(send)f(and)f(the)g(sender)h(in)g(a)f(receiv)o(e.)20 b(As)14 b(in)h(in)o(tra-comm)o(unication,)g(the)f(target)f(pro)q(cess)h (is)h(sp)q(eci\014ed)75 1389 y(using)k(a)g Fv(\(comm)m(unicato)o(r,)d (rank\))j Fx(pair.)30 b(Unlik)o(e)21 b(in)o(tra-comm)o(unication,)e (the)g(rank)g(is)g(relativ)o(e)g(to)f(a)75 1445 y(second,)d(remote)g (group.)166 1501 y(All)e(in)o(ter-comm)o(unicator)f(constructors)g(are) f(blo)q(c)o(king)j(and)e(require)h(that)e(the)h(lo)q(cal)h(and)g (remote)75 1558 y(groups)i(b)q(e)h(disjoin)o(t)g(in)g(order)f(to)f(a)o (v)o(oid)h(deadlo)q(c)o(k.)166 1614 y(Here)f(is)h(a)f(summary)g(of)g (the)g(prop)q(erties)h(of)f(in)o(ter-comm)o(unication)i(and)e(in)o (ter-comm)o(unicators:)143 1702 y Ft(\017)23 b Fx(The)c(syn)o(tax)g(of) g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o(unication)g(is)g(the)f(same)g (for)g(b)q(oth)h(in)o(ter-)f(and)h(in)o(tra-)189 1758 y(comm)o(unication.)26 b(The)17 b(same)g(comm)o(unicator)g(can)g(b)q(e) g(used)h(b)q(oth)f(for)g(send)g(and)h(for)e(receiv)o(e)189 1815 y(op)q(erations.)143 1906 y Ft(\017)23 b Fx(A)14 b(target)g(pro)q(cess)h(is)g(addressed)h(b)o(y)f(its)g(rank)f(in)i(the) f(remote)f(group,)g(b)q(oth)h(for)f(sends)i(and)f(for)189 1963 y(receiv)o(es.)143 2054 y Ft(\017)23 b Fx(Comm)o(unications)12 b(using)i(an)e(in)o(ter-comm)o(unicator)h(are)f(guaran)o(teed)g(not)g (to)g(con\015ict)h(with)g(an)o(y)189 2110 y(comm)o(unications)i(that)g (use)h(a)e(di\013eren)o(t)i(comm)o(unicator.)143 2202 y Ft(\017)23 b Fx(An)15 b(in)o(ter-comm)o(unicator)g(cannot)g(b)q(e)h (used)g(for)f(collectiv)o(e)i(comm)o(unication.)143 2293 y Ft(\017)23 b Fx(A)15 b(comm)o(unicator)g(will)i(pro)o(vide)e(either)h (in)o(tra-)f(or)g(in)o(ter-comm)o(unication,)h(nev)o(er)f(b)q(oth.)75 2380 y(The)i(routine)f Fv(MPI)p 412 2380 14 2 v 16 w(COMM)p 570 2380 V 17 w(TEST)p 701 2380 V 16 w(INTER)h Fx(ma)o(y)e(b)q(e)i (used)g(to)f(determine)h(if)g(a)f(comm)o(unicator)g(is)h(an)75 2437 y(in)o(ter-)12 b(or)f(in)o(tra-comm)o(unicator.)19 b(In)o(ter-comm)o(unicators)11 b(can)h(b)q(e)h(used)f(as)f(argumen)o (ts)g(to)g(some)h(of)f(the)75 2493 y(other)i(comm)o(unicator)h(access)g (routines.)20 b(In)o(ter-comm)o(unicators)13 b(cannot)h(b)q(e)g(used)h (as)e(input)i(to)e(some)75 2550 y(of)i(the)g(constructor)f(routines)i (for)f(in)o(tra-comm)o(unicators)f(\(for)h(instance,)g Fv(MPI)p 1476 2550 V 16 w(COMM)p 1634 2550 V 17 w(CREA)l(TE)p Fx(\).)189 2647 y Fl(A)n(dvic)n(e)g(to)i(implementors.)41 b Fx(F)l(or)15 b(the)g(purp)q(ose)h(of)f(p)q(oin)o(t-to-p)q(oin)o(t)h (comm)o(unication,)g(comm)o(u-)189 2704 y(nicators)f(can)g(b)q(e)h (represen)o(ted)g(in)g(eac)o(h)f(pro)q(cess)g(b)o(y)g(a)g(tuple)i (consisting)f(of:)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 154 162 154 161 bop 75 -100 a Fx(154)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)189 45 y Fq(group)189 118 y(send)p 295 118 16 2 v 18 w(con)o(text)189 191 y(receiv)o(e)p 350 191 V 18 w(con)o(text)189 264 y(source)189 358 y Fx(F)l(or)9 b(in)o(ter-comm)o(unicators,)i Fq(group)g Fx(describ)q(es)h(the)e(remote)g(group,)h(and)f Fq(source)g Fx(is)h(the)f(rank)g(of)189 415 y(the)k(pro)q(cess)h(in)g (the)g(lo)q(cal)g(group.)20 b(F)l(or)14 b(in)o(tra-comm)o(unicators,)f Fq(group)i Fx(is)g(the)f(comm)o(unicator)189 471 y(group)20 b(\(remote=lo)q(cal\),)j Fq(source)d Fx(is)i(the)f(rank)g(of)f(the)i (pro)q(cess)f(in)h(this)f(group,)h(and)f Fq(send)189 527 y(con)o(text)e Fx(and)g Fq(receiv)o(e)h(con)o(text)f Fx(are)g(iden)o(tical.)32 b(A)19 b(group)g(is)g(represen)o(ted)g(b)o(y) g(a)g(rank-to-)189 584 y(absolute-address)c(translation)h(table.)189 659 y(The)11 b(in)o(ter-comm)o(unicator)h(cannot)g(b)q(e)g(discussed)h (sensibly)g(without)f(considering)h(pro)q(cesses)f(in)189 716 y(b)q(oth)f(the)h(lo)q(cal)g(and)g(remote)e(groups.)19 b(Imagine)12 b(a)f(pro)q(cess)g Fq(P)g Fx(in)i(group)e Ft(P)t Fx(,)g(whic)o(h)h(has)f(an)h(in)o(ter-)189 772 y(comm)o(unicator)18 b Fq(C)527 779 y Fh(P)556 772 y Fx(,)i(and)f(a)f(pro)q(cess)h Fq(Q)g Fx(in)h(group)e Ft(Q)p Fx(,)i(whic)o(h)f(has)g(an)g(in)o(ter-comm)o(unicator)189 828 y Fq(C)227 835 y Fh(Q)257 828 y Fx(.)h(Then)243 922 y Ft(\017)j Fq(C)327 929 y Fh(P)356 922 y Fq(.group)14 b Fx(describ)q(es)j(the)f(group)e Ft(Q)i Fx(and)f Fq(C)1097 929 y Fh(Q)1128 922 y Fq(.group)f Fx(describ)q(es)j(the)e(group)g Ft(P)t Fx(.)243 995 y Ft(\017)23 b Fq(C)327 1002 y Fh(P)356 995 y Fq(.send)p 477 995 V 18 w(con)o(text)17 b(=)h Fx(C)770 1002 y Fh(Q)800 995 y Fq(.receiv)o(e)p 976 995 V 18 w(con)o(text)d Fx(and)h(the)f(con)o(text)g(is)g(unique)i(in)f Ft(Q)p Fx(;)289 1052 y Fq(C)327 1059 y Fh(P)356 1052 y Fq(.receiv)o(e)p 532 1052 V 18 w(con)o(text)h(=)35 b(C)847 1059 y Fh(Q)877 1052 y Fq(.send)p 998 1052 V 18 w(con)o(text)16 b Fx(and)f(this)h(con)o (text)e(is)i(unique)h(in)f Ft(P)t Fx(.)243 1125 y Ft(\017)23 b Fq(C)327 1132 y Fh(P)356 1125 y Fq(.source)14 b Fx(is)i(rank)f(of)f Fq(P)h Fx(in)h Ft(P)j Fx(and)c Fq(C)1013 1132 y Fh(Q)1044 1125 y Fq(.source)f Fx(is)i(rank)f(of)g Fq(Q)g Fx(in)h Ft(Q)p Fx(.)189 1219 y(Assume)j(that)f Fq(P)g Fx(sends)i(a)e(message)g (to)h Fq(Q)g Fx(using)g(the)g(in)o(ter-comm)o(unicator.)31 b(Then)19 b Fq(P)g Fx(uses)189 1275 y(the)g Fq(group)f Fx(table)i(to)e(\014nd)h(the)g(absolute)g(address)g(of)g Fq(Q)p Fx(;)f Fq(source)h Fx(and)g Fq(send)p 1611 1275 V 18 w(con)o(text)g Fx(are)189 1332 y(app)q(ended)d(to)f(the)g (message.)189 1407 y(Assume)22 b(that)f Fq(Q)h Fx(p)q(osts)f(a)h (receiv)o(e)g(with)h(an)e(explicit)j(source)e(argumen)o(t)f(using)i (the)f(in)o(ter-)189 1463 y(comm)o(unicator.)c(Then)c Fq(Q)f Fx(matc)o(hes)g Fq(receiv)o(e)p 1006 1463 V 18 w(con)o(text)g Fx(to)f(the)i(message)e(con)o(text)h(and)g(source)189 1520 y(argumen)o(t)h(to)h(the)g(message)g(source.)189 1595 y(The)g(same)g(algorithm)g(is)h(appropriate)f(for)g(in)o(tra-comm) o(unicators)f(as)h(w)o(ell.)189 1670 y(In)g(order)g(to)f(supp)q(ort)h (in)o(ter-comm)o(unicator)g(accessors)g(and)g(constructors,)f(it)h(is)g (necessary)h(to)189 1726 y(supplemen)o(t)j(this)f(mo)q(del)h(with)f (additional)i(structures,)e(that)f(store)g(information)h(ab)q(out)g (the)189 1783 y(lo)q(cal)i(comm)o(unication)f(group,)h(and)f (additional)h(safe)f(con)o(texts.)30 b(\()p Fl(End)19 b(of)h(advic)n(e)g(to)g(imple-)189 1839 y(mentors.)p Fx(\))75 1961 y Fj(5.6.1)49 b(Inter-comm)n(unicato)o(r)13 b(Accesso)o(rs)75 2141 y Fv(MPI)p 160 2141 14 2 v 16 w(COMM)p 318 2141 V 16 w(TEST)p 448 2141 V 17 w(INTER\(comm)m(,)f (\015ag\))117 2218 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))117 2294 y(OUT)108 b Fv(\015ag)518 b Fk(\(logical\))75 2418 y Fu(int)23 b(MPI)p 245 2418 15 2 v 17 w(Comm)p 358 2418 V 17 w(test)p 471 2418 V 16 w(inter\(MPI)p 703 2418 V 16 w(Comm)h(comm,)f(int)g(*flag\))75 2504 y(MPI)p 150 2504 V 17 w(COMM)p 263 2504 V 16 w(TEST)p 375 2504 V 17 w(INTER\(COMM,)f(FLAG,)i(IERROR\))170 2561 y(INTEGER)f(COMM,)h (IERROR)170 2617 y(LOGICAL)f(FLAG)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 155 163 155 162 bop 75 -100 a Fm(5.6.)34 b(INTER-COMMUNICA)l(TION)1047 b Fx(155)75 45 y(This)20 b(lo)q(cal)g(routine)g(allo)o(ws)f(the)g (calling)i(pro)q(cess)f(to)e(determine)i(if)g(a)f(comm)o(unicator)g(is) g(an)h(in)o(ter-)75 102 y(comm)o(unicator)g(or)h(an)f(in)o(tra-comm)o (unicator.)36 b(It)21 b(returns)g Fi(true)g Fx(if)g(it)g(is)h(an)e(in)o (ter-comm)o(unicator,)75 158 y(otherwise)15 b Fi(false)p Fx(.)166 214 y(When)j(an)f(in)o(ter-comm)o(unicator)h(is)g(used)g(as)f (an)g(input)i(argumen)o(t)d(to)h(the)h(comm)o(unicator)f(ac-)75 271 y(cessors)d(describ)q(ed)i(ab)q(o)o(v)o(e)e(under)h(in)o(tra-comm)o (unication,)f(the)h(follo)o(wing)f(table)h(describ)q(es)h(b)q(eha)o (vior.)p 270 365 1410 2 v 269 422 2 57 v 652 405 a Fv(MPI)p 737 405 14 2 v 16 w(COMM)p 895 405 V 17 w(*)e Fx(F)l(unction)j(Beha)o (vior)p 1679 422 2 57 v 269 478 V 655 461 a(\(in)f(In)o(ter-Comm)o (unication)f(Mo)q(de\))p 1679 478 V 270 480 1410 2 v 270 490 V 269 546 2 57 v 295 529 a Fv(MPI)p 380 529 14 2 v 16 w(COMM)p 538 529 V 16 w(SIZE)p 729 546 2 57 v 111 w Fx(returns)g(the)g(size)h(of)f(the)g(lo)q(cal)i(group.)p 1679 546 V 269 603 V 295 586 a Fv(MPI)p 380 586 14 2 v 16 w(COMM)p 538 586 V 16 w(GROUP)p 729 603 2 57 v 52 w Fx(returns)e(the)g(lo)q(cal)h(group.)p 1679 603 V 269 659 V 295 642 a Fv(MPI)p 380 642 14 2 v 16 w(COMM)p 538 642 V 16 w(RANK)p 729 659 2 57 v 81 w Fx(returns)f(the)g(rank)g(in)h (the)f(lo)q(cal)i(group)p 1679 659 V 270 661 1410 2 v 75 746 a(F)l(urthermore,)d(the)g(op)q(eration)g Fv(MPI)p 715 746 14 2 v 16 w(COMM)p 873 746 V 17 w(COMP)l(ARE)h Fx(is)g(v)m(alid)h(for)e(in)o(ter-comm)o(unicators.)19 b(Both)75 803 y(comm)o(unicators)13 b(m)o(ust)h(b)q(e)g(either)g(in)o (tra-)g(or)f(in)o(ter-comm)o(unicators,)h(or)f(else)i Fi(MPI)p 1515 803 13 2 v 14 w(UNEQUAL)e Fx(results.)75 859 y(Both)22 b(corresp)q(onding)g(lo)q(cal)h(and)f(remote)f(groups)h (m)o(ust)f(compare)g(correctly)h(to)f(get)h(the)g(results)75 916 y Fi(MPI)p 152 916 V 14 w(CONGRUENT)c Fx(and)g Fi(MPI)p 603 916 V 15 w(SIMILAR)p Fx(.)f(In)i(particular,)h(it)e(is)h(p)q (ossible)i(for)d Fi(MPI)p 1515 916 V 14 w(SIMILAR)g Fx(to)g(result)75 972 y(b)q(ecause)e(either)g(the)f(lo)q(cal)i(or)d(remote)h(groups)g(w)o (ere)g(similar)h(but)f(not)g(iden)o(tical.)166 1029 y(The)23 b(follo)o(wing)g(accessors)f(pro)o(vide)h(consisten)o(t)f(access)h(to)f (the)g(remote)g(group)g(of)g(an)h(in)o(ter-)75 1085 y(comm)o(unicator:) 166 1142 y(The)15 b(follo)o(wing)h(are)f(all)h(lo)q(cal)h(op)q (erations.)75 1293 y Fv(MPI)p 160 1293 14 2 v 16 w(COMM)p 318 1293 V 16 w(REMOTE)p 521 1293 V 17 w(SIZE\(comm)m(,)12 b(size\))117 1370 y Fk(IN)155 b Fv(comm)466 b Fk(in)o(ter-comm)o (unicator)11 b(\(handle\))117 1445 y(OUT)108 b Fv(size)519 b Fk(n)o(um)o(b)q(er)19 b(of)g(pro)q(cesses)j(in)d(the)h(remote)f (group)g(of)38 b Fi(comm)905 1501 y Fk(\(in)o(teger\))75 1626 y Fu(int)23 b(MPI)p 245 1626 15 2 v 17 w(Comm)p 358 1626 V 17 w(remote)p 519 1626 V 16 w(size\(MPI)p 727 1626 V 16 w(Comm)h(comm,)f(int)g(*size\))75 1712 y(MPI)p 150 1712 V 17 w(COMM)p 263 1712 V 16 w(REMOTE)p 423 1712 V 17 w(SIZE\(COMM,)f(SIZE,)h(IERROR\))170 1769 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 1950 y Fv(MPI)p 160 1950 14 2 v 16 w(COMM)p 318 1950 V 16 w(REMOTE)p 521 1950 V 17 w(GROUP\(comm)o(,)11 b(group\))117 2027 y Fk(IN)155 b Fv(comm)466 b Fk(in)o(ter-comm)o(unicator)11 b(\(handle\))117 2102 y(OUT)108 b Fv(group)479 b Fk(remote)14 b(group)f(corresp)q(onding)i(to)f Fi(comm)8 b Fk(\(handle\))75 2227 y Fu(int)23 b(MPI)p 245 2227 15 2 v 17 w(Comm)p 358 2227 V 17 w(remote)p 519 2227 V 16 w(group\(MPI)p 751 2227 V 16 w(Comm)g(comm,)h(MPI)p 1102 2227 V 16 w(Group)g(*group\)) 75 2313 y(MPI)p 150 2313 V 17 w(COMM)p 263 2313 V 16 w(REMOTE)p 423 2313 V 17 w(GROUP\(COMM,)e(GROUP,)h(IERROR\))170 2369 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)189 2506 y Fl(R)n(ationale.)83 b Fx(Symmetric)23 b(access)g(to)f(b)q(oth)g(the)h(lo)q(cal)g(and)g (remote)f(groups)g(of)g(an)h(in)o(ter-)189 2562 y(comm)o(unicator)17 b(is)h(imp)q(ortan)o(t,)g(so)f(this)h(function,)h(as)f(w)o(ell)g(as)g Fv(MPI)p 1407 2562 14 2 v 16 w(COMM)p 1565 2562 V 16 w(REMOTE)p 1768 2562 V 17 w(SIZE)189 2619 y Fx(ha)o(v)o(e)c(b)q(een)j (pro)o(vided.)j(\()p Fl(End)c(of)g(r)n(ationale.)p Fx(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 156 164 156 163 bop 75 -100 a Fx(156)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fj(5.6.2)49 b(Inter-comm)n(unicato)o(r)13 b(Op)q(erations)75 131 y Fx(This)18 b(section)g(in)o(tro)q(duces)g(four)f(blo)q(c)o(king)h (in)o(ter-comm)o(unicator)g(op)q(erations.)26 b Fv(MPI)p 1573 131 14 2 v 16 w(INTERCOMM-)p 75 187 V 89 187 a(CREA)l(TE)13 b Fx(is)f(used)h(to)f(bind)h(t)o(w)o(o)e(in)o(tra-comm)o(unicators)h (in)o(to)g(an)g(inter-comm)o(unicator;)h(the)f(function)75 244 y Fv(MPI)p 160 244 V 16 w(INTERCOMM)p 450 244 V 16 w(MERGE)h Fx(creates)f(an)g(in)o(tra-comm)o(unicator)g(b)o(y)g(merging) h(the)f(lo)q(cal)h(and)g(remote)75 300 y(groups)18 b(of)g(an)g(in)o (ter-comm)o(unicator.)30 b(The)19 b(functions)p Fv(MPI)p 1140 300 V 16 w(COMM)p 1298 300 V 17 w(DUP)f Fx(and)p Fv(MPI)p 1581 300 V 16 w(COMM)p 1739 300 V 17 w(FREE)p Fx(,)75 357 y(in)o(tro)q(duced)e(previously)l(,)h(duplicate)g(and)e (free)g(an)h(in)o(ter-comm)o(unicator,)e(resp)q(ectiv)o(ely)l(.)166 413 y(Ov)o(erlap)23 b(of)f(lo)q(cal)h(and)g(remote)f(groups)g(that)g (are)g(b)q(ound)h(in)o(to)g(an)f(in)o(ter-comm)o(unicator)g(is)75 470 y(prohibited.)h(If)15 b(there)h(is)g(o)o(v)o(erlap,)f(then)h(the)f (program)g(is)h(erroneous)f(and)h(is)g(lik)o(ely)h(to)e(deadlo)q(c)o (k.)22 b(\(If)75 526 y(a)16 b(pro)q(cess)g(is)g(m)o(ultithreaded,)h (and)f Fv(MPI)f Fx(calls)i(blo)q(c)o(k)g(only)f(a)g(thread,)f(rather)g (than)h(a)g(pro)q(cess,)g(then)75 583 y(\\dual)f(mem)o(b)q(ership")g (can)f(b)q(e)g(supp)q(orted.)20 b(It)14 b(is)h(then)f(the)g(user's)g (resp)q(onsibilit)o(y)j(to)c(mak)o(e)h(sure)g(that)75 639 y(calls)i(on)f(b)q(ehalf)i(of)e(the)g(t)o(w)o(o)f(\\roles")h(of)f (a)h(pro)q(cess)h(are)f(executed)h(b)o(y)f(t)o(w)o(o)f(indep)q(enden)o (t)j(threads.\))166 695 y(The)12 b(function)h Fv(MPI)p 516 695 V 16 w(INTERCOMM)p 806 695 V 17 w(CREA)l(TE)g Fx(can)g(b)q(e)f(used)h(to)f(create)g(an)g(in)o(ter-comm)o(unicator)75 752 y(from)19 b(t)o(w)o(o)g(existing)i(in)o(tra-comm)o(unicators,)g(in) g(the)f(follo)o(wing)h(situation:)30 b(A)o(t)20 b(least)g(one)h (selected)75 808 y(mem)o(b)q(er)f(from)g(eac)o(h)g(group)g(\(the)g (\\group)g(leader"\))g(has)g(the)h(abilit)o(y)g(to)f(comm)o(unicate)g (with)h(the)75 865 y(selected)14 b(mem)o(b)q(er)f(from)g(the)g(other)f (group;)h(that)g(is,)g(a)g(\\p)q(eer")g(comm)o(unicator)f(exists)i(to)e (whic)o(h)i(b)q(oth)75 921 y(leaders)e(b)q(elong,)g(and)g(eac)o(h)f (leader)h(kno)o(ws)e(the)i(rank)e(of)h(the)g(other)g(leader)h(in)g (this)g(p)q(eer)f(comm)o(unicator)75 978 y(\(the)18 b(t)o(w)o(o)e (leaders)j(could)g(b)q(e)g(the)f(same)f(pro)q(cess\).)28 b(F)l(urthermore,)18 b(mem)o(b)q(ers)g(of)g(eac)o(h)g(group)f(kno)o(w) 75 1034 y(the)e(rank)g(of)g(their)h(leader.)166 1091 y(Construction)10 b(of)h(an)f(in)o(ter-comm)o(unicator)h(from)f(t)o(w)o (o)f(in)o(tra-comm)o(unicators)h(requires)h(separate)75 1147 y(collectiv)o(e)16 b(op)q(erations)e(in)h(the)f(lo)q(cal)h(group)e (and)i(in)f(the)g(remote)g(group,)f(as)h(w)o(ell)h(as)f(a)f(p)q(oin)o (t-to-p)q(oin)o(t)75 1204 y(comm)o(unication)j(b)q(et)o(w)o(een)f(a)g (pro)q(cess)h(in)g(the)f(lo)q(cal)h(group)f(and)h(a)f(pro)q(cess)g(in)h (the)f(remote)g(group.)166 1260 y(In)g(standard)g Fv(MPI)f Fx(implemen)o(tations)i(\(with)f(static)g(pro)q(cess)g(allo)q(cation)h (at)f(initialization\),)i(the)75 1316 y Fv(MPI)p 160 1316 V 16 w(COMM)p 318 1316 V 16 w(W)o(ORLD)j Fx(comm)o(unicator)f (\(or)f(preferably)i(a)f(dedicated)i(duplicate)g(thereof)t(\))e(can)g (b)q(e)75 1373 y(this)e(p)q(eer)g(comm)o(unicator.)24 b(In)18 b(dynamic)f Fv(MPI)f Fx(implemen)o(tations,)i(where,)f(for)f (example,)h(a)g(pro)q(cess)75 1429 y(ma)o(y)g(spa)o(wn)h(new)g(c)o (hild)i(pro)q(cesses)f(during)g(an)f Fv(MPI)f Fx(execution,)j(the)e (paren)o(t)f(pro)q(cess)i(ma)o(y)e(b)q(e)i(the)75 1486 y(\\bridge")d(b)q(et)o(w)o(een)g(the)g(old)g(comm)o(unication)h(univ)o (erse)g(and)f(the)f(new)h(comm)o(unication)h(w)o(orld)f(that)75 1542 y(includes)h(the)f(paren)o(t)f(and)g(its)g(c)o(hildren.)166 1599 y(The)25 b(application)h(top)q(ology)e(functions)h(describ)q(ed)h (in)g(c)o(hapter)e(6)g(do)g(not)g(apply)i(to)e(in)o(ter-)75 1655 y(comm)o(unicators.)18 b(Users)11 b(that)g(require)h(this)g (capabilit)o(y)h(should)f(utilize)i Fv(MPI)p 1418 1655 V 16 w(INTERCOMM)p 1708 1655 V 16 w(MERGE)75 1712 y Fx(to)g(build)j(an) e(in)o(tra-comm)o(unicator,)f(then)i(apply)f(the)g(graph)g(or)g (cartesian)g(top)q(ology)f(capabilities)k(to)75 1768 y(that)i(in)o(tra-comm)o(unicator,)h(creating)f(an)h(appropriate)f(top) q(ology-orien)o(ted)h(in)o(tra-comm)o(unicator.)75 1825 y(Alternativ)o(ely)l(,)f(it)e(ma)o(y)f(b)q(e)h(reasonable)g(to)g (devise)h(one's)e(o)o(wn)g(application)j(top)q(ology)d(mec)o(hanisms)75 1881 y(for)e(this)g(case,)g(without)g(loss)h(of)e(generalit)o(y)l(.) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 157 165 157 164 bop 75 -100 a Fm(5.6.)34 b(INTER-COMMUNICA)l(TION)1047 b Fx(157)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(INTERCOMM)p 450 45 V 16 w(CREA)l(TE\(lo)q(cal)p 741 45 V 17 w(comm)n(,)44 b(lo)q(cal)p 1022 45 V 16 w(leader,)k(p)q(eer)p 1291 45 V 17 w(comm)m(,)d(remote)p 1618 45 V 14 w(leader,)i(tag,)75 102 y(newintercomm)n(\))117 179 y Fk(IN)155 b Fv(lo)q(cal)p 409 179 V 17 w(comm)361 b Fk(lo)q(cal)13 b(in)o(tra-comm)o(unicator)e (\(handle\))117 254 y(IN)155 b Fv(lo)q(cal)p 409 254 V 17 w(leader)370 b Fk(rank)14 b(of)f(lo)q(cal)g(group)h(leader)g(in)g Fi(lo)q(cal)p 1517 254 13 2 v 15 w(comm)8 b Fk(\(in)o(teger\))117 329 y(IN)155 b Fv(p)q(eer)p 401 329 14 2 v 18 w(comm)368 b Fk(\\p)q(eer")15 b(in)o(tra-comm)n(unicator;)c(signi\014can)o(t)i (only)g(at)g(the)h Fi(lo-)905 385 y(cal)p 955 385 13 2 v 15 w(leader)i Fk(\(handle\))117 461 y(IN)155 b Fv(remote)p 455 461 14 2 v 14 w(leader)327 b Fk(rank)12 b(of)f(remote)g(group)h (leader)g(in)23 b Fi(p)q(eer)p 1548 461 13 2 v 16 w(comm)n Fk(;)8 b(signi\014can)o(t)905 517 y(only)13 b(at)h(the)h Fi(lo)q(cal)p 1201 517 V 15 w(leader)g Fk(\(in)o(teger\))117 592 y(IN)155 b Fv(tag)526 b Fk(\\safe")14 b(tag)f(\(in)o(teger\))117 667 y(OUT)108 b Fv(newintercomm)306 b Fk(new)15 b(in)o(ter-comm)o (unicator)c(\(handle\))75 792 y Fu(int)23 b(MPI)p 245 792 15 2 v 17 w(Intercomm)p 478 792 V 16 w(create\(MPI)p 734 792 V 16 w(Comm)g(local)p 989 792 V 17 w(comm,)g(int)g(local)p 1364 792 V 17 w(leader,)393 848 y(MPI)p 468 848 V 17 w(Comm)g(peer)p 700 848 V 17 w(comm,)g(int)h(remote)p 1100 848 V 16 w(leader,)f(int)g(tag,)393 905 y(MPI)p 468 905 V 17 w(Comm)g(*newintercomm\))75 991 y(MPI)p 150 991 V 17 w(INTERCOMM)p 383 991 V 16 w(CREATE\(LOCAL)p 687 991 V 15 w(COMM,)g(LOCAL)p 965 991 V 17 w(LEADER,)g(PEER)p 1269 991 V 16 w(COMM,)h(REMOTE)p 1573 991 V 16 w(LEADER,)f(TAG,)393 1048 y(NEWINTERCOMM,)f(IERROR\))170 1104 y(INTEGER)h(LOCAL)p 484 1104 V 17 w(COMM,)g(LOCAL)p 764 1104 V 17 w(LEADER,)f(PEER)p 1067 1104 V 17 w(COMM,)h(REMOTE)p 1371 1104 V 17 w(LEADER,)g(TAG,)170 1161 y(NEWINTERCOMM,)g(IERROR)75 1247 y Fx(This)d(call)h(creates)e(an)h (in)o(ter-comm)o(unicator.)33 b(It)20 b(is)g(collectiv)o(e)i(o)o(v)o (er)d(the)g(union)i(of)e(the)h(lo)q(cal)h(and)75 1303 y(remote)d(groups.)28 b(Pro)q(cesses)18 b(should)h(pro)o(vide)g(iden)o (tical)h Fv(lo)q(cal)p 1197 1303 14 2 v 16 w(comm)12 b Fx(and)18 b Fv(lo)q(cal)p 1525 1303 V 17 w(leader)g Fx(argumen)o(ts)75 1360 y(within)e(eac)o(h)g(group.)j(Wildcards)e(are)d (not)h(p)q(ermitted)h(for)f Fv(remo)o(te)p 1255 1360 V 14 w(leader,)g(lo)q(cal)p 1497 1360 V 16 w(leader)p Fx(,)g(and)h Fv(tag)p Fx(.)166 1416 y(This)g(call)g(uses)f(p)q(oin)o (t-to-p)q(oin)o(t)h(comm)o(unication)g(with)f(comm)o(unicator)g Fv(p)q(eer)p 1542 1416 V 17 w(comm)m Fx(,)c(and)16 b(with)75 1473 y(tag)e Fv(tag)g Fx(b)q(et)o(w)o(een)h(the)g(leaders.)20 b(Th)o(us,)15 b(care)f(m)o(ust)g(b)q(e)h(tak)o(en)f(that)g(there)h(b)q (e)g(no)g(p)q(ending)h(comm)o(uni-)75 1529 y(cation)f(on)g Fv(p)q(eer)p 357 1529 V 18 w(comm)9 b Fx(that)14 b(could)j(in)o (terfere)e(with)h(this)f(comm)o(unication.)189 1636 y Fl(A)n(dvic)n(e)j(to)i(users.)60 b Fx(W)l(e)19 b(recommend)g(using)g(a) f(dedicated)i(p)q(eer)f(comm)o(unicator,)g(suc)o(h)g(as)f(a)189 1692 y(duplicate)e(of)e(MPI)p 528 1692 V 16 w(COMM)p 696 1692 V 16 w(W)o(ORLD,)h(to)e(a)o(v)o(oid)i(trouble)g(with)f(p)q (eer)h(comm)o(unicators.)20 b(\()p Fl(End)189 1748 y(of)c(advic)n(e)g (to)h(users.)p Fx(\))75 1949 y Fv(MPI)p 160 1949 V 16 w(INTERCOMM)p 450 1949 V 16 w(MERGE\(intercomm)o(,)11 b(high,)16 b(newintracomm)n(\))117 2026 y Fk(IN)155 b Fv(intercomm)380 b Fk(In)o(ter-Comm)o(unicator)11 b(\(handle\))117 2102 y(IN)155 b Fv(high)507 b Fk(\(logical\))117 2177 y(OUT)108 b Fv(newintracomm)304 b Fk(new)15 b(in)o(tra-comm)n(unicator) c(\(handle\))75 2301 y Fu(int)23 b(MPI)p 245 2301 15 2 v 17 w(Intercomm)p 478 2301 V 16 w(merge\(MPI)p 710 2301 V 16 w(Comm)g(intercomm,)g(int)g(high,)393 2358 y(MPI)p 468 2358 V 17 w(Comm)g(*newintracomm\))75 2444 y(MPI)p 150 2444 V 17 w(INTERCOMM)p 383 2444 V 16 w(MERGE\(INTERCOMM,)e (HIGH,)j(INTRACOMM,)e(IERROR\))170 2501 y(INTEGER)h(INTERCOMM,)g (INTRACOMM,)f(IERROR)170 2557 y(LOGICAL)h(HIGH)75 2643 y Fx(This)e(function)f(creates)g(an)g(in)o(tra-comm)o(unicator)g(from)f (the)h(union)h(of)f(the)g(t)o(w)o(o)e(groups)i(that)f(are)75 2700 y(asso)q(ciated)f(with)g Fv(intercomm)n Fx(.)24 b(All)19 b(pro)q(cesses)f(should)h(pro)o(vide)f(the)g(same)f Fv(high)h Fx(v)m(alue)h(within)g(eac)o(h)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 158 166 158 165 bop 75 -100 a Fx(158)370 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)383 4 y 18945146 4357381 0 0 24865505 5920358 startTexFig 383 4 a %%BeginDocument: context-fig-1.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def /col8 {.68 .85 .9 setrgbcolor} bind def /col9 {0 .39 0 setrgbcolor} bind def /col10 {.65 .17 .17 setrgbcolor} bind def /col11 {1 .51 0 setrgbcolor} bind def /col12 {.63 .13 .94 setrgbcolor} bind def /col13 {1 .75 .8 setrgbcolor} bind def /col14 {.7 .13 .13 setrgbcolor} bind def /col15 {1 .84 0 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -90.0 198.0 translate 0.900 -0.900 scale 0.500 setlinewidth n 199 219 m 199 119 l 99 119 l 99 219 l clp gs col-1 s gr n 359 219 m 359 119 l 259 119 l 259 219 l clp gs col-1 s gr n 519 219 m 519 119 l 419 119 l 419 219 l clp gs col-1 s gr 1.000 setlinewidth n 220.000 173.000 m 204.000 169.000 l 220.000 165.000 l gs 2 setlinejoin col-1 s gr n 204 169 m 254 169 l gs col-1 s gr n 238.000 165.000 m 254.000 169.000 l 238.000 173.000 l gs 2 setlinejoin col-1 s gr n 380.000 173.000 m 364.000 169.000 l 380.000 165.000 l gs 2 setlinejoin col-1 s gr n 364 169 m 414 169 l gs col-1 s gr n 398.000 165.000 m 414.000 169.000 l 398.000 173.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 16.00 scalefont setfont 279 174 m gs 1 -1 scale (Group 1) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 439 174 m gs 1 -1 scale (Group 2) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 174 m gs 1 -1 scale (Group 0) col-1 show gr $F2psEnd %%EndDocument endTexFig 644 378 a Fx(Figure)g(5.1:)k(Three-group)c(pip)q(eline)q(.)75 509 y(of)h(the)g(t)o(w)o(o)e(groups.)22 b(If)16 b(pro)q(cesses)h(in)g (one)f(group)f(pro)o(vided)i(the)f(v)m(alue)h Fv(high)g(=)f(false)g Fx(and)h(pro)q(cesses)75 566 y(in)h(the)g(other)f(group)g(pro)o(vided)i (the)e(v)m(alue)i Fv(high)f(=)g(true)g Fx(then)g(the)f(union)i(orders)e (the)g(\\lo)o(w")g(group)75 622 y(b)q(efore)g(the)g(\\high")g(group.)23 b(If)17 b(all)h(pro)q(cesses)f(pro)o(vided)g(the)g(same)g Fv(high)g Fx(argumen)o(t)f(then)g(the)h(order)75 679 y(of)g(the)h(union)h(is)f(arbitrary)l(.)27 b(This)18 b(call)h(is)f(blo)q(c)o(king)h(and)f(collectiv)o(e)i(within)f(the)e (union)i(of)f(the)f(t)o(w)o(o)75 735 y(groups.)189 839 y Fl(A)n(dvic)n(e)27 b(to)h(implementors.)118 b Fx(The)29 b(implemen)o(tation)g(of)f Fv(MPI)p 1405 839 14 2 v 16 w(INTERCOMM)p 1695 839 V 17 w(MERGE)p Fx(,)189 895 y Fv(MPI)p 274 895 V 15 w(COMM)p 431 895 V 17 w(FREE)k Fx(and)g Fv(MPI)p 776 895 V 16 w(COMM)p 934 895 V 16 w(DUP)g Fx(are)f(similar)i(to)e(the)h(implemen)o(tation)h(of)189 952 y Fv(MPI)p 274 952 V 15 w(INTERCOMM)p 563 952 V 17 w(CREA)l(TE)p Fx(,)13 b(except)h(that)e(con)o(texts)f(priv)m(ate)j(to)e (the)h(input)g(inter-comm)o(un-)189 1008 y(icator)h(are)h(used)h(for)e (comm)o(unication)i(b)q(et)o(w)o(een)g(group)e(leaders)i(rather)f(than) g(con)o(texts)f(inside)189 1065 y(a)h(bridge)h(comm)o(unicator.)j(\()p Fl(End)d(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))75 1186 y Fj(5.6.3)49 b(Inter-Comm)n(unication)14 b(Examples)75 1272 y Fv(Example)f(1:)19 b(Three-Group)e(\\Pip)q(eline")75 1358 y Fx(Groups)f(0)g(and)h(1)f(comm)o(unicate.)24 b(Groups)16 b(1)h(and)f(2)g(comm)o(unicate.)24 b(Therefore,)17 b(group)f(0)g (requires)75 1414 y(one)f(in)o(ter-comm)o(unicator,)f(group)g(1)g (requires)i(t)o(w)o(o)d(in)o(ter-comm)o(unicators,)h(and)h(group)f(2)g (requires)h(1)75 1471 y(in)o(ter-comm)o(unicator.)147 1574 y Fu(main\(int)22 b(argc,)i(char)f(**argv\))147 1631 y({)194 1687 y(MPI_Comm)71 b(myComm;)166 b(/*)23 b(intra-communicator)f(of)h(local)h(sub-group)e(*/)194 1744 y(MPI_Comm)71 b(myFirstComm;)46 b(/*)23 b(inter-communicator)f(*/) 194 1800 y(MPI_Comm)71 b(mySecondComm;)22 b(/*)h(second)h (inter-communicator)d(\(group)i(1)h(only\))f(*/)194 1857 y(int)h(membershipKey;)194 1913 y(int)g(rank;)194 2026 y(MPI_Init\(&argc,)e(&argv\);)194 2083 y(MPI_Comm_rank\(MPI_COMM_WORLD) o(,)f(&rank\);)194 2195 y(/*)j(User)f(code)h(must)f(generate)g (membershipKey)f(in)h(the)h(range)f([0,)g(1,)h(2])g(*/)194 2252 y(membershipKey)e(=)i(rank)f(\045)h(3;)194 2365 y(/*)g(Build)f(intra-communicator)e(for)j(local)f(sub-group)g(*/)194 2421 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e(membershipKey,)h(rank,)h (&myComm\);)194 2534 y(/*)h(Build)f(inter-communicators.)45 b(Tags)23 b(are)h(hard-coded.)e(*/)194 2591 y(if)i(\(membershipKey)e (==)h(0\))194 2647 y({)501 b(/*)24 b(Group)f(0)h(communicates)e(with)h (group)h(1.)f(*/)242 2704 y(MPI_Intercomm_create\()e(myComm,)i(0,)h (MPI_COMM_WORLD,)d(1,)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 159 167 159 166 bop 75 -100 a Fm(5.6.)29 b(INTER-COMMUNICA)l(TION)1052 b Fx(159)383 4 y 18945146 5115185 0 0 28417720 7696465 startTexFig 383 4 a %%BeginDocument: context-fig-2.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def /col8 {.68 .85 .9 setrgbcolor} bind def /col9 {0 .39 0 setrgbcolor} bind def /col10 {.65 .17 .17 setrgbcolor} bind def /col11 {1 .51 0 setrgbcolor} bind def /col12 {.63 .13 .94 setrgbcolor} bind def /col13 {1 .75 .8 setrgbcolor} bind def /col14 {.7 .13 .13 setrgbcolor} bind def /col15 {1 .84 0 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -63.0 198.0 translate 0.900 -0.900 scale 0.500 setlinewidth n 199 219 m 199 119 l 99 119 l 99 219 l clp gs col-1 s gr n 359 219 m 359 119 l 259 119 l 259 219 l clp gs col-1 s gr n 519 219 m 519 119 l 419 119 l 419 219 l clp gs col-1 s gr 1.000 setlinewidth n 220.000 173.000 m 204.000 169.000 l 220.000 165.000 l gs 2 setlinejoin col-1 s gr n 204 169 m 254 169 l gs col-1 s gr n 238.000 165.000 m 254.000 169.000 l 238.000 173.000 l gs 2 setlinejoin col-1 s gr n 380.000 173.000 m 364.000 169.000 l 380.000 165.000 l gs 2 setlinejoin col-1 s gr n 364 169 m 414 169 l gs col-1 s gr n 398.000 165.000 m 414.000 169.000 l 398.000 173.000 l gs 2 setlinejoin col-1 s gr n 78.000 165.000 m 94.000 169.000 l 78.000 173.000 l gs 2 setlinejoin col-1 s gr n 94 169 m 69 169 l 69 89 l 549 89 l 549 169 l 524 169 l gs col-1 s gr n 540.000 173.000 m 524.000 169.000 l 540.000 165.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 16.00 scalefont setfont 279 174 m gs 1 -1 scale (Group 1) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 439 174 m gs 1 -1 scale (Group 2) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 174 m gs 1 -1 scale (Group 0) col-1 show gr $F2psEnd %%EndDocument endTexFig 682 426 a Fx(Figure)16 b(5.2:)i(Three-group)e(ring.)743 559 y Fu(1,)24 b(&myFirstComm\);)194 615 y(})194 672 y(else)g(if)f(\(membershipKey)f(==)i(1\))194 728 y({)334 b(/*)24 b(Group)f(1)h(communicates)e(with)h(groups)g(0)h(and)g(2.)f(*/) 242 785 y(MPI_Intercomm_create\()e(myComm,)i(0,)h(MPI_COMM_WORLD,)d(0,) 743 841 y(1,)j(&myFirstComm\);)242 898 y(MPI_Intercomm_create\()d (myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 954 y(12,)j(&mySecondComm\);) 194 1011 y(})194 1067 y(else)g(if)f(\(membershipKey)f(==)i(2\))194 1123 y({)501 b(/*)24 b(Group)f(2)h(communicates)e(with)h(group)h(1.)f (*/)242 1180 y(MPI_Intercomm_create\()e(myComm,)i(0,)h(MPI_COMM_WORLD,) d(1,)743 1236 y(12,)j(&myFirstComm\);)194 1293 y(})194 1406 y(/*)g(Do)f(work)h(...)f(*/)194 1519 y(switch\(membershipKey\))45 b(/*)24 b(free)f(communicators)f(appropriately)g(*/)194 1575 y({)194 1632 y(case)i(1:)266 1688 y(MPI_Comm_free\(&mySecondCo)o (mm\);)194 1744 y(case)g(0:)194 1801 y(case)g(2:)266 1857 y(MPI_Comm_free\(&myFirstCom)o(m\);)266 1914 y(break;)194 1970 y(})194 2083 y(MPI_Finalize\(\);)147 2140 y(})75 2260 y Fv(Example)13 b(2:)19 b(Three-Group)e(\\Ring")75 2346 y Fx(Groups)12 b(0)g(and)h(1)f(comm)o(unicate.)19 b(Groups)13 b(1)f(and)g(2)h(comm)o(unicate.)19 b(Groups)12 b(0)g(and)h(2)f(comm)o(unicate.)75 2402 y(Therefore,)j(eac)o(h)g (requires)h(t)o(w)o(o)e(in)o(ter-comm)o(unicators.)147 2508 y Fu(main\(int)22 b(argc,)i(char)f(**argv\))147 2565 y({)194 2621 y(MPI_Comm)71 b(myComm;)142 b(/*)24 b(intra-communicator)d(of)j(local)f(sub-group)f(*/)194 2678 y(MPI_Comm)71 b(myFirstComm;)22 b(/*)i(inter-communicators)d(*/) -32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 160 168 160 167 bop 75 -100 a Fx(160)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)194 45 y Fu(MPI_Comm)71 b(mySecondComm;)194 102 y(MPI_Status)23 b(status;)194 158 y(int)h(membershipKey;)194 214 y(int)g(rank;)194 327 y(MPI_Init\(&argc,)e(&argv\);)194 384 y (MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&rank\);)194 440 y(...)194 553 y(/*)j(User)f(code)h(must)f(generate)g(membershipKey)f (in)h(the)h(range)f([0,)g(1,)h(2])g(*/)194 610 y(membershipKey)e(=)i (rank)f(\045)h(3;)194 723 y(/*)g(Build)f(intra-communicator)e(for)j (local)f(sub-group)g(*/)194 779 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e (membershipKey,)h(rank,)h(&myComm\);)194 892 y(/*)h(Build)f (inter-communicators.)45 b(Tags)23 b(are)h(hard-coded.)e(*/)194 948 y(if)i(\(membershipKey)e(==)h(0\))194 1005 y({)310 b(/*)24 b(Group)f(0)h(communicates)e(with)h(groups)g(1)h(and)g(2.)f(*/) 242 1061 y(MPI_Intercomm_create\()e(myComm,)i(0,)h(MPI_COMM_WORLD,)d (1,)743 1118 y(1,)j(&myFirstComm\);)242 1174 y(MPI_Intercomm_create\()d (myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 1231 y(2,)j(&mySecondComm\);) 194 1287 y(})194 1344 y(else)g(if)f(\(membershipKey)f(==)i(1\))194 1400 y({)215 b(/*)24 b(Group)f(1)g(communicates)g(with)g(groups)g(0)h (and)f(2.)h(*/)242 1456 y(MPI_Intercomm_create\()d(myComm,)i(0,)h (MPI_COMM_WORLD,)d(0,)743 1513 y(1,)j(&myFirstComm\);)242 1569 y(MPI_Intercomm_create\()d(myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 1626 y(12,)j(&mySecondComm\);)194 1682 y(})194 1739 y(else)g(if)f (\(membershipKey)f(==)i(2\))194 1795 y({)191 b(/*)24 b(Group)f(2)h(communicates)e(with)h(groups)g(0)h(and)f(1.)h(*/)242 1852 y(MPI_Intercomm_create\()d(myComm,)i(0,)h(MPI_COMM_WORLD,)d(0,)743 1908 y(2,)j(&myFirstComm\);)242 1965 y(MPI_Intercomm_create\()d (myComm,)i(0,)h(MPI_COMM_WORLD,)d(1,)743 2021 y(12,)j(&mySecondComm\);) 194 2077 y(})194 2190 y(/*)g(Do)f(some)h(work)f(...)h(*/)194 2303 y(/*)g(Then)f(free)h(communicators)e(before)h(terminating...)f(*/) 194 2360 y(MPI_Comm_free\(&myFirstComm\);)194 2416 y (MPI_Comm_free\(&mySecondComm\))o(;)194 2473 y (MPI_Comm_free\(&myComm\);)194 2529 y(MPI_Finalize\(\);)147 2586 y(})1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 161 169 161 168 bop 75 -100 a Fm(5.6.)29 b(INTER-COMMUNICA)l(TION)1052 b Fx(161)75 45 y Fv(Example)13 b(3:)19 b(Building)e(Name)12 b(Service)j(fo)o(r)f(Intercomm)n(unication)75 131 y Fx(The)i(follo)o (wing)g(pro)q(cedures)h(exemplify)g(the)f(pro)q(cess)g(b)o(y)f(whic)o (h)i(a)e(user)h(could)h(create)e(name)h(service)75 187 y(for)g(building)j(in)o(tercomm)o(unicators)e(via)g(a)f(rendezv)o(ous)h (in)o(v)o(olving)h(a)f(serv)o(er)f(comm)o(unicator,)g(and)h(a)75 244 y(tag)d(name)h(selected)i(b)o(y)e(b)q(oth)g(groups.)166 300 y(After)k(all)h Fv(MPI)f Fx(pro)q(cesses)h(execute)g Fv(MPI)p 910 300 14 2 v 15 w(INIT)p Fx(,)f(ev)o(ery)g(pro)q(cess)h (calls)g(the)f(example)h(function,)75 357 y Fv(Init)p 141 357 V 17 w(server\(\))p Fx(,)c(de\014ned)i(b)q(elo)o(w.)26 b(Then,)18 b(if)g(the)f Fv(new)p 975 357 V 17 w(w)o(o)o(rld)f Fx(returned)h(is)h(NULL,)g(the)f(pro)q(cess)g(getting)75 413 y(NULL)e(is)g(required)h(to)e(implemen)o(t)i(a)e(serv)o(er)g (function,)h(in)g(a)f(reactiv)o(e)h(lo)q(op,)g Fv(Do)p 1502 413 V 16 w(server\(\))p Fx(.)j(Ev)o(ery)o(one)75 470 y(else)g(just)f(do)q(es)g(their)h(prescrib)q(ed)g(computation,)f (using)h Fv(new)p 1168 470 V 17 w(w)o(o)o(rld)e Fx(as)h(the)g(new)h (e\013ectiv)o(e)f(\\global")75 526 y(comm)o(unicator.)k(One)16 b(designated)h(pro)q(cess)e(calls)i Fv(Undo)p 1076 526 V 17 w(Server\(\))e Fx(to)g(get)g(rid)h(of)g(the)f(serv)o(er)h(when)g (it)75 583 y(is)g(not)f(needed)h(an)o(y)f(longer.)166 639 y(F)l(eatures)g(of)g(this)g(approac)o(h)g(include:)143 745 y Ft(\017)23 b Fx(Supp)q(ort)15 b(for)g(m)o(ultiple)i(name)e(serv)o (ers)143 839 y Ft(\017)23 b Fx(Abilit)o(y)16 b(to)f(scop)q(e)h(the)f (name)g(serv)o(ers)g(to)f(sp)q(eci\014c)k(pro)q(cesses)143 933 y Ft(\017)23 b Fx(Abilit)o(y)16 b(to)f(mak)o(e)g(suc)o(h)g(serv)o (ers)g(come)g(and)g(go)g(as)g(desired.)75 1039 y Fu(#define)23 b(INIT_SERVER_TAG_1)e(666)75 1096 y(#define)i(UNDO_SERVER_TAG_1)93 b(777)75 1209 y(static)23 b(int)g(server_key_val;)75 1321 y(/*)h(for)f(attribute)g(management)f(for)i(server_comm,)46 b(copy)23 b(callback:)g(*/)75 1378 y(void)g(handle_copy_fn\(MPI_Comm)e (*oldcomm,)i(int)g(*keyval,)g(void)g(*extra_state,)75 1434 y(void)g(*attribute_val_in,)f(void)h(**attribute_val_out,)e(int)j (*flag\))75 1491 y({)147 1547 y(/*)f(copy)g(the)h(handle)f(*/)147 1604 y(*attribute_val_out)e(=)j(attribute_val_in;)147 1660 y(*flag)f(=)g(1;)h(/*)g(indicate)e(that)i(copy)f(to)h(happen)f(*/) 75 1717 y(})75 1830 y(int)g(Init_server\(peer_comm,)e(rank_of_server,)h (server_comm,)g(new_world\))75 1886 y(MPI_Comm)h(peer_comm;)75 1942 y(int)g(rank_of_server;)75 1999 y(MPI_Comm)g(*server_comm;)75 2055 y(MPI_Comm)g(*new_world;)94 b(/*)23 b(new)h(effective)e(world,)h (sans)h(server)f(*/)75 2112 y({)170 2168 y(MPI_Comm)g(temp_comm,)g (lone_comm;)170 2225 y(MPI_Group)g(peer_group,)f(temp_group;)170 2281 y(int)i(rank_in_peer_comm,)d(size,)i(color,)h(key)f(=)h(0;)170 2338 y(int)g(peer_leader,)e(peer_leader_rank_in_temp_co)o(mm;)170 2451 y(MPI_Comm_rank\(peer_comm,)f(&rank_in_peer_comm\);)170 2507 y(MPI_Comm_size\(peer_comm,)g(&size\);)170 2620 y(if)j(\(\(size)f(<)h(2\))f(||)h(\(0)g(>)f(rank_of_server\))f(||)i (\(rank_of_server)e(>=)h(size\)\))266 2676 y(return)g (\(MPI_ERR_OTHER\);)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 162 170 162 169 bop 75 -100 a Fx(162)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)170 102 y Fu(/*)24 b(create)f(two)h(communicators,)d(by)j(splitting)f (peer_comm)242 158 y(into)g(the)h(server)f(process,)g(and)g(everyone)g (else)g(*/)170 271 y(peer_leader)g(=)h(\(rank_of_server)d(+)j(1\))g (\045)f(size;)47 b(/*)24 b(arbitrary)f(choice)g(*/)170 384 y(if)h(\(\(color)f(=)h(\(rank_in_peer_comm)d(==)j (rank_of_server\)\)\))170 440 y({)266 497 y(MPI_Comm_split\(peer_comm,) c(color,)j(key,)h(&lone_comm\);)266 610 y(MPI_Intercomm_create\(lone)o (_comm,)d(0,)i(peer_comm,)g(peer_leader,)719 666 y(INIT_SERVER_TAG_1,)f (server_comm\);)266 779 y(MPI_Comm_free\(&lone_comm\))o(;)266 835 y(*new_world)g(=)i(\(MPI_Comm\))f(0;)170 892 y(})170 948 y(else)170 1005 y({)266 1061 y(MPI_Comm_Split\(peer_comm,)d(color,) j(key,)h(&temp_comm\);)266 1174 y(MPI_Comm_group\(peer_comm,)c (&peer_group\);)266 1231 y(MPI_Comm_group\(temp_comm,)g(&temp_group\);) 266 1287 y(MPI_Group_translate_ranks)o(\(peer_gr)o(oup,)h(1,)i (&peer_leader,)123 1344 y(temp_group,)f(&peer_leader_rank_in_temp_c)o (omm\);)266 1456 y(MPI_Intercomm_create\(temp)o(_comm,)f (peer_leader_rank_in_tem)o(p_comm,)719 1513 y(peer_comm,)i (rank_of_server,)719 1569 y(INIT_SERVER_TAG_1,)f(server_comm\);)266 1682 y(/*)h(attach)h(new_world)e(communication)g(attribute)h(to)g (server_comm:)g(*/)266 1795 y(/*)g(CRITICAL)g(SECTION)g(FOR)h (MULTITHREADING)e(*/)266 1852 y(if\(server_keyval)g(==)h (MPI_KEYVAL_INVALID\))266 1908 y({)361 1965 y(/*)h(acquire)f(the)g (process-local)f(name)i(for)f(the)h(server)f(keyval)g(*/)361 2021 y(MPI_Attr_keyval_create\(handle)o(_copy_f)o(n,)e(NULL,)1197 2077 y(&server_keyval,)g(NULL\);)266 2134 y(})266 2247 y(*new_world)h(=)i(temp_comm;)266 2360 y(/*)f(Cache)h(handle)f(of)g (intra-communicator)f(on)h(inter-communicator:)f(*/)266 2416 y(MPI_Attr_put\(server_comm,)e(server_keyval,)i(\(void)h (*\)\(*new_world\)\);)170 2473 y(})170 2586 y(return)g (\(MPI_SUCCESS\);)75 2642 y(})1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 163 171 163 170 bop 75 -100 a Fm(5.6.)29 b(INTER-COMMUNICA)l(TION)1052 b Fx(163)166 45 y(The)15 b(actual)h(serv)o(er)f(pro)q(cess)g(w)o(ould)h (commit)f(to)f(running)j(the)e(follo)o(wing)h(co)q(de:)75 139 y Fu(int)23 b(Do_server\(server_comm\))75 195 y(MPI_Comm)g (server_comm;)75 252 y({)170 308 y(void)h(init_queue\(\);)170 365 y(int)g(en_queue\(\),)e(de_queue\(\);)h(/*)g(keep)h(triplets)e(of)i (integers)910 421 y(for)g(later)f(matching)g(\(fns)g(not)g(shown\))g (*/)170 534 y(MPI_Comm)g(comm;)170 591 y(MPI_Status)g(status;)170 647 y(int)h(client_tag,)e(client_source;)170 703 y(int)i (client_rank_in_new_world,)c(pairs_rank_in_new_world;)170 760 y(int)k(buffer[10],)e(count)h(=)h(1;)170 873 y(void)g(*queue;)170 929 y(init_queue\(&queue\);)170 1099 y(for)g(\(;;\))170 1155 y({)266 1212 y(MPI_Recv\(buffer,)e(count,)h(MPI_INT,)f (MPI_ANY_SOURCE,)g(MPI_ANY_TAG,)481 1268 y(server_comm,)g(&status\);)h (/*)g(accept)g(from)g(any)h(client)f(*/)266 1381 y(/*)g(determine)g (client:)g(*/)266 1437 y(client_tag)f(=)i(status.MPI_TAG;)266 1494 y(client_source)e(=)i(status.MPI_SOURCE;)266 1550 y(client_rank_in_new_world)c(=)k(buffer[0];)266 1663 y(if)f(\(client_tag)g(==)g(UNDO_SERVER_TAG_1\))165 b(/*)24 b(client)f(that)1292 1720 y(terminates)g(server)g(*/)266 1776 y({)361 1833 y(while)h(\(de_queue\(queue,)d(MPI_ANY_TAG,)h (&pairs_rank_in_new_world,)743 1889 y(&pairs_rank_in_server\)\))457 1945 y(;)361 2058 y(MPI_Intercomm_free\(&server_co)o(mm\);)361 2115 y(break;)266 2171 y(})266 2284 y(if)h(\(de_queue\(queue,)f (client_tag,)h(&pairs_rank_in_new_worl)o(d,)648 2341 y(&pairs_rank_in_server\)\))266 2397 y({)361 2454 y(/*)h(matched)f (pair)g(with)g(same)h(tag,)f(tell)g(them)433 2510 y(about)g(each)g (other!)g(*/)361 2566 y(buffer[0])g(=)h(pairs_rank_in_new_world;)361 2623 y(MPI_Send\(buffer,)e(1,)i(MPI_INT,)e(client_src,)h(client_tag,) 1340 2679 y(server_comm\);)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 164 172 164 171 bop 75 -100 a Fx(164)370 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)361 102 y Fu(buffer[0])23 b(=)h(client_rank_in_new_world)o(;)361 158 y(MPI_Send\(buffer,)e(1,)i(MPI_INT,)e(pairs_rank_in_server,)f (client_tag,)576 214 y(server_comm\);)266 271 y(})266 327 y(else)361 384 y(en_queue\(queue,)h(client_tag,)h(client_source,) 1030 440 y(client_rank_in_new_worl)o(d\);)170 553 y(})75 610 y(})166 703 y Fx(A)18 b(particular)h(pro)q(cess)g(w)o(ould)f(b)q(e) h(resp)q(onsible)h(for)e(ending)h(the)g(serv)o(er)f(when)g(it)h(is)f (no)h(longer)75 760 y(needed.)i(Its)15 b(call)i(to)d Fv(Undo)p 553 760 14 2 v 18 w(server)h Fx(w)o(ould)g(terminate)g(serv)o (er)g(function.)75 854 y Fu(int)23 b(Undo_server\(server_comm\))116 b(/*)24 b(example)f(client)g(that)g(ends)h(server)f(*/)75 910 y(MPI_Comm)g(*server_comm;)75 967 y({)170 1023 y(int)h(buffer)f(=)h (0;)170 1080 y(MPI_Send\(&buffer,)e(1,)i(MPI_INT,)e(0,)i (UNDO_SERVER_TAG_1,)d(*server_comm\);)170 1136 y (MPI_Intercomm_free\(server_com)o(m\);)75 1192 y(})166 1286 y Fx(The)15 b(follo)o(wing)g(is)g(a)f(blo)q(c)o(king)i (name-service)f(for)f(in)o(ter-comm)o(unication,)i(with)f(same)f(seman) o(tic)75 1343 y(restrictions)k(as)g Fv(MPI)p 457 1343 V 16 w(Intercomm)p 679 1343 V 10 w(create)p Fx(,)h(but)f(simpli\014ed)i (syn)o(tax.)27 b(It)18 b(uses)g(the)g(functionalit)o(y)h(just)75 1399 y(de\014ned)e(to)d(create)h(the)g(name)h(service.)75 1493 y Fu(int)23 b(Intercomm_name_create\(local_c)o(omm,)e (server_comm,)h(tag,)h(comm\))75 1549 y(MPI_Comm)g(local_comm,)f (server_comm;)75 1606 y(int)h(tag;)75 1662 y(MPI_Comm)g(*comm;)75 1719 y({)170 1775 y(int)h(error;)170 1832 y(int)g(found;)71 b(/*)23 b(attribute)g(acquisition)f(mgmt)i(for)f(new_world)g(*/)481 1888 y(/*)g(comm)h(in)f(server_comm)g(*/)170 1945 y(void)h(*val;)170 2058 y(MPI_Comm)f(new_world;)170 2170 y(int)h(buffer[10],)e(rank;)170 2227 y(int)i(local_leader)e(=)i(0;)170 2340 y (MPI_Attr_get\(server_comm,)d(server_keyval,)h(&val,)h(&found\);)170 2396 y(new_world)g(=)h(\(MPI_Comm\)val;)e(/*)h(retrieve)g(cached)g (handle)g(*/)170 2509 y(MPI_Comm_rank\(server_comm,)e(&rank\);)47 b(/*)23 b(rank)g(in)h(local)f(group)g(*/)170 2622 y(if)h(\(rank)f(==)h (local_leader\))170 2679 y({)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 165 173 165 172 bop 75 -100 a Fm(5.7.)34 b(CA)o(CHING)1407 b Fx(165)266 45 y Fu(buffer[0])23 b(=)g(rank;)266 102 y (MPI_Send\(&buffer,)e(1,)j(MPI_INT,)f(0,)g(tag,)h(server_comm\);)266 158 y(MPI_Recv\(&buffer,)d(1,)j(MPI_INT,)f(0,)g(tag,)h(server_comm\);) 170 214 y(})170 327 y(error)g(=)f(MPI_Intercomm_create\(local_le)o (ader,)e(local_comm,)h(buffer[0],)862 384 y(new_world,)h(tag,)g (comm\);)170 497 y(return\(error\);)75 553 y(})75 696 y Fn(5.7)59 b(Caching)75 798 y Fv(MPI)18 b Fx(pro)o(vides)g(a)g(\\cac)o (hing")g(facilit)o(y)h(that)f(allo)o(ws)g(an)g(application)i(to)d (attac)o(h)g(arbitrary)h(pieces)h(of)75 854 y(information,)24 b(called)g Fq(attributes)p Fx(,)g(to)e(comm)o(unicators.)40 b(More)22 b(precisely)l(,)k(the)c(cac)o(hing)h(facilit)o(y)75 910 y(allo)o(ws)15 b(a)g(p)q(ortable)h(library)g(to)e(do)i(the)f(follo) o(wing:)143 1003 y Ft(\017)23 b Fx(pass)14 b(information)g(b)q(et)o(w)o (een)h(calls)g(b)o(y)f(asso)q(ciating)h(it)f(with)h(an)f Fv(MPI)f Fx(in)o(tra-)i(or)e(inter-comm)o(un-)189 1059 y(icator,)143 1152 y Ft(\017)23 b Fx(quic)o(kly)16 b(retriev)o(e)g (that)e(information,)h(and)143 1245 y Ft(\017)23 b Fx(b)q(e)18 b(guaran)o(teed)e(that)h(out-of-date)g(information)g(is)h(nev)o(er)g (retriev)o(ed,)f(ev)o(en)h(if)g(the)f(comm)o(uni-)189 1302 y(cator)d(is)i(freed)f(and)h(its)f(handle)h(subsequen)o(tly)h (reused)f(b)o(y)f Fv(MPI)p Fx(.)166 1394 y(The)i(cac)o(hing)h (capabilities,)h(in)f(some)e(form,)g(are)h(required)h(b)o(y)e(built-in) k Fv(MPI)c Fx(routines)h(suc)o(h)g(as)75 1451 y(collectiv)o(e)22 b(comm)o(unication)e(and)g(application)i(top)q(ology)l(.)33 b(De\014ning)21 b(an)f(in)o(terface)g(to)f(these)h(capa-)75 1507 y(bilities)h(as)c(part)h(of)f(the)h Fv(MPI)g Fx(standard)f(is)i(v) m(aluable)h(b)q(ecause)f(it)f(p)q(ermits)h(routines)f(lik)o(e)h (collectiv)o(e)75 1563 y(comm)o(unication)h(and)f(application)h(top)q (ologies)g(to)e(b)q(e)h(implemen)o(ted)i(as)d(p)q(ortable)i(co)q(de,)g (and)f(also)75 1620 y(b)q(ecause)h(it)g(mak)o(es)e Fv(MPI)h Fx(more)g(extensible)i(b)o(y)e(allo)o(wing)h(user-written)g(routines)f (to)g(use)g(standard)75 1676 y Fv(MPI)c Fx(calling)i(sequences.)189 1780 y Fl(A)n(dvic)n(e)d(to)i(users.)40 b Fx(The)15 b(comm)o(unicator)f Fi(MPI)p 1009 1780 13 2 v 15 w(COMM)p 1154 1780 V 14 w(SELF)h Fx(is)g(a)f(suitable)i(c)o(hoice)g(for)e(p)q(osting)189 1837 y(pro)q(cess-lo)q(cal)21 b(attributes,)e(via)h(this)g (attributing-cac)o(hing)g(mec)o(hanism.)33 b(\()p Fl(End)20 b(of)g(advic)n(e)g(to)189 1893 y(users.)p Fx(\))75 2015 y Fj(5.7.1)49 b(F)o(unctionalit)o(y)75 2100 y Fx(A)o(ttributes)15 b(are)g(attac)o(hed)g(to)g(comm)o(unicators.)20 b(A)o(ttributes)c(are)f (lo)q(cal)h(to)f(the)h(pro)q(cess)f(and)h(sp)q(eci\014c)75 2157 y(to)h(the)i(comm)o(unicator)e(to)h(whic)o(h)h(they)f(are)g(attac) o(hed.)28 b(A)o(ttributes)18 b(are)f(not)h(propagated)g(b)o(y)g Fv(MPI)75 2213 y Fx(from)k(one)h(comm)o(unicator)f(to)g(another)g (except)h(when)g(the)g(comm)o(unicator)f(is)i(duplicated)g(using)75 2270 y Fv(MPI)p 160 2270 14 2 v 16 w(COMM)p 318 2270 V 16 w(DUP)e Fx(\(and)f(ev)o(en)h(then)f(the)h(application)h(m)o(ust)e (giv)o(e)g(sp)q(eci\014c)i(p)q(ermission)g(through)75 2326 y(callbac)o(k)16 b(functions)g(for)f(the)g(attribute)g(to)g(b)q(e) h(copied\).)189 2430 y Fl(A)n(dvic)n(e)e(to)i(implementors.)39 b Fx(A)o(ttributes)15 b(are)f(scalar)g(v)m(alues,)h(equal)g(in)g(size)h (to,)d(or)h(larger)g(than)189 2487 y(a)i(C-language)g(p)q(oin)o(ter.)24 b(A)o(ttributes)17 b(can)f(alw)o(a)o(ys)g(hold)h(an)g Fv(MPI)e Fx(handle.)25 b(\()p Fl(End)17 b(of)g(advic)n(e)h(to)189 2543 y(implementors.)p Fx(\))166 2647 y(The)12 b(cac)o(hing)g(in)o (terface)g(de\014ned)g(here)g(represen)o(ts)g(that)e(attributes)i(b)q (e)g(stored)f(b)o(y)g Fv(MPI)g Fx(opaquely)75 2704 y(within)16 b(a)f(comm)o(unicator.)20 b(Accessor)15 b(functions)h(include)i(the)d (follo)o(wing:)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 166 174 166 173 bop 75 -100 a Fx(166)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)143 45 y Ft(\017)23 b Fx(obtain)16 b(a)f(k)o(ey)h(v)m(alue)h(\(used)f(to)g (iden)o(tify)h(an)f(attribute\);)f(the)h(user)g(sp)q(eci\014es)i (\\callbac)o(k")e(func-)189 102 y(tions)h(b)o(y)g(whic)o(h)h Fv(MPI)f Fx(informs)g(the)h(application)h(when)f(the)f(comm)o(unicator) g(is)h(destro)o(y)o(ed)e(or)189 158 y(copied.)143 252 y Ft(\017)23 b Fx(store)14 b(and)i(retriev)o(e)f(the)g(v)m(alue)i(of)d (an)i(attribute;)189 358 y Fl(A)n(dvic)n(e)10 b(to)i(implementors.)37 b Fx(Cac)o(hing)10 b(and)h(callbac)o(k)g(functions)g(are)f(only)g (called)i(sync)o(hronously)l(,)189 415 y(in)k(resp)q(onse)f(to)g (explicit)j(application)e(requests.)k(This)c(a)o(v)o(oid)f(problems)h (that)f(result)g(from)g(re-)189 471 y(p)q(eated)h(crossings)h(b)q(et)o (w)o(een)f(user)h(and)f(system)g(space.)24 b(\(This)16 b(sync)o(hronous)g(calling)i(rule)f(is)g(a)189 527 y(general)e(prop)q (ert)o(y)g(of)g Fv(MPI)p Fx(.\))189 603 y(The)20 b(c)o(hoice)h(of)f(k)o (ey)g(v)m(alues)i(is)f(under)g(con)o(trol)f(of)f Fv(MPI)p Fx(.)h(This)h(allo)o(ws)f Fv(MPI)g Fx(to)f(optimize)j(its)189 659 y(implemen)o(tation)14 b(of)e(attribute)h(sets.)19 b(It)12 b(also)h(a)o(v)o(oids)g(con\015ict)h(b)q(et)o(w)o(een)f(indep)q (enden)o(t)i(mo)q(dules)189 716 y(cac)o(hing)h(information)f(on)g(the)g (same)g(comm)o(unicators.)189 791 y(A)g(m)o(uc)o(h)g(smaller)h(in)o (terface,)f(consisting)h(of)e(just)h(a)g(callbac)o(k)h(facilit)o(y)l(,) g(w)o(ould)f(allo)o(w)g(the)g(en)o(tire)189 847 y(cac)o(hing)f(facilit) o(y)h(to)e(b)q(e)h(implemen)o(ted)h(b)o(y)f(p)q(ortable)g(co)q(de.)20 b(Ho)o(w)o(ev)o(er,)13 b(with)h(the)f(minimal)j(call-)189 904 y(bac)o(k)c(in)o(terface,)h(some)f(form)g(of)g(table)h(searc)o (hing)g(is)g(implied)i(b)o(y)d(the)h(need)g(to)f(handle)i(arbitrary)189 960 y(comm)o(unicators.)23 b(In)17 b(con)o(trast,)e(the)i(more)f (complete)h(in)o(terface)g(de\014ned)g(here)g(p)q(ermits)g(rapid)189 1016 y(access)11 b(to)f(attributes)h(through)f(the)h(use)g(of)g(p)q (oin)o(ters)g(in)h(comm)o(unicators)f(\(to)e(\014nd)j(the)f(attribute) 189 1073 y(table\))h(and)h(clev)o(erly)g(c)o(hosen)g(k)o(ey)f(v)m (alues)i(\(to)d(retriev)o(e)i(individual)j(attributes\).)i(In)13 b(ligh)o(t)g(of)f(the)189 1129 y(e\016ciency)j(\\hit")e(inheren)o(t)h (in)h(the)e(minimal)i(in)o(terface,)f(the)g(more)f(complete)h(in)o (terface)g(de\014ned)189 1186 y(here)h(is)h(seen)g(to)e(b)q(e)i(sup)q (erior.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 1292 y Fv(MPI)f Fx(pro)o(vides)g(the)h(follo)o(wing)f (services)i(related)e(to)g(cac)o(hing.)20 b(They)c(are)f(all)h(pro)q (cess)f(lo)q(cal.)75 1443 y Fv(MPI)p 160 1443 14 2 v 16 w(KEYV)l(AL)p 346 1443 V 17 w(CREA)l(TE\(cop)o(y)p 636 1443 V 17 w(fn,)g(delete)p 828 1443 V 18 w(fn,)g(k)o(eyval,)g (extra)p 1150 1443 V 16 w(state\))117 1520 y Fk(IN)155 b Fv(cop)o(y)p 407 1520 V 17 w(fn)447 b Fk(Cop)o(y)14 b(callbac)o(k)f(function)g(for)h Fi(k)o(eyval)117 1595 y Fk(IN)155 b Fv(delete)p 431 1595 V 18 w(fn)422 b Fk(Delete)15 b(callbac)o(k)e(function)g(for)h Fi(k)o(eyval)117 1671 y Fk(OUT)108 b Fv(k)o(eyval)471 b Fk(k)o(ey)14 b(v)n(alue)f(for)h (future)g(access)i(\(in)o(teger\))117 1746 y(IN)155 b Fv(extra)p 416 1746 V 17 w(state)384 b Fk(Extra)14 b(state)h(for)f (callbac)o(k)f(functions)75 1870 y Fu(int)23 b(MPI)p 245 1870 15 2 v 17 w(Keyval)p 406 1870 V 17 w(create\(MPI)p 663 1870 V 15 w(Copy)p 774 1870 V 17 w(function)g(*copy)p 1126 1870 V 16 w(fn,)h(MPI)p 1310 1870 V 17 w(Delete)p 1471 1870 V 16 w(function)393 1927 y(*delete)p 564 1927 V 16 w(fn,)g(int)f(*keyval,)g(void*)g(extra)p 1249 1927 V 17 w(state\))75 2013 y(MPI)p 150 2013 V 17 w(KEYVAL)p 311 2013 V 16 w(CREATE\(COPY)p 591 2013 V 16 w(FN,)g(DELETE)p 846 2013 V 17 w(FN,)g(KEYVAL,)g(EXTRA)p 1269 2013 V 16 w(STATE,)h(IERROR\))170 2069 y(EXTERNAL)f(COPY)p 484 2069 V 17 w(FN,)g(DELETE)p 740 2069 V 17 w(FN)170 2126 y(INTEGER)g(KEYVAL,)g(EXTRA)p 675 2126 V 17 w(STATE,)g(IERROR)166 2212 y Fx(Generates)18 b(a)g(new)g(attribute)h(k)o(ey)l(.)29 b(Keys)19 b(are)f(lo)q(cally)i(unique)g(in)f(a)f(pro)q(cess,)h(and)f (opaque)h(to)75 2269 y(user,)g(though)g(they)f(are)h(explicitly)i (stored)d(in)i(in)o(tegers.)30 b(Once)19 b(allo)q(cated,)i(the)d(k)o (ey)h(v)m(alue)h(can)e(b)q(e)75 2325 y(used)e(to)e(asso)q(ciate)i (attributes)e(and)i(access)f(them)g(on)h(an)o(y)e(lo)q(cally)j (de\014ned)g(comm)o(unicator.)166 2382 y(The)10 b Fv(cop)o(y)p 343 2382 14 2 v 17 w(fn)g Fx(function)h(is)g(in)o(v)o(ok)o(ed)f(when)h (a)f(comm)o(unicator)f(is)i(duplicated)h(b)o(y)e Fv(MPI)p 1617 2382 V 16 w(COMM)p 1775 2382 V 17 w(DUP)p Fx(.)75 2438 y Fv(cop)o(y)p 164 2438 V 16 w(fn)16 b Fx(should)g(b)q(e)g(of)f(t) o(yp)q(e)g Fi(MPI)p 667 2438 13 2 v 14 w(Cop)o(y)p 768 2438 V 15 w(function)p Fx(,)h(whic)o(h)g(is)g(de\014ned)g(as)f(follo)o (ws:)75 2532 y Fu(typedef)23 b(int)g(MPI_Copy_function\(MPI_Comm)e (*oldcomm,)h(int)i(*keyval,)791 2588 y(void)f(*extra_state,)f(void)i (*attribute_val_in,)791 2645 y(void)f(**attribute_val_out,)e(int)j (*flag\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 167 175 167 174 bop 75 -100 a Fm(5.7.)34 b(CA)o(CHING)1407 b Fx(167)166 45 y(A)15 b(F)l(ortran)f(declaration)i(for)f(suc)o(h)g(a)g (function)h(is)g(as)f(follo)o(ws:)75 102 y Fu(FUNCTION)23 b(COPY)p 389 102 15 2 v 16 w(FUNCTION\(OLDCOMM,)f(KEYVAL,)h(EXTRA)p 1146 102 V 16 w(STATE,)g(ATTRIBUTE)p 1545 102 V 16 w(VAL)p 1633 102 V 17 w(IN,)393 158 y(ATTRIBUTE)p 612 158 V 16 w(VAL)p 700 158 V 17 w(OUT,)g(FLAG\))170 214 y(INTEGER)g(OLDCOMM,)g (KEYVAL,)g(EXTRA)p 890 214 V 17 w(STATE,)g(ATTRIBUTE)p 1290 214 V 16 w(VAL)p 1378 214 V 16 w(IN,)170 271 y(ATTRIBUTE)p 389 271 V 16 w(VAL)p 477 271 V 17 w(OUT)170 327 y(LOGICAL)g(FLAG)166 414 y Fx(The)13 b(cop)o(y)g(callbac)o(k)i(function)f(is)f(in)o(v)o(ok)o (ed)h(for)e(eac)o(h)h(k)o(ey)g(v)m(alue)i(in)f Fv(oldcomm)7 b Fx(in)14 b(arbitrary)f(order.)75 470 y(Eac)o(h)18 b(call)i(to)e(the)g (cop)o(y)g(callbac)o(k)i(is)f(made)f(with)h(a)f(k)o(ey)g(v)m(alue)i (and)e(its)h(corresp)q(onding)g(attribute.)75 527 y(If)f(it)g(returns)g Fi(\015ag)e(=)g(0)p Fx(,)i(then)g(the)g(attribute)f(is)h(deleted)h(in)g (the)f(duplicated)i(comm)o(unicator.)27 b(Oth-)75 583 y(erwise)17 b(\()p Fi(\015ag)e(=)h(1)p Fx(\),)g(the)g(new)h(attribute)g (v)m(alue)h(is)f(set)g(via)g Fv(attribute)p 1274 583 14 2 v 18 w(val)p 1346 583 V 16 w(out)p Fx(.)25 b(The)17 b(function)g(returns)75 640 y Fi(MPI)p 152 640 13 2 v 14 w(SUCCESS)d Fx(on)h(success)h(and)f(an)g(error)f(co)q(de)i(on)f (failure)h(\(in)g(whic)o(h)g(case)f Fv(MPI)p 1525 640 14 2 v 16 w(COMM)p 1683 640 V 16 w(DUP)g Fx(will)75 696 y(fail\).)166 753 y Fv(cop)o(y)p 255 753 V 16 w(fn)g Fx(ma)o(y)g(b)q(e)g(sp)q(eci\014ed)i(as)d Fv(MPI)p 803 753 V 16 w(NULL)p 932 753 V 16 w(FN)h Fx(from)f(either)i(C)e(or)g(F)o (OR)l(TRAN,)i(in)f(whic)o(h)h(case)75 809 y(no)k(cop)o(y)h(callbac)o(k) g(o)q(ccurs)g(for)e Fv(k)o(eyval)p Fx(;)k Fv(MPI)p 891 809 V 16 w(NULL)p 1020 809 V 16 w(FN)d Fx(is)h(a)f(function)h(that)f (do)q(es)g(nothing)h(other)75 865 y(than)c(returning)g Fi(\015ag)e(=)h(0)p Fx(.)24 b(In)18 b(C,)e(the)h(NULL)h(function)f(p)q (oin)o(ter)g(has)g(the)g(same)f(b)q(eha)o(vior)i(as)e(using)75 922 y Fv(MPI)p 160 922 V 16 w(NULL)p 289 922 V 16 w(FN)p Fx(.)i(As)h(a)f(further)h(con)o(v)o(enience,)h Fv(MPI)p 1010 922 V 16 w(DUP)p 1119 922 V 17 w(FN)e Fx(is)h(a)g(simple-minded)i (cop)o(y)e(callbac)o(k)75 978 y(a)o(v)m(ailable)g(from)d(C)h(and)h(F)o (OR)l(TRAN;)f(it)h(sets)e Fi(\015ag)g(=)g(1)p Fx(,)h(and)g(returns)g (the)h(v)m(alue)g(of)f Fv(attribute)p 1756 978 V 18 w(val)p 1828 978 V 16 w(in)75 1035 y Fx(in)f Fv(attribute)p 294 1035 V 18 w(val)p 366 1035 V 16 w(out)p Fx(.)166 1091 y(Note)f(that)h(the)g(C)f(v)o(ersion)h(of)g(this)g Fv(MPI)p 885 1091 V 16 w(COMM)p 1043 1091 V 16 w(DUP)g Fx(assumes)g(that)f(the)h (callbac)o(k)h(functions)75 1148 y(follo)o(w)11 b(the)g(C)g(protot)o (yp)q(e,)f(while)j(the)e(corresp)q(onding)h(F)o(OR)l(TRAN)f(v)o(ersion) g(assumes)g(the)g(F)o(OR)l(TRAN)75 1204 y(protot)o(yp)q(e.)189 1310 y Fl(A)n(dvic)n(e)19 b(to)i(users.)69 b Fx(A)20 b(v)m(alid)i(cop)o(y)d(function)i(is)g(one)f(that)f(completely)i (duplicates)h(the)e(in-)189 1367 y(formation)d(b)o(y)i(making)f(a)g (full)i(duplicate)g(cop)o(y)e(of)g(the)g(data)g(structures)g(implied)j (b)o(y)d(an)g(at-)189 1423 y(tribute;)13 b(another)g(migh)o(t)f(just)g (mak)o(e)h(another)f(reference)h(to)f(that)g(data)g(structure,)h(while) h(using)189 1480 y(a)i(reference-coun)o(t)i(mec)o(hanism.)27 b(Other)17 b(t)o(yp)q(es)g(of)g(attributes)g(migh)o(t)g(not)g(cop)o(y)g (at)f(all)i(\(they)189 1536 y(migh)o(t)d(b)q(e)h(sp)q(eci\014c)h(to)d Fv(oldcomm)c Fx(only\).)20 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g (users.)p Fx(\))166 1643 y(Analogous)g(to)g Fv(cop)o(y)p 534 1643 V 17 w(fn)h Fx(is)g(a)f(callbac)o(k)i(deletion)g(function,)f (de\014ned)h(as)e(follo)o(ws.)24 b(The)17 b Fv(delete)p 1823 1643 V 18 w(fn)75 1699 y Fx(function)h(is)g(in)o(v)o(ok)o(ed)g (when)g(a)g(comm)o(unicator)f(is)h(deleted)h(b)o(y)e Fv(MPI)p 1289 1699 V 16 w(COMM)p 1447 1699 V 16 w(FREE)h Fx(or)g(when)g(a)f(call)75 1755 y(is)j(made)f(explicitly)k(to)18 b Fv(MPI)p 596 1755 V 16 w(A)l(TTR)p 729 1755 V 17 w(DELETE)p Fx(.)h Fv(delete)p 1058 1755 V 18 w(fn)h Fx(should)g(b)q(e)g(of)f(t)o (yp)q(e)h Fi(MPI)p 1584 1755 13 2 v 14 w(Delete)p 1707 1755 V 16 w(function)p Fx(,)75 1812 y(whic)o(h)c(is)g(de\014ned)g(as)f (follo)o(ws:)75 1918 y Fu(typedef)23 b(int)g (MPI_Delete_function\(MPI_Comm)d(*comm,)j(int)h(*keyval,)75 1975 y(void)f(*attribute_val,)f(void)h(*extra_state\);)166 2081 y Fx(A)15 b(F)l(ortran)f(declaration)i(for)f(suc)o(h)g(a)g (function)h(is)g(as)f(follo)o(ws:)75 2137 y Fu(FUNCTION)23 b(DELETE)p 437 2137 15 2 v 16 w(FUNCTION\(COMM,)f(KEYVAL,)h(ATTRIBUTE)p 1218 2137 V 16 w(VAL,)g(EXTRA)p 1473 2137 V 17 w(STATE\))170 2194 y(INTEGER)g(COMM,)h(KEYVAL,)e(ATTRIBUTE)p 914 2194 V 16 w(VAL,)i(EXTRA)p 1170 2194 V 16 w(STATE)166 2280 y Fx(This)16 b(function)h(is)f(called)h(b)o(y)f Fv(MPI)p 773 2280 14 2 v 16 w(COMM)p 931 2280 V 16 w(FREE)g Fx(and)g Fv(MPI)p 1243 2280 V 16 w(A)l(TTR)p 1376 2280 V 17 w(DELETE)g Fx(to)f(do)h(whatev)o(er)75 2337 y(is)h(needed)h(to)e(remo)o(v)o(e)g (an)g(attribute.)24 b(It)17 b(ma)o(y)e(b)q(e)j(sp)q(eci\014ed)g(as)f (the)f(n)o(ull)i(function)g(p)q(oin)o(ter)f(in)g(C)f(or)75 2393 y(as)f Fv(MPI)p 216 2393 V 16 w(NULL)p 345 2393 V 16 w(FN)h Fx(from)f(either)h(C)f(or)g(F)o(OR)l(TRAN,)h(in)h(whic)o(h) f(case)g(no)f(delete)i(callbac)o(k)g(o)q(ccurs)f(for)75 2450 y Fv(k)o(eyval)p Fx(.)166 2506 y(The)10 b(sp)q(ecial)i(k)o(ey)e(v) m(alue)i Fi(MPI)p 664 2506 13 2 v 14 w(KEYV)m(AL)p 835 2506 V 14 w(INV)m(ALID)c Fx(is)j(nev)o(er)f(returned)h(b)o(y)f Fv(MPI)p 1493 2506 14 2 v 16 w(KEYV)l(AL)p 1679 2506 V 17 w(CREA)l(TE)p Fx(.)75 2563 y(Therefore,)15 b(it)g(can)g(b)q(e)h (used)g(for)f(static)g(initialization)j(of)d(k)o(ey)g(v)m(alues.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 168 176 168 175 bop 75 -100 a Fx(168)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(KEYV)l(AL)p 346 45 V 17 w(FREE\(k)o(eyval\))117 122 y Fk(INOUT)62 b Fv(k)o(eyval)471 b Fk(F)m(rees)15 b(the)g(in)o(teger)f(k)o(ey)g(v)n(alue)f(\(in)o (teger\))75 247 y Fu(int)23 b(MPI)p 245 247 15 2 v 17 w(Keyval)p 406 247 V 17 w(free\(int)f(*keyval\))75 333 y(MPI)p 150 333 V 17 w(KEYVAL)p 311 333 V 16 w(FREE\(KEYVAL,)g (IERROR\))170 390 y(INTEGER)h(KEYVAL,)g(IERROR)166 476 y Fx(F)l(rees)37 b(an)g(extan)o(t)g(attribute)g(k)o(ey)l(.)87 b(This)37 b(function)i(sets)e(the)g(v)m(alue)i(of)e Fv(k)o(eyval)g Fx(to)75 533 y Fi(MPI)p 152 533 13 2 v 14 w(KEYV)m(AL)p 323 533 V 14 w(INV)m(ALID)p Fx(.)15 b(Note)h(that)f(it)i(is)g(not)f (erroneous)g(to)f(free)i(an)f(attribute)g(k)o(ey)g(that)g(is)h(in)g (use,)75 589 y(b)q(ecause)e(the)e(actual)h(free)f(do)q(es)h(not)f (transpire)h(un)o(til)h(after)e(all)h(references)g(\(in)g(other)f(comm) o(unicators)75 645 y(on)20 b(the)g(pro)q(cess\))f(to)h(the)g(k)o(ey)f (ha)o(v)o(e)h(b)q(een)h(freed.)34 b(These)20 b(references)g(need)h(to)e (b)q(e)i(explictly)h(freed)75 702 y(b)o(y)d(the)h(program,)f(either)h (via)f(calls)i(to)e Fv(MPI)p 887 702 14 2 v 15 w(A)l(TTR)p 1019 702 V 18 w(DELETE)g Fx(that)f(free)i(one)f(attribute)h(instance,) 75 758 y(or)c(b)o(y)g(calls)h(to)e Fv(MPI)p 439 758 V 16 w(COMM)p 597 758 V 17 w(FREE)h Fx(that)g(free)g(all)h(attribute)f (instances)h(asso)q(ciated)f(with)g(the)h(freed)75 815 y(comm)o(unicator.)189 921 y Fl(A)n(dvic)n(e)h(to)h(implementors.)30 b Fx(The)18 b(function)i Fv(MPI)p 1068 921 V 16 w(NULL)p 1197 921 V 16 w(FN)e Fx(need)h(not)f(b)q(e)h(aliased)h(to)e Fu(\(void)189 978 y(\(*\)\)0)i Fx(in)j(C,)e(though)g(this)h(is)g (\014ne.)40 b(It)22 b(could)g(b)q(e)g(a)g(legitimately)h(callable)g (function)g(that)189 1034 y(pro\014les)15 b(and)g(so)g(on.)k(F)l(or)14 b(F)o(OR)l(TRAN,)i(it)f(is)g(most)f(con)o(v)o(enien)o(t)i(to)e(ha)o(v)o (e)g Fv(MPI)p 1576 1034 V 16 w(NULL)p 1705 1034 V 16 w(FN)h Fx(b)q(e)g(a)189 1090 y(legitimate)h(do-nothing)g(function)g (call.\()p Fl(End)g(of)g(advic)n(e)g(to)h(implementors.)p Fx(\))75 1291 y Fv(MPI)p 160 1291 V 16 w(A)l(TTR)p 293 1291 V 17 w(PUT\(comm)m(,)12 b(k)o(eyval,)i(attribute)p 868 1291 V 18 w(val\))117 1368 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)7 b(to)i(whic)o(h)g(attribute)h(will)e(b)q(e)h (attac)o(hed)h(\(han-)905 1425 y(dle\))117 1500 y(IN)155 b Fv(k)o(eyval)471 b Fk(k)o(ey)14 b(v)n(alue,)f(as)h(returned)h(b)o(y) 905 1557 y Fi(MPI)p 982 1557 13 2 v 15 w(KEYV)m(AL)p 1154 1557 V 13 w(CREA)m(TE)d Fk(\(in)o(teger\))117 1632 y(IN)155 b Fv(attribute)p 484 1632 14 2 v 18 w(val)352 b Fk(attribute)15 b(v)n(alue)75 1756 y Fu(int)23 b(MPI)p 245 1756 15 2 v 17 w(Attr)p 358 1756 V 17 w(put\(MPI)p 543 1756 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void*)g(attribute)p 1467 1756 V 16 w(val\))75 1843 y(MPI)p 150 1843 V 17 w(ATTR)p 263 1843 V 16 w(PUT\(COMM,)g(KEYVAL,)g(ATTRIBUTE)p 925 1843 V 16 w(VAL,)g(IERROR\))170 1899 y(INTEGER)g(COMM,)h(KEYVAL,)e (ATTRIBUTE)p 914 1899 V 16 w(VAL,)i(IERROR)166 1985 y Fx(This)14 b(function)g(stores)f(the)g(stipulated)i(attribute)e(v)m (alue)i Fv(attribute)p 1331 1985 14 2 v 18 w(val)e Fx(for)g(subsequen)o (t)h(retriev)m(al)75 2042 y(b)o(y)j Fv(MPI)p 225 2042 V 16 w(A)l(TTR)p 358 2042 V 17 w(GET)p Fx(.)h(If)g(the)f(v)m(alue)i(is) f(already)g(presen)o(t,)g(then)f(the)h(outcome)f(is)h(as)f(if)h Fv(MPI)p 1729 2042 V 16 w(A)l(TTR-)p 75 2098 V 89 2098 a(DELETE)f Fx(w)o(as)f(\014rst)h(called)i(to)d(delete)j(the)e(previous) h(v)m(alue)g(\(and)f(the)h(callbac)o(k)g(function)g Fv(delete)p 1823 2098 V 18 w(fn)75 2155 y Fx(w)o(as)f(executed\),)i(and)f(a)f(new)i (v)m(alue)g(w)o(as)e(next)h(stored.)27 b(The)18 b(call)h(is)g (erroneous)f(if)g(there)g(is)g(no)g(k)o(ey)75 2211 y(with)e(v)m(alue)g Fv(k)o(eyval)p Fx(;)f(in)h(particular)f Fi(MPI)p 779 2211 13 2 v 15 w(KEYV)m(AL)p 951 2211 V 13 w(INV)m(ALID)f Fx(is)i(an)f(erroneous)g(k)o(ey)g(v)m(alue.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 169 177 169 176 bop 75 -100 a Fm(5.7.)29 b(CA)o(CHING)1412 b Fx(169)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(A)l(TTR)p 293 45 V 17 w(GET\(comm)m(,)12 b(k)o(eyval,)i(attribute)p 865 45 V 18 w(val,)h(\015ag\))117 122 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)10 b(to)i(whic)o(h)g(attribute)h(is)f(attac)o(hed) h(\(handle\))117 197 y(IN)155 b Fv(k)o(eyval)471 b Fk(k)o(ey)14 b(v)n(alue)f(\(in)o(teger\))117 273 y(OUT)108 b Fv(attribute)p 484 273 V 18 w(val)352 b Fk(attribute)15 b(v)n(alue,)d(unless)j Fi(\015ag)f Fk(=)g(false)117 348 y(OUT)108 b Fv(\015ag)518 b Fb(true)14 b Fk(if)h(an)f(attribute)i(v)n(alue)e(w)o(as)h(extracted;) i Fb(false)d Fk(if)g(no)905 404 y(attribute)h(is)e(asso)q(ciated)i (with)f(the)g(k)o(ey)75 529 y Fu(int)23 b(MPI)p 245 529 15 2 v 17 w(Attr)p 358 529 V 17 w(get\(MPI)p 543 529 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void)g(**attribute)p 1491 529 V 16 w(val,)g(int)h(*flag\))75 615 y(MPI)p 150 615 V 17 w(ATTR)p 263 615 V 16 w(GET\(COMM,)f(KEYVAL,)g(ATTRIBUTE)p 925 615 V 16 w(VAL,)g(FLAG,)g(IERROR\))170 672 y(INTEGER)g(COMM,)h (KEYVAL,)e(ATTRIBUTE)p 914 672 V 16 w(VAL,)i(IERROR)170 728 y(LOGICAL)f(FLAG)166 814 y Fx(Retriev)o(es)c(attribute)g(v)m(alue)h (b)o(y)f(k)o(ey)l(.)30 b(The)19 b(call)h(is)f(erroneous)g(if)g(there)g (is)g(no)g(k)o(ey)g(with)g(v)m(alue)75 871 y Fv(k)o(eyval)p Fx(.)30 b(On)19 b(the)g(other)f(hand,)h(the)g(call)h(is)f(correct)f(if) h(the)g(k)o(ey)f(v)m(alue)i(exists,)g(but)e(no)h(attribute)f(is)75 927 y(attac)o(hed)e(on)h Fu(comm)f Fx(for)g(that)g(k)o(ey;)g(in)i(suc)o (h)f(case,)g(the)f(call)i(returns)f Fu(flag)23 b(=)h(false)p Fx(.)f(In)17 b(particular)75 984 y Fi(MPI)p 152 984 13 2 v 14 w(KEYV)m(AL)p 323 984 V 14 w(INV)m(ALID)d Fx(is)i(an)f (erroneous)g(k)o(ey)g(v)m(alue.)75 1135 y Fv(MPI)p 160 1135 14 2 v 16 w(A)l(TTR)p 293 1135 V 17 w(DELETE\(com)o(m)n(,)c(k)o (eyval\))117 1212 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)10 b(to)i(whic)o(h)g(attribute)h(is)f(attac)o(hed)h(\(handle\))117 1287 y(IN)155 b Fv(k)o(eyval)471 b Fk(The)15 b(k)o(ey)e(v)n(alue)h(of)f (the)i(deleted)f(attribute)h(\(in)o(teger\))75 1412 y Fu(int)23 b(MPI)p 245 1412 15 2 v 17 w(Attr)p 358 1412 V 17 w(delete\(MPI)p 615 1412 V 16 w(Comm)g(comm,)g(int)h(keyval\))75 1498 y(MPI)p 150 1498 V 17 w(ATTR)p 263 1498 V 16 w(DELETE\(COMM,)f (KEYVAL,)g(IERROR\))170 1554 y(INTEGER)g(COMM,)h(KEYVAL,)e(IERROR)166 1641 y Fx(Delete)14 b(attribute)g(from)f(cac)o(he)h(b)o(y)f(k)o(ey)l(.) 20 b(This)14 b(function)g(in)o(v)o(ok)o(es)g(the)g(attribute)f(delete)i (function)75 1697 y Fv(delete)p 188 1697 14 2 v 18 w(fn)g Fx(sp)q(eci\014ed)j(when)d(the)h Fv(k)o(eyval)e Fx(w)o(as)h(created.) 166 1754 y(Whenev)o(er)h(a)g(comm)o(unicator)f(is)h(replicated)i(using) e(the)g(function)h Fv(MPI)p 1435 1754 V 16 w(COMM)p 1593 1754 V 16 w(DUP)p Fx(,)f(all)h(call-)75 1810 y(bac)o(k)h(cop)o(y)f (functions)i(for)e(attributes)g(that)g(are)h(curren)o(tly)g(set)f(are)h (in)o(v)o(ok)o(ed)g(\(in)g(arbitrary)f(order\).)75 1867 y(Whenev)o(er)23 b(a)f(comm)o(unicator)g(is)g(deleted)i(using)f(the)g (function)g Fv(MPI)p 1343 1867 V 15 w(COMM)p 1500 1867 V 17 w(FREE)g Fx(all)g(callbac)o(k)75 1923 y(delete)16 b(functions)g(for)f(attributes)g(that)f(are)h(curren)o(tly)h(set)f(are) g(in)o(v)o(ok)o(ed.)75 2045 y Fj(5.7.2)49 b(A)o(ttributes)14 b(Example)189 2131 y Fl(R)n(ationale.)40 b Fx(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))189 2237 y Fl(A)n(dvic)n(e)22 b(to)h(users.)83 b Fx(This)23 b(example)h(sho)o(ws)e(ho)o(w)g(to)f (write)i(a)f(collectiv)o(e)i(comm)o(unication)189 2293 y(op)q(eration)16 b(that)f(uses)i(cac)o(hing)g(to)e(b)q(e)i(more)f (e\016cien)o(t)h(after)e(the)i(\014rst)e(call.)24 b(The)17 b(co)q(ding)g(st)o(yle)189 2350 y(assumes)11 b(that)h Fv(MPI)f Fx(function)i(results)f(return)g(only)h(error)e(statuses.)18 b(\()p Fl(End)13 b(of)g(advic)n(e)g(to)h(users.)p Fx(\))147 2456 y Fu(/*)23 b(key)h(for)f(this)g(module's)g(stuff:)g(*/)147 2513 y(static)g(int)g(gop_key)g(=)h(MPI_KEYVAL_INVALID;)147 2626 y(typedef)e(struct)147 2682 y({)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 170 178 170 177 bop 75 -100 a Fx(170)370 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)218 45 y Fu(int)24 b(ref_count;)237 b(/*)23 b(reference)g(count)g(*/)218 102 y(/*)h(other)f(stuff,)g(whatever)g(else)g(we)h(want)f(*/)147 158 y(})g(gop_stuff_type;)147 271 y(Efficient_Collective_Op)d(\(comm,)j (...\))147 327 y(MPI_Comm)f(comm;)147 384 y({)194 440 y(gop_stuff_type)g(*gop_stuff;)194 497 y(MPI_Group)166 b(group;)194 553 y(int)310 b(foundflag;)194 666 y (MPI_Comm_group\(comm,)21 b(&group\);)194 779 y(if)j(\(gop_key)f(==)g (MPI_KEYVAL_INVALID\))f(/*)h(get)h(a)f(key)h(on)f(first)h(call)f(ever)g (*/)194 835 y({)242 892 y(if)h(\()f(!)h(MPI_Attr_keyval_create\()d (gop_stuff_copier,)839 948 y(gop_stuff_destructor,)839 1005 y(&gop_key,)h(\(void)h(*\)0\)\);)242 1061 y(/*)h(get)f(the)h(key)f (while)g(assigning)g(its)g(copy)h(and)f(delete)g(callback)314 1118 y(behavior.)f(*/)242 1231 y(MPI_Abort)h(\("Insufficient)f(keys)h (available"\);)194 1287 y(})194 1400 y(MPI_Attr_get)g(\(comm,)g (gop_key,)f(&gop_stuff,)h(&foundflag\);)194 1456 y(if)h(\(foundflag\)) 194 1513 y({)g(/*)g(This)f(module)g(has)g(executed)g(in)h(this)f(group) g(before.)314 1569 y(We)g(will)h(use)f(the)g(cached)g(information)g(*/) 194 1626 y(})194 1682 y(else)194 1739 y({)h(/*)g(This)f(is)h(a)f(group) g(that)h(we)f(have)h(not)f(yet)h(cached)f(anything)f(in.)314 1795 y(We)h(will)h(now)f(do)h(so.)242 1852 y(*/)242 1965 y(/*)g(First,)f(allocate)f(storage)h(for)h(the)f(stuff)g(we)h(want,)314 2021 y(and)f(initialize)g(the)g(reference)g(count)g(*/)242 2134 y(gop_stuff)g(=)g(\(gop_stuff_type)f(*\))i(malloc)f (\(sizeof\(gop_stuff_type\)\);)242 2190 y(if)h(\(gop_stuff)e(==)i (NULL\))f({)h(/*)f(abort)g(on)h(out-of-memory)e(error)h(*/)h(})242 2303 y(gop_stuff)f(->)g(ref_count)g(=)h(1;)242 2416 y(/*)g(Second,)f (fill)g(in)g(*gop_stuff)g(with)g(whatever)g(we)h(want.)314 2473 y(This)f(part)g(isn't)g(shown)h(here)f(*/)242 2586 y(/*)h(Third,)f(store)g(gop_stuff)f(as)i(the)g(attribute)e(value)h(*/) 242 2642 y(MPI_Attr_put)f(\()i(comm,)f(gop_key,)g(gop_stuff\);)194 2698 y(})1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 171 179 171 178 bop 75 -100 a Fm(5.8.)34 b(F)o(ORMALIZING)16 b(THE)f(LOOSEL)l(Y)i(SYNCHR)o(ONOUS)g(MODEL)394 b Fx(171)194 45 y Fu(/*)24 b(Then,)f(in)h(any)f(case,)g(use)h(contents)f(of)g (*gop_stuff)266 102 y(to)g(do)h(the)g(global)f(op)g(...)h(*/)147 158 y(})147 271 y(/*)f(The)h(following)e(routine)h(is)h(called)f(by)g (MPI)h(when)f(a)h(group)f(is)h(freed)f(*/)147 384 y (gop_stuff_destructor)e(\(comm,)i(keyval,)g(gop_stuff,)f(extra\))147 440 y(MPI_Comm)g(comm;)147 497 y(int)h(keyval;)147 553 y(gop_stuff_type)f(*gop_stuff;)147 610 y(void)h(*extra;)147 666 y({)194 723 y(if)h(\(keyval)f(!=)g(gop_key\))g({)h(/*)f(abort)h(--) f(programming)g(error)g(*/)g(})194 835 y(/*)h(The)f(group's)g(being)g (freed)h(removes)f(one)g(reference)g(to)g(gop_stuff)g(*/)194 892 y(gop_stuff)g(->)h(ref_count)e(-=)i(1;)194 1005 y(/*)g(If)f(no)h (references)f(remain,)f(then)i(free)f(the)h(storage)e(*/)194 1061 y(if)i(\(gop_stuff)e(->)i(ref_count)f(==)g(0\))h({)242 1118 y(free\(\(void)f(*\)gop_stuff\);)194 1174 y(})147 1231 y(})147 1344 y(/*)g(The)h(following)e(routine)h(is)h(called)f(by)g (MPI)h(when)f(a)h(group)f(is)h(copied)f(*/)147 1400 y(gop_stuff_copier) e(\(comm,)i(keyval,)g(gop_stuff,)g(extra\))147 1456 y(MPI_Comm)f(comm;) 147 1513 y(int)h(keyval;)147 1569 y(gop_stuff_type)f(*gop_stuff;)147 1626 y(void)h(*extra;)147 1682 y({)194 1739 y(if)h(\(keyval)f(!=)g (gop_key\))g({)h(/*)f(abort)h(--)f(programming)g(error)g(*/)g(})194 1852 y(/*)h(The)f(new)h(group)f(adds)g(one)h(reference)e(to)i(this)f (gop_stuff)g(*/)194 1908 y(gop_stuff)g(->)h(ref_count)e(+=)i(1;)147 1965 y(})75 2108 y Fn(5.8)59 b(F)n(o)n(rmalizing)21 b(the)e(Lo)r(osely) g(Synchronous)g(Mo)r(del)75 2209 y Fx(In)i(this)h(section,)g(w)o(e)f (mak)o(e)f(further)h(statemen)o(ts)e(ab)q(out)i(the)g(lo)q(osely)h (sync)o(hronous)f(mo)q(del,)i(with)75 2266 y(particular)16 b(atten)o(tion)f(to)f(in)o(tra-comm)o(unication.)75 2387 y Fj(5.8.1)49 b(Basic)17 b(Stateme)o(nts)75 2473 y Fx(When)e(a)g (caller)h(passes)e(a)h(comm)o(unicator)f(\(that)g(con)o(tains)h(a)g (con)o(text)f(and)h(group\))f(to)g(a)h(callee,)h(that)75 2530 y(comm)o(unicator)g(m)o(ust)h(b)q(e)g(free)g(of)f(side)i (e\013ects)e(throughout)g(execution)i(of)f(the)g(subprogram:)22 b(there)75 2586 y(should)16 b(b)q(e)f(no)g(activ)o(e)g(op)q(erations)g (on)g(that)f(comm)o(unicator)g(that)g(migh)o(t)h(in)o(v)o(olv)o(e)g (the)g(pro)q(cess.)20 b(This)75 2643 y(pro)o(vides)i(one)g(mo)q(del)h (in)f(whic)o(h)h(libraries)g(can)f(b)q(e)h(written,)g(and)f(w)o(ork)f (\\safely)l(.")39 b(F)l(or)21 b(libraries)75 2699 y(so)e(designated,)h (the)g(callee)h(has)e(p)q(ermission)h(to)f(do)g(whatev)o(er)g(comm)o (unication)h(it)f(lik)o(es)h(with)g(the)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 172 180 172 179 bop 75 -100 a Fx(172)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fx(comm)o(unicator,)j(and)h(under)g(the)f(ab)q(o)o(v)o(e)g(guaran) o(tee)g(kno)o(ws)g(that)f(no)h(other)g(comm)o(unications)h(will)75 102 y(in)o(terfere.)36 b(Since)22 b(w)o(e)f(p)q(ermit)g(go)q(o)q(d)f (implemen)o(tations)i(to)e(create)g(new)h(comm)o(unicators)f(without)75 158 y(sync)o(hronization)14 b(\(suc)o(h)g(as)f(b)o(y)g(preallo)q(cated) i(con)o(texts)e(on)g(comm)o(unicators\),)g(this)g(do)q(es)h(not)f(imp)q (ose)75 214 y(a)i(signi\014can)o(t)h(o)o(v)o(erhead.)166 271 y(This)k(form)f(of)g(safet)o(y)f(is)i(analogous)f(to)g(other)g (common)g(computer-science)j(usages,)e(suc)o(h)f(as)75 327 y(passing)c(a)f(descriptor)h(of)f(an)h(arra)o(y)e(to)h(a)g(library) i(routine.)k(The)15 b(library)g(routine)g(has)g(ev)o(ery)f(righ)o(t)h (to)75 384 y(exp)q(ect)h(suc)o(h)f(a)g(descriptor)h(to)f(b)q(e)g(v)m (alid)i(and)f(mo)q(di\014able.)75 506 y Fj(5.8.2)49 b(Mo)q(dels)17 b(of)g(Execution)75 591 y Fx(In)12 b(the)g(lo)q(osely)h(sync)o(hronous) f(mo)q(del,)h(transfer)e(of)h(con)o(trol)f(to)h(a)f Fq(parallel)16 b(pro)q(cedure)c Fx(is)g(e\013ected)g(b)o(y)75 648 y(ha)o(ving)f(eac)o (h)g(executing)g(pro)q(cess)g(in)o(v)o(ok)o(e)g(the)f(pro)q(cedure.)19 b(The)11 b(in)o(v)o(o)q(cation)g(is)g(a)g(collectiv)o(e)h(op)q (eration:)75 704 y(it)i(is)h(executed)g(b)o(y)f(all)h(pro)q(cesses)f (in)h(the)g(execution)g(group,)e(and)i(in)o(v)o(o)q(cations)f(are)g (similarly)i(ordered)75 761 y(at)f(all)h(pro)q(cesses.)k(Ho)o(w)o(ev)o (er,)14 b(the)h(in)o(v)o(o)q(cation)h(need)g(not)f(b)q(e)h(sync)o (hronized.)166 817 y(W)l(e)d(sa)o(y)g(that)g(a)g(parallel)i(pro)q (cedure)f(is)g Fl(active)f Fx(in)h(a)g(pro)q(cess)f(if)h(the)f(pro)q (cess)h(b)q(elongs)g(to)f(a)g(group)75 874 y(that)j(ma)o(y)h(collectiv) o(ely)i(execute)f(the)f(pro)q(cedure,)h(and)f(some)f(mem)o(b)q(er)h(of) g(that)f(group)h(is)g(curren)o(tly)75 930 y(executing)c(the)f(pro)q (cedure)h(co)q(de.)19 b(If)13 b(a)e(parallel)j(pro)q(cedure)f(is)f (activ)o(e)g(in)h(a)f(pro)q(cess,)g(then)g(this)h(pro)q(cess)75 987 y(ma)o(y)e(b)q(e)i(receiving)g(messages)e(p)q(ertaining)i(to)f (this)g(pro)q(cedure,)h(ev)o(en)f(if)g(it)g(do)q(es)h(not)e(curren)o (tly)h(execute)75 1043 y(the)j(co)q(de)h(of)f(this)g(pro)q(cedure.)75 1163 y Fv(Static)h(comm)m(unicato)o(r)d(allo)q(cation)75 1249 y Fx(This)22 b(co)o(v)o(ers)f(the)h(case)g(where,)h(at)e(an)o(y)g (p)q(oin)o(t)h(in)h(time,)g(at)e(most)g(one)h(in)o(v)o(o)q(cation)g(of) f(a)g(parallel)75 1305 y(pro)q(cedure)c(can)f(b)q(e)h(activ)o(e)f(at)f (an)o(y)h(pro)q(cess,)g(and)g(the)g(group)f(of)h(executing)h(pro)q (cesses)f(is)h(\014xed.)23 b(F)l(or)75 1362 y(example,)d(all)g(in)o(v)o (o)q(cations)f(of)g(parallel)h(pro)q(cedures)g(in)o(v)o(olv)o(e)f(all)h (pro)q(cesses,)g(pro)q(cesses)f(are)f(single-)75 1418 y(threaded,)d(and)h(there)f(are)g(no)g(recursiv)o(e)h(in)o(v)o(o)q (cations.)166 1475 y(In)k(suc)o(h)g(a)g(case,)g(a)g(comm)o(unicator)f (can)h(b)q(e)g(statically)h(allo)q(cated)f(to)g(eac)o(h)f(pro)q (cedure.)35 b(The)75 1531 y(static)18 b(allo)q(cation)h(can)f(b)q(e)h (done)g(in)g(a)e(pream)o(ble,)j(as)d(part)h(of)g(initialization)j(co)q (de.)29 b(If)18 b(the)g(parallel)75 1588 y(pro)q(cedures)g(can)f(b)q(e) g(organized)g(in)o(to)g(libraries,)h(so)f(that)f(only)h(one)g(pro)q (cedure)h(of)e(eac)o(h)h(library)h(can)75 1644 y(b)q(e)e(concurren)o (tly)g(activ)o(e)g(in)g(eac)o(h)f(pro)q(cessor,)g(then)h(it)g(is)g (su\016cien)o(t)g(to)f(allo)q(cate)h(one)f(comm)o(unicator)75 1701 y(p)q(er)h(library)l(.)75 1821 y Fv(Dynamic)c(comm)n(unicato)o(r)g (allo)q(cation)75 1906 y Fx(Calls)j(of)g(parallel)h(pro)q(cedures)g (are)e(w)o(ell-nested)i(if)g(a)e(new)h(parallel)h(pro)q(cedure)g(is)f (alw)o(a)o(ys)f(in)o(v)o(ok)o(ed)h(in)75 1963 y(a)j(subset)h(of)e(a)h (group)g(executing)i(the)e(same)g(parallel)i(pro)q(cedure.)30 b(Th)o(us,)19 b(pro)q(cesses)g(that)e(execute)75 2019 y(the)e(same)g(parallel)i(pro)q(cedure)f(ha)o(v)o(e)f(the)g(same)g (execution)h(stac)o(k.)166 2076 y(In)i(suc)o(h)g(a)g(case,)f(a)h(new)g (comm)o(unicator)f(needs)h(to)f(b)q(e)i(dynamically)g(allo)q(cated)g (for)e(eac)o(h)h(new)75 2132 y(in)o(v)o(o)q(cation)11 b(of)f(a)g(parallel)i(pro)q(cedure.)19 b(The)11 b(allo)q(cation)g(is)g (done)g(b)o(y)f(the)h(caller.)19 b(A)11 b(new)f(comm)o(unicator)75 2189 y(can)k(b)q(e)h(generated)g(b)o(y)f(a)g(call)h(to)f Fv(MPI)p 746 2189 14 2 v 15 w(COMM)p 903 2189 V 17 w(DUP)p Fx(,)g(if)g(the)h(callee)g(execution)h(group)e(is)g(iden)o(tical)j(to) 75 2245 y(the)f(caller)h(execution)g(group,)e(or)g(b)o(y)h(a)f(call)i (to)e Fv(MPI)p 1005 2245 V 16 w(COMM)p 1163 2245 V 16 w(SPLIT)g Fx(if)i(the)f(caller)g(execution)h(group)75 2302 y(is)f(split)h(in)o(to)f(sev)o(eral)f(subgroups)h(executing)h (distinct)g(parallel)g(routines.)22 b(The)15 b(new)h(comm)o(unicator)75 2358 y(is)g(passed)f(as)g(an)g(argumen)o(t)g(to)f(the)h(in)o(v)o(ok)o (ed)h(routine.)166 2414 y(The)h(need)h(for)e(generating)h(a)f(new)h (comm)o(unicator)f(at)h(eac)o(h)f(in)o(v)o(o)q(cation)i(can)f(b)q(e)g (alleviated)h(or)75 2471 y(a)o(v)o(oided)e(altogether)e(in)j(some)e (cases:)20 b(If)15 b(the)h(execution)g(group)f(is)h(not)f(split,)h (then)f(one)h(can)f(allo)q(cate)75 2527 y(a)g(stac)o(k)g(of)g(comm)o (unicators)h(in)g(a)f(pream)o(ble,)h(and)g(next)g(manage)f(the)h(stac)o (k)f(in)h(a)f(w)o(a)o(y)g(that)g(mimics)75 2584 y(the)g(stac)o(k)g(of)f (recursiv)o(e)i(calls.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 173 181 173 180 bop 75 -100 a Fm(5.8.)34 b(F)o(ORMALIZING)16 b(THE)f(LOOSEL)l(Y)i(SYNCHR)o(ONOUS)g(MODEL)394 b Fx(173)166 45 y(One)17 b(can)f(also)g(tak)o(e)f(adv)m(an)o(tage)g(of)h(the)g(w)o (ell-ordering)h(prop)q(ert)o(y)f(of)f(comm)o(unication)i(to)e(a)o(v)o (oid)75 102 y(confusing)g(caller)g(and)f(callee)h(comm)o(unication,)f (ev)o(en)h(if)f(b)q(oth)g(use)g(the)g(same)f(comm)o(unicator.)19 b(T)l(o)14 b(do)75 158 y(so,)g(one)i(needs)g(to)e(abide)j(b)o(y)e(the)g (follo)o(wing)h(t)o(w)o(o)e(rules:)143 252 y Ft(\017)23 b Fx(messages)13 b(sen)o(t)g(b)q(efore)h(a)f(pro)q(cedure)h(call)h (\(or)e(b)q(efore)h(a)f(return)g(from)g(the)h(pro)q(cedure\))g(are)f (also)189 308 y(receiv)o(ed)j(b)q(efore)f(the)h(matc)o(hing)f(call)h (\(or)f(return\))g(at)f(the)h(receiving)i(end;)143 402 y Ft(\017)23 b Fx(messages)14 b(are)h(alw)o(a)o(ys)g(selected)h(b)o(y)f (source)g(\(no)g(use)h(is)g(made)f(of)g Fi(MPI)p 1430 402 13 2 v 14 w(ANY)p 1529 402 V 15 w(SOURCE)p Fx(\).)75 522 y Fv(The)h(General)f(case)75 608 y Fx(In)22 b(the)g(general)g (case,)h(there)e(ma)o(y)g(b)q(e)h(m)o(ultiple)i(concurren)o(tly)e (activ)o(e)g(in)o(v)o(o)q(cations)g(of)f(the)g(same)75 664 y(parallel)i(pro)q(cedure)g(within)g(the)f(same)f(group;)j(in)o(v)o (o)q(cations)e(ma)o(y)f(not)h(b)q(e)g(w)o(ell-nested.)41 b(A)22 b(new)75 721 y(comm)o(unicator)11 b(needs)i(to)e(b)q(e)i (created)e(for)h(eac)o(h)f(in)o(v)o(o)q(cation.)20 b(It)11 b(is)i(the)f(user's)f(resp)q(onsibilit)o(y)j(to)d(mak)o(e)75 777 y(sure)18 b(that,)f(should)h(t)o(w)o(o)e(distinct)j(parallel)g(pro) q(cedures)f(b)q(e)g(in)o(v)o(ok)o(ed)g(concurren)o(tly)g(on)f(o)o(v)o (erlapping)75 834 y(sets)e(of)g(pro)q(cesses,)g(then)g(comm)o(unicator) g(creation)h(b)q(e)f(prop)q(erly)i(co)q(ordinated.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 174 182 174 181 bop 75 356 a Fo(Chapter)34 b(6)75 564 y Fr(Pro)s(cess)40 b(T)-10 b(op)s(ologi)q(es)75 805 y Fn(6.1)59 b(Intro)r(duction)75 906 y Fx(This)22 b(c)o(hapter)g(discusses)h(the)f Fv(MPI)f Fx(top)q(ology)g(mec)o(hanism.)40 b(A)22 b(top)q(ology)f(is)h(an)g (extra,)g(optional)75 963 y(attribute)14 b(that)g(one)g(can)h(giv)o(e)g (to)e(an)i(in)o(tra-comm)o(unicator;)f(top)q(ologies)g(cannot)h(b)q(e)g (added)g(to)f(in)o(ter-)75 1019 y(comm)o(unicators.)19 b(A)d(top)q(ology)e(can)i(pro)o(vide)f(a)g(con)o(v)o(enien)o(t)h (naming)f(mec)o(hanism)h(for)e(the)i(pro)q(cesses)75 1076 y(of)j(a)f(group)h(\(within)h(a)e(comm)o(unicator\),)h(and)g (additionally)l(,)j(ma)o(y)c(assist)h(the)g(run)o(time)h(system)e(in)75 1132 y(mapping)e(the)f(pro)q(cesses)h(on)o(to)e(hardw)o(are.)166 1188 y(As)19 b(stated)g(in)h(c)o(hapter)f(5,)h(a)f(pro)q(cess)h(group)f (in)h Fv(MPI)e Fx(is)i(a)f(collection)i(of)e Fu(n)g Fx(pro)q(cesses.)33 b(Eac)o(h)75 1245 y(pro)q(cess)17 b(in)h(the)e(group)h(is)g(assigned)g (a)g(rank)f(b)q(et)o(w)o(een)h Fu(0)g Fx(and)g Fu(n-1)p Fx(.)24 b(In)17 b(man)o(y)f(parallel)i(applications)75 1301 y(a)c(linear)h(ranking)g(of)f(pro)q(cesses)h(do)q(es)f(not)g (adequately)h(re\015ect)g(the)f(logical)h(comm)o(unication)g(pattern)75 1358 y(of)21 b(the)g(pro)q(cesses)g(\(whic)o(h)h(is)g(usually)g (determined)g(b)o(y)f(the)h(underlying)h(problem)f(geometry)e(and)75 1414 y(the)f(n)o(umerical)h(algorithm)g(used\).)31 b(Often)20 b(the)f(pro)q(cesses)g(are)g(arranged)g(in)h(top)q(ological)f(patterns) 75 1471 y(suc)o(h)c(as)g(t)o(w)o(o-)e(or)i(three-dimensional)i(grids.)j (More)14 b(generally)l(,)i(the)f(logical)h(pro)q(cess)g(arrangemen)o(t) d(is)75 1527 y(describ)q(ed)19 b(b)o(y)f(a)f(graph.)25 b(In)18 b(this)g(c)o(hapter)f(w)o(e)g(will)i(refer)f(to)e(this)i (logical)g(pro)q(cess)g(arrangemen)o(t)e(as)75 1584 y(the)f(\\virtual)h (top)q(ology)l(.")166 1640 y(A)11 b(clear)g(distinction)i(m)o(ust)e(b)q (e)g(made)g(b)q(et)o(w)o(een)g(the)h(virtual)f(pro)q(cess)g(top)q (ology)g(and)g(the)g(top)q(ology)75 1697 y(of)j(the)h(underlying,)i(ph) o(ysical)f(hardw)o(are.)j(The)c(virtual)g(top)q(ology)f(can)h(b)q(e)h (exploited)g(b)o(y)f(the)f(system)75 1753 y(in)j(the)f(assignmen)o(t)h (of)e(pro)q(cesses)i(to)f(ph)o(ysical)h(pro)q(cessors,)f(if)h(this)g (helps)g(to)f(impro)o(v)o(e)g(the)g(comm)o(u-)75 1810 y(nication)i(p)q(erformance)f(on)g(a)g(giv)o(en)h(mac)o(hine.)26 b(Ho)o(w)17 b(this)g(mapping)h(is)g(done,)f(ho)o(w)o(ev)o(er,)g(is)g (outside)75 1866 y(the)e(scop)q(e)g(of)f Fv(MPI)p Fx(.)g(The)g (description)j(of)d(the)g(virtual)i(top)q(ology)l(,)e(on)h(the)f(other) g(hand,)h(dep)q(ends)h(only)75 1922 y(on)i(the)g(application,)i(and)e (is)h(mac)o(hine-indep)q(endent.)30 b(The)19 b(functions)f(that)g(are)f (prop)q(osed)i(in)g(this)75 1979 y(c)o(hapter)c(deal)h(only)g(with)f (mac)o(hine-indep)q(en)q(den)o(t)j(mapping.)189 2067 y Fl(R)n(ationale.)55 b Fx(Though)18 b(ph)o(ysical)h(mapping)g(is)f (not)f(discussed,)j(the)e(existence)h(of)e(the)h(virtual)189 2123 y(top)q(ology)d(information)i(ma)o(y)e(b)q(e)i(used)f(as)g(advice) h(b)o(y)f(the)g(run)o(time)h(system.)22 b(There)16 b(are)g(w)o(ell-)189 2179 y(kno)o(wn)h(tec)o(hniques)h(for)f(mapping)h(grid/torus)f (structures)g(to)g(hardw)o(are)g(top)q(ologies)h(suc)o(h)f(as)189 2236 y(h)o(yp)q(ercub)q(es)k(or)f(grids.)35 b(F)l(or)20 b(more)g(complicated)h(graph)f(structures)g(go)q(o)q(d)h(heuristics)g (often)189 2292 y(yield)e(nearly)e(optimal)h(results)g([20)o(].)25 b(On)18 b(the)f(other)g(hand,)h(if)g(there)f(is)h(no)f(w)o(a)o(y)f(for) h(the)g(user)189 2349 y(to)12 b(sp)q(ecify)j(the)f(logical)g(pro)q (cess)g(arrangemen)o(t)e(as)h(a)g(\\virtual)h(top)q(ology)l(,")f(a)g (random)g(mapping)189 2405 y(is)k(most)f(lik)o(ely)j(to)d(result.)26 b(On)17 b(some)g(mac)o(hines,)g(this)h(will)g(lead)g(to)e(unnecessary)i (con)o(ten)o(tion)189 2462 y(in)g(the)f(in)o(terconnection)i(net)o(w)o (ork.)25 b(Some)17 b(details)h(ab)q(out)f(predicted)i(and)e(measured)h (p)q(erfor-)189 2518 y(mance)d(impro)o(v)o(emen)o(ts)f(that)g(result)i (from)e(go)q(o)q(d)h(pro)q(cess-to-pro)q(cessor)f(mapping)i(on)f(mo)q (dern)189 2575 y(w)o(ormhole-routing)g(arc)o(hitectures)g(can)h(b)q(e)f (found)h(in)g([10)o(,)f(9].)189 2647 y(Besides)22 b(p)q(ossible)g(p)q (erformance)f(b)q(ene\014ts,)h(the)f(virtual)g(top)q(ology)g(can)g (function)g(as)f(a)h(con-)189 2704 y(v)o(enien)o(t,)16 b(pro)q(cess-naming)h(structure,)f(with)g(tremendous)g(b)q(ene\014ts)h (for)f(program)f(readabilit)o(y)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 175 183 175 182 bop 75 -100 a Fm(6.2.)34 b(VIR)l(TUAL)17 b(TOPOLOGIES)1096 b Fx(175)189 45 y(and)15 b(notational)g(p)q(o)o(w)o(er)g(in)h (message-passing)f(programming.)20 b(\()p Fl(End)15 b(of)h(r)n (ationale.)p Fx(\))75 188 y Fn(6.2)59 b(Virtual)20 b(T)-5 b(op)r(ologies)75 290 y Fx(The)22 b(comm)o(unication)h(pattern)e(of)g (a)h(set)g(of)f(pro)q(cesses)h(can)g(b)q(e)h(represen)o(ted)f(b)o(y)g (a)f(graph.)40 b(The)75 346 y(no)q(des)14 b(stand)f(for)f(the)i(pro)q (cesses,)f(and)g(the)h(edges)f(connect)h(pro)q(cesses)f(that)g(comm)o (unicate)g(with)h(eac)o(h)75 403 y(other.)37 b Fv(MPI)21 b Fx(pro)o(vides)h(message-passing)f(b)q(et)o(w)o(een)h(an)o(y)e(pair)i (of)f(pro)q(cesses)g(in)h(a)f(group.)38 b(There)75 459 y(is)23 b(no)g(requiremen)o(t)g(for)f(op)q(ening)i(a)e(c)o(hannel)i (explicitly)l(.)46 b(Therefore,)24 b(a)e(\\missing)h(link")h(in)g(the) 75 516 y(user-de\014ned)19 b(pro)q(cess)e(graph)g(do)q(es)g(not)g(prev) o(en)o(t)g(the)g(corresp)q(onding)h(pro)q(cesses)g(from)e(exc)o (hanging)75 572 y(messages.)26 b(It)17 b(means)g(rather)g(that)g(this)h (connection)g(is)g(neglected)h(in)f(the)f(virtual)h(top)q(ology)l(.)26 b(This)75 629 y(strategy)21 b(implies)k(that)c(the)i(top)q(ology)f(giv) o(es)h(no)f(con)o(v)o(enien)o(t)h(w)o(a)o(y)f(of)g(naming)g(this)h (path)o(w)o(a)o(y)e(of)75 685 y(comm)o(unication.)28 b(Another)17 b(p)q(ossible)j(consequence)f(is)f(that)f(an)g(automatic)g (mapping)i(to)q(ol)e(\(if)h(one)75 741 y(exists)12 b(for)g(the)g(run)o (time)h(en)o(vironmen)o(t\))f(will)i(not)e(tak)o(e)f(accoun)o(t)h(of)g (this)g(edge)h(when)f(mapping.)20 b(Edges)75 798 y(in)13 b(the)g(comm)o(unication)g(graph)g(are)f(not)g(w)o(eigh)o(ted,)h(so)f (that)g(pro)q(cesses)h(are)f(either)h(simply)h(connected)75 854 y(or)h(not)f(connected)j(at)d(all.)189 961 y Fl(R)n(ationale.)55 b Fx(Exp)q(erience)20 b(with)e(similar)h(tec)o(hniques)g(in)g(P)l(ARMA) o(CS)f([5,)f(8])g(sho)o(w)g(that)h(this)189 1017 y(information)i(is)h (usually)h(su\016cien)o(t)f(for)e(a)h(go)q(o)q(d)g(mapping.)36 b(Additionally)l(,)24 b(a)c(more)g(precise)189 1074 y(sp)q (eci\014cation)e(is)f(more)f(di\016cult)i(for)e(the)g(user)h(to)f(set)g (up,)h(and)g(it)f(w)o(ould)h(mak)o(e)f(the)h(in)o(terface)189 1130 y(functions)f(substan)o(tially)g(more)f(complicated.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fx(\))166 1236 y(Sp)q(ecifying)22 b(the)e(virtual)g(top)q(ology)g(in)g(terms)g(of)f(a) g(graph)h(is)g(su\016cien)o(t)h(for)e(all)i(applications.)75 1293 y(Ho)o(w)o(ev)o(er,)12 b(in)i(man)o(y)e(applications)j(the)e (graph)g(structure)g(is)g(regular,)g(and)g(the)g(detailed)i(set-up)e (of)g(the)75 1349 y(graph)d(w)o(ould)i(b)q(e)f(incon)o(v)o(enien)o(t)h (for)e(the)h(user)g(and)g(migh)o(t)g(b)q(e)g(less)g(e\016cien)o(t)h(at) e(run)h(time.)19 b(A)10 b(large)h(frac-)75 1406 y(tion)j(of)g(all)h (parallel)h(applications)g(use)e(pro)q(cess)h(top)q(ologies)g(lik)o(e)g (rings,)f(t)o(w)o(o-)f(or)h(higher-dimensional)75 1462 y(grids,)19 b(or)f(tori.)28 b(These)19 b(structures)f(are)g(completely) h(de\014ned)h(b)o(y)e(the)h(n)o(um)o(b)q(er)f(of)g(dimensions)i(and)75 1519 y(the)15 b(n)o(um)o(b)q(ers)h(of)f(pro)q(cesses)g(in)h(eac)o(h)g (co)q(ordinate)f(direction.)22 b(Also,)15 b(the)g(mapping)h(of)f(grids) h(and)f(tori)75 1575 y(is)h(generally)h(an)f(easier)g(problem)g(then)g (that)f(of)g(general)h(graphs.)21 b(Th)o(us,)16 b(it)f(is)i(desirable)g (to)e(address)75 1631 y(these)g(cases)h(explicitly)l(.)166 1688 y(Pro)q(cess)i(co)q(ordinates)g(in)h(a)f(cartesian)g(structure)g (b)q(egin)h(their)f(n)o(um)o(b)q(ering)h(at)f(0.)28 b(Ro)o(w-ma)s(jor) 75 1744 y(n)o(um)o(b)q(ering)18 b(is)g(alw)o(a)o(ys)e(used)i(for)e(the) h(pro)q(cesses)h(in)f(a)g(cartesian)g(structure.)26 b(This)17 b(means)g(that,)g(for)75 1801 y(example,)i(the)f(relation)g(b)q(et)o(w) o(een)g(group)f(rank)h(and)g(co)q(ordinates)g(for)f(four)g(pro)q (cesses)h(in)h(a)e(\(2)11 b Ft(\002)i Fx(2\))75 1857 y(grid)j(is)f(as)g(follo)o(ws.)206 1949 y(co)q(ord)g(\(0,0\):)48 b(rank)15 b(0)206 2005 y(co)q(ord)g(\(0,1\):)48 b(rank)15 b(1)206 2062 y(co)q(ord)g(\(1,0\):)48 b(rank)15 b(2)206 2118 y(co)q(ord)g(\(1,1\):)48 b(rank)15 b(3)75 2249 y Fn(6.3)59 b(Emb)r(edding)18 b(in)i(MPI)75 2351 y Fx(The)13 b(supp)q(ort)f(for)g(virtual)h(top)q(ologies)g(as)f(de\014ned)h(in)h (this)e(c)o(hapter)h(is)f(consisten)o(t)h(with)g(other)f(parts)f(of)75 2407 y Fv(MPI)p Fx(,)k(and,)h(whenev)o(er)g(p)q(ossible,)h(mak)o(es)f (use)g(of)f(functions)i(that)e(are)h(de\014ned)h(elsewhere.)23 b(T)l(op)q(ology)75 2464 y(information)e(is)g(asso)q(ciated)h(with)f (comm)o(unicators.)36 b(It)21 b(is)h(added)f(to)g(comm)o(unicators)f (using)i(the)75 2520 y(cac)o(hing)16 b(mec)o(hanism)g(describ)q(ed)h (in)f(Chapter)f(5.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 176 184 176 183 bop 75 -100 a Fx(176)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fn(6.4)59 b(Overview)19 b(of)h(the)f(F)n(unctions)75 147 y Fx(The)h(functions)h Fv(MPI)p 459 147 14 2 v 15 w(GRAPH)p 624 147 V 18 w(CREA)l(TE)g Fx(and)f Fv(MPI)p 1007 147 V 16 w(CART)p 1142 147 V 17 w(CREA)l(TE)h Fx(are)e(used)i(to)e(create)g(general)75 203 y(\(graph\))e(virtual)h(top)q(ologies)g(and)g(cartesian)g(top)q (ologies,)g(resp)q(ectiv)o(ely)l(.)29 b(These)18 b(top)q(ology)g (creation)75 259 y(functions)13 b(are)g(collectiv)o(e.)21 b(As)12 b(with)h(other)g(collectiv)o(e)h(calls,)g(the)f(program)f(m)o (ust)g(b)q(e)h(written)g(to)f(w)o(ork)75 316 y(correctly)l(,)j(whether) h(the)f(call)h(sync)o(hronizes)g(or)f(not.)166 372 y(The)21 b(top)q(ology)f(creation)h(functions)g(tak)o(e)f(as)g(input)i(an)e (existing)i(comm)o(unicator)e Fv(comm)p 1795 372 V 10 w(old)p Fx(,)75 429 y(whic)o(h)14 b(de\014nes)h(the)e(set)g(of)g(pro)q (cesses)h(on)g(whic)o(h)g(the)f(top)q(ology)g(is)h(to)f(b)q(e)h(mapp)q (ed.)20 b(A)14 b(new)f(comm)o(uni-)75 485 y(cator)e Fv(comm)p 309 485 V 11 w(top)q(ol)h Fx(is)h(created)f(that)f(carries)h(the)g(top) q(ological)h(structure)f(as)f(cac)o(hed)i(information)f(\(see)75 542 y(Chapter)f(5\).)18 b(In)12 b(analogy)f(to)g(function)i Fv(MPI)p 849 542 V 15 w(COMM)p 1006 542 V 17 w(CREA)l(TE)p Fx(,)f(no)f(cac)o(hed)h(information)g(propagates)75 598 y(from)i Fv(comm)p 303 598 V 11 w(old)h Fx(to)g Fv(comm)p 563 598 V 10 w(top)q(ol)p Fx(.)166 655 y Fv(MPI)p 251 655 V 16 w(CART)p 386 655 V 17 w(CREA)l(TE)j Fx(can)g(b)q(e)g(used)g (to)f(describ)q(e)i(cartesian)f(structures)f(of)g(arbitrary)g(dimen-)75 711 y(sion.)i(F)l(or)12 b(eac)o(h)h(co)q(ordinate)g(direction)g(one)g (sp)q(eci\014es)h(whether)f(the)f(pro)q(cess)h(structure)f(is)h(p)q (erio)q(dic)h(or)75 768 y(not.)19 b(Note)c(that)f(an)h Fp(n)p Fx(-dimensional)j(h)o(yp)q(ercub)q(e)e(is)g(an)e Fp(n)p Fx(-dimensional)k(torus)c(with)i(2)e(pro)q(cesses)h(p)q(er)75 824 y(co)q(ordinate)h(direction.)22 b(Th)o(us,)15 b(sp)q(ecial)i(supp)q (ort)e(for)g(h)o(yp)q(ercub)q(e)i(structures)e(is)h(not)f(necessary)l (.)21 b(The)75 880 y(lo)q(cal)13 b(auxiliary)h(function)f Fv(MPI)p 626 880 V 15 w(DIMS)p 752 880 V 16 w(CREA)l(TE)h Fx(can)e(b)q(e)h(used)f(to)g(compute)g(a)g(balanced)h(distribution)75 937 y(of)i(pro)q(cesses)g(among)g(a)g(giv)o(en)g(n)o(um)o(b)q(er)h(of)f (dimensions.)189 1043 y Fl(R)n(ationale.)26 b Fx(Similar)18 b(functions)g(are)f(con)o(tained)h(in)g(EXPRESS)g([22)o(])f(and)h(P)l (ARMA)o(CS.)f(\()p Fl(End)189 1100 y(of)f(r)n(ationale.)p Fx(\))166 1206 y(The)h(function)g Fv(MPI)p 525 1206 V 16 w(TOPO)p 667 1206 V 17 w(TEST)f Fx(can)h(b)q(e)g(used)g(to)f (inquire)i(ab)q(out)e(the)h(top)q(ology)f(asso)q(ciated)75 1262 y(with)g(a)f(comm)o(unicator.)20 b(The)c(top)q(ological)g (information)f(can)h(b)q(e)g(extracted)f(from)g(the)g(comm)o(unica-)75 1319 y(tor)h(using)h(the)g(functions)h Fv(MPI)p 633 1319 V 15 w(GRAPHDIMS)p 909 1319 V 17 w(GET)f Fx(and)g Fv(MPI)p 1203 1319 V 16 w(GRAPH)p 1369 1319 V 18 w(GET)p Fx(,)f(for)g(general)i (graphs,)75 1375 y(and)e Fv(MPI)p 249 1375 V 15 w(CARTDIM)p 469 1375 V 17 w(GET)f Fx(and)h Fv(MPI)p 760 1375 V 16 w(CART)p 895 1375 V 17 w(GET)p Fx(,)f(for)g(cartesian)g(top)q(ologies.) 21 b(Sev)o(eral)16 b(additional)75 1432 y(functions)f(are)f(pro)o (vided)h(to)f(manipulate)i(cartesian)e(top)q(ologies:)20 b(the)15 b(functions)g Fv(MPI)p 1603 1432 V 16 w(CART)p 1738 1432 V 17 w(RANK)75 1488 y Fx(and)f Fv(MPI)p 247 1488 V 16 w(CART)p 382 1488 V 17 w(COORDS)h Fx(translate)e(cartesian)h (co)q(ordinates)f(in)o(to)h(a)f(group)h(rank,)f(and)h(vice-v)o(ersa;)75 1545 y(the)20 b(function)h Fv(MPI)p 426 1545 V 16 w(CART)p 561 1545 V 17 w(SUB)g Fx(can)f(b)q(e)h(used)g(to)f(extract)f(a)h (cartesian)g(subspace)h(\(analogous)f(to)75 1601 y Fv(MPI)p 160 1601 V 16 w(COMM)p 318 1601 V 16 w(SPLIT)p Fx(\).)e(The)i(function) g Fv(MPI)p 868 1601 V 16 w(CART)p 1003 1601 V 17 w(SHIFT)f Fx(pro)o(vides)g(the)h(information)f(needed)i(to)75 1658 y(comm)o(unicate)46 b(with)h(neigh)o(b)q(ors)f(in)h(a)f(cartesian)g (dimension.)114 b(The)47 b(t)o(w)o(o)d(functions)75 1714 y Fv(MPI)p 160 1714 V 16 w(GRAPH)p 326 1714 V 17 w(NEIGHBORS)p 594 1714 V 19 w(COUNT)19 b Fx(and)h Fv(MPI)p 963 1714 V 15 w(GRAPH)p 1128 1714 V 18 w(NEIGHBORS)h Fx(can)e(b)q(e)h(used)f(to) g(extract)75 1770 y(the)h(neigh)o(b)q(ors)g(of)f(a)g(no)q(de)h(in)h(a)e (graph.)33 b(The)19 b(function)i Fv(MPI)p 1204 1770 V 16 w(CART)p 1339 1770 V 17 w(SUB)f Fx(is)g(collectiv)o(e)h(o)o(v)o(er)e (the)75 1827 y(input)d(comm)o(unicator's)f(group;)f(all)i(other)f (functions)h(are)f(lo)q(cal.)166 1883 y(Tw)o(o)c(additional)i (functions,)g Fv(MPI)p 767 1883 V 16 w(GRAPH)p 933 1883 V 18 w(MAP)f Fx(and)g Fv(MPI)p 1229 1883 V 16 w(CART)p 1364 1883 V 17 w(MAP)g Fx(are)g(presen)o(ted)g(in)h(the)75 1940 y(last)f(section.)19 b(In)12 b(general)g(these)g(functions)h(are)e (not)g(called)i(b)o(y)f(the)g(user)f(directly)l(.)21 b(Ho)o(w)o(ev)o(er,)11 b(together)75 1996 y(with)16 b(the)h(comm)o (unicator)e(manipulation)j(functions)f(presen)o(ted)f(in)h(Chapter)f (5,)f(they)i(are)e(su\016cien)o(t)75 2053 y(to)g(implemen)o(t)h(all)g (other)f(top)q(ology)g(functions.)20 b(Section)d(6.5.7)c(outlines)k (suc)o(h)e(an)g(implemen)o(tation.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 177 185 177 184 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fx(177)75 45 y Fn(6.5)59 b(T)-5 b(op)r(ology)19 b(Constructo)n(rs)75 148 y Fj(6.5.1)49 b(Ca)o(rtesian)16 b(Constructo)o(r)75 329 y Fv(MPI)p 160 329 14 2 v 16 w(CART)p 295 329 V 17 w(CREA)l(TE\(comm)p 617 329 V 11 w(old,)f(ndims,)e(dims,)g(p)q(erio)q (ds,)j(reo)o(rder,)e(comm)p 1412 329 V 10 w(ca)o(rt\))117 406 y Fk(IN)171 b Fv(comm)p 454 406 V 10 w(old)383 b Fk(input)14 b(comm)o(unicator)d(\(handle\))117 481 y(IN)171 b Fv(ndims)458 b Fk(n)o(um)o(b)q(er)13 b(of)h(dimensions)e(of)i (cartesian)g(grid)g(\(in)o(teger\))117 556 y(IN)171 b Fv(dims)481 b Fk(in)o(teger)19 b(arra)o(y)f(of)f(size)i Fi(ndims)d Fk(sp)q(ecifying)i(the)g(n)o(um)o(b)q(er)g(of)905 612 y(pro)q(cesses)f(in)c(eac)o(h)i(dimension)117 688 y(IN)171 b Fv(p)q(erio)q(ds)436 b Fk(logical)12 b(arra)o(y)h(of)g(size) h Fi(ndims)d Fk(sp)q(ecifying)i(whether)h(the)g(grid)905 744 y(is)g(p)q(erio)q(dic)g(\()p Fi(true)p Fk(\))h(or)f(not)g(\()p Fi(false)p Fk(\))g(in)g(eac)o(h)g(dimension)117 819 y(IN)171 b Fv(reo)o(rder)438 b Fk(ranking)10 b(ma)o(y)g(b)q(e)h(reordered)i(\()p Fi(true)p Fk(\))f(or)f(not)g(\()p Fi(false)p Fk(\))g(\(logical\))117 894 y(OUT)124 b Fv(comm)p 454 894 V 10 w(ca)o(rt)367 b Fk(comm)o(unicator)11 b(with)j(new)g(cartesian)h(top)q(ology)d (\(handle\))75 1019 y Fu(int)23 b(MPI)p 245 1019 15 2 v 17 w(Cart)p 358 1019 V 17 w(create\(MPI)p 615 1019 V 16 w(Comm)g(comm)p 846 1019 V 17 w(old,)g(int)g(ndims,)g(int)h (*dims,)f(int)g(*periods,)393 1075 y(int)h(reorder,)e(MPI)p 778 1075 V 17 w(Comm)h(*comm)p 1034 1075 V 17 w(cart\))75 1162 y(MPI)p 150 1162 V 17 w(CART)p 263 1162 V 16 w(CREATE\(COMM)p 543 1162 V 16 w(OLD,)g(NDIMS,)g(DIMS,)h(PERIODS,)e(REORDER,)h(COMM)p 1514 1162 V 17 w(CART,)g(IERROR\))170 1218 y(INTEGER)g(COMM)p 460 1218 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g(COMM)p 1074 1218 V 17 w(CART,)g(IERROR)170 1275 y(LOGICAL)g(PERIODS\(*\),)g (REORDER)166 1361 y Fv(MPI)p 251 1361 14 2 v 16 w(CART)p 386 1361 V 17 w(CREA)l(TE)18 b Fx(returns)f(a)g(handle)i(to)e(a)g(new)g (comm)o(unicator)g(to)g(whic)o(h)h(the)g(cartesian)75 1418 y(top)q(ology)13 b(information)g(is)g(attac)o(hed.)19 b(If)13 b Fv(reo)o(rder)f(=)h(false)h Fx(then)f(the)g(rank)g(of)f(eac)o (h)h(pro)q(cess)h(in)g(the)f(new)75 1474 y(group)g(is)g(iden)o(tical)i (to)d(its)h(rank)g(in)h(the)f(old)h(group.)k(Otherwise,)c(the)f (function)h(ma)o(y)e(reorder)h(the)g(pro-)75 1530 y(cesses)h(\(p)q (ossibly)i(so)d(as)h(to)g(c)o(ho)q(ose)g(a)f(go)q(o)q(d)h(em)o(b)q (edding)i(of)e(the)g(virtual)h(top)q(ology)e(on)o(to)g(the)i(ph)o (ysical)75 1587 y(mac)o(hine\).)k(If)12 b(the)g(total)f(size)i(of)e (the)h(cartesian)g(grid)g(is)g(smaller)h(than)f(the)f(size)i(of)f(the)f (group)h(of)f Fv(comm)m Fx(,)75 1643 y(then)k(some)g(pro)q(cesses)h (are)f(returned)g Fi(MPI)p 826 1643 13 2 v 15 w(COMM)p 971 1643 V 14 w(NULL)p Fx(,)h(in)g(analogy)f(to)f Fv(MPI)p 1475 1643 14 2 v 16 w(COMM)p 1633 1643 V 17 w(SPLIT)p Fx(.)g(The)75 1700 y(call)i(is)g(erroneous)f(if)h(it)f(sp)q(eci\014es)i (a)e(grid)h(that)e(is)i(larger)f(than)g(the)g(group)g(size.)75 1822 y Fj(6.5.2)49 b(Ca)o(rtesian)16 b(Convenience)g(F)o(unction:)21 b(MPI)p 996 1822 15 2 v 18 w(DIMS)p 1131 1822 V 18 w(CREA)l(TE)75 1907 y Fx(F)l(or)15 b(cartesian)h(top)q(ologies,)g(the)g(function)g Fv(MPI)p 919 1907 14 2 v 16 w(DIMS)p 1046 1907 V 16 w(CREA)l(TE)h Fx(helps)g(the)f(user)g(select)g(a)g(balanced)75 1964 y(distribution)k(of)f(pro)q(cesses)g(p)q(er)g(co)q(ordinate)g (direction,)i(dep)q(ending)g(on)d(the)h(n)o(um)o(b)q(er)g(of)g(pro)q (cesses)75 2020 y(in)i(the)f(group)g(to)f(b)q(e)i(balanced)h(and)e (optional)h(constrain)o(ts)e(that)h(can)g(b)q(e)h(sp)q(eci\014ed)h(b)o (y)e(the)g(user.)75 2077 y(One)f(use)g(is)h(to)e(partition)h(all)g(the) g(pro)q(cesses)g(\(the)f(size)i(of)e Fi(MPI)p 1227 2077 13 2 v 15 w(COMM)p 1372 2077 V 14 w(W)o(ORLD)p Fx('s)g(group\))g(in)o (to)h(an)75 2133 y Fp(n)p Fx(-dimensional)f(top)q(ology)l(.)75 2284 y Fv(MPI)p 160 2284 14 2 v 16 w(DIMS)p 287 2284 V 16 w(CREA)l(TE\(nno)q(des,)f(ndims,)c(dims\))117 2361 y Fk(IN)171 b Fv(nno)q(des)441 b Fk(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g(a)f(grid)h(\(in)o(teger\))117 2436 y(IN)171 b Fv(ndims)458 b Fk(n)o(um)o(b)q(er)13 b(of)h(cartesian)g(dimensions)f(\(in)o(teger\))117 2512 y(INOUT)78 b Fv(dims)481 b Fk(in)o(teger)18 b(arra)o(y)f(of)f(size)i Fb(ndims)e Fk(sp)q(ecifying)h(the)h(n)o(um)o(b)q(er)f(of)905 2568 y(no)q(des)e(in)e(eac)o(h)i(dimension)75 2692 y Fu(int)23 b(MPI)p 245 2692 15 2 v 17 w(Dims)p 358 2692 V 17 w(create\(int)f(nnodes,)h(int)h(ndims,)f(int)g(*dims\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 178 186 178 185 bop 75 -100 a Fx(178)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(DIMS)p 263 45 V 16 w(CREATE\(NNODES,)22 b(NDIMS,)h(DIMS,)h (IERROR\))170 102 y(INTEGER)f(NNODES,)g(NDIMS,)g(DIMS\(*\),)g(IERROR) 166 188 y Fx(The)13 b(en)o(tries)h(in)f(the)g(arra)o(y)f Fv(dims)f Fx(are)i(set)f(to)h(describ)q(e)i(a)d(cartesian)h(grid)h (with)f Fv(ndims)e Fx(dimensions)75 244 y(and)i(a)g(total)g(of)f Fv(nno)q(des)k Fx(no)q(des.)j(The)14 b(dimensions)h(are)d(set)h(to)g(b) q(e)h(as)e(close)i(to)f(eac)o(h)g(other)g(as)g(p)q(ossible,)75 301 y(using)i(an)g(appropriate)g(divisibili)q(t)o(y)i(algorithm.)i(The) c(caller)h(ma)o(y)e(further)g(constrain)h(the)g(op)q(eration)75 357 y(of)e(this)h(routine)g(b)o(y)g(sp)q(ecifying)h(elemen)o(ts)f(of)f (arra)o(y)g Fu(dims)p Fx(.)19 b(If)13 b Fu(dims[i])g Fx(is)h(set)f(to)g(a)g(p)q(ositiv)o(e)i(n)o(um)o(b)q(er,)75 414 y(the)i(routine)g(will)h(not)e(mo)q(dify)i(the)e(n)o(um)o(b)q(er)h (of)f(no)q(des)i(in)f(dimension)h Fu(i)p Fx(;)f(only)g(those)g(en)o (tries)g(where)75 470 y Fu(dims[i])23 b(=)h(0)15 b Fx(are)g(mo)q (di\014ed)h(b)o(y)f(the)h(call.)166 527 y(Negativ)o(e)e(input)i(v)m (alues)f(of)f Fu(dims[i])f Fx(are)h(erroneous.)20 b(An)15 b(error)e(will)j(o)q(ccur)f(if)g Fu(nnodes)e Fx(is)i(not)f(a)75 583 y(m)o(ultiple)j(of)366 543 y Fe(Y)305 637 y Ff(i;dims)p Fg([)p Ff(i)p Fg(])p Fh(6)p Fg(=0)489 583 y Fp(dims)p Fx([)p Fp(i)p Fx(].)166 685 y(F)l(or)g Fu(dims[i])g Fx(set)g(b)o(y)g (the)h(call,)h Fu(dims[i])e Fx(will)i(b)q(e)f(ordered)g(in)h (non-increasing)g(order.)27 b(Arra)o(y)75 741 y Fu(dims)20 b Fx(is)i(suitable)g(for)f(use)g(as)g(input)h(to)e(routine)i Fv(MPI)p 1076 741 14 2 v 16 w(CART)p 1211 741 V 17 w(CREA)l(TE)p Fx(.)g Fv(MPI)p 1514 741 V 15 w(DIMS)p 1640 741 V 16 w(CREA)l(TE)g Fx(is)75 798 y(lo)q(cal.)75 1055 y Fq(Example)c(6.1)p 380 873 1247 2 v 379 930 2 57 v 405 913 a Fu(dims)p 634 930 V 159 w Fx(function)e(call)p 1303 930 V 424 w Fu(dims)p 1626 930 V 379 986 V 405 969 a Fx(b)q(efore)g(call)p 634 986 V 1303 986 V 720 w(on)f(return)p 1626 986 V 380 988 1247 2 v 379 1044 2 57 v 405 1027 a(\(0,0\))p 634 1044 V 160 w Fv(MPI)p 745 1027 14 2 v 16 w(DIMS)p 872 1027 V 15 w(CREA)l(TE\(6,)h(2,)e(dims\))p 1303 1044 2 57 v 48 w Fx(\(3,2\))p 1626 1044 V 379 1101 V 405 1084 a(\(0,0\))p 634 1101 V 160 w Fv(MPI)p 745 1084 14 2 v 16 w(DIMS)p 872 1084 V 15 w(CREA)l(TE\(7,)i(2,)e(dims\))p 1303 1101 2 57 v 48 w Fx(\(7,1\))p 1626 1101 V 379 1157 V 405 1140 a(\(0,3,0\))p 634 1157 V 124 w Fv(MPI)p 745 1140 14 2 v 16 w(DIMS)p 872 1140 V 15 w(CREA)l(TE\(6,)i(3,)e(dims\))p 1303 1157 2 57 v 48 w Fx(\(2,3,1\))p 1626 1157 V 379 1214 V 405 1197 a(\(0,3,0\))p 634 1214 V 124 w Fv(MPI)p 745 1197 14 2 v 16 w(DIMS)p 872 1197 V 15 w(CREA)l(TE\(7,)i(3,)e (dims\))p 1303 1214 2 57 v 48 w Fx(erroneous)h(call)p 1626 1214 V 380 1215 1247 2 v 75 1399 a Fj(6.5.3)49 b(General)16 b(\(Graph\))f(Constructo)o(r)75 1579 y Fv(MPI)p 160 1579 14 2 v 16 w(GRAPH)p 326 1579 V 17 w(CREA)l(TE\(comm)p 649 1579 V 12 w(old,)g(nno)q(des,)h(index,)g(edges,)g(reo)o(rder,)e (comm)p 1444 1579 V 10 w(graph\))117 1656 y Fk(IN)171 b Fv(comm)p 454 1656 V 10 w(old)383 b Fk(input)14 b(comm)o(unicator)d (without)j(top)q(ology)e(\(handle\))117 1731 y(IN)171 b Fv(nno)q(des)441 b Fk(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g (graph)g(\(in)o(teger\))117 1806 y(IN)171 b Fv(index)473 b Fk(arra)o(y)14 b(of)f(in)o(tegers)i(describing)f(no)q(de)h(degrees)g (\(see)g(b)q(elo)o(w\))117 1882 y(IN)171 b Fv(edges)468 b Fk(arra)o(y)14 b(of)f(in)o(tegers)i(describing)f(graph)g(edges)h (\(see)g(b)q(elo)o(w\))117 1957 y(IN)171 b Fv(reo)o(rder)438 b Fk(ranking)10 b(ma)o(y)g(b)q(e)h(reordered)i(\()p Fi(true)p Fk(\))f(or)f(not)g(\()p Fi(false)p Fk(\))g(\(logical\))117 2032 y(OUT)124 b Fv(comm)p 454 2032 V 10 w(graph)333 b Fk(comm)o(unicator)11 b(with)j(graph)g(top)q(ology)e(added)i (\(handle\))75 2156 y Fu(int)23 b(MPI)p 245 2156 15 2 v 17 w(Graph)p 382 2156 V 17 w(create\(MPI)p 639 2156 V 16 w(Comm)g(comm)p 870 2156 V 16 w(old,)h(int)f(nnodes,)g(int)h (*index,)f(int)g(*edges,)393 2213 y(int)h(reorder,)e(MPI)p 778 2213 V 17 w(Comm)h(*comm)p 1034 2213 V 17 w(graph\))75 2299 y(MPI)p 150 2299 V 17 w(GRAPH)p 287 2299 V 16 w(CREATE\(COMM)p 567 2299 V 16 w(OLD,)g(NNODES,)g(INDEX,)g(EDGES,)g(REORDER,)g(COMM)p 1538 2299 V 17 w(GRAPH,)393 2356 y(IERROR\))170 2412 y(INTEGER)g(COMM)p 460 2412 V 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g (EDGES\(*\),)f(COMM)p 1360 2412 V 17 w(GRAPH,)h(IERROR)170 2469 y(LOGICAL)g(REORDER)166 2555 y Fv(MPI)p 251 2555 14 2 v 16 w(GRAPH)p 417 2555 V 17 w(CREA)l(TE)f Fx(returns)e(a)g (handle)h(to)f(a)g(new)g(comm)o(unicator)g(to)g(whic)o(h)h(the)f(graph) 75 2611 y(top)q(ology)e(information)h(is)g(attac)o(hed.)30 b(If)19 b Fv(reo)o(rder)f(=)h(false)g Fx(then)g(the)g(rank)f(of)h(eac)o (h)f(pro)q(cess)h(in)h(the)75 2668 y(new)13 b(group)g(is)h(iden)o (tical)h(to)d(its)i(rank)f(in)h(the)f(old)g(group.)19 b(Otherwise,)14 b(the)g(function)g(ma)o(y)e(reorder)h(the)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 179 187 179 186 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fx(179)75 45 y(pro)q(cesses.)25 b(If)17 b(the)g(size,)h Fv(nno)q(des)p Fx(,)h(of)e(the)g(graph)f(is)i(smaller)f(than)g(the)g (size)h(of)e(the)h(group)g(of)f Fv(comm)m Fx(,)75 102 y(then)f(some)f(pro)q(cesses)h(are)f(returned)g Fi(MPI)p 822 102 13 2 v 15 w(COMM)p 967 102 V 14 w(NULL)p Fx(,)h(in)h(analogy)e (to)f Fv(MPI)p 1468 102 14 2 v 16 w(CART)p 1603 102 V 17 w(CREA)l(TE)j Fx(and)75 158 y Fv(MPI)p 160 158 V 16 w(COMM)p 318 158 V 16 w(SPLIT)p Fx(.)c(The)h(call)h(is)g(erroneous)e (if)i(it)f(sp)q(eci\014es)h(a)f(graph)f(that)h(is)g(larger)g(than)f (the)h(group)75 214 y(size)j(of)f(the)g(input)h(comm)o(unicator.)166 271 y(The)d(three)h(parameters)e Fv(nno)q(des,)j(index)f Fx(and)g Fv(edges)g Fx(de\014ne)g(the)f(graph)g(structure.)19 b Fv(nno)q(des)c Fx(is)f(the)75 327 y(n)o(um)o(b)q(er)f(of)g(no)q(des)g (of)f(the)h(graph.)19 b(The)13 b(no)q(des)h(are)e(n)o(um)o(b)q(ered)i (from)e Fu(0)h Fx(to)f Fu(nnodes-1)p Fx(.)18 b(The)13 b Fu(i)p Fx(th)f(en)o(try)75 384 y(of)k(arra)o(y)e Fv(index)j Fx(stores)f(the)g(total)f(n)o(um)o(b)q(er)h(of)g(neigh)o(b)q(ors)h(of)e (the)h(\014rst)g Fu(i)g Fx(graph)g(no)q(des.)22 b(The)17 b(lists)f(of)75 440 y(neigh)o(b)q(ors)d(of)g(no)q(des)g Fu(0,)24 b(1,)f Fp(:)8 b(:)g(:)e Fu(,)24 b(nnodes-1)11 b Fx(are)h(stored)h(in)g(consecutiv)o(e)h(lo)q(cations)f(in)h(arra)o(y) d Fv(edges)p Fx(.)75 497 y(The)16 b(arra)o(y)e Fv(edges)i Fx(is)g(a)f(\015attened)h(represen)o(tation)f(of)g(the)h(edge)f(lists.) 21 b(The)16 b(total)f(n)o(um)o(b)q(er)h(of)f(en)o(tries)75 553 y(in)h Fv(index)g Fx(is)g Fv(nno)q(des)h Fx(and)e(the)g(total)g(n)o (um)o(b)q(er)g(of)g(en)o(tries)h(in)g Fv(edges)g Fx(is)f(equal)h(to)f (the)g(n)o(um)o(b)q(er)h(of)e(graph)75 610 y(edges.)166 666 y(The)19 b(de\014nitions)h(of)e(the)h(argumen)o(ts)f Fu(nnodes)p Fx(,)g Fu(index)p Fx(,)h(and)g Fu(edges)f Fx(are)g(illustrated)i(with)f(the)75 723 y(follo)o(wing)d(simple)h (example.)75 827 y Fq(Example)h(6.2)k Fx(Assume)g(there)g(are)g(four)f (pro)q(cesses)i(0,)g(1,)f(2,)h(3)f(with)g(the)g(follo)o(wing)h (adjacency)75 884 y(matrix:)p 181 929 433 2 v 180 985 2 57 v 206 968 a(pro)q(cess)p 373 985 V 50 w(neigh)o(b)q(ors)p 613 985 V 181 987 433 2 v 180 1043 2 57 v 266 1026 a(0)p 373 1043 V 110 w(1,)15 b(3)p 613 1043 V 180 1100 V 266 1083 a(1)p 373 1100 V 110 w(0)p 613 1100 V 180 1156 V 266 1139 a(2)p 373 1156 V 110 w(3)p 613 1156 V 180 1213 V 266 1196 a(3)p 373 1213 V 110 w(0,)g(2)p 613 1213 V 181 1214 433 2 v 166 1289 a(Then,)g(the)h(input)g(argumen)o(ts)e(are:) 206 1381 y(nno)q(des)i(=)50 b(4)206 1437 y(index)16 b(=)81 b(2,)15 b(3,)f(4,)h(6)206 1494 y(edges)g(=)82 b(1,)15 b(3,)f(0,)h(3,)f(0,)h(2)166 1586 y(Th)o(us,)h(in)h(C,)e Fu(index[0])g Fx(is)i(the)f(degree)g(of)g(no)q(de)h(zero,)f(and)g Fu(index[i])23 b(-)h(index[i-1])14 b Fx(is)j(the)75 1642 y(degree)22 b(of)g(no)q(de)g Fu(i,)i(i=1,)f Fp(:)8 b(:)g(:)e Fu(,)23 b(nnodes-1)p Fx(;)h(the)e(list)h(of)e(neigh)o(b)q(ors)i(of)e (no)q(de)i(zero)e(is)i(stored)e(in)75 1699 y Fu(edges[j])p Fx(,)14 b(for)h Fu(0)e Ft(\024)h Fu(j)f Ft(\024)h Fu(index)p Fx([)p Fu(0)p Fx(])9 b Ft(\000)h Fu(1)16 b Fx(and)g(the)f(list)i(of)e (neigh)o(b)q(ors)h(of)f(no)q(de)i Fu(i)p Fx(,)e Fu(i)e Fp(>)h Fu(0)p Fx(,)h(is)h(stored)f(in)75 1755 y Fu(edges[j])p Fx(,)f Fu(index)p Fx([)p Fu(i)9 b Ft(\000)h Fu(1)p Fx(])i Ft(\024)h Fu(j)g Ft(\024)g Fu(index)p Fx([)p Fu(i)p Fx(])8 b Ft(\000)j Fu(1)p Fx(.)166 1811 y(In)17 b(F)l(ortran,)f Fu(index\(1\))f Fx(is)j(the)e(degree)i(of)e(no)q(de)h(zero,)g(and)g Fu(index\(i+1\))22 b(-)i(index\(i\))16 b Fx(is)h(the)75 1868 y(degree)22 b(of)g(no)q(de)g Fu(i,)i(i=1,)f Fp(:)8 b(:)g(:)e Fu(,)23 b(nnodes-1)p Fx(;)h(the)e(list)h(of)e(neigh)o(b)q (ors)i(of)e(no)q(de)i(zero)e(is)i(stored)e(in)75 1924 y Fu(edges\(j\))p Fx(,)d(for)g Fu(1)f Ft(\024)i Fu(j)f Ft(\024)g Fu(index)p Fx(\()p Fu(1)p Fx(\))f(and)i(the)f(list)h(of)f (neigh)o(b)q(ors)h(of)f(no)q(de)i Fu(i)p Fx(,)e Fu(i)g Fp(>)h Fu(0)p Fx(,)f(is)h(stored)f(in)75 1981 y Fu(edges\(j\))p Fx(,)c Fu(index)p Fx(\()p Fu(i)p Fx(\))8 b(+)j Fu(1)h Ft(\024)h Fu(j)g Ft(\024)g Fu(index)p Fx(\()p Fu(i)c Fx(+)h Fu(1)p Fx(\).)189 2086 y Fl(A)n(dvic)n(e)17 b(to)i (implementors.)56 b Fx(The)19 b(follo)o(wing)f(top)q(ology)g (information)g(is)g(lik)o(ely)i(to)d(b)q(e)h(stored)189 2142 y(with)d(a)g(comm)o(unicator:)243 2236 y Ft(\017)23 b Fx(T)o(yp)q(e)15 b(of)g(top)q(ology)g(\(cartesian/graph\),)243 2308 y Ft(\017)23 b Fx(F)l(or)14 b(a)h(cartesian)g(top)q(ology:)316 2381 y(1.)22 b Fu(ndims)14 b Fx(\(n)o(um)o(b)q(er)h(of)g(dimensions\),) 316 2445 y(2.)22 b Fu(dims)14 b Fx(\(n)o(um)o(b)q(ers)h(of)g(pro)q (cesses)h(p)q(er)f(co)q(ordinate)h(direction\),)316 2510 y(3.)22 b Fu(periods)14 b Fx(\(p)q(erio)q(dicit)o(y)j(information\),) 316 2574 y(4.)22 b Fu(own_position)12 b Fx(\(o)o(wn)h(p)q(osition)i(in) f(grid,)h(could)f(also)g(b)q(e)h(computed)f(from)f(rank)h(and)374 2631 y(dims\))243 2704 y Ft(\017)23 b Fx(F)l(or)14 b(a)h(graph)g(top)q (ology:)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 180 188 180 187 bop 75 -100 a Fx(180)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)316 45 y Fx(1.)22 b Fu(index)p Fx(,)316 110 y(2.)g Fu(edges)p Fx(,)289 183 y(whic)o(h)16 b(are)f(the)g(v)o(ectors)f(de\014ning)j(the)e(graph)g(structure.)189 277 y(F)l(or)k(a)g(graph)h(structure)f(the)h(n)o(um)o(b)q(er)g(of)f(no) q(des)i(is)f(equal)h(to)e(the)h(n)o(um)o(b)q(er)g(of)f(pro)q(cesses)h (in)189 333 y(the)h(group.)36 b(Therefore,)22 b(the)f(n)o(um)o(b)q(er)g (of)g(no)q(des)g(do)q(es)g(not)g(ha)o(v)o(e)f(to)g(b)q(e)i(stored)e (explicitly)l(.)189 390 y(An)c(additional)i(zero)f(en)o(try)f(at)f(the) i(start)e(of)h(arra)o(y)f Fv(index)j Fx(simpli\014es)g(access)f(to)f (the)g(top)q(ology)189 446 y(information.)k(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fx(\))75 568 y Fj(6.5.4)49 b(T)l(op)q(ology)19 b(inquiry)e(functions)75 654 y Fx(If)11 b(a)f(top)q(ology)g(has)h(b)q(een)g(de\014ned)h(with)f (one)g(of)f(the)g(ab)q(o)o(v)o(e)g(functions,)i(then)f(the)g(top)q (ology)f(information)75 710 y(can)15 b(b)q(e)h(lo)q(ok)o(ed)g(up)g (using)f(inquiry)i(functions.)k(They)15 b(all)h(are)f(lo)q(cal)i (calls.)75 861 y Fv(MPI)p 160 861 14 2 v 16 w(TOPO)p 302 861 V 17 w(TEST\(comm)m(,)12 b(status\))117 938 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(\(handle\))117 1013 y(OUT)124 b Fv(status)460 b Fk(top)q(ology)13 b(t)o(yp)q(e)h(of)g (comm)o(uni)o(cator)e Fb(comm)h Fk(\(c)o(hoice\))75 1138 y Fu(int)23 b(MPI)p 245 1138 15 2 v 17 w(Topo)p 358 1138 V 17 w(test\(MPI)p 567 1138 V 16 w(Comm)g(comm,)g(int)h(*status\))75 1224 y(MPI)p 150 1224 V 17 w(TOPO)p 263 1224 V 16 w(TEST\(COMM,)f (STATUS,)g(IERROR\))170 1281 y(INTEGER)g(COMM,)h(STATUS,)e(IERROR)166 1367 y Fx(The)h(function)g Fv(MPI)p 537 1367 14 2 v 16 w(TOPO)p 679 1367 V 17 w(TEST)g Fx(returns)f(the)h(t)o(yp)q(e)g(of)f (top)q(ology)g(that)g(is)h(assigned)g(to)f(a)75 1424 y(comm)o(unicator.)166 1480 y(The)15 b(output)g(v)m(alue)i Fu(status)d Fx(is)i(one)f(of)g(the)g(follo)o(wing:)117 1584 y Fi(MPI)p 194 1584 13 2 v 14 w(GRAPH)602 b Fx(graph)15 b(top)q(ology)117 1638 y Fi(MPI)p 194 1638 V 14 w(CART)631 b Fx(cartesian)15 b(top)q(ology)117 1693 y Fi(MPI)p 194 1693 V 14 w(UNDEFINED)508 b Fx(no)15 b(top)q(ology)75 1870 y Fv(MPI)p 160 1870 14 2 v 16 w(GRAPHDIMS)p 437 1870 V 17 w(GET\(comm)m(,)d(nno)q(des,)17 b(nedges\))117 1947 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)7 b(for)i(group)h(with)f(graph)h(structure)i(\(handle\))117 2022 y(OUT)124 b Fv(nno)q(des)441 b Fk(n)o(um)o(b)q(er)15 b(of)f(no)q(des)i(in)f(graph)g(\(in)o(teger\))h(\(same)e(as)i(n)o(um)o (b)q(er)905 2079 y(of)d(pro)q(cesses)k(in)c(the)i(group\))117 2154 y(OUT)124 b Fv(nedges)445 b Fk(n)o(um)o(b)q(er)13 b(of)h(edges)h(in)e(graph)h(\(in)o(teger\))75 2278 y Fu(int)23 b(MPI)p 245 2278 15 2 v 17 w(Graphdims)p 478 2278 V 16 w(get\(MPI)p 662 2278 V 16 w(Comm)h(comm,)f(int)g(*nnodes,)g (int)h(*nedges\))75 2365 y(MPI)p 150 2365 V 17 w(GRAPHDIMS)p 383 2365 V 16 w(GET\(COMM,)e(NNODES,)h(NEDGES,)g(IERROR\))170 2421 y(INTEGER)g(COMM,)h(NNODES,)e(NEDGES,)h(IERROR)166 2508 y Fx(F)l(unctions)c Fv(MPI)p 462 2508 14 2 v 16 w(GRAPHDIMS)p 739 2508 V 17 w(GET)g Fx(and)g Fv(MPI)p 1037 2508 V 16 w(GRAPH)p 1203 2508 V 18 w(GET)g Fx(retriev)o(e)g(the)f (graph-top)q(ology)75 2564 y(information)d(that)g(w)o(as)f(asso)q (ciated)i(with)f(a)g(comm)o(unicator)g(b)o(y)g Fv(MPI)p 1310 2564 V 16 w(GRAPH)p 1476 2564 V 17 w(CREA)l(TE)p Fx(.)166 2621 y(The)21 b(information)g(pro)o(vided)h(b)o(y)f Fv(MPI)p 864 2621 V 16 w(GRAPHDIMS)p 1141 2621 V 17 w(GET)g Fx(can)g(b)q(e)g(used)h(to)e(dimension)j(the)75 2677 y(v)o(ectors)14 b Fu(index)h Fx(and)g Fu(edges)g Fx(correctly)g(for)g (the)g(follo)o(wing)h(call)g(to)f Fv(MPI)p 1339 2677 V 16 w(GRAPH)p 1505 2677 V 17 w(GET)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 181 189 181 188 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fx(181)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GRAPH)p 326 45 V 17 w(GET\(comm)n(,)11 b(maxindex,)i(maxedges,)g(index,)j (edges\))117 122 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(graph)g(structure)i(\(handle\))117 197 y(IN)171 b Fv(maxi)o(ndex)392 b Fk(length)14 b(of)f(v)o(ector)i Fi(index)g Fk(in)f(the)g(calling)f(program)905 254 y(\(in)o(teger\))117 329 y(IN)171 b Fv(maxedges)386 b Fk(length)14 b(of)f(v)o(ector)i Fi(edges)g Fk(in)e(the)i(calling)d(program)905 385 y(\(in)o(teger\))117 461 y(OUT)124 b Fv(index)473 b Fk(arra)o(y)17 b(of)g(in)o(tegers)i(con) o(taining)d(the)i(graph)g(structure)h(\(for)905 517 y(details)14 b(see)h(the)g(de\014nition)e(of)g Fi(MPI)p 1484 517 13 2 v 15 w(GRAPH)p 1638 517 V 14 w(CREA)m(TE)p Fk(\))117 592 y(OUT)124 b Fv(edges)468 b Fk(arra)o(y)14 b(of)f(in)o(tegers)i(con) o(taining)e(the)h(graph)g(structure)75 717 y Fu(int)23 b(MPI)p 245 717 15 2 v 17 w(Graph)p 382 717 V 17 w(get\(MPI)p 567 717 V 16 w(Comm)g(comm,)g(int)h(maxindex,)f(int)g(maxedges,)g(int)g (*index,)393 773 y(int)h(*edges\))75 860 y(MPI)p 150 860 V 17 w(GRAPH)p 287 860 V 16 w(GET\(COMM,)f(MAXINDEX,)g(MAXEDGES,)f (INDEX,)h(EDGES,)g(IERROR\))170 916 y(INTEGER)g(COMM,)h(MAXINDEX,)e (MAXEDGES,)h(INDEX\(*\),)f(EDGES\(*\),)h(IERROR)75 1097 y Fv(MPI)p 160 1097 14 2 v 16 w(CARTDIM)p 381 1097 V 16 w(GET\(comm)m(,)12 b(ndims\))117 1174 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j (\(handle\))117 1249 y(OUT)124 b Fv(ndims)458 b Fk(n)o(um)o(b)q(er)12 b(of)f(dimensions)g(of)h(the)h(cartesian)g(structure)h(\(in)o(te-)905 1306 y(ger\))75 1430 y Fu(int)23 b(MPI)p 245 1430 15 2 v 17 w(Cartdim)p 430 1430 V 16 w(get\(MPI)p 614 1430 V 17 w(Comm)g(comm,)g(int)h(*ndims\))75 1517 y(MPI)p 150 1517 V 17 w(CARTDIM)p 335 1517 V 16 w(GET\(COMM,)f(NDIMS,)g (IERROR\))170 1573 y(INTEGER)g(COMM,)h(NDIMS,)f(IERROR)166 1660 y Fx(The)14 b(functions)h Fv(MPI)p 538 1660 14 2 v 16 w(CARTDIM)p 759 1660 V 16 w(GET)f Fx(and)g Fv(MPI)p 1046 1660 V 16 w(CART)p 1181 1660 V 17 w(GET)g Fx(return)g(the)g (cartesian)g(top)q(ology)75 1716 y(information)h(that)g(w)o(as)f(asso)q (ciated)i(with)f(a)g(comm)o(unicator)g(b)o(y)g Fv(MPI)p 1310 1716 V 16 w(CART)p 1445 1716 V 17 w(CREA)l(TE)p Fx(.)75 1867 y Fv(MPI)p 160 1867 V 16 w(CART)p 295 1867 V 17 w(GET\(comm)m(,)d(maxdim)m(s,)g(dims,)h(p)q(erio)q(ds,)j(co)q(o)o (rds\))117 1944 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j(\(handle\))117 2019 y(IN)171 b Fv(maxdim)m(s)402 b Fk(length)16 b(of)f(v)o(ectors)i Fb(dims,)k(periods)p Fk(,)15 b(and)g Fb(coords)g Fk(in)g(the)905 2076 y(calling)e(program)f(\(in)o(teger\))117 2151 y(OUT)124 b Fv(dims)481 b Fk(n)o(um)o(b)q(er)13 b(of)g(pro)q(cesses)i(for)e(eac)o (h)h(cartesian)g(dimension)e(\(ar-)905 2207 y(ra)o(y)i(of)f(in)o (teger\))117 2283 y(OUT)124 b Fv(p)q(erio)q(ds)436 b Fk(p)q(erio)q(dicit)o(y)21 b(\()p Fi(true)p Fk(/)p Fi(false)p Fk(\))g(for)g(eac)o(h)g(cartesian)h(dimension)905 2339 y(\(arra)o(y)14 b(of)f(logical\))117 2414 y(OUT)124 b Fv(co)q(o)o(rds)449 b Fk(co)q(ordinates)21 b(of)f(calling)e(pro)q(cess) k(in)e(cartesian)h(structure)905 2471 y(\(arra)o(y)14 b(of)f(in)o(teger\))75 2595 y Fu(int)23 b(MPI)p 245 2595 15 2 v 17 w(Cart)p 358 2595 V 17 w(get\(MPI)p 543 2595 V 16 w(Comm)g(comm,)h(int)f(maxdims,)g(int)g(*dims,)g(int)h(*periods,) 393 2651 y(int)g(*coords\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 182 190 182 189 bop 75 -100 a Fx(182)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(CART)p 263 45 V 16 w(GET\(COMM,)23 b(MAXDIMS,)g(DIMS,)g (PERIODS,)g(COORDS,)g(IERROR\))170 102 y(INTEGER)g(COMM,)h(MAXDIMS,)e (DIMS\(*\),)h(COORDS\(*\),)g(IERROR)170 158 y(LOGICAL)g(PERIODS\(*\))75 339 y Fv(MPI)p 160 339 14 2 v 16 w(CART)p 295 339 V 17 w(RANK\(comm)m(,)12 b(co)q(o)o(rds,)j(rank\))117 416 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(cartesian)g (structure)j(\(handle\))117 491 y(IN)171 b Fv(co)q(o)o(rds)449 b Fk(in)o(teger)17 b(arra)o(y)f(\(of)g(size)h Fb(ndims)p Fk(\))e(sp)q(ecifying)h(the)h(cartesian)905 548 y(co)q(ordinates)e(of)e (a)h(pro)q(cess)117 623 y(OUT)124 b Fv(rank)488 b Fk(rank)14 b(of)f(sp)q(eci\014ed)j(pro)q(cess)f(\(in)o(teger\))75 747 y Fu(int)23 b(MPI)p 245 747 15 2 v 17 w(Cart)p 358 747 V 17 w(rank\(MPI)p 567 747 V 16 w(Comm)g(comm,)g(int)h(*coords,)f (int)g(*rank\))75 834 y(MPI)p 150 834 V 17 w(CART)p 263 834 V 16 w(RANK\(COMM,)g(COORDS,)g(RANK,)g(IERROR\))170 890 y(INTEGER)g(COMM,)h(COORDS\(*\),)e(RANK,)h(IERROR)166 977 y Fx(F)l(or)12 b(a)h(pro)q(cess)g(group)g(with)g(cartesian)g (structure,)g(the)g(function)g Fv(MPI)p 1397 977 14 2 v 16 w(CART)p 1532 977 V 17 w(RANK)g Fx(translates)75 1033 y(the)22 b(logical)h(pro)q(cess)f(co)q(ordinates)g(to)g(pro)q (cess)g(ranks)f(as)h(they)g(are)f(used)i(b)o(y)e(the)h(p)q(oin)o (t-to-p)q(oin)o(t)75 1090 y(routines.)166 1146 y(F)l(or)f(dimension)i Fu(i)e Fx(with)h Fu(periods\(i\))h(=)h(true)p Fx(,)e(if)g(the)f(co)q (ordinate,)j Fu(coords\(i\))p Fx(,)d(is)h(out)f(of)75 1203 y(range,)14 b(that)g(is,)h Fu(coords\(i\))22 b Fp(<)j Fu(0)14 b Fx(or)g Fu(coords\(i\))23 b Ft(\025)h Fu(dims\(i\))p Fx(,)13 b(it)i(is)g(shifted)h(bac)o(k)e(to)g(the)h(in)o(terv)m(al)75 1259 y Fu(0)24 b Ft(\024)g Fu(coords\(i\))f Fp(<)h Fu(dims\(i\))e Fx(automatically)l(.)46 b(Out-of-range)23 b(co)q(ordinates)h(are)f (erroneous)h(for)75 1315 y(non-p)q(erio)q(dic)18 b(dimensions.)75 1466 y Fv(MPI)p 160 1466 V 16 w(CART)p 295 1466 V 17 w(COORDS\(comm)n(,)12 b(rank,)j(maxdim)m(s,)d(co)q(o)o(rds\))117 1544 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j(\(handle\))117 1619 y(IN)171 b Fv(rank)488 b Fk(rank)14 b(of)f(a)h(pro)q(cess)i(within)d (group)g(of)h Fi(comm)8 b Fk(\(in)o(teger\))117 1694 y(IN)171 b Fv(maxdim)m(s)402 b Fk(length)11 b(of)f(v)o(ector)i Fb(coord)d Fk(in)h(the)i(calling)d(program)g(\(in)o(teger\))117 1769 y(OUT)124 b Fv(co)q(o)o(rds)449 b Fk(in)o(teger)16 b(arra)o(y)e(\(of)h(size)h Fb(ndims)p Fk(\))e(con)o(taining)g(the)h (cartesian)905 1825 y(co)q(ordinates)g(of)e(sp)q(eci\014ed)j(pro)q (cess)f(\(in)o(teger\))75 1950 y Fu(int)23 b(MPI)p 245 1950 15 2 v 17 w(Cart)p 358 1950 V 17 w(coords\(MPI)p 615 1950 V 16 w(Comm)g(comm,)g(int)h(rank,)f(int)g(maxdims,)g(int)g (*coords\))75 2036 y(MPI)p 150 2036 V 17 w(CART)p 263 2036 V 16 w(COORDS\(COMM,)g(RANK,)g(MAXDIMS,)g(COORDS,)f(IERROR\))170 2093 y(INTEGER)h(COMM,)h(RANK,)f(MAXDIMS,)f(COORDS\(*\),)h(IERROR)166 2179 y Fx(The)28 b(in)o(v)o(erse)g(mapping,)k(rank-to-co)q(ordinates)c (translation)f(is)i(pro)o(vided)g(b)o(y)e Fv(MPI)p 1727 2179 14 2 v 16 w(CART-)p 75 2236 V 89 2236 a(COORDS)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 183 191 183 190 bop 75 -100 a Fm(6.5.)29 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)963 b Fx(183)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GRAPH)p 326 45 V 17 w(NEIGHBORS)p 594 45 V 19 w(COUNT\(comm)m(,)12 b(rank,)j(nneighb)q(o)o(rs\))117 122 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(graph)g(top)q(ology)e(\(handle\))117 196 y(IN)171 b Fv(rank)488 b Fk(rank)14 b(of)f(pro)q(cess)j(in)d(group) h(of)f Fi(comm)8 b Fk(\(in)o(teger\))117 270 y(OUT)124 b Fv(nneighb)q(o)o(rs)369 b Fk(n)o(um)o(b)q(er)13 b(of)h(neigh)o(b)q (ors)g(of)f(sp)q(eci\014ed)j(pro)q(cess)f(\(in)o(teger\))75 394 y Fu(int)23 b(MPI)p 245 394 15 2 v 17 w(Graph)p 382 394 V 17 w(neighbors)p 615 394 V 16 w(count\(MPI)p 847 394 V 16 w(Comm)g(comm,)g(int)h(rank,)f(int)g(*nneighbors\))75 481 y(MPI)p 150 481 V 17 w(GRAPH)p 287 481 V 16 w(NEIGHBORS)p 519 481 V 16 w(COUNT\(COMM,)g(RANK,)g(NNEIGHBORS,)f(IERROR\))170 537 y(INTEGER)h(COMM,)h(RANK,)f(NNEIGHBORS,)f(IERROR)166 624 y Fv(MPI)p 251 624 14 2 v 16 w(GRAPH)p 417 624 V 17 w(NEIGHBORS)p 685 624 V 18 w(COUNT)h Fx(and)e Fv(MPI)p 1058 624 V 16 w(GRAPH)p 1224 624 V 18 w(NEIGHBORS)i Fx(pro)o(vide)f (adjacency)75 680 y(information)15 b(for)g(a)g(general,)g(graph)g(top)q (ology)l(.)75 831 y Fv(MPI)p 160 831 V 16 w(GRAPH)p 326 831 V 17 w(NEIGHBORS\(comm)o(,)d(rank,)j(maxneighb)q(o)o(rs,)d(neighb)q (o)o(rs\))117 908 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(graph)g(top)q(ology)e(\(handle\))117 982 y(IN)171 b Fv(rank)488 b Fk(rank)14 b(of)f(pro)q(cess)j(in)d(group)h(of)f Fi(comm)8 b Fk(\(in)o(teger\))117 1056 y(IN)171 b Fv(maxneighb)q(o)o (rs)310 b Fk(size)15 b(of)e(arra)o(y)h Fi(neighb)q(o)o(rs)h Fk(\(in)o(teger\))117 1130 y(OUT)124 b Fv(neighb)q(o)o(rs)392 b Fk(ranks)13 b(of)g(pro)q(cesses)i(that)e(are)h(neigh)o(b)q(ors)f(to)g (sp)q(eci\014ed)h(pro-)905 1187 y(cess)i(\(arra)o(y)d(of)h(in)o (teger\))75 1311 y Fu(int)23 b(MPI)p 245 1311 15 2 v 17 w(Graph)p 382 1311 V 17 w(neighbors\(MPI)p 711 1311 V 15 w(Comm)g(comm,)h(int)f(rank,)g(int)h(maxneighbors,)393 1367 y(int)g(*neighbors\))75 1454 y(MPI)p 150 1454 V 17 w(GRAPH)p 287 1454 V 16 w(NEIGHBORS\(COMM,)e(RANK,)h(MAXNEIGHBORS,)f (NEIGHBORS,)h(IERROR\))170 1510 y(INTEGER)g(COMM,)h(RANK,)f (MAXNEIGHBORS,)f(NEIGHBORS\(*\),)g(IERROR)75 1638 y Fq(Example)c(6.3)k Fx(Supp)q(ose)14 b(that)d Fu(comm)h Fx(is)g(a)g(comm)o(unicator)g(with) g(a)g(sh)o(u\017e-exc)o(hange)h(top)q(ology)l(.)19 b(The)75 1694 y(group)h(has)f(2)317 1678 y Ff(n)360 1694 y Fx(mem)o(b)q(ers.)34 b(Eac)o(h)19 b(pro)q(cess)h(is)h(lab)q(eled)h(b)o(y)d Fp(a)1166 1701 y Fg(1)1186 1694 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)1312 1701 y Ff(n)1354 1694 y Fx(with)21 b Fp(a)1487 1701 y Ff(i)1521 1694 y Ft(2)f(f)p Fx(0)p Fp(;)8 b Fx(1)p Ft(g)p Fx(,)19 b(and)h(has)75 1751 y(three)d(neigh)o(b)q(ors:)23 b(exc)o(hange\()p Fp(a)639 1758 y Fg(1)659 1751 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)785 1758 y Ff(n)808 1751 y Fx(\))14 b(=)i Fp(a)915 1758 y Fg(1)935 1751 y Fp(;)8 b(:)g(:)g(:)t(;)g(a)1060 1758 y Ff(n)p Fh(\000)p Fg(1)1128 1751 y Fp(;)g Fx(\026)-23 b Fp(a)1173 1758 y Ff(n)1213 1751 y Fx(\()q(\026)f Fp(a)14 b Fx(=)i(1)11 b Ft(\000)g Fp(a)p Fx(\),)17 b(sh)o(u\017e\()p Fp(a)1639 1758 y Fg(1)1658 1751 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)1784 1758 y Ff(n)1807 1751 y Fx(\))15 b(=)75 1807 y Fp(a)99 1814 y Fg(2)119 1807 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)245 1814 y Ff(n)267 1807 y Fp(;)g(a)312 1814 y Fg(1)331 1807 y Fx(,)24 b(and)e(unsh)o(u\017e\()p Fp(a)680 1814 y Fg(1)700 1807 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)826 1814 y Ff(n)849 1807 y Fx(\))24 b(=)g Fp(a)974 1814 y Ff(n)998 1807 y Fp(;)8 b(a)1043 1814 y Fg(1)1062 1807 y Fp(;)g(:)g(:)g(:)t(;)g(a)1187 1814 y Ff(n)p Fh(\000)p Fg(1)1255 1807 y Fx(.)41 b(The)22 b(graph)g(adjacency)h(list)f(is)75 1864 y(illustrated)17 b(b)q(elo)o(w)e(for)g Fp(n)e Fx(=)g(3.)p 75 1928 1119 2 v 74 1984 2 57 v 133 1968 a Fq(no)q(de)p 300 1984 V 104 w(exc)o(hange)122 b(sh)o(u\017e)h(unsh)o(u\017e)p 1193 1984 V 74 2041 V 300 2041 V 326 2024 a Fx(neigh)o(b)q(ors\(1\))49 b(neigh)o(b)q(ors\(2\))h(neigh)o(b)q(ors\(3\))p 1193 2041 V 75 2043 1119 2 v 74 2099 2 57 v 100 2082 a(0)f(\(000\))p 300 2099 V 161 w(1)275 b(0)g(0)p 1193 2099 V 74 2156 V 100 2139 a(1)49 b(\(001\))p 300 2156 V 161 w(0)275 b(2)g(4)p 1193 2156 V 74 2212 V 100 2195 a(2)49 b(\(010\))p 300 2212 V 161 w(3)275 b(4)g(1)p 1193 2212 V 74 2268 V 100 2251 a(3)49 b(\(011\))p 300 2268 V 161 w(2)275 b(6)g(5)p 1193 2268 V 74 2325 V 100 2308 a(4)49 b(\(100\))p 300 2325 V 161 w(5)275 b(1)g(2)p 1193 2325 V 74 2381 V 100 2364 a(5)49 b(\(101\))p 300 2381 V 161 w(4)275 b(3)g(6)p 1193 2381 V 74 2438 V 100 2421 a(6)49 b(\(110\))p 300 2438 V 161 w(7)275 b(5)g(3)p 1193 2438 V 74 2494 V 100 2477 a(7)49 b(\(111\))p 300 2494 V 161 w(6)275 b(7)g(7)p 1193 2494 V 75 2496 1119 2 v 166 2591 a(Supp)q(ose)15 b(that)e(the)g(comm)o(unicator)h Fu(comm)f Fx(has)g(this)h(top)q(ology) f(asso)q(ciated)h(with)g(it.)20 b(The)14 b(follo)o(w-)75 2647 y(ing)g(co)q(de)h(fragmen)o(t)e(cycles)h(through)g(the)g(three)g (t)o(yp)q(es)g(of)f(neigh)o(b)q(ors)i(and)f(p)q(erforms)f(an)h (appropriate)75 2704 y(p)q(erm)o(utation)h(for)g(eac)o(h.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 184 192 184 191 bop 75 -100 a Fx(184)887 b Fm(CHAPTER)15 b(6.)30 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fu(C)48 b(assume:)22 b(each)i(process)f(has)g(stored)g(a)h(real)f(number)g(A.)75 102 y(C)48 b(extract)22 b(neighborhood)h(information)218 158 y(CALL)g(MPI_COMM_RANK\(comm,)f(myrank,)h(ierr\))218 214 y(CALL)g(MPI_GRAPH_NEIGHBORS\(comm,)e(myrank,)i(3,)g(neighbors,)g (ierr\))75 271 y(C)48 b(perform)22 b(exchange)h(permutation)218 327 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e (neighbors\(1\),)h(0,)194 384 y(+)119 b(neighbors\(1\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 440 y(C)48 b(perform)22 b(shuffle)h(permutation)218 497 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e(1,) j(MPI_REAL,)e(neighbors\(2\),)h(0,)194 553 y(+)119 b(neighbors\(3\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 610 y(C)48 b(perform)22 b(unshuffle)h(permutation)218 666 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e (1,)j(MPI_REAL,)e(neighbors\(3\),)h(0,)194 723 y(+)119 b(neighbors\(2\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 844 y Fj(6.5.5)49 b(Ca)o(rtesian)16 b(Shift)g(Co)q(o)o(rdinates)75 930 y Fx(If)f(the)g(pro)q(cess)f(top)q(ology)h(is)g(a)f(cartesian)h (structure,)f(a)g Fv(MPI)p 1145 930 14 2 v 16 w(SENDRECV)i Fx(op)q(eration)f(is)g(lik)o(ely)h(to)e(b)q(e)75 986 y(used)19 b(along)g(a)f(co)q(ordinate)h(direction)h(to)e(p)q(erform)g (a)g(shift)h(of)f(data.)29 b(As)19 b(input,)h Fv(MPI)p 1629 986 V 16 w(SENDRECV)75 1043 y Fx(tak)o(es)14 b(the)h(rank)g(of)f (a)h(source)f(pro)q(cess)i(for)e(the)h(receiv)o(e,)g(and)g(the)g(rank)g (of)f(a)h(destination)g(pro)q(cess)h(for)75 1099 y(the)11 b(send.)18 b(If)11 b(the)g(function)g Fv(MPI)p 641 1099 V 16 w(CART)p 776 1099 V 17 w(SHIFT)f Fx(is)h(called)h(for)e(a)h (cartesian)f(pro)q(cess)h(group,)g(it)f(pro)o(vides)75 1155 y(the)j(calling)h(pro)q(cess)e(with)h(the)g(ab)q(o)o(v)o(e)f(iden) o(ti\014ers,)i(whic)o(h)f(then)g(can)g(b)q(e)g(passed)f(to)g Fv(MPI)p 1616 1155 V 16 w(SENDRECV)p Fx(.)75 1212 y(The)17 b(user)h(sp)q(eci\014es)h(the)e(co)q(ordinate)h(direction)g(and)f(the)h (size)g(of)f(the)g(step)g(\(p)q(ositiv)o(e)h(or)f(negativ)o(e\).)75 1268 y(The)e(function)h(is)g(lo)q(cal.)75 1419 y Fv(MPI)p 160 1419 V 16 w(CART)p 295 1419 V 17 w(SHIFT\(com)o(m)n(,)11 b(direction,)16 b(disp,)g(rank)p 977 1419 V 16 w(source,)g(rank)p 1224 1419 V 16 w(dest\))117 1497 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j(\(handle\))117 1571 y(IN)171 b Fv(direction)408 b Fk(co)q(ordinate)14 b(dimension)f(of)g(shift)h(\(in)o(teger\))117 1646 y(IN)171 b Fv(disp)497 b Fk(displacemen)o(t)18 b(\()p Fa(>)h Fk(0:)27 b(up)o(w)o(ards)18 b(shift,)h Fa(<)g Fk(0:)27 b(do)o(wn)o(w)o(ards)905 1702 y(shift\))14 b(\(in)o(teger\))117 1777 y(OUT)124 b Fv(rank)p 420 1777 V 16 w(source)353 b Fk(rank)14 b(of)f(source)j (pro)q(cess)f(\(in)o(teger\))117 1852 y(OUT)124 b Fv(rank)p 420 1852 V 16 w(dest)396 b Fk(rank)14 b(of)f(destination)h(pro)q(cess)i (\(in)o(teger\))75 1976 y Fu(int)23 b(MPI)p 245 1976 15 2 v 17 w(Cart)p 358 1976 V 17 w(shift\(MPI)p 591 1976 V 16 w(Comm)g(comm,)g(int)h(direction,)e(int)i(disp,)f(int)g(*rank)p 1681 1976 V 17 w(source,)393 2033 y(int)h(*rank)p 612 2033 V 16 w(dest\))75 2119 y(MPI)p 150 2119 V 17 w(CART)p 263 2119 V 16 w(SHIFT\(COMM,)f(DIRECTION,)f(DISP,)h(RANK)p 1067 2119 V 17 w(SOURCE,)g(RANK)p 1371 2119 V 17 w(DEST,)g(IERROR\))170 2175 y(INTEGER)g(COMM,)h(DIRECTION,)e(DISP,)h(RANK)p 1009 2175 V 17 w(SOURCE,)g(RANK)p 1313 2175 V 16 w(DEST,)h(IERROR)166 2262 y Fx(Dep)q(ending)18 b(on)e(the)g(p)q(erio)q(dicit)o(y)i(of)e(the) g(cartesian)h(group)f(in)h(the)f(sp)q(eci\014ed)i(co)q(ordinate)f (direc-)75 2318 y(tion,)c Fv(MPI)p 264 2318 14 2 v 16 w(CART)p 399 2318 V 17 w(SHIFT)g Fx(pro)o(vides)h(the)f(iden)o (ti\014ers)i(for)e(a)g(circular)h(or)f(an)g(end-o\013)g(shift.)20 b(In)14 b(the)f(case)75 2375 y(of)j(an)g(end-o\013)g(shift,)h(the)f(v)m (alue)i Fi(MPI)p 735 2375 13 2 v 14 w(PROC)p 861 2375 V 13 w(NULL)f Fx(ma)o(y)f(b)q(e)h(returned)f(in)h Fv(rank)p 1479 2375 14 2 v 17 w(source)g Fx(or)e Fv(rank)p 1771 2375 V 17 w(dest)p Fx(,)75 2431 y(indicating)i(that)e(the)g(source)g (or)g(the)g(destination)h(for)f(the)g(shift)h(is)f(out)g(of)g(range.)75 2534 y Fq(Example)j(6.4)k Fx(The)14 b(comm)o(unicator,)e Fv(comm)n Fx(,)e(has)i(a)h(t)o(w)o(o-dimensional,)h(p)q(erio)q(dic,)h (cartesian)f(top)q(ol-)75 2591 y(ogy)f(asso)q(ciated)h(with)g(it.)19 b(A)14 b(t)o(w)o(o-dimensional)g(arra)o(y)f(of)g Fu(REAL)p Fx(s)g(is)h(stored)f(one)h(elemen)o(t)g(p)q(er)g(pro)q(cess,)75 2647 y(in)h(v)m(ariable)h Fu(A)p Fx(.)e(One)h(wishes)g(to)e(sk)o(ew)h (this)h(arra)o(y)l(,)e(b)o(y)i(shifting)g(column)g Fu(i)f Fx(\(v)o(ertically)l(,)h(i.e.,)g(along)f(the)75 2704 y(column\))i(b)o(y)f Fu(i)g Fx(steps.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 185 193 185 192 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fx(185)75 45 y Fu(....)75 102 y(C)24 b(find)f(process)g(rank)218 158 y(CALL)g(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\)\))75 214 y(C)h(find)f(cartesian)g(coordinates)218 271 y(CALL)g (MPI_CART_COORDS\(comm,)e(rank,)j(maxdims,)e(coords,)h(ierr\))75 327 y(C)h(compute)f(shift)g(source)g(and)g(destination)218 384 y(CALL)g(MPI_CART_SHIFT\(comm,)f(1,)h(coords\(2\),)g(source,)g (dest,)g(ierr\))75 440 y(C)h(skew)f(array)218 497 y(CALL)g (MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e(dest,)i(0,)f(source,)g(0,) h(comm,)194 553 y(+)621 b(status,)23 b(ierr\))75 674 y Fj(6.5.6)49 b(P)o(a)o(rtitioning)17 b(of)g(Ca)o(rtesian)e(structures) 75 854 y Fv(MPI)p 160 854 14 2 v 16 w(CART)p 295 854 V 17 w(SUB\(comm)m(,)d(remain)p 690 854 V 14 w(dims,)g(new)o(comm)n(\)) 117 931 y Fk(IN)171 b Fv(comm)450 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j(\(handle\))117 1005 y(IN)171 b Fv(rem)o(ain)p 467 1005 V 14 w(dims)337 b Fk(the)24 b Fb(i)p Fk(th)f(en)o(try)h(of)e Fi(remain)p 1365 1005 13 2 v 13 w(dims)e Fk(sp)q(eci\014es)25 b(whether)f(the)905 1062 y Fb(i)p Fk(th)11 b(dimension)f(is)h(k)o(ept)h(in)f(the)h(subgrid) f(\()p Fb(true)p Fk(\))g(or)g(is)g(drop-)905 1118 y(p)q(ed)k(\()p Fb(false)p Fk(\))e(\(logical)f(v)o(ector\))117 1192 y(OUT)124 b Fv(new)o(comm)377 b Fk(comm)o(unicator)20 b(con)o(taining)h(the)i (subgrid)f(that)g(includes)905 1249 y(the)15 b(calling)d(pro)q(cess)k (\(handle\))75 1373 y Fu(int)23 b(MPI)p 245 1373 15 2 v 17 w(Cart)p 358 1373 V 17 w(sub\(MPI)p 543 1373 V 16 w(Comm)g(comm,)h(int)f(*remain)p 1085 1373 V 16 w(dims,)g(MPI)p 1316 1373 V 17 w(Comm)h(*newcomm\))75 1459 y(MPI)p 150 1459 V 17 w(CART)p 263 1459 V 16 w(SUB\(COMM,)f(REMAIN)p 662 1459 V 16 w(DIMS,)h(NEWCOMM,)e(IERROR\))170 1516 y(INTEGER)h(COMM,)h(NEWCOMM,)e(IERROR)170 1572 y(LOGICAL)h(REMAIN)p 508 1572 V 17 w(DIMS\(*\))166 1659 y Fx(If)28 b(a)f(cartesian)g(top)q (ology)g(has)h(b)q(een)g(created)g(with)g Fv(MPI)p 1263 1659 14 2 v 15 w(CART)p 1397 1659 V 17 w(CREA)l(TE)p Fx(,)h(the)e(function)75 1715 y Fv(MPI)p 160 1715 V 16 w(CART)p 295 1715 V 17 w(SUB)c Fx(can)f(b)q(e)h(used)f(to)g(partition)g (the)g(comm)o(unicator)g(group)g(in)o(to)g(subgroups)g(that)75 1772 y(form)17 b(lo)o(w)o(er-dimensional)j(cartesian)f(subgrids,)g(and) f(to)g(build)i(for)d(eac)o(h)h(subgroup)h(a)e(comm)o(unica-)75 1828 y(tor)k(with)h(the)g(asso)q(ciated)f(subgrid)i(cartesian)f(top)q (ology)l(.)39 b(\(This)22 b(function)g(is)h(closely)f(related)h(to)75 1885 y Fv(MPI)p 160 1885 V 16 w(COMM)p 318 1885 V 16 w(SPLIT)p Fx(.\))75 1983 y Fq(Example)18 b(6.5)k Fx(Assume)12 b(that)g Fv(MPI)p 726 1983 V 15 w(CART)p 860 1983 V 17 w(CREA)l(TE)p Fu(\(...,)25 b(comm\))11 b Fx(has)h(de\014ned)h(a)e(\(2)s Ft(\002)s Fx(3)s Ft(\002)s Fx(4\))i(grid.)75 2039 y(Let)i Fu(remain)p 303 2039 15 2 v 17 w(dims)23 b(=)h(\(true,)f(false,)g (true\))p Fx(.)c(Then)d(a)f(call)h(to,)194 2127 y Fu (MPI_CART_SUB\(comm,)22 b(remain_dims,)g(comm_new\),)75 2215 y Fx(will)k(create)d(three)i(comm)o(unicators)e(eac)o(h)h(with)h (eigh)o(t)f(pro)q(cesses)g(in)h(a)f(2)15 b Ft(\002)i Fx(4)23 b(cartesian)i(top)q(ol-)75 2271 y(ogy)l(.)36 b(If)21 b Fu(remain)p 388 2271 V 16 w(dims)j(=)f(\(false,)g(false,)g (true\))d Fx(then)h(the)g(call)h(to)e Fv(MPI)p 1494 2271 14 2 v 16 w(CART)p 1629 2271 V 17 w(SUB\(comm)m(,)75 2328 y(remain)p 209 2328 V 13 w(dims,)i(comm)p 465 2328 V 10 w(new\))h Fx(will)h(create)e(six)g(non-o)o(v)o(erlapping)h(comm)o (unicators,)h(eac)o(h)e(with)g(four)75 2384 y(pro)q(cesses,)15 b(in)h(a)f(one-dimensional)j(cartesian)d(top)q(ology)l(.)75 2505 y Fj(6.5.7)49 b(Lo)o(w-level)18 b(top)q(ology)g(functions)75 2591 y Fx(The)d(t)o(w)o(o)e(additional)k(functions)e(in)o(tro)q(duced)h (in)g(this)f(section)h(can)e(b)q(e)i(used)f(to)g(implemen)o(t)h(all)f (other)75 2647 y(top)q(ology)g(functions.)21 b(In)15 b(general)h(they)f(will)i(not)e(b)q(e)h(called)h(b)o(y)e(the)g(user)h (directly)l(,)g(unless)g(he)g(or)f(she)75 2704 y(is)h(creating)f (additional)i(virtual)f(top)q(ology)e(capabilit)o(y)j(other)e(than)g (that)f(pro)o(vided)i(b)o(y)g Fv(MPI)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 186 194 186 193 bop 75 -100 a Fx(186)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(CART)p 295 45 V 17 w(MAP\(comm)m(,)d(ndims,)h(dims,)f(p)q (erio)q(ds,)k(newrank\))117 122 y Fk(IN)171 b Fv(comm)450 b Fk(input)14 b(comm)o(unicator)d(\(handle\))117 197 y(IN)171 b Fv(ndims)458 b Fk(n)o(um)o(b)q(er)13 b(of)h(dimensions)e(of) i(cartesian)g(structure)i(\(in)o(teger\))117 273 y(IN)171 b Fv(dims)481 b Fk(in)o(teger)18 b(arra)o(y)f(of)f(size)i Fb(ndims)e Fk(sp)q(ecifying)h(the)h(n)o(um)o(b)q(er)f(of)905 329 y(pro)q(cesses)g(in)c(eac)o(h)i(co)q(ordinate)f(direction)117 404 y(IN)171 b Fv(p)q(erio)q(ds)436 b Fk(logical)16 b(arra)o(y)h(of)g (size)h Fb(ndims)f Fk(sp)q(ecifying)g(the)h(p)q(erio)q(dicit)o(y)905 461 y(sp)q(eci\014cation)d(in)e(eac)o(h)i(co)q(ordinate)f(direction)117 536 y(OUT)124 b Fv(newrank)414 b Fk(reordered)11 b(rank)f(of)e(the)i (calling)e(pro)q(cess;)k Fi(MPI)p 1643 536 13 2 v 15 w(UNDEFINED)905 592 y Fk(if)h(calling)g(pro)q(cess)i(do)q(es)g(not)f(b) q(elong)f(to)h(grid)g(\(in)o(teger\))75 717 y Fu(int)23 b(MPI)p 245 717 15 2 v 17 w(Cart)p 358 717 V 17 w(map\(MPI)p 543 717 V 16 w(Comm)g(comm,)h(int)f(ndims,)g(int)h(*dims,)f(int)g (*periods,)393 773 y(int)h(*newrank\))75 860 y(MPI)p 150 860 V 17 w(CART)p 263 860 V 16 w(MAP\(COMM,)f(NDIMS,)g(DIMS,)g (PERIODS,)g(NEWRANK,)g(IERROR\))170 916 y(INTEGER)g(COMM,)h(NDIMS,)f (DIMS\(*\),)f(NEWRANK,)h(IERROR)170 972 y(LOGICAL)g(PERIODS\(*\))166 1059 y Fv(MPI)p 251 1059 14 2 v 16 w(CART)p 386 1059 V 17 w(MAP)12 b Fx(computes)h(an)f(\\optimal")g(placemen)o(t)i(for)d (the)i(calling)h(pro)q(cess)f(on)f(the)h(ph)o(ys-)75 1115 y(ical)i(mac)o(hine.)20 b(A)13 b(p)q(ossible)i(implemen)o(tation)g (of)e(this)h(function)g(is)g(to)e(alw)o(a)o(ys)h(return)g(the)h(rank)f (of)g(the)75 1172 y(calling)k(pro)q(cess,)e(that)g(is,)g(not)g(to)f(p)q (erform)h(an)o(y)g(reordering.)189 1278 y Fl(A)n(dvic)n(e)22 b(to)h(implementors.)84 b Fx(The)22 b(function)i Fv(MPI)p 1138 1278 V 15 w(CART)p 1272 1278 V 17 w(CREA)l(TE\(comm)n(,)d(ndims,)h (dims,)189 1335 y(p)q(erio)q(ds,)h(reo)o(rder,)d(comm)p 647 1335 V 11 w(ca)o(rt\))p Fx(,)h(with)g Fu(reorder)i(=)h(true)c Fx(can)h(b)q(e)g(implemen)o(ted)i(b)o(y)d(calling)189 1391 y Fv(MPI)p 274 1391 V 15 w(CART)p 408 1391 V 17 w(MAP\(comm)m(,)71 b(ndims,)g(dims,)g(p)q(erio)q(ds,)j(newrank\))p Fx(,)g(then)62 b(calling)189 1447 y Fv(MPI)p 274 1447 V 15 w(COMM)p 431 1447 V 17 w(SPLIT\(com)o(m)m(,)26 b(colo)o(r,)i(k)o (ey)l(,)h(comm)p 1096 1447 V 11 w(ca)o(rt\))p Fx(,)f(with)f Fu(color)d(=)f(0)k Fx(if)g Fu(newrank)c Ft(6)p Fx(=)189 1504 y Fu(MPI)p 264 1504 15 2 v 16 w(UNDEFINED)p Fx(,)14 b Fu(color)23 b(=)h(MPI)p 786 1504 V 17 w(UNDEFINED)14 b Fx(otherwise,)h(and)g Fu(key)23 b(=)h(newrank)p Fx(.)189 1579 y(The)c(function)h Fv(MPI)p 555 1579 14 2 v 15 w(CART)p 689 1579 V 17 w(SUB\(comm)n(,)d(rema)o(in)p 1090 1579 V 14 w(dims,)h(comm)p 1344 1579 V 10 w(new\))i Fx(can)f(b)q(e)g (implemen)o(ted)189 1636 y(b)o(y)c(a)g(call)i(to)e Fv(MPI)p 519 1636 V 16 w(COMM)p 677 1636 V 16 w(SPLIT\(comm)m(,)d(colo)o(r,)j(k) o(ey)l(,)h(comm)p 1304 1636 V 10 w(new\))p Fx(,)g(using)g(a)g(single)h (n)o(um)o(b)q(er)189 1692 y(enco)q(ding)11 b(of)f(the)g(lost)g (dimensions)i(as)e Fu(color)g Fx(and)g(a)g(single)h(n)o(um)o(b)q(er)g (enco)q(ding)h(of)d(the)i(preserv)o(ed)189 1748 y(dimensions)16 b(as)f Fu(key)p Fx(.)189 1824 y(All)f(other)g(cartesian)f(top)q(ology)g (functions)i(can)e(b)q(e)i(implemen)o(ted)g(lo)q(cally)l(,)g(using)g (the)e(top)q(ology)189 1880 y(information)h(that)g(is)i(cac)o(hed)f (with)g(the)g(comm)o(unicator.)k(\()p Fl(End)c(of)h(advic)n(e)g(to)g (implementors.)p Fx(\))166 1986 y(The)f(corresp)q(onding)i(new)e (function)h(for)f(general)g(graph)g(structures)g(is)h(as)f(follo)o(ws.) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 187 195 187 194 bop 75 -100 a Fm(6.6.)34 b(AN)15 b(APPLICA)l(TION)i(EXAMPLE)974 b Fx(187)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(GRAPH)p 326 45 V 17 w(MAP\(comm)m(,)12 b(nno)q(des,)17 b(index,)f(edges,)g (newrank\))117 122 y Fk(IN)171 b Fv(comm)450 b Fk(input)14 b(comm)o(unicator)d(\(handle\))117 197 y(IN)171 b Fv(nno)q(des)441 b Fk(n)o(um)o(b)q(er)13 b(of)h(graph)f(no)q(des)i(\(in)o(teger\))117 273 y(IN)171 b Fv(index)473 b Fk(in)o(teger)30 b(arra)o(y)g(sp)q (ecifying)f(the)i(graph)e(structure,)36 b(see)905 329 y Fi(MPI)p 982 329 13 2 v 15 w(GRAPH)p 1136 329 V 14 w(CREA)m(TE)117 404 y Fk(IN)171 b Fv(edges)468 b Fk(in)o(teger)15 b(arra)o(y)e(sp)q(ecifying)h(the)g(graph)g(structure)117 479 y(OUT)124 b Fv(newrank)414 b Fk(reordered)11 b(rank)f(of)e(the)i (calling)e(pro)q(cess;)k Fi(MPI)p 1643 479 V 15 w(UNDEFINED)905 536 y Fk(if)j(the)i(calling)d(pro)q(cess)k(do)q(es)e(not)g(b)q(elong)g (to)f(graph)h(\(in)o(te-)905 592 y(ger\))75 717 y Fu(int)23 b(MPI)p 245 717 15 2 v 17 w(Graph)p 382 717 V 17 w(map\(MPI)p 567 717 V 16 w(Comm)g(comm,)g(int)h(nnodes,)f(int)g(*index,)g(int)h (*edges,)393 773 y(int)g(*newrank\))75 860 y(MPI)p 150 860 V 17 w(GRAPH)p 287 860 V 16 w(MAP\(COMM,)f(NNODES,)g(INDEX,)g (EDGES,)g(NEWRANK,)g(IERROR\))170 916 y(INTEGER)g(COMM,)h(NNODES,)e (INDEX\(*\),)h(EDGES\(*\),)g(NEWRANK,)f(IERROR)189 1052 y Fl(A)n(dvic)n(e)17 b(to)h(implementors.)53 b Fx(The)18 b(function)g Fv(MPI)p 1087 1052 14 2 v 16 w(GRAPH)p 1253 1052 V 17 w(CREA)l(TE\(comm)n(,)d(nno)q(des,)k(index,)189 1109 y(edges,)j(reo)o(rder,)e(comm)p 614 1109 V 11 w(graph\))p Fx(,)h(with)g Fu(reorder)i(=)h(true)c Fx(can)g(b)q(e)h(implemen)o(ted)i (b)o(y)d(calling)189 1165 y Fv(MPI)p 274 1165 V 15 w(GRAPH)p 439 1165 V 18 w(MAP\(comm)m(,)65 b(nno)q(des,)70 b(index,)e(edges,)h (newrank\))p Fx(,)f(then)58 b(calling)189 1222 y Fv(MPI)p 274 1222 V 15 w(COMM)p 431 1222 V 17 w(SPLIT\(com)o(m)m(,)21 b(colo)o(r,)h(k)o(ey)l(,)i(comm)p 1080 1222 V 11 w(graph\))p Fx(,)f(with)g Fu(color)g(=)h(0)e Fx(if)h Fu(newrank)g Ft(6)p Fx(=)189 1278 y Fu(MPI)p 264 1278 15 2 v 16 w(UNDEFINED)p Fx(,)14 b Fu(color)23 b(=)h(MPI)p 786 1278 V 17 w(UNDEFINED)14 b Fx(otherwise,)h(and)g Fu(key)23 b(=)h(newrank)p Fx(.)189 1353 y(All)c(other)f(graph)g(top)q(ology)g(functions)h(can)f(b)q(e)h (implemen)o(ted)h(lo)q(cally)l(,)h(using)e(the)f(top)q(ology)189 1410 y(information)14 b(that)g(is)i(cac)o(hed)f(with)g(the)g(comm)o (unicator.)k(\()p Fl(End)c(of)h(advic)n(e)g(to)g(implementors.)p Fx(\))75 1553 y Fn(6.6)59 b(An)20 b(Application)f(Example)75 1654 y Fq(Example)f(6.6)k Fx(The)11 b(example)h(in)f(\014gure)g(6.1)f (sho)o(ws)g(ho)o(w)g(the)h(grid)g(de\014nition)h(and)f(inquiry)h (functions)75 1711 y(can)20 b(b)q(e)g(used)h(in)f(an)g(application)h (program.)33 b(A)19 b(partial)h(di\013eren)o(tial)h(equation,)g(for)e (instance)i(the)75 1767 y(P)o(oisson)e(equation,)g(is)g(to)g(b)q(e)g (solv)o(ed)g(on)g(a)f(rectangular)h(domain.)31 b(First,)19 b(the)g(pro)q(cesses)g(organize)75 1824 y(themselv)o(es)e(in)h(a)e(t)o (w)o(o-dimensional)h(structure.)24 b(Eac)o(h)16 b(pro)q(cess)h(then)g (inquires)h(ab)q(out)f(the)f(ranks)g(of)75 1880 y(its)e(neigh)o(b)q (ors)h(in)g(the)f(four)g(directions)h(\(up,)f(do)o(wn,)g(righ)o(t,)g (left\).)19 b(The)c(n)o(umerical)g(problem)g(is)g(solv)o(ed)75 1937 y(b)o(y)g(an)g(iterativ)o(e)h(metho)q(d,)f(the)g(details)h(of)f (whic)o(h)h(are)f(hidden)i(in)f(the)f(subroutine)h Fu(relax)p Fx(.)166 1993 y(In)e(eac)o(h)f(relaxation)g(step)g(eac)o(h)g(pro)q (cess)h(computes)f(new)g(v)m(alues)h(for)f(the)g(solution)h(grid)f (function)75 2050 y(at)h(all)i(p)q(oin)o(ts)f(o)o(wned)g(b)o(y)g(the)g (pro)q(cess.)20 b(Then)15 b(the)g(v)m(alues)h(at)e(in)o(ter-pro)q(cess) i(b)q(oundaries)g(ha)o(v)o(e)e(to)g(b)q(e)75 2106 y(exc)o(hanged)e (with)h(neigh)o(b)q(oring)g(pro)q(cesses.)19 b(F)l(or)11 b(example,)i(the)f(exc)o(hange)g(subroutine)h(migh)o(t)f(con)o(tain)75 2162 y(a)i(call)i(lik)o(e)g Fv(MPI)p 363 2162 14 2 v 16 w(SEND\(...,neigh)p 666 2162 V 15 w(rank\(1\),...\))i Fx(to)c(send)h(up)q(dated)g(v)m(alues)h(to)e(the)h(left-hand)g(neigh)o (b)q(or)75 2219 y Fu(\(i-1,j\))p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 188 196 188 195 bop 75 -100 a Fx(188)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)194 163 y Fu(integer)23 b(ndims,)g(num)p 627 163 15 2 v 17 w(neigh)194 219 y(logical)g(reorder) 194 276 y(parameter)g(\(ndims=2,)g(num)p 747 276 V 16 w(neigh=4,)g(reorder=.true.\))194 332 y(integer)g(comm,)g(comm)p 627 332 V 17 w(cart,)g(dims\(ndims\),)f(neigh)p 1217 332 V 17 w(def\(ndims\),)g(ierr)194 388 y(integer)h(neigh)p 508 388 V 17 w(rank\(num)p 717 388 V 16 w(neigh\),)g(own)p 996 388 V 16 w(position\(ndims\),)f(i,)i(j)194 445 y(logical)f (periods\(ndims\))194 501 y(real)p Ft(\003)p Fu(8)g(u\(0:101,0:101\),)f (f\(0:101,0:101\))194 558 y(data)i(dims)f(/)h(ndims)f Ft(\003)g Fu(0)h(/)194 614 y(comm)g(=)f(MPI)p 436 614 V 17 w(COMM)p 549 614 V 17 w(WORLD)75 671 y(C)107 b(Set)23 b(process)g(grid)g(size)h(and)f(periodicity)194 727 y(call)h(MPI)p 389 727 V 16 w(DIMS)p 501 727 V 17 w(CREATE\(comm,)e(ndims,)h (dims,ierr\))194 784 y(periods\(1\))g(=)h(.TRUE.)194 840 y(periods\(2\))f(=)h(.TRUE.)75 897 y(C)107 b(Create)23 b(a)g(grid)h(structure)e(in)i(WORLD)f(group)g(and)h(inquire)f(about)g (own)g(position)194 953 y(call)h(MPI)p 389 953 V 16 w(CART)p 501 953 V 17 w(CREATE)f(\(comm,)g(ndims,)g(dims,)g(periods,)g(reorder,) g(comm)p 1688 953 V 17 w(cart,ierr\))194 1009 y(call)h(MPI)p 389 1009 V 16 w(CART)p 501 1009 V 17 w(GET)f(\(comm)p 733 1009 V 17 w(cart,)g(ndims,)g(dims,)g(periods,)g(own)p 1490 1009 V 17 w(position,ierr\))75 1066 y(C)107 b(Look)23 b(up)h(the)f(ranks)g(for)h(the)f(neighbors.)46 b(Own)24 b(process)f(coordinates)f(are)i(\(i,j\).)75 1122 y(C)107 b(Neighbors)22 b(are)i(\(i-1,j\),)f(\(i+1,j\),)f(\(i,j-1\),)h (\(i,j+1\))194 1179 y(i)h(=)g(own)p 365 1179 V 17 w(position\(1\))194 1235 y(j)g(=)g(own)p 365 1235 V 17 w(position\(2\))194 1292 y(neigh)p 317 1292 V 17 w(def\(1\))f(=)h(i-1)194 1348 y(neigh)p 317 1348 V 17 w(def\(2\))f(=)h(j)194 1405 y(call)g(MPI)p 389 1405 V 16 w(CART)p 501 1405 V 17 w(RANK)f(\(comm)p 757 1405 V 17 w(cart,)g(neigh)p 1037 1405 V 17 w(def,)g(neigh)p 1293 1405 V 16 w(rank\(1\),ierr\))194 1461 y(neigh)p 317 1461 V 17 w(def\(1\))g(=)h(i+1)194 1518 y(neigh)p 317 1518 V 17 w(def\(2\))f(=)h(j)194 1574 y(call)g(MPI)p 389 1574 V 16 w(CART)p 501 1574 V 17 w(RANK)f(\(comm)p 757 1574 V 17 w(cart,)g(neigh)p 1037 1574 V 17 w(def,)g(neigh)p 1293 1574 V 16 w(rank\(2\),ierr\))194 1630 y(neigh)p 317 1630 V 17 w(def\(1\))g(=)h(i)194 1687 y(neigh)p 317 1687 V 17 w(def\(2\))f(=)h(j-1)194 1743 y(call)g(MPI)p 389 1743 V 16 w(CART)p 501 1743 V 17 w(RANK)f(\(comm)p 757 1743 V 17 w(cart,)g(neigh)p 1037 1743 V 17 w(def,)g(neigh)p 1293 1743 V 16 w(rank\(3\),ierr\))194 1800 y(neigh)p 317 1800 V 17 w(def\(1\))g(=)h(i)194 1856 y(neigh)p 317 1856 V 17 w(def\(2\))f(=)h(j+1)194 1913 y(call)g(MPI)p 389 1913 V 16 w(CART)p 501 1913 V 17 w(RANK)f(\(comm)p 757 1913 V 17 w(cart,)g(neigh)p 1037 1913 V 17 w(def,)g(neigh)p 1293 1913 V 16 w(rank\(4\),ierr\))75 1969 y(C)107 b(Initialize)22 b(the)i(grid)f(functions)g(and)g(start)g(the)h(iteration)194 2026 y(call)g(init)f(\(u,)g(f\))194 2082 y(do)h(10)f(it=1,100)242 2139 y(call)g(relax)h(\(u,)f(f\))75 2195 y(C)107 b(Exchange)22 b(data)i(with)f(neighbor)g(processes)242 2251 y(call)g(exchange)g(\(u,) h(comm)p 771 2251 V 16 w(cart,)f(neigh)p 1050 2251 V 17 w(rank,)g(num)p 1282 2251 V 17 w(neigh\))75 2308 y(10)71 b(continue)194 2364 y(call)24 b(output)f(\(u\))194 2421 y(end)154 2625 y Fx(Figure)16 b(6.1:)j(Set-up)d(of)e(pro)q(cess)i (structure)f(for)f(t)o(w)o(o-dimensional)i(parallel)h(P)o(oisson)e (solv)o(er.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 189 197 189 196 bop 75 356 a Fo(Chapter)34 b(7)75 564 y Fr(MPI)40 b(En)m(vironmen)m(tal)h(Managemen)m(t)75 786 y Fx(This)16 b(c)o(hapter)g(discusses)h(routines)g(for)e(getting)h(and,)f(where)h (appropriate,)g(setting)g(v)m(arious)g(param-)75 843 y(eters)g(that)f(relate)h(to)f(the)h Fv(MPI)f Fx(implemen)o(tation)i (and)f(the)g(execution)g(en)o(vironmen)o(t)h(\(suc)o(h)e(as)h(error)75 899 y(handling\).)21 b(The)13 b(pro)q(cedures)h(for)e(en)o(tering)i (and)f(lea)o(ving)h(the)f Fv(MPI)g Fx(execution)h(en)o(vironmen)o(t)f (are)g(also)75 956 y(describ)q(ed)k(here.)75 1099 y Fn(7.1)59 b(Implementation)17 b(info)n(rmation)75 1202 y Fj(7.1.1)49 b(Environmental)15 b(Inquiries)75 1288 y Fx(A)k(set)g(of)f(attributes)h (that)f(describ)q(e)i(the)f(execution)h(en)o(vironmen)o(t)f(are)g (attac)o(hed)f(to)g(the)h(comm)o(u-)75 1344 y(nicator)f Fi(MPI)p 309 1344 13 2 v 14 w(COMM)p 453 1344 V 15 w(W)o(ORLD)g Fx(when)h Fv(MPI)e Fx(is)i(initialized.)32 b(The)19 b(v)m(alue)g(of)f (these)g(attributes)g(can)g(b)q(e)75 1401 y(inquired)h(b)o(y)d(using)h (the)g(function)h Fv(MPI)p 784 1401 14 2 v 15 w(A)l(TTR)p 916 1401 V 17 w(GET)f Fx(describ)q(ed)i(in)e(Chapter)g(5.)23 b(It)17 b(is)g(erroneous)g(to)75 1457 y(delete)f(these)g(attributes)f (or)f(free)i(their)f(k)o(eys.)166 1514 y(The)g(list)h(of)f (prede\014ned)i(attribute)e(k)o(eys)g(include)75 1608 y Fi(MPI)p 152 1608 13 2 v 14 w(T)m(A)o(G)p 246 1608 V 14 w(UB)22 b Fx(Upp)q(er)16 b(b)q(ound)g(for)f(tag)f(v)m(alue.)75 1701 y Fi(MPI)p 152 1701 V 14 w(HOST)23 b Fx(Host)15 b(pro)q(cess)g(rank,)g(if)h(suc)o(h)f(exists,)g Fi(MPI)p 1041 1701 V 15 w(PROC)p 1168 1701 V 13 w(NULL)p Fx(,)h(otherwise.)75 1795 y Fi(MPI)p 152 1795 V 14 w(IO)22 b Fx(rank)14 b(of)f(a)g(no)q(de)h (that)f(has)h(regular)g(I/O)g(facilities)i(\(p)q(ossibly)f(m)o (yrank\).)j(No)q(des)c(in)h(the)e(same)189 1852 y(comm)o(unicator)h(ma) o(y)h(return)g(di\013eren)o(t)h(v)m(alues)g(for)e(this)i(parameter.)166 1946 y(V)l(endors)c(ma)o(y)g(add)g(implemen)o(tation)h(sp)q(eci\014c)h (parameters)d(\(suc)o(h)i(as)e(no)q(de)i(n)o(um)o(b)q(er,)g(real)f (mem-)75 2002 y(ory)j(size,)g(virtual)h(memory)f(size,)h(etc.\))166 2058 y(The)f(required)i(parameter)d(v)m(alues)i(are)f(discussed)i(in)f (more)f(detail)h(b)q(elo)o(w:)75 2179 y Fv(T)l(ag)f(values)75 2264 y Fx(T)l(ag)h(v)m(alues)h(range)f(from)f Fu(0)h Fx(to)f(the)i(v)m(alue)g(returned)f(for)g Fi(MPI)p 1162 2264 V 14 w(T)m(A)o(G)p 1256 2264 V 14 w(UB)f Fx(inclusiv)o(e.)26 b(These)16 b(v)m(alues)h(are)75 2321 y(guaran)o(teed)c(to)g(b)q(e)i (unc)o(hanging)f(during)h(the)f(execution)h(of)e(an)h Fv(MPI)f Fx(program.)18 b(In)c(addition,)h(the)f(tag)75 2377 y(upp)q(er)20 b(b)q(ound)g(v)m(alue)g(m)o(ust)f(b)q(e)g Fl(at)h(le)n(ast)j Fx(32767.)30 b(An)19 b Fv(MPI)f Fx(implemen)o (tation)i(is)g(free)f(to)f(mak)o(e)h(the)75 2434 y(v)m(alue)c(of)f Fi(MPI)p 319 2434 V 14 w(T)m(A)o(G)p 413 2434 V 14 w(UB)f Fx(larger)h(than)g(this;)h(for)e(example,)i(the)f(v)m(alue)h(2)1303 2417 y Fg(30)1348 2434 y Ft(\000)8 b Fx(1)14 b(is)h(also)f(a)f(legal)i (v)m(alue)h(for)75 2490 y Fi(MPI)p 152 2490 V 14 w(T)m(A)o(G)p 246 2490 V 14 w(UB)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 190 198 190 197 bop 75 -100 a Fx(190)532 b Fm(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y Fv(Host)g(rank)75 131 y Fx(The)e(v)m(alue)h(returned)g(for)e Fi(MPI)p 612 131 13 2 v 14 w(HOST)i Fx(gets)e(the)h(rank)g(of)f(the)i Fu(HOST)e Fx(pro)q(cess)h(in)h(the)f(group)g(asso)q(ciated)75 187 y(with)22 b(comm)o(unicator)f Fi(MPI)p 565 187 V 14 w(COMM)p 709 187 V 15 w(W)o(ORLD)p Fx(,)g(if)h(there)g(is)g(suc)o (h.)39 b Fi(MPI)p 1343 187 V 15 w(PROC)p 1470 187 V 13 w(NULL)22 b Fx(is)h(returned)e(if)75 244 y(there)16 b(is)h(no)f(host.) 21 b Fv(MPI)16 b Fx(do)q(es)g(not)g(sp)q(ecify)h(what)f(it)g(means)g (for)f(a)h(pro)q(cess)g(to)f(b)q(e)i(a)f Fu(HOST)p Fx(,)f(nor)h(do)q (es)75 300 y(it)f(requires)h(that)f(a)g Fu(HOST)f Fx(exists.)75 420 y Fv(IO)h(rank)75 506 y Fx(The)e(v)m(alue)h(returned)f(for)f Fi(MPI)p 607 506 V 14 w(IO)f Fx(is)j(the)e(rank)h(of)f(a)g(pro)q (cessor)g(that)g(can)h(pro)o(vide)g(language-standard)75 563 y(I/O)i(facilities.)21 b(F)l(or)14 b(F)l(ortran,)e(this)j(means)f (that)f(all)i(of)f(the)g(F)l(ortran)f(I/O)h(op)q(erations)h(are)e(supp) q(orted)75 619 y(\(e.g.,)i Fu(OPEN)p Fx(,)h Fu(REWIND)p Fx(,)f Fu(WRITE)p Fx(\).)g(F)l(or)h(C,)g(this)h(means)g(that)f(all)h (of)f(the)h(ANSI-C)h(I/O)f(op)q(erations)f(are)75 676 y(supp)q(orted)g(\(e.g.,)d Fu(fopen)p Fx(,)h Fu(fprintf)p Fx(,)g Fu(lseek)p Fx(\).)166 732 y(If)i(ev)o(ery)g(pro)q(cess)h(can)f (pro)o(vide)h(language-standard)f(I/O,)g(then)h(the)f(v)m(alue)i Fi(MPI)p 1603 732 V 14 w(ANY)p 1702 732 V 15 w(SOURCE)75 788 y Fx(m)o(ust)f(b)q(e)i(returned.)29 b(If)18 b(no)g(pro)q(cess)g (can)g(pro)o(vide)h(language-standard)f(I/O,)g(then)g(the)h(v)m(alue)g Fi(MPI-)p 75 845 V 87 845 a(PROC)p 201 845 V 14 w(NULL)14 b Fx(m)o(ust)f(b)q(e)h(returned.)20 b(If)14 b(sev)o(eral)g(pro)q (cesses)g(can)f(pro)o(vide)h(I/O,)g(then)g(an)o(y)f(them)g(ma)o(y)g(b)q (e)75 901 y(returned.)20 b(The)c(same)f(v)m(alue)h(\(rank\))e(need)i (not)f(b)q(e)h(returned)g(b)o(y)f(all)h(pro)q(cesses.)75 1052 y Fv(MPI)p 160 1052 14 2 v 16 w(GET)p 264 1052 V 17 w(PROCESSOR)p 540 1052 V 18 w(NAME\()g(name,)c(resultlen)17 b(\))117 1129 y Fk(OUT)108 b Fv(name)483 b Fk(A)17 b(unique)g(sp)q (eci\014er)i(for)e(the)g(actual)g(\(as)g(opp)q(osed)h(to)e(vir-)905 1186 y(tual\))e(no)q(de.)117 1261 y(OUT)108 b Fv(resultlen)430 b Fk(Length)12 b(\(in)g(prin)o(table)f(c)o(haracters\))i(of)f(the)g (result)g(returned)905 1318 y(in)i Fb(name)75 1442 y Fu(int)23 b(MPI)p 245 1442 15 2 v 17 w(Get)p 334 1442 V 17 w(processor)p 567 1442 V 16 w(name\(char)g(*name,)g(int)g (*resultlen\))75 1528 y(MPI)p 150 1528 V 17 w(GET)p 239 1528 V 17 w(PROCESSOR)p 472 1528 V 15 w(NAME\()h(NAME,)f(RESULTLEN,)f (IERROR\))170 1585 y(CHARACTER*\(*\))h(NAME)170 1641 y(INTEGER)g(RESULTLEN,IERROR)166 1728 y Fx(This)17 b(routine)f(returns) g(the)g(name)h(of)e(the)i(pro)q(cessor)e(on)i(whic)o(h)g(it)f(w)o(as)f (called)j(at)d(the)i(momen)o(t)75 1784 y(of)j(the)g(call.)35 b(The)20 b(name)g(is)h(a)e(c)o(haracter)h(string)g(for)f(maxim)o(um)h (\015exibilit)o(y)l(.)37 b(F)l(rom)19 b(this)h(v)m(alue)i(it)75 1841 y(m)o(ust)14 b(b)q(e)h(p)q(ossible)h(to)e(iden)o(tify)i(a)e(sp)q (eci\014c)j(piece)f(of)e(hardw)o(are;)g(p)q(ossible)i(v)m(alues)g (include)g(\\pro)q(cessor)75 1897 y(9)k(in)h(rac)o(k)e(4)h(of)f (mpp.cs.org")g(and)h(\\231")f(\(where)h(231)f(is)i(the)f(actual)g(pro)q (cessor)g(n)o(um)o(b)q(er)g(in)h(the)75 1954 y(running)16 b(homogeneous)f(system\).)k(The)c(argumen)o(t)f Fv(name)e Fx(m)o(ust)j(represen)o(t)f(storage)g(that)g(is)i(at)e(least)75 2010 y Fi(MPI)p 152 2010 13 2 v 14 w(MAX)p 258 2010 V 15 w(PROCESSOR)p 514 2010 V 13 w(NAME)g Fx(c)o(haracters)f(long.)20 b Fv(MPI)p 1073 2010 14 2 v 16 w(GET)p 1177 2010 V 17 w(PROCESSOR)p 1453 2010 V 18 w(NAME)15 b Fx(ma)o(y)e(write)h(up)75 2067 y(to)h(this)g(man)o(y)g(c)o(haracters)f(in)o(to)i Fv(name)p Fx(.)166 2123 y(The)11 b(n)o(um)o(b)q(er)h(of)e(c)o (haracters)h(actually)h(written)f(is)g(returned)h(in)g(the)f(output)g (argumen)o(t,)g Fv(resultlen)p Fx(.)189 2229 y Fl(R)n(ationale.)72 b Fx(This)21 b(function)h(allo)o(ws)f Fv(MPI)f Fx(implemen)o(tations)i (that)d(do)i(pro)q(cess)g(migration)189 2286 y(to)d(return)i(the)f (curren)o(t)g(pro)q(cessor.)32 b(Note)19 b(that)g(nothing)h(in)g Fv(MPI)f Fl(r)n(e)n(quir)n(es)f Fx(or)h(de\014nes)h(pro-)189 2342 y(cess)g(migration;)i(this)e(de\014nition)i(of)d Fv(MPI)p 953 2342 V 16 w(GET)p 1057 2342 V 17 w(PROCESSOR)p 1333 2342 V 18 w(NAME)h Fx(simply)i(allo)o(ws)e(suc)o(h)189 2399 y(an)15 b(implemen)o(tation.)21 b(\()p Fl(End)16 b(of)g(r)n(ationale.)p Fx(\))189 2505 y Fl(A)n(dvic)n(e)g(to)h(users.) 47 b Fx(The)16 b(user)g(m)o(ust)g(pro)o(vide)h(at)e(least)i Fi(MPI)p 1262 2505 13 2 v 14 w(MAX)p 1368 2505 V 15 w(PROCESSOR)p 1624 2505 V 12 w(NAME)g Fx(space)189 2561 y(to)g(write)i(the)f(pro)q (cessor)g(name)g(|)h(pro)q(cessor)f(names)g(can)g(b)q(e)h(this)g(long.) 29 b(The)19 b(user)f(should)189 2618 y(examine)g(the)f(ouput)h(argumen) o(t,)e Fv(resultlen)p Fx(,)j(to)e(determine)h(the)g(actual)f(length)h (of)f(the)g(name.)189 2674 y(\()p Fl(End)e(of)i(advic)n(e)f(to)g (users.)p Fx(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 191 199 191 198 bop 75 -100 a Fm(7.2.)34 b(ERR)o(OR)17 b(HANDLING)1194 b Fx(191)75 45 y Fn(7.2)59 b(Erro)n(r)21 b(handling)75 147 y Fx(An)14 b Fv(MPI)f Fx(implemen)o(tation)i(cannot)f(or)f(ma)o(y)h (c)o(ho)q(ose)f(not)h(to)f(handle)i(some)f(errors)f(that)g(o)q(ccur)h (during)75 203 y Fv(MPI)e Fx(calls.)21 b(These)13 b(can)g(include)i (errors)e(that)f(generate)h(exceptions)h(or)e(traps,)h(suc)o(h)g(as)g (\015oating)f(p)q(oin)o(t)75 259 y(errors)17 b(or)g(access)g (violations.)27 b(The)18 b(set)f(of)g(errors)g(that)f(are)i(handled)g (b)o(y)g Fv(MPI)f Fx(is)h(implemen)o(tation-)75 316 y(dep)q(enden)o(t.) j(Eac)o(h)15 b(suc)o(h)h(error)e(generates)h(an)g Fq(MPI)i(exception)p Fx(.)166 372 y(A)f(user)h(can)f(asso)q(ciate)h(an)f(error)g(handler)h (with)g(a)f(comm)o(unicator.)23 b(The)16 b(sp)q(eci\014ed)j(error)d (han-)75 429 y(dling)21 b(routine)f(will)h(b)q(e)f(used)h(for)e(an)o(y) g Fv(MPI)g Fx(exception)h(that)f(o)q(ccurs)h(during)h(a)e(call)h(to)f Fv(MPI)g Fx(for)g(a)75 485 y(comm)o(unication)c(with)g(this)g(comm)o (unicator.)k Fv(MPI)14 b Fx(calls)i(that)d(are)h(not)h(related)f(to)g (an)o(y)g(comm)o(unica-)75 542 y(tor)d(are)h(considered)i(to)d(b)q(e)i (attac)o(hed)f(to)f(the)h(comm)o(unicator)g Fi(MPI)p 1230 542 13 2 v 15 w(COMM)p 1375 542 V 14 w(W)o(ORLD)p Fx(.)g(The)g(attac)o(hmen)o(t)75 598 y(of)i(error)g(handlers)i(to)e (comm)o(unicators)g(is)i(purely)f(lo)q(cal:)21 b(di\013eren)o(t)15 b(pro)q(cesses)g(ma)o(y)f(attac)o(h)g(di\013eren)o(t)75 655 y(error)g(handlers)j(to)d(the)i(same)e(comm)o(unicator.)166 711 y(A)j(newly)i(created)e(comm)o(unicator)h(inherits)g(the)g(error)f (handler)h(that)f(is)h(asso)q(ciated)g(with)g(the)75 768 y(\\paren)o(t")h(comm)o(unicator.)32 b(In)20 b(particular,)h(the)f (user)g(can)f(sp)q(ecify)i(a)e(\\global")h(error)f(handler)i(for)75 824 y(all)d(comm)o(unicators)e(b)o(y)h(asso)q(ciating)g(this)g(handler) h(with)f(the)g(comm)o(unicator)f Fi(MPI)p 1570 824 V 14 w(COMM)p 1714 824 V 15 w(W)o(ORLD)75 880 y Fx(immediately)h(after)d (initialization.)166 937 y(Sev)o(eral)i(prede\014ned)g(error)f (handlers)h(are)f(a)o(v)m(ailable)i(in)f Fv(MPI)p Fx(:)75 1026 y Fi(MPI)p 152 1026 V 14 w(ERRORS)p 326 1026 V 14 w(ARE)p 420 1026 V 14 w(F)m(A)m(T)m(AL)21 b Fx(The)15 b(handler,)f(when)h(called,)g(causes)f(the)g(program)f(to)g(ab)q(ort)h (on)g(all)g(exe-)189 1082 y(cuting)f(pro)q(cesses.)20 b(This)13 b(has)g(the)g(same)g(e\013ect)g(as)f(if)i Fv(MPI)p 1203 1082 14 2 v 16 w(ABORT)g Fx(w)o(as)e(called)j(b)o(y)e(the)g(pro)q (cess)189 1139 y(that)h(in)o(v)o(ok)o(ed)i(the)f(handler.)75 1231 y Fi(MPI)p 152 1231 13 2 v 14 w(ERRORS)p 326 1231 V 14 w(RETURN)22 b Fx(The)16 b(handler)g(has)f(no)g(e\013ect.)166 1319 y(Implemen)o(tations)k(ma)o(y)f(pro)o(vide)h(additional)h (prede\014ned)g(error)d(handlers)j(and)e(programmers)75 1376 y(can)d(co)q(de)h(their)g(o)o(wn)f(error)f(handlers.)166 1432 y(The)19 b(error)f(handler)i Fi(MPI)p 623 1432 V 15 w(ERRORS)p 798 1432 V 13 w(ARE)p 891 1432 V 15 w(F)m(A)m(T)m(AL)d Fx(is)i(asso)q(ciated)g(b)o(y)g(default)h(with)f Fi(MPI)p 1719 1432 V 14 w(COMM-)p 75 1489 V 87 1489 a(W)o(ORLD)e Fx(after)f(initialization.)27 b(Th)o(us,)17 b(if)g(the)g(user)f(c)o(ho) q(oses)h(not)f(to)g(con)o(trol)h(error)f(handling,)i(ev)o(ery)75 1545 y(error)13 b(that)f Fv(MPI)h Fx(handles)h(is)g(treated)f(as)g (fatal.)19 b(Since)14 b(\(almost\))e(all)j Fv(MPI)d Fx(calls)j(return)e (an)g(error)g(co)q(de,)75 1602 y(a)g(user)h(ma)o(y)g(c)o(ho)q(ose)f(to) g(handle)j(errors)d(in)h(its)g(main)g(co)q(de,)h(b)o(y)e(testing)h(the) g(return)g(co)q(de)g(of)g Fv(MPI)f Fx(calls)75 1658 y(and)k(executing)h (a)e(suitable)i(reco)o(v)o(ery)e(co)q(de)i(when)f(the)g(call)g(w)o(as)f (not)h(successful.)26 b(In)17 b(this)g(case,)g(the)75 1715 y(error)g(handler)h Fi(MPI)p 432 1715 V 15 w(ERRORS)p 607 1715 V 13 w(RETURN)f Fx(will)i(b)q(e)f(used.)28 b(Usually)18 b(it)g(is)g(more)f(con)o(v)o(enien)o(t)h(and)f(more)75 1771 y(e\016cien)o(t)i(not)f(to)f(test)h(for)g(errors)f(after)h(eac)o (h)g Fv(MPI)g Fx(call,)h(and)g(ha)o(v)o(e)f(suc)o(h)g(error)g(handled)i (b)o(y)e(a)g(non)75 1827 y(trivial)e Fv(MPI)f Fx(error)f(handler.)166 1884 y(After)h(an)g(error)f(is)i(detected,)g(the)f(state)f(of)h Fv(MPI)g Fx(is)h(unde\014ned.)21 b(That)15 b(is,)g(using)h(a)f (user-de\014ned)75 1940 y(error)h(handler,)i(or)e Fi(MPI)p 501 1940 V 14 w(ERRORS)p 675 1940 V 14 w(RETURN)p Fx(,)g(do)q(es)h Fl(not)k Fx(necessarily)d(allo)o(w)f(the)f(user)h(to)f(con)o(tin)o(ue)i (to)75 1997 y(use)c Fv(MPI)g Fx(after)f(an)h(error)f(is)h(detected.)20 b(The)15 b(purp)q(ose)f(of)g(these)g(error)f(handlers)i(is)f(to)g(allo) o(w)g(a)g(user)g(to)75 2053 y(issue)f(user-de\014ned)i(error)d (messages)g(and)g(to)g(tak)o(e)g(actions)h(unrelated)g(to)f Fv(MPI)g Fx(\(suc)o(h)h(as)f(\015ushing)i(I/O)75 2110 y(bu\013ers\))j(b)q(efore)g(a)g(program)f(exits.)27 b(An)17 b Fv(MPI)g Fx(implemen)o(tation)h(is)g(free)f(to)g(allo)o(w)g Fv(MPI)g Fx(to)f(con)o(tin)o(ue)75 2166 y(after)e(an)i(error)e(but)h (is)h(not)f(required)h(to)f(do)g(so.)189 2266 y Fl(A)n(dvic)n(e)i(to)i (implementors.)56 b Fx(A)17 b(go)q(o)q(d)h(qualit)o(y)g(implemen)o (tation)h(will,)h(to)d(the)h(greatest)f(p)q(os-)189 2322 y(sible,)25 b(exten)o(t,)e(circumscrib)q(e)i(the)d(impact)g(of)g(an)g (error,)h(so)f(that)g(normal)g(pro)q(cessing)h(can)189 2378 y(con)o(tin)o(ue)14 b(after)f(an)h(error)g(handler)h(w)o(as)e(in)o (v)o(ok)o(ed.)19 b(The)c(implemen)o(tation)g(do)q(cumen)o(tation)f (will)189 2435 y(pro)o(vide)k(information)g(on)g(the)g(p)q(ossible)i (e\013ect)e(of)f(eac)o(h)h(class)h(of)e(errors.)28 b(\()p Fl(End)18 b(of)h(advic)n(e)g(to)189 2491 y(implementors.)p Fx(\))166 2591 y(An)f Fv(MPI)f Fx(error)g(handler)i(is)f(an)g(opaque)g (ob)s(ject,)f(whic)o(h)i(is)f(accessed)h(b)o(y)e(a)h(handle.)29 b Fv(MPI)17 b Fx(calls)75 2647 y(are)d(pro)o(vided)h(to)f(create)h(new) f(error)g(handlers,)h(to)f(asso)q(ciate)h(error)e(handlers)j(with)f (comm)o(unicators,)75 2704 y(and)g(to)g(test)g(whic)o(h)h(error)e (handler)i(is)g(asso)q(ciated)g(with)f(a)g(comm)o(unicator.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 192 200 192 199 bop 75 -100 a Fx(192)532 b Fm(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(ERRHANDLER)p 469 45 V 18 w(CREA)l(TE\()g(function,)g(errhandler)f(\))117 122 y Fk(IN)155 b Fv(function)434 b Fk(user)15 b(de\014ned)g(error)g (handling)e(pro)q(cedure)117 197 y(OUT)108 b Fv(errhandler)397 b Fk(MPI)14 b(error)h(handler)f(\(handle\))75 322 y Fu(int)23 b(MPI)p 245 322 15 2 v 17 w(Errhandler)p 502 322 V 16 w(create\(MPI)p 758 322 V 16 w(Handler)p 942 322 V 16 w(function)g(*function,)393 378 y(MPI)p 468 378 V 17 w(Errhandler)f(*errhandler\))75 465 y(MPI)p 150 465 V 17 w(ERRHANDLER)p 407 465 V 16 w(CREATE\(FUNCTION,)f(HANDLER,)i (IERROR\))170 521 y(EXTERNAL)g(FUNCTION)170 578 y(INTEGER)g (ERRHANDLER,)g(IERROR)166 664 y Fx(Register)d(the)g(user)g(routine)h Fv(function)g Fx(for)f(use)g(as)f(an)h Fv(MPI)g Fx(exception)h (handler.)35 b(Returns)20 b(in)75 721 y Fv(errhandler)15 b Fx(a)g(handle)i(to)d(the)h(registered)h(exception)g(handler.)189 827 y Fl(A)n(dvic)n(e)g(to)h(implementors.)46 b Fx(The)16 b(handle)h(returned)g(ma)o(y)e(con)o(tain)h(the)h(address)f(of)f(the)i (error)189 883 y(handling)f(routine.)k(This)c(call)g(is)f(sup)q (er\015uous)h(in)f(C,)g(whic)o(h)g(has)g(a)f(referencing)i(op)q (erator,)e(but)189 940 y(is)h(necessary)h(in)g(F)l(ortran.)i(\()p Fl(End)e(of)g(advic)n(e)h(to)f(implementors.)p Fx(\))166 1046 y(The)g(user)h(routine)f(should)i(b)q(e)f(a)e(C)h(function)h(of)f (t)o(yp)q(e)h Fi(MPI)p 1224 1046 13 2 v 14 w(Handler)p 1371 1046 V 17 w(function)p Fx(,)g(whic)o(h)g(is)g(de\014ned)75 1103 y(as:)75 1209 y Fu(typedef)23 b(void)g (\(MPI_Handler_function\)\(MPI_C)o(omm)e(*,)i(int)h(*,)f(...\);)75 1315 y Fx(The)14 b(\014rst)f(argumen)o(t)f(is)i(the)g(comm)o(unicator)f (in)h(use.)19 b(The)14 b(second)g(is)g(the)f(error)g(co)q(de)h(to)f(b)q (e)h(returned)75 1372 y(b)o(y)19 b(the)g Fv(MPI)g Fx(routine.)32 b(The)19 b(remaining)i(argumen)o(ts)d(are)h(\\)p Fu(stdargs)p Fx(")e(argumen)o(ts)i(whose)g(n)o(um)o(b)q(er)75 1428 y(and)g(meaning)h(is)g(implemen)o(tation-dep)q(enden)o(t.)34 b(An)20 b(implemen)o(tation)g(should)g(clearly)h(do)q(cumen)o(t)75 1484 y(these)15 b(argumen)o(ts.)k(Addresses)d(are)f(used)h(so)f(that)f (the)h(handler)i(ma)o(y)d(b)q(e)i(written)f(in)h(F)l(ortran.)189 1591 y Fl(R)n(ationale.)72 b Fx(The)21 b(v)m(ariable)h(argumen)o(t)e (list)h(is)g(pro)o(vided)h(b)q(ecause)f(it)g(pro)o(vides)g(an)g(ANSI-) 189 1647 y(standard)13 b(ho)q(ok)h(for)g(pro)o(viding)h(additional)g (information)f(to)g(the)g(error)f(handler;)i(without)g(this)189 1704 y(ho)q(ok,)f(ANSI)i(C)f(prohibits)i(additional)f(argumen)o(ts.)j (\()p Fl(End)d(of)g(r)n(ationale.)p Fx(\))75 1904 y Fv(MPI)p 160 1904 14 2 v 16 w(ERRHANDLER)p 469 1904 V 18 w(SET\()f(comm)m(,)d (errhandler)j(\))117 1982 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(to)j(set)h(the)f(error)h(handler)f(for)g (\(handle\))117 2057 y(IN)155 b Fv(errhandler)397 b Fk(new)15 b(MPI)f(error)g(handler)h(for)e(comm)o(unicator)e(\(handle\))75 2181 y Fu(int)23 b(MPI)p 245 2181 15 2 v 17 w(Errhandler)p 502 2181 V 16 w(set\(MPI)p 686 2181 V 16 w(Comm)h(comm,)f(MPI)p 1037 2181 V 17 w(Errhandler)f(errhandler\))75 2268 y(MPI)p 150 2268 V 17 w(ERRHANDLER)p 407 2268 V 16 w(SET\(COMM,)g(ERRHANDLER,)h (IERROR\))170 2324 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)166 2411 y Fx(Asso)q(ciates)16 b(the)g(new)g(error)f(handler)h Fv(erro)o(rhandler)f Fx(with)h(comm)o(unicator)g Fv(comm)9 b Fx(at)15 b(the)h(calling)75 2467 y(pro)q(cess.)k(Note)15 b(that)f(an)i(error)e(handler)i(is)g(alw)o(a)o(ys)f(asso)q(ciated)g (with)g(the)h(comm)o(unicator.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 193 201 193 200 bop 75 -100 a Fm(7.2.)34 b(ERR)o(OR)17 b(HANDLING)1194 b Fx(193)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(ERRHANDLER)p 469 45 V 18 w(GET\()15 b(comm)m(,)d(errhandler)j(\))117 122 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(to)j(get)g(the)h(error)g(handler)f(from)e(\(handle\))117 197 y(OUT)108 b Fv(errhandler)397 b Fk(MPI)16 b(error)g(handler)g (curren)o(tly)g(asso)q(ciated)g(with)f(comm)o(u-)905 254 y(nicator)f(\(handle\))75 378 y Fu(int)23 b(MPI)p 245 378 15 2 v 17 w(Errhandler)p 502 378 V 16 w(get\(MPI)p 686 378 V 16 w(Comm)h(comm,)f(MPI)p 1037 378 V 17 w(Errhandler)f (*errhandler\))75 465 y(MPI)p 150 465 V 17 w(ERRHANDLER)p 407 465 V 16 w(GET\(COMM,)g(ERRHANDLER,)h(IERROR\))170 521 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)166 608 y Fx(Returns)16 b(in)g Fv(errhandler)g Fx(\(a)f(handle)i(to\))e(the)g (error)g(handler)i(that)e(is)h(curren)o(tly)g(asso)q(ciated)g(with)75 664 y(comm)o(unicator)f Fv(comm)m Fx(.)166 720 y(Example:)22 b(A)16 b(library)h(function)g(ma)o(y)f(register)g(at)f(its)i(en)o(try)f (p)q(oin)o(t)g(the)g(curren)o(t)g(error)g(handler)75 777 y(for)h(a)h(comm)o(unicator,)f(set)h(its)g(o)o(wn)f(priv)m(ate)i (error)e(handler)i(for)e(this)h(comm)o(unicator,)g(and)g(restore)75 833 y(b)q(efore)d(exiting)i(the)e(previous)h(error)e(handler.)75 984 y Fv(MPI)p 160 984 14 2 v 16 w(ERRHANDLER)p 469 984 V 18 w(FREE\()h(errhandler)g(\))117 1061 y Fk(IN)155 b Fv(errhandler)397 b Fk(MPI)14 b(error)h(handler)f(\(handle\))75 1186 y Fu(int)23 b(MPI)p 245 1186 15 2 v 17 w(Errhandler)p 502 1186 V 16 w(free\(MPI)p 710 1186 V 16 w(Errhandler)g(*errhandler\)) 75 1272 y(MPI)p 150 1272 V 17 w(ERRHANDLER)p 407 1272 V 16 w(FREE\(ERRHANDLER,)e(IERROR\))170 1329 y(INTEGER)i(ERRHANDLER,)g (IERROR)166 1415 y Fx(Marks)14 b(the)h(error)g(handler)h(asso)q(ciated) g(with)f Fv(errhandler)g Fx(for)g(deallo)q(cation)i(and)e(sets)g Fv(errhandler)75 1472 y Fx(to)h Fi(MPI)p 209 1472 13 2 v 14 w(ERRHANDLER)p 492 1472 V 15 w(NULL)p Fx(.)h(The)g(error)f (handler)i(will)g(b)q(e)g(deallo)q(cated)g(after)e(all)h(comm)o (unicators)75 1528 y(asso)q(ciated)e(with)h(it)f(ha)o(v)o(e)g(b)q(een)i (deallo)q(cated.)75 1679 y Fv(MPI)p 160 1679 14 2 v 16 w(ERROR)p 323 1679 V 18 w(STRING\()e(erro)o(rco)q(de,)f(string,)h (resultlen)i(\))117 1756 y Fk(IN)155 b Fv(erro)o(rco)q(de)410 b Fk(Error)15 b(co)q(de)g(returned)g(b)o(y)f(an)f(MPI)i(routine)117 1831 y(OUT)108 b Fv(string)481 b Fk(T)m(ext)14 b(that)g(corresp)q(onds) i(to)e(the)g Fi(erro)o(rco)q(de)117 1906 y Fk(OUT)108 b Fv(resultlen)430 b Fk(Length)12 b(\(in)g(prin)o(table)f(c)o (haracters\))i(of)f(the)g(result)g(returned)905 1963 y(in)i Fi(string)75 2087 y Fu(int)23 b(MPI)p 245 2087 15 2 v 17 w(Error)p 382 2087 V 17 w(string\(int)f(errorcode,)h(char)g (*string,)g(int)g(*resultlen\))75 2174 y(MPI)p 150 2174 V 17 w(ERROR)p 287 2174 V 16 w(STRING\(ERRORCODE,)f(STRING,)h (RESULTLEN,)f(IERROR\))170 2230 y(INTEGER)h(ERRORCODE,)g(RESULTLEN,)f (IERROR)170 2287 y(CHARACTER*\(*\))h(STRING)166 2373 y Fx(Returns)f(the)f(error)g(string)g(asso)q(ciated)g(with)h(an)f (error)f(co)q(de.)39 b(The)21 b(argumen)o(t)g Fv(string)h Fx(m)o(ust)75 2429 y(represen)o(t)15 b(storage)f(that)h(is)g(at)g (least)g Fi(MPI)p 811 2429 13 2 v 15 w(MAX)p 918 2429 V 14 w(ERROR)p 1069 2429 V 14 w(STRING)g Fx(c)o(haracters)f(long.)166 2486 y(The)d(n)o(um)o(b)q(er)h(of)e(c)o(haracters)h(actually)h(written) f(is)g(returned)h(in)g(the)f(output)g(argumen)o(t,)g Fv(resultlen)p Fx(.)189 2591 y Fl(R)n(ationale.)38 b Fx(The)14 b(form)e(of)h(this)g(function)h(w)o(as)e(c)o(hosen)i(to)e (mak)o(e)h(the)g(F)l(ortran)f(and)h(C)g(bindings)189 2647 y(similar.)30 b(A)18 b(v)o(ersion)g(that)g(returns)g(a)g(p)q(oin)o (ter)g(to)g(a)g(string)g(has)g(t)o(w)o(o)f(di\016culties.)31 b(First,)18 b(the)189 2704 y(return)10 b(string)h(m)o(ust)g(b)q(e)g (statically)h(allo)q(cated)g(and)f(di\013eren)o(t)g(for)f(eac)o(h)h (error)f(message)g(\(allo)o(wing)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 194 202 194 201 bop 75 -100 a Fx(194)532 b Fm(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)189 45 y Fx(the)d(p)q(oin)o(ters)g(returned)g(b)o(y)g(successiv)o(e)h(calls)g (to)e Fi(MPI)p 1117 45 13 2 v 14 w(ERROR)p 1268 45 V 14 w(STRING)h Fx(to)f(p)q(oin)o(t)h(to)f(the)h(correct)189 102 y(message\).)25 b(Second,)19 b(in)f(F)l(ortran,)e(a)i(function)g (declared)g(as)f(returning)h Fi(CHARA)o(CTER*\(*\))e Fx(can)189 158 y(not)e(b)q(e)i(referenced)g(in,)g(for)f(example,)g(a)g Fi(PRINT)g Fx(statemen)o(t.)k(\()p Fl(End)c(of)i(r)n(ationale.)p Fx(\))75 301 y Fn(7.3)59 b(Erro)n(r)21 b(co)r(des)e(and)g(classes)75 403 y Fx(The)13 b(error)f(co)q(des)h(returned)f(b)o(y)h Fv(MPI)f Fx(are)g(left)h(en)o(tirely)g(to)f(the)h(implemen)o(tation)h (\(with)e(the)h(exception)75 459 y(of)h Fi(MPI)p 203 459 V 15 w(SUCCESS)p Fx(\).)e(This)j(is)h(done)f(to)f(allo)o(w)h(an)f (implemen)o(tation)i(to)f(pro)o(vide)g(as)f(m)o(uc)o(h)h(information)75 516 y(as)g(p)q(ossible)i(in)f(the)f(error)g(co)q(de)g(\(for)g(use)g (with)h Fv(MPI)p 1004 516 14 2 v 15 w(ERROR)p 1166 516 V 19 w(STRING)p Fx(\).)166 572 y(T)l(o)d(mak)o(e)f(it)h(p)q(ossible)i (for)e(an)f(application)j(to)d(in)o(terpret)i(an)f(error)f(co)q(de,)i (the)f(routine)g Fv(MPI)p 1761 572 V 16 w(ERR-)75 629 y(OR)p 140 629 V 17 w(CLASS)19 b Fx(con)o(v)o(erts)e(an)i(error)e(co)q (de)i(in)o(to)f(one)h(of)f(a)g(small)h(set)f(of)g(sp)q(eci\014ed)i(v)m (alues,)g(called)g Fl(err)n(or)75 685 y(classes)p Fx(.)e(V)l(alid)f (error)d(classes)i(include)384 778 y Fi(MPI)p 461 778 13 2 v 14 w(SUCCESS)191 b Fx(No)15 b(error)384 835 y Fi(MPI)p 461 835 V 14 w(ERR)p 554 835 V 15 w(BUFFER)117 b Fx(In)o(v)m(alid)17 b(bu\013er)e(p)q(oin)o(ter)384 891 y Fi(MPI)p 461 891 V 14 w(ERR)p 554 891 V 15 w(COUNT)130 b Fx(In)o(v)m(alid)17 b(coun)o(t)e(argumen)o(t)384 948 y Fi(MPI)p 461 948 V 14 w(ERR)p 554 948 V 15 w(TYPE)166 b Fx(In)o(v)m(alid)17 b(datat)o(yp)q(e)e(argumen)o(t)384 1004 y Fi(MPI)p 461 1004 V 14 w(ERR)p 554 1004 V 15 w(T)m(A)o(G)194 b Fx(In)o(v)m(alid)17 b(tag)d(argumen)o(t)384 1061 y Fi(MPI)p 461 1061 V 14 w(ERR)p 554 1061 V 15 w(COMM)144 b Fx(In)o(v)m(alid)17 b(comm)o(unicator)384 1117 y Fi(MPI)p 461 1117 V 14 w(ERR)p 554 1117 V 15 w(RANK)161 b Fx(In)o(v)m(alid)17 b(rank)384 1174 y Fi(MPI)p 461 1174 V 14 w(ERR)p 554 1174 V 15 w(REQUEST)86 b Fx(In)o(v)m(alid)17 b(request)e(\(handle\))384 1230 y Fi(MPI)p 461 1230 V 14 w(ERR)p 554 1230 V 15 w(ROOT)157 b Fx(In)o(v)m(alid)17 b(ro)q(ot)384 1287 y Fi(MPI)p 461 1287 V 14 w(ERR)p 554 1287 V 15 w(GROUP)132 b Fx(In)o(v)m(alid)17 b(group)384 1343 y Fi(MPI)p 461 1343 V 14 w(ERR)p 554 1343 V 15 w(OP)216 b Fx(In)o(v)m(alid)17 b(op)q(eration)384 1399 y Fi(MPI)p 461 1399 V 14 w(ERR)p 554 1399 V 15 w(TOPOLOGY)48 b Fx(In)o(v)m(alid)17 b(top)q(ology)384 1456 y Fi(MPI)p 461 1456 V 14 w(ERR)p 554 1456 V 15 w(DIMS)173 b Fx(In)o(v)m(alid)17 b(dimension)g(argumen)o(t)384 1512 y Fi(MPI)p 461 1512 V 14 w(ERR)p 554 1512 V 15 w(ARG)191 b Fx(In)o(v)m(alid)17 b(argumen)o(t)d(of)h(some)g(other)g(kind)384 1569 y Fi(MPI)p 461 1569 V 14 w(ERR)p 554 1569 V 15 w(UNKNO)o(WN)60 b Fx(Unkno)o(wn)15 b(error)384 1625 y Fi(MPI)p 461 1625 V 14 w(ERR)p 554 1625 V 15 w(TRUNCA)m(TE)56 b Fx(Message)14 b(truncated)h(on)h(receiv)o(e)384 1682 y Fi(MPI)p 461 1682 V 14 w(ERR)p 554 1682 V 15 w(OTHER)134 b Fx(Kno)o(wn)15 b(error)g(not)f(in)i(this)g(list)384 1738 y Fi(MPI)p 461 1738 V 14 w(ERR)p 554 1738 V 15 w(INTERN)124 b Fx(In)o(ternal)16 b(MPI)f(error)384 1795 y Fi(MPI)p 461 1795 V 14 w(ERR)p 554 1795 V 15 w(LASTCODE)60 b Fx(Last)15 b(standard)f(error)h(co)q(de) 166 1897 y(An)20 b(implemen)o(tation)h(is)f(free)g(to)e(de\014ne)j (more)e(error)g(classes;)j(ho)o(w)o(ev)o(er,)e(the)f(standard)h(error) 75 1954 y(classes)c(m)o(ust)e(b)q(e)i(used)g(where)f(appropriate.)20 b(The)c(error)e(classes)i(satisfy)l(,)389 2056 y(0)d(=)g Fv(MPI)p 557 2056 V 14 w(SUCCESS)h Fp(<)f Fv(MPI)p 906 2056 V 14 w(ERR)p 1005 2056 V 16 w Fp(:::)e Ft(\024)i Fv(MPI)p 1201 2056 V 15 w(ERR)p 1301 2056 V 16 w(LASTCODE)p Fp(:)189 2162 y Fl(R)n(ationale.)42 b Fx(The)16 b(di\013erence)h(b)q (et)o(w)o(een)e Fi(MPI)p 983 2162 V 15 w(ERR)p 1077 2162 V 14 w(UNKNO)o(WN)h Fx(and)g Fi(MPI)p 1485 2162 V 14 w(ERR)p 1578 2162 V 14 w(OTHER)g Fx(is)g(that)189 2218 y Fv(MPI)p 274 2218 14 2 v 15 w(ERROR)p 436 2218 V 18 w(STRING)g Fx(can)g(return)f(useful)h(information)f(ab)q(out)g Fi(MPI)p 1433 2218 13 2 v 15 w(ERR)p 1527 2218 V 14 w(OTHER)p Fx(.)189 2294 y(Note)h(that)g Fi(MPI)p 477 2294 V 14 w(SUCCESS)f Fx(=)g(0)i(is)g(necessary)f(to)g(b)q(e)i(consisten)o(t)e (with)h(C)g(practice;)g(the)g(sepa-)189 2350 y(ration)g(of)g(error)f (classes)i(and)g(error)e(co)q(des)i(allo)o(ws)g(us)f(to)g(de\014ne)h (the)g(error)e(classes)i(this)g(w)o(a)o(y)l(.)189 2406 y(Ha)o(ving)d(a)g(kno)o(wn)g Fi(LASTCODE)f Fx(is)i(often)f(a)g(nice)h (sanit)o(y)f(c)o(hec)o(k)h(as)f(w)o(ell.)20 b(\()p Fl(End)c(of)g(r)n (ationale.)p Fx(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 195 203 195 202 bop 75 -100 a Fm(7.4.)34 b(TIMERS)1446 b Fx(195)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(ERROR)p 323 45 V 18 w(CLASS\()15 b(erro)o(rco)q(de,)f(erro)o(rclass)g(\))117 122 y Fk(IN)155 b Fv(erro)o(rco)q(de)410 b Fk(Error)15 b(co)q(de)g(returned)g(b)o(y)f(an)f(MPI)i(routine)117 197 y(OUT)108 b Fv(erro)o(rclass)410 b Fk(Error)15 b(class)f(asso)q (ciated)h(with)e Fi(erro)o(rco)q(de)75 322 y Fu(int)23 b(MPI)p 245 322 15 2 v 17 w(Error)p 382 322 V 17 w(class\(int)f (errorcode,)h(int)g(*errorclass\))75 408 y(MPI)p 150 408 V 17 w(ERROR)p 287 408 V 16 w(CLASS\(ERRORCODE,)f(ERRORCLASS,)g (IERROR\))170 465 y(INTEGER)h(ERRORCODE,)g(ERRORCLASS,)f(IERROR)75 638 y Fn(7.4)59 b(Timers)75 739 y Fv(MPI)15 b Fx(de\014nes)h(a)f (timer.)21 b(A)15 b(timer)h(is)f(sp)q(eci\014ed)j(ev)o(en)e(though)f (it)g(is)h(not)f(\\message-passing,")g(b)q(ecause)75 796 y(timing)21 b(parallel)g(programs)e(is)i(imp)q(ortan)o(t)e(in)i (\\p)q(erformance)f(debugging")h(and)f(b)q(ecause)h(existing)75 852 y(timers)16 b(\(b)q(oth)f(in)i(POSIX)f(1003.1-1988)d(and)j(1003.4D) e(14.1)g(and)i(in)h(F)l(ortran)d(90\))h(are)g(either)i(incon-)75 909 y(v)o(enien)o(t)f(or)f(do)g(not)f(pro)o(vide)i(adequate)f(access)h (to)e(high-resolution)j(timers.)75 1060 y Fv(MPI)p 160 1060 14 2 v 16 w(WTIME\(\))75 1164 y Fu(double)23 b(MPI)p 317 1164 15 2 v 17 w(Wtime\(void\))75 1250 y(DOUBLE)g(PRECISION)g(MPI)p 556 1250 V 16 w(WTIME\(\))166 1336 y Fv(MPI)p 251 1336 14 2 v 16 w(WTIME)10 b Fx(returns)g(a)h(\015oating-p)q(oin)o(t)g(n)o (um)o(b)q(er)g(of)f(seconds,)i(represen)o(ting)f(elapsed)h(w)o(all-clo) q(c)o(k)75 1393 y(time)k(since)g(some)f(time)g(in)h(the)g(past.)166 1449 y(The)21 b(\\time)g(in)h(the)f(past")g(is)g(guaran)o(teed)g(not)f (to)h(c)o(hange)g(during)h(the)f(life)h(of)f(the)g(pro)q(cess.)75 1506 y(The)d(user)g(is)g(resp)q(onsible)h(for)e(con)o(v)o(erting)h (large)f(n)o(um)o(b)q(ers)h(of)f(seconds)h(to)f(other)g(units)i(if)f (they)f(are)75 1562 y(preferred.)166 1619 y(This)j(function)h(is)f(p)q (ortable)g(\(it)g(returns)f(seconds,)i(not)e(\\tic)o(ks"\),)h(it)g (allo)o(ws)g(high-resolution,)75 1675 y(and)15 b(carries)h(no)f (unnecessary)h(baggage.)j(One)d(w)o(ould)f(use)h(it)f(lik)o(e)i(this:) 75 1781 y Fu({)147 1838 y(double)23 b(starttime,)f(endtime;)147 1894 y(starttime)g(=)i(double)f(MPI_Wtime\(\);)170 1951 y(....)48 b(stuff)23 b(to)g(be)h(timed)47 b(...)147 2007 y(endtime)70 b(=)24 b(double)f(MPI_Wtime\(\);)147 2064 y(printf\("That)f(took)h(\045f)h(seconds\\n",endtime-start)o(time\);)75 2120 y(})166 2226 y Fx(The)18 b(times)g(returned)h(are)e(lo)q(cal)i(to) f(the)g(no)q(de)g(that)f(called)j(them.)28 b(There)18 b(is)h(no)e(requiremen)o(t)75 2283 y(that)d(di\013eren)o(t)i(no)q(des)g (return)f(\\the)g(same)g(time.")75 2434 y Fv(MPI)p 160 2434 V 16 w(WTICK\(\))75 2538 y Fu(double)23 b(MPI)p 317 2538 15 2 v 17 w(Wtick\(void\))75 2624 y(DOUBLE)g(PRECISION)g(MPI)p 556 2624 V 16 w(WTICK\(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 196 204 196 203 bop 75 -100 a Fx(196)532 b Fm(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)166 45 y Fv(MPI)p 251 45 14 2 v 16 w(WTICK)k Fx(returns)h(the)g(resolution)h (of)f Fv(MPI)p 1038 45 V 15 w(WTIME)g Fx(in)h(seconds.)38 b(That)20 b(is,)j(it)e(returns,)75 102 y(as)g(a)f(double)j(precision)f (v)m(alue,)h(the)f(n)o(um)o(b)q(er)f(of)g(seconds)g(b)q(et)o(w)o(een)g (successiv)o(e)h(clo)q(c)o(k)g(tic)o(ks.)38 b(F)l(or)75 158 y(example,)22 b(if)g(the)e(clo)q(c)o(k)i(is)f(implemen)o(ted)h(b)o (y)f(the)g(hardw)o(are)f(as)g(a)g(coun)o(ter)h(that)f(is)h(incremen)o (ted)75 214 y(ev)o(ery)15 b(millisecond,)j(the)d(v)m(alue)i(returned)e (b)o(y)g Fv(MPI)p 971 214 V 16 w(WTICK)f Fx(should)i(b)q(e)g(10)1398 198 y Fh(\000)p Fg(3)1445 214 y Fx(.)75 358 y Fn(7.5)59 b(Sta)n(rtup)75 459 y Fx(One)11 b(goal)f(of)g Fv(MPI)f Fx(is)i(to)f(ac)o(hiev)o(e)g Fl(sour)n(c)n(e)i(c)n(o)n(de)f(p)n (ortability)p Fx(.)18 b(By)11 b(this)f(w)o(e)g(mean)g(that)g(a)g (program)f(written)75 516 y(using)16 b Fv(MPI)e Fx(and)i(complying)g (with)g(the)f(relev)m(an)o(t)h(language)f(standards)g(is)h(p)q(ortable) f(as)g(written,)g(and)75 572 y(m)o(ust)h(not)g(require)i(an)o(y)e (source)h(co)q(de)g(c)o(hanges)g(when)g(mo)o(v)o(ed)f(from)g(one)h (system)f(to)g(another.)24 b(This)75 629 y(explicitly)d(do)q(es)e Fl(not)j Fx(sa)o(y)c(an)o(ything)g(ab)q(out)h(ho)o(w)e(an)i Fv(MPI)e Fx(program)h(is)h(started)e(or)h(launc)o(hed)i(from)75 685 y(the)c(command)g(line,)h(nor)f(what)f(the)h(user)g(m)o(ust)f(do)h (to)f(set)h(up)g(the)g(en)o(vironmen)o(t)g(in)h(whic)o(h)g(an)f Fv(MPI)75 741 y Fx(program)h(will)j(run.)28 b(Ho)o(w)o(ev)o(er,)18 b(an)g(implemen)o(tation)h(ma)o(y)f(require)g(some)g(setup)h(to)e(b)q (e)i(p)q(erformed)75 798 y(b)q(efore)14 b(other)g Fv(MPI)f Fx(routines)h(ma)o(y)g(b)q(e)g(called.)21 b(T)l(o)14 b(pro)o(vide)g(for)f(this,)i Fv(MPI)e Fx(includes)j(an)e (initialization)75 854 y(routine)i Fv(MPI)p 317 854 V 15 w(INIT)p Fx(.)75 1005 y Fv(MPI)p 160 1005 V 16 w(INIT\(\))75 1109 y Fu(int)23 b(MPI)p 245 1109 15 2 v 17 w(Init\(int)g(*argc,)g (char)g(***argv\))75 1195 y(MPI)p 150 1195 V 17 w(INIT\(IERROR\))170 1252 y(INTEGER)g(IERROR)166 1338 y Fx(This)17 b(routine)h(m)o(ust)e(b)q (e)i(called)g(b)q(efore)f(an)o(y)g(other)f Fv(MPI)h Fx(routine.)25 b(It)17 b(m)o(ust)f(b)q(e)i(called)h(at)d(most)75 1395 y(once;)f(subsequen)o(t)h(calls)g(are)f(erroneous)g(\(see)g Fv(MPI)p 982 1395 14 2 v 16 w(INITIALIZED)p Fx(\).)166 1451 y(All)k Fv(MPI)d Fx(programs)g(m)o(ust)h(con)o(tain)h(a)f(call)h (to)f Fv(MPI)p 1089 1451 V 16 w(init)p Fx(;)h(this)g(routine)g(m)o(ust) f(b)q(e)h(called)h(b)q(efore)75 1508 y(an)o(y)f(other)g Fv(MPI)f Fx(routine)i(\(apart)e(from)g Fu(MPI)p 872 1508 15 2 v 17 w(INITIALIZED)p Fx(\))f(is)j(called.)30 b(The)18 b(v)o(ersion)h(for)e(ANSI)i(C)75 1564 y(accepts)c(the)h Fi(a)o(rgc)f Fx(and)g Fi(a)o(rgv)g Fx(that)g(are)g(pro)o(vided)h(b)o(y) f(the)g(argumen)o(ts)f(to)h Fu(main)p Fx(:)75 1658 y Fu(MPI_init\()23 b(argc,)g(argv)g(\);)75 1752 y Fx(The)15 b(F)l(ortran)f(v)o(ersion)i(tak)o(es)e(only)i Fi(IERROR)p Fx(.)75 1903 y Fv(MPI)p 160 1903 14 2 v 16 w(FINALIZE\(\))75 2006 y Fu(int)23 b(MPI)p 245 2006 15 2 v 17 w(Finalize\(void\))75 2093 y(MPI)p 150 2093 V 17 w(FINALIZE\(IERROR\))170 2149 y(INTEGER)g(IERROR)166 2236 y Fx(This)14 b(routines)f(cleans)h(up)g (all)g Fv(MPI)f Fx(state.)18 b(Once)c(this)g(routine)f(is)h(called,)h (no)e Fv(MPI)f Fx(routine)i(\(ev)o(en)75 2292 y Fv(MPI)p 160 2292 14 2 v 16 w(INIT)p Fx(\))e(ma)o(y)h(b)q(e)h(called.)21 b(The)14 b(user)g(m)o(ust)f(ensure)h(that)f(all)h(p)q(ending)i(comm)o (unications)e(in)o(v)o(olving)75 2349 y(a)h(pro)q(cess)g(complete)h(b)q (efore)g(the)f(pro)q(cess)g(calls)i Fv(MPI)p 1023 2349 V 16 w(FINALIZE)p Fx(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 197 205 197 204 bop 75 -100 a Fm(7.5.)34 b(ST)l(AR)l(TUP)1411 b Fx(197)75 45 y Fv(MPI)p 160 45 14 2 v 16 w(INITIALIZED\()13 b(\015ag)i(\))117 122 y Fk(OUT)108 b Fv(\015ag)518 b Fk(F)p Fx(lag)15 b(is)f(true)h(if)f Fv(MPI)p 1270 122 V 16 w(INIT)g Fx(has)g(b)q(een)h(called)h(and)f(false)905 179 y(otherwise.)75 303 y Fu(int)23 b(MPI)p 245 303 15 2 v 17 w(Initialized\(int)f(*flag\))75 390 y(MPI)p 150 390 V 17 w(INITIALIZED\(FLAG,)f(IERROR\))170 446 y(LOGICAL)i(FLAG)170 503 y(INTEGER)g(IERROR)166 589 y Fx(This)c(routine)f(ma)o(y)g(b)q(e)h (used)g(to)e(determine)i(whether)g Fv(MPI)p 1242 589 14 2 v 16 w(INIT)e Fx(has)h(b)q(een)h(called.)31 b(It)18 b(is)h(the)75 645 y Fl(only)c Fx(routine)g(that)g(ma)o(y)f(b)q(e)i (called)h(b)q(efore)e Fv(MPI)p 938 645 V 16 w(INIT)g Fx(is)g(called.)75 796 y Fv(MPI)p 160 796 V 16 w(ABORT\()h(comm)m(,)c (erro)o(rco)q(de)i(\))117 874 y Fk(IN)155 b Fv(comm)466 b Fk(comm)o(unicator)11 b(of)j(tasks)g(to)g(ab)q(ort)117 949 y(IN)155 b Fv(erro)o(rco)q(de)410 b Fk(error)15 b(co)q(de)g(to)e (return)j(to)d(in)o(v)o(oking)f(en)o(vironmen)o(t)75 1073 y Fu(int)23 b(MPI)p 245 1073 15 2 v 17 w(Abort\(MPI)p 478 1073 V 16 w(Comm)g(comm,)h(int)f(errorcode\))75 1160 y(MPI)p 150 1160 V 17 w(ABORT\(COMM,)f(ERRORCODE,)h(IERROR\))170 1216 y(INTEGER)g(COMM,)h(ERRORCODE,)e(IERROR)166 1303 y Fx(This)d(routine)f(mak)o(es)g(a)g(\\b)q(est)g(attempt")f(to)h(ab)q (ort)g(all)h(tasks)e(in)i(the)g(group)e(of)h Fv(comm)m Fx(.)26 b(This)75 1359 y(function)21 b(do)q(es)g(not)f(require)h(that)f (the)g(in)o(v)o(oking)h(en)o(vironmen)o(t)f(tak)o(e)g(an)o(y)g(action)h (with)f(the)h(error)75 1416 y(co)q(de.)g(Ho)o(w)o(ev)o(er,)14 b(a)h(Unix)i(or)e(POSIX)h(en)o(vironmen)o(t)g(should)g(handle)h(this)f (as)f(a)g Fu(return)23 b(errorcode)75 1472 y Fx(from)14 b(the)i(main)f(program)f(or)h(an)g Fu(abort\(errorcode\))p Fx(.)166 1528 y Fv(MPI)f Fx(implemen)o(tations)i(are)e(required)i(to)d (de\014ne)j(the)f(b)q(eha)o(vior)g(of)f Fv(MPI)p 1441 1528 14 2 v 16 w(ABORT)i Fx(at)e(least)g(for)g(a)75 1585 y Fu(comm)i Fx(of)g Fi(MPI)p 317 1585 13 2 v 14 w(COMM)p 461 1585 V 15 w(W)o(ORLD)p Fx(.)f Fv(MPI)h Fx(implemen)o(tations)i(ma)o (y)d(ignore)i(the)g Fu(comm)e Fx(argumen)o(t)h(and)g(act)75 1641 y(as)f(if)g(the)h Fu(comm)e Fx(w)o(as)h Fi(MPI)p 526 1641 V 14 w(COMM)p 670 1641 V 15 w(W)o(ORLD)p Fx(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 198 206 198 205 bop 75 356 a Fo(Chapter)34 b(8)75 564 y Fr(Pro\014ling)42 b(In)m(terface)75 805 y Fn(8.1)59 b(Requirements)75 906 y Fx(T)l(o)15 b(meet)g(the)g Fv(MPI)g Fx(pro\014ling)h(in)o(terface,)f (an)h(implemen)o(tation)g(of)f(the)g Fv(MPI)g Fx(functions)h Fl(must)131 997 y Fx(1.)22 b(pro)o(vide)14 b(a)g(mec)o(hanism)g (through)g(whic)o(h)h(all)g(of)e(the)h Fv(MPI)f Fx(de\014ned)j (functions)f(ma)o(y)e(b)q(e)h(accessed)189 1054 y(with)e(a)h(name)f (shift.)19 b(Th)o(us)13 b(all)g(of)f(the)h Fv(MPI)f Fx(functions)h (\(whic)o(h)g(normally)g(start)e(with)i(the)g(pre\014x)189 1110 y(\\)p Fu(MPI)p 287 1110 15 2 v 16 w Fx("\))i(should)h(also)f(b)q (e)h(accessible)h(with)e(the)h(pre\014x)f(\\)p Fu(PMPI)p 1289 1110 V 17 w Fx(".)131 1203 y(2.)22 b(ensure)d(that)e(those)i Fv(MPI)e Fx(functions)j(whic)o(h)f(are)f(not)g(replaced)i(ma)o(y)e (still)h(b)q(e)h(link)o(ed)g(in)o(to)e(an)189 1260 y(executable)e (image)f(without)h(causing)f(name)h(clashes.)131 1352 y(3.)22 b(do)q(cumen)o(t)c(the)h(implemen)o(tation)g(of)f(di\013eren)o (t)h(language)f(bindings)i(of)e(the)g Fv(MPI)g Fx(in)o(terface)g(if)189 1409 y(they)f(are)h(la)o(y)o(ered)g(on)f(top)g(of)h(eac)o(h)f(other,)h (so)f(that)g(the)h(pro\014ler)g(dev)o(elop)q(er)h(kno)o(ws)e(whether) 189 1465 y(she)d(m)o(ust)f(implemen)o(t)i(the)e(pro\014le)i(in)o (terface)f(for)f(eac)o(h)h(binding,)h(or)e(can)h(economise)h(b)o(y)e (imple-)189 1522 y(men)o(ting)i(it)h(only)f(for)g(the)g(lo)o(w)o(est)g (lev)o(el)h(routines.)131 1615 y(4.)22 b(where)15 b(the)g(implemen)o (tation)i(of)d(di\013eren)o(t)i(language)f(bindings)i(is)f(is)g(done)f (through)g(a)g(la)o(y)o(ered)189 1671 y(approac)o(h)h(\(e.g.)24 b(the)17 b(F)l(ortran)f(binding)j(is)e(a)f(set)h(of)g(\\wrapp)q(er")f (functions)i(whic)o(h)f(call)h(the)f(C)189 1727 y(implemen)o(tation\),) g(ensure)g(that)f(these)g(wrapp)q(er)h(functions)g(are)f(separable)h (from)f(the)g(rest)g(of)189 1784 y(the)f(library)l(.)189 1859 y(This)k(is)h(necessary)f(to)f(allo)o(w)h(a)g(separate)f (pro\014ling)j(library)e(to)g(b)q(e)g(correctly)h(implemen)o(ted,)189 1915 y(since)g(\(at)f(least)g(with)h(Unix)g(link)o(er)h(seman)o(tics\)) e(the)h(pro\014ling)g(library)g(m)o(ust)f(con)o(tain)h(these)189 1971 y(wrapp)q(er)14 b(functions)h(if)g(it)f(is)h(to)f(p)q(erform)g(as) f(exp)q(ected.)21 b(This)15 b(requiremen)o(t)g(allo)o(ws)f(the)h(p)q (erson)189 2028 y(who)c(builds)i(the)e(pro\014ling)h(library)g(to)f (extract)f(these)i(functions)g(from)e(the)h(original)i Fv(MPI)d Fx(library)189 2084 y(and)k(add)g(them)g(in)o(to)g(the)f (pro\014ling)j(library)e(without)g(bringing)h(along)f(an)o(y)g(other)f (unnecessary)189 2141 y(co)q(de.)131 2234 y(5.)22 b(pro)o(vide)15 b(a)g(no-op)h(routine)f Fv(MPI)p 756 2234 14 2 v 16 w(PCONTROL)h Fx(in)g(the)g Fv(MPI)e Fx(library)l(.)75 2376 y Fn(8.2)59 b(Discussion)75 2478 y Fx(The)14 b(ob)s(jectiv)o(e)f(of)g(the)h Fv(MPI)f Fx(pro\014ling)h(in)o(terface)g(is)g(to)f(ensure)h(that)f(it)h (is)g(relativ)o(ely)g(easy)g(for)f(authors)75 2534 y(of)19 b(pro\014ling)h(\(and)f(other)g(similar\))h(to)q(ols)g(to)e(in)o (terface)i(their)f(co)q(des)h(to)f Fv(MPI)g Fx(implemen)o(tations)h(on) 75 2591 y(di\013eren)o(t)15 b(mac)o(hines.)166 2647 y(Since)20 b Fv(MPI)f Fx(is)g(a)g(mac)o(hine)h(indep)q(enden)o(t)h(standard)e (with)g(man)o(y)g(di\013eren)o(t)g(implemen)o(tations,)75 2704 y(it)e(is)g(unreasonable)g(to)f(exp)q(ect)i(that)e(the)g(authors)g (of)g(pro\014ling)i(to)q(ols)f(for)e Fv(MPI)h Fx(will)j(ha)o(v)o(e)d (access)h(to)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 199 207 199 206 bop 75 -100 a Fm(8.3.)34 b(LOGIC)16 b(OF)f(THE)g(DESIGN)1092 b Fx(199)75 45 y(the)14 b(source)g(co)q(de)g(whic)o(h)h(implemen)o(ts)g Fv(MPI)e Fx(on)h(an)o(y)f(particular)i(mac)o(hine.)20 b(It)14 b(is)g(therefore)g(necessary)75 102 y(to)21 b(pro)o(vide)i(a)f (mec)o(hanism)g(b)o(y)g(whic)o(h)h(the)f(implemen)o(tors)h(of)f(suc)o (h)g(to)q(ols)g(can)g(collect)i(whatev)o(er)75 158 y(p)q(erformance)15 b(information)h(they)f(wish)h Fl(without)g Fx(access)f(to)g(the)g (underlying)i(implemen)o(tation.)166 214 y(W)l(e)e(b)q(eliev)o(e)h (that)e(ha)o(ving)h(suc)o(h)g(an)f(in)o(terface)h(is)g(imp)q(ortan)o(t) f(if)i Fv(MPI)d Fx(is)j(to)e(b)q(e)h(attractiv)o(e)e(to)h(end)75 271 y(users,)h(since)i(the)e(a)o(v)m(ailabilit)o(y)j(of)d(man)o(y)g (di\013eren)o(t)h(to)q(ols)f(will)i(b)q(e)g(a)e(signi\014can)o(t)h (factor)f(in)h(attracting)75 327 y(users)f(to)g(the)g Fv(MPI)g Fx(standard.)166 384 y(The)e(pro\014ling)h(in)o(terface)e(is)h (just)g(that,)f(an)g(in)o(terface.)20 b(It)12 b(sa)o(ys)g Fl(nothing)g Fx(ab)q(out)g(the)h(w)o(a)o(y)f(in)h(whic)o(h)75 440 y(it)h(is)h(used.)20 b(There)14 b(is)h(therefore)f(no)g(attempt)f (to)g(la)o(y)h(do)o(wn)g(what)g(information)g(is)h(collected)g(through) 75 497 y(the)g(in)o(terface,)g(or)g(ho)o(w)g(the)g(collected)i (information)e(is)h(sa)o(v)o(ed,)e(\014ltered,)i(or)f(displa)o(y)o(ed.) 166 553 y(While)g(the)g(initial)h(imp)q(etus)f(for)f(the)g(dev)o (elopmen)o(t)h(of)e(this)i(in)o(terface)f(arose)g(from)f(the)h(desire)i (to)75 610 y(p)q(ermit)g(the)f(implemen)o(tation)i(of)d(pro\014ling)j (to)q(ols,)d(it)i(is)f(clear)h(that)f(an)g(in)o(terface)g(lik)o(e)h (that)f(sp)q(eci\014ed)75 666 y(ma)o(y)j(also)g(pro)o(v)o(e)g(useful)h (for)f(other)g(purp)q(oses,)i(suc)o(h)e(as)h(\\in)o(ternet)o(w)o (orking")e(m)o(ultiple)j Fv(MPI)e Fx(imple-)75 723 y(men)o(tations.)j (Since)c(all)f(that)f(is)h(de\014ned)h(is)f(an)g(in)o(terface,)f(there) h(is)g(no)f(ob)s(jection)h(to)f(its)h(b)q(eing)g(used)75 779 y(wherev)o(er)f(it)h(is)f(useful.)166 835 y(As)j(the)g(issues)h(b)q (eing)g(addressed)f(here)h(are)f(in)o(timately)h(tied)f(up)h(with)f (the)g(w)o(a)o(y)f(in)i(whic)o(h)g(ex-)75 892 y(ecutable)j(images)e (are)g(built,)j(whic)o(h)e(ma)o(y)f(di\013er)h(greatly)f(on)g (di\013eren)o(t)h(mac)o(hines,)h(the)e(examples)75 948 y(giv)o(en)d(b)q(elo)o(w)f(should)i(b)q(e)e(treated)g(solely)h(as)f (one)g(w)o(a)o(y)f(of)h(implemen)o(ting)i(the)e(ob)s(jectiv)o(e)h(of)e (the)i Fv(MPI)75 1005 y Fx(pro\014ling)g(in)o(terface.)k(The)16 b(actual)g(requiremen)o(ts)g(made)g(of)f(an)h(implemen)o(tation)h(are)e (those)g(detailed)75 1061 y(in)i(the)f(Requiremen)o(ts)i(section)e(ab)q (o)o(v)o(e,)g(the)g(whole)h(of)f(the)g(rest)g(of)f(this)i(c)o(hapter)f (is)g(only)h(presen)o(t)f(as)75 1118 y(justi\014cation)g(and)f (discussion)i(of)e(the)g(logic)h(for)f(those)g(requiremen)o(ts.)166 1174 y(The)i(examples)g(b)q(elo)o(w)h(sho)o(w)e(one)h(w)o(a)o(y)f(in)h (whic)o(h)h(an)f(implemen)o(tation)h(could)f(b)q(e)h(constructed)75 1231 y(to)h(meet)h(the)g(requiremen)o(ts)g(on)g(a)g(Unix)h(system)e (\(there)h(are)f(doubtless)i(others)e(whic)o(h)i(w)o(ould)f(b)q(e)75 1287 y(equally)d(v)m(alid\).)75 1428 y Fn(8.3)59 b(Logic)20 b(of)g(the)f(design)75 1529 y Fx(Pro)o(vided)d(that)g(an)g Fv(MPI)f Fx(implemen)o(tation)i(meets)f(the)g(requiremen)o(ts)h(ab)q(o) o(v)o(e,)e(it)h(is)h(p)q(ossible)h(for)d(the)75 1586 y(implemen)o(tor)20 b(of)f(the)h(pro\014ling)g(system)f(to)g(in)o (tercept)h(all)g(of)f(the)h Fv(MPI)f Fx(calls)h(whic)o(h)g(are)g(made)f (b)o(y)75 1642 y(the)g(user)h(program.)30 b(She)20 b(can)f(then)h (collect)g(whatev)o(er)f(information)g(she)h(requires)g(b)q(efore)f (calling)75 1698 y(the)d(underlying)h Fv(MPI)e Fx(implemen)o(tation)h (\(through)f(its)h(name)f(shifted)h(en)o(try)f(p)q(oin)o(ts\))h(to)f (ac)o(hiev)o(e)h(the)75 1755 y(desired)g(e\013ects.)75 1874 y Fj(8.3.1)49 b(Miscellaneous)17 b(control)g(of)f(p)o(ro\014ling) 75 1960 y Fx(There)e(is)g(a)g(clear)g(requiremen)o(t)h(for)e(the)h (user)g(co)q(de)g(to)f(b)q(e)i(able)f(to)f(con)o(trol)h(the)g (pro\014ler)g(dynamically)75 2016 y(at)h(run)g(time.)20 b(This)c(is)g(normally)g(used)f(for)g(\(at)f(least\))h(the)g(purp)q (oses)h(of)143 2096 y Ft(\017)23 b Fx(Enabling)16 b(and)g(disabling)h (pro\014ling)f(dep)q(ending)i(on)d(the)g(state)f(of)h(the)g (calculation.)143 2184 y Ft(\017)23 b Fx(Flushing)16 b(trace)f(bu\013ers)g(at)g(non-critical)i(p)q(oin)o(ts)e(in)h(the)g (calculation)143 2272 y Ft(\017)23 b Fx(Adding)16 b(user)f(ev)o(en)o (ts)g(to)g(a)g(trace)f(\014le.)166 2351 y(These)h(requiremen)o(ts)h (are)f(met)g(b)o(y)g(use)h(of)e(the)i Fv(MPI)p 1092 2351 14 2 v 15 w(PCONTROL)p Fx(.)75 2502 y Fv(MPI)p 160 2502 V 16 w(PCONTROL\(level,)f Fp(:)8 b(:)g(:)e Fv(\))117 2579 y Fk(IN)155 b Fv(level)504 b Fk(Pro\014ling)13 b(lev)o(el)75 2704 y Fu(int)23 b(MPI)p 245 2704 15 2 v 17 w(Pcontrol\(const)f(int)i (level,)f Fp(:)8 b(:)g(:)d Fu(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 200 208 200 207 bop 75 -100 a Fx(200)878 b Fm(CHAPTER)15 b(8.)30 b(PR)o(OFILING)16 b(INTERF)-5 b(A)o(CE)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(PCONTROL\(level\))170 102 y(INTEGER)23 b(LEVEL,)g Fp(:)8 b(:)g(:)166 188 y Fv(MPI)16 b Fx(libraries)j (themselv)o(es)f(mak)o(e)e(no)h(use)h(of)e(this)i(routine,)g(and)f (simply)h(return)f(immediately)75 244 y(to)d(the)g(user)h(co)q(de.)20 b(Ho)o(w)o(ev)o(er)13 b(the)i(presence)g(of)f(calls)i(to)d(this)i (routine)g(allo)o(ws)g(a)f(pro\014ling)h(pac)o(k)m(age)g(to)75 301 y(b)q(e)h(explicitly)i(called)f(b)o(y)e(the)g(user.)166 357 y(Since)k Fv(MPI)f Fx(has)g(no)g(con)o(trol)g(of)f(the)i(implemen)o (tation)g(of)f(the)g(pro\014ling)h(co)q(de,)g(w)o(e)f(are)g(unable)75 414 y(to)13 b(sp)q(ecify)i(precisely)h(the)e(seman)o(tics)g(whic)o(h)h (will)g(b)q(e)g(pro)o(vided)f(b)o(y)g(calls)h(to)e Fv(MPI)p 1505 414 14 2 v 16 w(PCONTROL)p Fx(.)h(This)75 470 y(v)m(agueness)i (extends)f(to)g(the)g(n)o(um)o(b)q(er)h(of)f(argumen)o(ts)f(to)h(the)g (function,)g(and)h(their)g(datat)o(yp)q(es.)166 527 y(Ho)o(w)o(ev)o(er) 11 b(to)g(pro)o(vide)h(some)g(lev)o(el)h(of)e(p)q(ortabilit)o(y)i(of)e (user)h(co)q(des)h(to)e(di\013eren)o(t)h(pro\014ling)h(libraries,)75 583 y(w)o(e)i(request)g(the)g(follo)o(wing)h(meanings)g(for)f(certain)g (v)m(alues)i(of)d(lev)o(el.)143 677 y Ft(\017)23 b Fu(level==0)14 b Fx(Pro\014ling)i(is)g(disabled.)143 771 y Ft(\017)23 b Fu(level==1)14 b Fx(Pro\014ling)i(is)g(enabled)g(at)f(a)g(normal)g (default)h(lev)o(el)g(of)f(detail.)143 865 y Ft(\017)23 b Fu(level==2)14 b Fx(Pro\014le)h(bu\013ers)g(are)g(\015ushed.)21 b(\(This)16 b(ma)o(y)e(b)q(e)i(a)f(no-op)g(in)h(some)f(pro\014lers\).) 143 958 y Ft(\017)23 b Fx(All)13 b(other)f(v)m(alues)h(of)f Fu(level)f Fx(ha)o(v)o(e)h(pro\014le)h(library)g(de\014ned)g(e\013ects) f(and)h(additional)g(argumen)o(ts.)166 1052 y(W)l(e)18 b(also)f(request)h(that)f(the)h(default)g(state)f(after)g Fv(MPI)p 1147 1052 V 16 w(INIT)g Fx(has)g(b)q(een)i(called)g(is)g(for)e (pro\014ling)75 1109 y(to)e(b)q(e)h(enabled)h(at)e(the)h(normal)f (default)h(lev)o(el.)22 b(\(i.e.)f(as)15 b(if)h Fv(MPI)p 1204 1109 V 16 w(PCONTROL)h Fx(had)e(just)h(b)q(een)g(called)75 1165 y(with)h(the)f(argumen)o(t)g(1\).)23 b(This)17 b(allo)o(ws)f (users)h(to)e(link)j(with)f(a)f(pro\014ling)i(library)f(and)g(obtain)f (pro\014le)75 1222 y(output)f(without)g(ha)o(ving)h(to)e(mo)q(dify)i (their)g(source)f(co)q(de)h(at)e(all.)166 1278 y(The)h(pro)o(vision)g (of)f Fv(MPI)p 589 1278 V 16 w(PCONTROL)i Fx(as)e(a)g(no-op)h(in)g(the) g(standard)f Fv(MPI)g Fx(library)h(allo)o(ws)g(them)75 1335 y(to)g(mo)q(dify)h(their)g(source)f(co)q(de)h(to)f(obtain)h(more)f (detailed)i(pro\014ling)f(information,)g(but)f(still)i(b)q(e)f(able)75 1391 y(to)f(link)h(exactly)g(the)f(same)g(co)q(de)h(against)f(the)g (standard)g Fv(MPI)f Fx(library)l(.)75 1534 y Fn(8.4)59 b(Examples)75 1637 y Fj(8.4.1)49 b(Pro\014ler)16 b(implem)o(ent)o (ation)75 1723 y Fx(Supp)q(ose)25 b(that)e(the)h(pro\014ler)h(wishes)g (to)e(accum)o(ulate)i(the)f(total)f(amoun)o(t)g(of)h(data)f(sen)o(t)h (b)o(y)g(the)75 1780 y(MPI)p 167 1780 V 16 w(SEND)18 b(function,)g(along)f(with)h(the)f(total)g(elapsed)h(time)f(sp)q(en)o (t)h(in)g(the)f(function.)27 b(This)18 b(could)75 1836 y(trivially)f(b)q(e)f(ac)o(hiev)o(ed)g(th)o(us)75 1942 y Fu(static)23 b(int)g(totalBytes;)75 1999 y(static)g(double)g (totalTime;)75 2112 y(int)g(MPI_SEND\(void)g(*)g(buffer,)g(const)g(int) h(count,)f(MPI_Datatype)f(datatype,)385 2168 y(int)i(dest,)f(int)g (tag,)h(MPI_comm)e(comm\))75 2225 y({)147 2281 y(double)h(tstart)g(=)g (MPI_Wtime\(\);)166 b(/*)23 b(Pass)h(on)f(all)h(the)f(arguments)g(*/) 147 2338 y(int)g(extent;)147 2394 y(int)g(result)95 b(=)23 b(PMPI_Send\(buffer,count,dataty)o(pe,dest)o(,tag,com)o(m\);)982 2507 y(/*)g(Accumulate)g(byte)g(count)g(*/)147 2563 y(totalBytes)f(+=)i (count)f(*)h(MPI_Type_size\(datatype,)o(&extent)o(\);)982 2620 y(/*)f(and)h(time)f(*/)147 2676 y(totalTime)46 b(+=)24 b(MPI_Wtime\(\))e(-)i(tstart;)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 201 209 201 208 bop 75 -100 a Fm(8.4.)34 b(EXAMPLES)1370 b Fx(201)147 102 y Fu(return)23 b(result;)75 158 y(})75 280 y Fj(8.4.2)49 b(MPI)17 b(lib)o(ra)o(ry)f(implem)o(ent)o(ation)75 366 y Fx(On)i(a)g(Unix)g(system,)g(in)h(whic)o(h)f(the)g Fv(MPI)f Fx(library)i(is)f(implemen)o(ted)h(in)g(C,)e(then)h(there)g (are)f(v)m(arious)75 422 y(p)q(ossible)22 b(options,)f(of)e(whic)o(h)i (t)o(w)o(o)e(of)g(the)h(most)g(ob)o(vious)g(are)g(presen)o(ted)g(here.) 35 b(Whic)o(h)20 b(is)h(b)q(etter)75 478 y(dep)q(ends)c(on)e(whether)g (the)h(link)o(er)g(and)f(compiler)i(supp)q(ort)e(w)o(eak)g(sym)o(b)q (ols.)75 599 y Fv(Systems)f(with)i(w)o(eak)f(symb)q(ols)75 684 y Fx(If)k(the)f(compiler)i(and)f(link)o(er)h(supp)q(ort)f(w)o(eak)f (external)h(sym)o(b)q(ols)f(\(e.g.)30 b(Solaris)19 b(2.x,)f(other)g (system)75 741 y(V.4)13 b(mac)o(hines\),)i(then)f(only)g(a)g(single)h (library)g(is)f(required)h(through)f(the)g(use)g(of)g Fu(#pragma)23 b(weak)13 b Fx(th)o(us)75 847 y Fu(#pragma)23 b(weak)g(MPI_Example)g(=)g(PMPI_Example)75 960 y(int)g (PMPI_Example\(/*)f(appropriate)h(args)g(*/\))75 1016 y({)170 1073 y(/*)h(Useful)f(content)g(*/)75 1129 y(})166 1236 y Fx(The)d(e\013ect)g(of)g(this)g Fu(#pragma)f Fx(is)i(to)e (de\014ne)i(the)f(external)h(sym)o(b)q(ol)f Fu(MPI)p 1472 1236 15 2 v 17 w(Example)f Fx(as)h(a)g(w)o(eak)75 1292 y(de\014nition.)h(This)15 b(means)f(that)g(the)g(link)o(er)h(will) h(not)e(complain)i(if)e(there)h(is)f(another)g(de\014nition)i(of)e(the) 75 1348 y(sym)o(b)q(ol)h(\(for)e(instance)i(in)g(the)f(pro\014ling)i (library\),)e(ho)o(w)o(ev)o(er)f(if)i(no)f(other)g(de\014nition)i (exists,)e(then)h(the)75 1405 y(link)o(er)h(will)h(use)f(the)f(w)o(eak) g(de\014nition.)75 1525 y Fv(Systems)f(without)j(w)o(eak)e(symb)q(ols) 75 1611 y Fx(In)k(the)g(absence)g(of)f(w)o(eak)g(sym)o(b)q(ols)h(then)g (one)f(p)q(ossible)j(solution)e(w)o(ould)g(b)q(e)g(to)f(use)h(the)f(C)h (macro)75 1667 y(pre-pro)q(cessor)c(th)o(us)75 1774 y Fu(#ifdef)23 b(PROFILELIB)75 1830 y(#)95 b(ifdef)23 b(__STDC__)75 1886 y(#)191 b(define)23 b(FUNCTION\(name\))f(P##name)75 1943 y(#)95 b(else)75 1999 y(#)191 b(define)23 b(FUNCTION\(name\))f (P/**/name)75 2056 y(#)95 b(endif)75 2112 y(#else)75 2169 y(#)g(define)23 b(FUNCTION\(name\))f(name)75 2225 y(#endif)166 2331 y Fx(Eac)o(h)15 b(of)g(the)g(user)g(visible)j (functions)e(in)g(the)f(library)h(w)o(ould)g(then)f(b)q(e)h(declared)g (th)o(us)75 2438 y Fu(int)23 b(FUNCTION\(MPI_Example\)\(/*)e (appropriate)h(args)i(*/\))75 2494 y({)170 2551 y(/*)g(Useful)f (content)g(*/)75 2607 y(})-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 202 210 202 209 bop 75 -100 a Fx(202)873 b Fm(CHAPTER)15 b(8.)35 b(PR)o(OFILING)16 b(INTERF)-5 b(A)o(CE)166 45 y Fx(The)20 b(same)g(source)g(\014le)h(can)g(then)f(b)q(e)h(compiled)h(to)d(pro)q (duce)i(b)q(oth)f(v)o(ersions)g(of)g(the)g(library)l(,)75 102 y(dep)q(ending)d(on)f(the)f(state)f(of)h(the)g Fu(PROFILELIB)f Fx(macro)h(sym)o(b)q(ol.)166 158 y(It)e(is)h(required)g(that)f(the)g (standard)g Fv(MPI)f Fx(library)i(b)q(e)g(built)h(in)f(suc)o(h)f(a)g(w) o(a)o(y)f(that)g(the)i(inclusion)h(of)75 214 y Fv(MPI)g Fx(functions)h(can)f(b)q(e)h(ac)o(hiev)o(ed)h(one)e(at)g(a)g(time.)20 b(This)c(is)g(a)f(somewhat)g(unpleasan)o(t)h(requiremen)o(t,)75 271 y(since)22 b(it)f(ma)o(y)f(mean)h(that)f(eac)o(h)g(external)i (function)f(has)g(to)f(b)q(e)h(compiled)i(from)d(a)g(separate)g (\014le.)75 327 y(Ho)o(w)o(ev)o(er)14 b(this)i(is)g(necessary)g(so)f (that)g(the)g(author)g(of)g(the)h(pro\014ling)g(library)g(need)h(only)f (de\014ne)g(those)75 384 y Fv(MPI)f Fx(functions)g(whic)o(h)h(she)g (wishes)g(to)e(in)o(tercept,)i(references)g(to)e(an)o(y)h(others)g(b)q (eing)h(ful\014lled)i(b)o(y)d(the)75 440 y(normal)g Fv(MPI)g Fx(library)l(.)21 b(Therefore)15 b(the)g(link)i(step)e(can)g(lo)q(ok)h (something)f(lik)o(e)h(this)75 547 y Fu(\045)24 b(cc)f(...)h(-lmyprof)f (-lpmpi)g(-lmpi)166 653 y Fx(Here)f Fu(libmyprof.a)d Fx(con)o(tains)j(the)f(pro\014ler)h(functions)h(whic)o(h)f(in)o (tercept)g(some)f(of)g(the)h Fv(MPI)75 709 y Fx(functions.)k Fu(libpmpi.a)16 b Fx(con)o(tains)h(the)g(\\name)g(shifted")g Fv(MPI)g Fx(functions,)h(and)f Fu(libmpi.a)f Fx(con)o(tains)75 766 y(the)f(normal)g(de\014nitions)i(of)e(the)g Fv(MPI)g Fx(functions.)75 887 y Fj(8.4.3)49 b(Complications)75 973 y Fv(Multiple)16 b(counting)75 1059 y Fx(Since)g(parts)e(of)g(the)g Fv(MPI)g Fx(library)i(ma)o(y)d(themselv)o(es)i(b)q(e)h(implemen)o(ted)g (using)f(more)f(basic)h Fv(MPI)f Fx(func-)75 1116 y(tions)h(\(e.g.)k(a) c(p)q(ortable)h(implemen)o(tation)g(of)f(the)g(collectiv)o(e)i(op)q (erations)f(implemen)o(ted)h(using)e(p)q(oin)o(t)75 1172 y(to)e(p)q(oin)o(t)h(comm)o(unications\),)g(there)g(is)h(p)q(oten)o (tial)f(for)f(pro\014ling)i(functions)g(to)e(b)q(e)h(called)i(from)d (within)75 1228 y(an)19 b Fv(MPI)g Fx(function)h(whic)o(h)g(w)o(as)e (called)j(from)e(a)g(pro\014ling)h(function.)33 b(This)20 b(could)g(lead)g(to)f(\\double)75 1285 y(coun)o(ting")d(of)g(the)g (time)h(sp)q(en)o(t)f(in)h(the)f(inner)i(routine.)23 b(Since)17 b(this)g(e\013ect)f(could)h(actually)g(b)q(e)g(useful)75 1341 y(under)e(some)e(circumstances)i(\(e.g.)k(it)14 b(migh)o(t)f(allo)o(w)i(one)f(to)f(answ)o(er)g(the)h(question)h(\\Ho)o (w)e(m)o(uc)o(h)h(time)75 1398 y(is)h(sp)q(en)o(t)g(in)h(the)e(p)q(oin) o(t)h(to)f(p)q(oin)o(t)i(routines)f(when)g(they're)f(called)i(from)e (collectiv)o(e)j(functions)e(?"\),)f(w)o(e)75 1454 y(ha)o(v)o(e)h (decided)j(not)d(to)g(enforce)h(an)o(y)g(restrictions)g(on)f(the)h (author)f(of)h(the)f Fv(MPI)h Fx(library)g(whic)o(h)h(w)o(ould)75 1511 y(o)o(v)o(ercome)11 b(this.)19 b(Therefore)12 b(the)g(author)g(of) f(the)h(pro\014ling)i(library)e(should)h(b)q(e)g(a)o(w)o(are)e(of)g (this)i(problem,)75 1567 y(and)f(guard)g(against)g(it)h(herself.)19 b(In)13 b(a)f(single)i(threaded)e(w)o(orld)g(this)h(is)g(easily)g(ac)o (hiev)o(ed)g(through)f(use)g(of)75 1624 y(a)j(static)f(v)m(ariable)j (in)f(the)f(pro\014ling)h(co)q(de)f(whic)o(h)h(remem)o(b)q(ers)f(if)h (y)o(ou)e(are)h(already)g(inside)i(a)e(pro\014ling)75 1680 y(routine.)25 b(It)17 b(b)q(ecomes)g(more)f(complex)i(in)f(a)g(m)o (ulti-threaded)h(en)o(vironmen)o(t)f(\(as)f(do)q(es)h(the)f(meaning)75 1737 y(of)f(the)g(times)h(recorded)f(!\))75 1857 y Fv(Link)o(er)g(o)q (ddities)75 1942 y Fx(The)g(Unix)g(link)o(er)g(traditionally)h(op)q (erates)e(in)i(one)e(pass)g(:)20 b(the)14 b(e\013ect)g(of)g(this)h(is)g (that)f(functions)h(from)75 1999 y(libraries)f(are)e(only)h(included)j (in)d(the)g(image)f(if)h(they)g(are)f(needed)i(at)e(the)h(time)g(the)f (library)h(is)h(scanned.)75 2055 y(When)j(com)o(bined)g(with)g(w)o(eak) f(sym)o(b)q(ols,)g(or)g(m)o(ultiple)i(de\014nitions)g(of)e(the)h(same)f (function,)h(this)g(can)75 2112 y(cause)e(o)q(dd)h(\(and)f(unexp)q (ected\))i(e\013ects.)166 2168 y(Consider,)24 b(for)e(instance,)j(an)d (implemen)o(tation)i(of)e Fv(MPI)g Fx(in)h(whic)o(h)g(the)g(F)l(ortran) e(binding)j(is)75 2225 y(ac)o(hiev)o(ed)19 b(b)o(y)e(using)i(wrapp)q (er)f(functions)g(on)g(top)g(of)f(the)h(C)f(implemen)o(tation.)29 b(The)18 b(author)g(of)f(the)75 2281 y(pro\014le)e(library)h(then)e (assumes)h(that)f(it)g(is)h(reasonable)g(only)g(to)f(pro)o(vide)h (pro\014le)g(functions)h(for)e(the)g(C)75 2338 y(binding,)20 b(since)e(F)l(ortran)e(will)j(ev)o(en)o(tually)g(call)f(these,)g(and)g (the)f(cost)g(of)g(the)h(wrapp)q(ers)f(is)h(assumed)75 2394 y(to)f(b)q(e)g(small.)27 b(Ho)o(w)o(ev)o(er,)16 b(if)i(the)f(wrapp)q(er)g(functions)h(are)f(not)g(in)h(the)f (pro\014ling)i(library)l(,)f(then)g(none)75 2451 y(of)c(the)g (pro\014led)i(en)o(try)e(p)q(oin)o(ts)g(will)i(b)q(e)f(unde\014ned)h (when)f(the)g(pro\014ling)g(library)g(is)g(called.)21 b(Therefore)75 2507 y(none)d(of)f(the)g(pro\014ling)i(co)q(de)f(will)h (b)q(e)f(included)i(in)e(the)g(image.)26 b(When)18 b(the)f(standard)g Fv(MPI)g Fx(library)75 2563 y(is)f(scanned,)f(the)g(F)l(ortran)f(wrapp) q(ers)h(will)i(b)q(e)e(resolv)o(ed,)h(and)f(will)h(also)f(pull)i(in)f (the)f(base)g(v)o(ersions)g(of)75 2620 y(the)g Fv(MPI)g Fx(functions.)21 b(The)15 b(o)o(v)o(erall)h(e\013ect)f(is)h(that)e(the) h(co)q(de)h(will)h(link)g(successfully)l(,)g(but)e(will)i(not)e(b)q(e) 75 2676 y(pro\014led.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 203 211 203 210 bop 75 -100 a Fm(8.5.)34 b(MUL)l(TIPLE)16 b(LEVELS)g(OF)f (INTER)o(CEPTION)739 b Fx(203)166 45 y(T)l(o)17 b(o)o(v)o(ercome)f (this)h(w)o(e)g(m)o(ust)f(ensure)h(that)g(the)g(F)l(ortran)e(wrapp)q (er)i(functions)h(are)f(included)i(in)75 102 y(the)d(pro\014ling)i(v)o (ersion)e(of)g(the)g(library)l(.)23 b(W)l(e)17 b(ensure)f(that)g(this)g (is)h(p)q(ossible)h(b)o(y)e(requiring)h(that)f(these)75 158 y(b)q(e)g(separable)f(from)g(the)g(rest)f(of)h(the)g(base)g Fv(MPI)g Fx(library)l(.)21 b(This)15 b(allo)o(ws)g(them)g(to)g(b)q(e)g Fu(ar)p Fx(ed)h(out)e(of)h(the)75 214 y(base)g(library)h(and)g(in)o(to) f(the)g(pro\014ling)i(one.)75 358 y Fn(8.5)59 b(Multiple)20 b(levels)f(of)h(interception)75 459 y Fx(The)15 b(sc)o(heme)h(giv)o(en) g(here)f(do)q(es)h(not)f(directly)h(supp)q(ort)f(the)h(nesting)g(of)e (pro\014ling)j(functions,)e(since)i(it)75 516 y(pro)o(vides)f(only)g(a) g(single)g(alternativ)o(e)g(name)g(for)f(eac)o(h)h Fv(MPI)f Fx(function.)22 b(Consideration)16 b(w)o(as)e(giv)o(en)j(to)75 572 y(an)e(implemen)o(tation)i(whic)o(h)f(w)o(ould)g(allo)o(w)g(m)o (ultiple)h(lev)o(els)g(of)e(call)h(in)o(terception,)h(ho)o(w)o(ev)o(er) d(w)o(e)h(w)o(ere)75 629 y(unable)j(to)e(construct)g(an)g(implemen)o (tation)i(of)e(this)h(whic)o(h)h(did)f(not)g(ha)o(v)o(e)f(the)g(follo)o (wing)i(disadv)m(an-)75 685 y(tages)143 779 y Ft(\017)23 b Fx(assuming)15 b(a)g(particular)h(implemen)o(tation)g(language.)143 873 y Ft(\017)23 b Fx(imp)q(osing)16 b(a)f(run)g(time)h(cost)f(ev)o(en) g(when)h(no)f(pro\014ling)h(w)o(as)f(taking)g(place.)75 966 y(Since)i(one)f(of)g(the)g(ob)s(jectiv)o(es)g(of)f Fv(MPI)h Fx(is)g(to)f(p)q(ermit)i(e\016cien)o(t,)f(lo)o(w)g(latency)h (implemen)o(tations,)g(and)75 1023 y(it)j(is)f(not)g(the)h(business)g (of)f(a)g(standard)g(to)g(require)h(a)f(particular)h(implemen)o(tation) g(language,)g(w)o(e)75 1079 y(decided)d(to)e(accept)g(the)g(sc)o(heme)h (outlined)h(ab)q(o)o(v)o(e.)166 1136 y(Note,)f(ho)o(w)o(ev)o(er,)f (that)h(it)h(is)f(p)q(ossible)j(to)c(use)i(the)g(sc)o(heme)f(ab)q(o)o (v)o(e)g(to)g(implemen)o(t)i(a)e(m)o(ulti-lev)o(el)75 1192 y(system,)k(since)g(the)g(function)g(called)i(b)o(y)d(the)h(user)f (ma)o(y)g(call)i(man)o(y)e(di\013eren)o(t)g(pro\014ling)i(functions)75 1249 y(b)q(efore)15 b(calling)i(the)f(underlying)h Fv(MPI)d Fx(function.)166 1305 y(Unfortunately)g(suc)o(h)h(an)f(implemen)o (tation)i(ma)o(y)d(require)j(more)d(co)q(op)q(eration)i(b)q(et)o(w)o (een)g(the)f(dif-)75 1362 y(feren)o(t)g(pro\014ling)h(libraries)h(than) e(is)g(required)h(for)f(the)g(single)h(lev)o(el)h(implemen)o(tation)f (detailed)h(ab)q(o)o(v)o(e.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 204 212 204 211 bop 75 377 a Fr(Biblio)q(graph)m(y)98 600 y Fx([1])21 b(V.)14 b(Bala)h(and)f(S.)h(Kipnis.)20 b(Pro)q(cess)14 b(groups:)19 b(a)14 b(mec)o(hanism)h(for)f(the)g(co)q(ordination)i(of)e (and)g(com-)168 656 y(m)o(unication)i(among)e(pro)q(cesses)i(in)f(the)g (Ven)o(us)h(collectiv)o(e)g(comm)o(unication)g(library)l(.)k(T)l(ec)o (hnical)168 713 y(rep)q(ort,)15 b(IBM)g(T.)g(J.)g(W)l(atson)g(Researc)o (h)g(Cen)o(ter,)g(Octob)q(er)g(1992.)k(Preprin)o(t.)98 806 y([2])i(V.)h(Bala,)h(S.)e(Kipnis,)k(L.)d(Rudolph,)j(and)c(Marc)g (Snir.)40 b(Designing)23 b(e\016cien)o(t,)h(scalable,)g(and)168 863 y(p)q(ortable)17 b(collectiv)o(e)i(comm)o(unication)e(libraries.)26 b(T)l(ec)o(hnical)18 b(rep)q(ort,)e(IBM)h(T.)f(J.)g(W)l(atson)g(Re-)168 919 y(searc)o(h)f(Cen)o(ter,)g(Octob)q(er)h(1992.)i(Preprin)o(t.)98 1013 y([3])j(Purushotham)16 b(V.)f(Bangalore,)g(Nathan)g(E.)g(Doss,)f (and)i(An)o(thon)o(y)f(Skjellum.)22 b(MPI++:)f(Issues)168 1070 y(and)16 b(Features.)j(In)d Fl(OON-SKI)f('94)p Fx(,)g(page)g(in)h (press,)f(1994.)98 1163 y([4])21 b(A.)14 b(Beguelin,)j(J.)d(Dongarra,)e (A.)i(Geist,)h(R.)f(Manc)o(hek,)g(and)g(V.)g(Sunderam.)19 b(Visualization)e(and)168 1220 y(debugging)g(in)f(a)f(heterogeneous)g (en)o(vironmen)o(t.)20 b Fl(IEEE)15 b(Computer)p Fx(,)h(26\(6\):88{95,) 11 b(June)17 b(1993.)98 1314 y([5])k(Luc)14 b(Bomans)f(and)h(Rolf)f (Hemp)q(el.)18 b(The)c(Argonne/GMD)e(macros)g(in)i(F)o(OR)l(TRAN)g(for) e(p)q(ortable)168 1370 y(parallel)20 b(programming)e(and)h(their)f (implemen)o(tation)i(on)e(the)h(In)o(tel)g(iPSC/2.)29 b Fl(Par)n(al)r(lel)19 b(Com-)168 1427 y(puting)p Fx(,)c(15:119{132,)d (1990.)98 1520 y([6])21 b(R.)15 b(Butler)g(and)f(E.)g(Lusk.)k(User's)c (guide)h(to)f(the)g(p4)g(programming)g(system.)k(T)l(ec)o(hnical)e(Rep) q(ort)168 1577 y(TM-ANL{92/17,)e(Argonne)h(National)h(Lab)q(oratory)l (,)e(1992.)98 1671 y([7])21 b(Ralph)j(Butler)f(and)g(Ewing)g(Lusk.)41 b(Monitors,)24 b(messages,)f(and)g(clusters:)35 b(the)22 b(p4)g(parallel)168 1727 y(programming)15 b(system.)21 b Fl(Journal)c(of)g(Par)n(al)r(lel)f(Computing)p Fx(,)f(1994.)20 b(to)15 b(app)q(ear)h(\(Also)g(Argonne)168 1784 y(National)11 b(Lab)q(oratory)e(Mathematics)h(and)g(Computer)g(Science)i(Division)g (preprin)o(t)f(P362-0493\).)98 1877 y([8])21 b(Robin)d(Calkin,)e(Rolf)h (Hemp)q(el,)g(Hans-Christian)g(Hopp)q(e,)g(and)f(P)o(eter)f(Wypior.)22 b(P)o(ortable)16 b(pro-)168 1934 y(gramming)11 b(with)h(the)f(parmacs)g (message{passing)g(library)l(.)k Fl(Par)n(al)r(lel)d(Computing,)h(Sp)n (e)n(cial)f(issue)168 1990 y(on)k(message{p)n(assing)f(interfac)n(es)p Fx(,)f(to)g(app)q(ear.)98 2084 y([9])21 b(S.)c(Chittor)g(and)h(R.)f(J.) g(En)o(b)q(o)q(dy)l(.)27 b(P)o(erformance)16 b(ev)m(aluation)j(of)d (mesh{connected)i(w)o(ormhole{)168 2141 y(routed)f(net)o(w)o(orks)e (for)i(in)o(terpro)q(cessor)f(comm)o(unication)i(in)f(m)o (ulticomputers.)25 b(In)17 b Fl(Pr)n(o)n(c)n(e)n(e)n(dings)168 2197 y(of)g(the)f(1990)i(Sup)n(er)n(c)n(omputing)e(Confer)n(enc)n(e)p Fx(,)c(pages)j(647{656,)e(1990.)75 2291 y([10])21 b(S.)f(Chittor)g(and) g(R.)g(J.)f(En)o(b)q(o)q(dy)l(.)35 b(Predicting)21 b(the)f(e\013ect)g (of)f(mapping)i(on)e(the)h(comm)o(unica-)168 2347 y(tion)g(p)q (erformance)g(of)f(large)g(m)o(ulticomputers.)34 b(In)20 b Fl(Pr)n(o)n(c)n(e)n(e)n(dings)d(of)k(the)f(1991)h(International)168 2404 y(Confer)n(enc)n(e)15 b(on)h(Par)n(al)r(lel)f(Pr)n(o)n(c)n (essing,)f(vol.)i(II)f(\(Softwar)n(e\))p Fx(,)f(pages)h(I)q(I{1)h({)f (I)q(I{4,)g(1991.)75 2498 y([11])21 b(J.)e(Dongarra,)f(A.)g(Geist,)h (R.)g(Manc)o(hek,)g(and)g(V.)f(Sunderam.)31 b(In)o(tegrated)18 b(PVM)g(framew)o(ork)168 2554 y(supp)q(orts)c(heterogeneous)g(net)o(w)o (ork)f(computing.)18 b Fl(Computers)e(in)e(Physics)p Fx(,)f(7\(2\):166{75,)e(April)168 2610 y(1993.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 205 213 205 212 bop 75 -100 a Fm(BIBLIOGRAPHY)1371 b Fx(205)75 45 y([12])21 b(J.)c(J.)g(Dongarra,)f(R.)h(Hemp)q(el,)i(A.)e(J.)g(G.)f (Hey)l(,)i(and)f(D.)f(W.)h(W)l(alk)o(er.)25 b(A)17 b(prop)q(osal)g(for) g(a)g(user-)168 102 y(lev)o(el,)c(message)d(passing)h(in)o(terface)g (in)h(a)e(distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec)o(hnical)g (Rep)q(ort)168 158 y(TM-12231,)g(Oak)j(Ridge)g(National)g(Lab)q (oratory)l(,)e(F)l(ebruary)h(1993.)75 249 y([13])21 b(Nathan)e(Doss,)f (William)i(Gropp,)e(Ewing)h(Lusk,)h(and)e(An)o(thon)o(y)g(Skjellum.)31 b(A)19 b(mo)q(del)g(imple-)168 305 y(men)o(tation)c(of)g(MPI.)20 b(T)l(ec)o(hnical)d(rep)q(ort,)d(Argonne)i(National)f(Lab)q(oratory)l (,)f(1993.)75 396 y([14])21 b(Edin)o(burgh)g(P)o(arallel)f(Computing)g (Cen)o(tre,)g(Univ)o(ersit)o(y)g(of)f(Edin)o(burgh.)34 b Fl(CHIMP)19 b(Conc)n(epts)p Fx(,)168 452 y(June)e(1991.)75 543 y([15])k(Edin)o(burgh)c(P)o(arallel)g(Computing)f(Cen)o(tre,)f (Univ)o(ersit)o(y)h(of)g(Edin)o(burgh.)22 b Fl(CHIMP)15 b(V)m(ersion)h(1.0)168 599 y(Interfac)n(e)p Fx(,)e(Ma)o(y)g(1992.)75 690 y([16])21 b(D.)d(F)l(eitelson.)28 b(Comm)o(unicators:)d(Ob)s (ject-based)18 b(m)o(ultipart)o(y)g(in)o(teractions)g(for)g(parallel)h (pro-)168 747 y(gramming.)j(T)l(ec)o(hnical)c(Rep)q(ort)f(91-12,)e (Dept.)g(Computer)h(Science,)i(The)e(Hebrew)g(Univ)o(ersit)o(y)168 803 y(of)f(Jerusalem,)h(No)o(v)o(em)o(b)q(er)f(1991.)75 894 y([17])21 b(Hub)q(ertus)i(F)l(rank)o(e,)f(P)o(eter)f(Ho)q(c)o(hsc)o (hild,)k(Pratap)c(P)o(attnaik,)h(and)g(Marc)e(Snir.)40 b(An)22 b(e\016cien)o(t)168 950 y(implemen)o(tation)i(of)f(MPI.)42 b(In)23 b Fl(1994)h(International)e(Confer)n(enc)n(e)f(on)i(Par)n(al)r (lel)g(Pr)n(o)n(c)n(essing)p Fx(,)168 1007 y(1994.)75 1097 y([18])e(G.)14 b(A.)g(Geist,)g(M.)g(T.)f(Heath,)h(B.)g(W.)g(P)o (eyton,)f(and)i(P)l(.)f(H.)g(W)l(orley)l(.)19 b(A)14 b(user's)g(guide)h(to)f(PICL:)g(a)168 1154 y(p)q(ortable)f(instrumen)o (ted)f(comm)o(unication)h(library)l(.)i(T)l(ec)o(hnical)f(Rep)q(ort)e (TM-11616,)f(Oak)h(Ridge)168 1210 y(National)k(Lab)q(oratory)l(,)e (Octob)q(er)i(1990.)75 1301 y([19])21 b(William)j(D.)c(Gropp)h(and)h (Barry)e(Smith.)39 b(Chameleon)22 b(parallel)h(programming)e(to)q(ols)g (users)168 1357 y(man)o(ual.)f(T)l(ec)o(hnical)d(Rep)q(ort)f (ANL-93/23,)e(Argonne)h(National)h(Lab)q(oratory)l(,)e(Marc)o(h)h (1993.)75 1448 y([20])21 b(O.)c(Kr\177)-23 b(amer)16 b(and)h(H.)g(M)q(\177)-24 b(uhlen)o(b)q(ein.)26 b(Mapping)17 b(strategies)g(in)g(message{based)f(m)o(ultipro)q(cessor)168 1504 y(systems.)k Fl(Par)n(al)r(lel)15 b(Computing)p Fx(,)g(9:213{225,)d(1989.)75 1595 y([21])21 b(nCUBE)16 b(Corp)q(oration.)j Fl(nCUBE)c(2)i(Pr)n(o)n(gr)n(ammers)e(Guide,)i (r2.0)p Fx(,)f(Decem)o(b)q(er)f(1990.)75 1686 y([22])21 b(P)o(arasoft)10 b(Corp)q(oration,)g(P)o(asadena,)h(CA.)i Fl(Expr)n(ess)f(User's)f(Guide)p Fx(,)h(v)o(ersion)f(3.2.5)f(edition,)i (1992.)75 1776 y([23])21 b(P)o(aul)c(Pierce.)23 b(The)16 b(NX/2)g(op)q(erating)g(system.)22 b(In)17 b Fl(Pr)n(o)n(c)n(e)n(e)n (dings)e(of)i(the)g(Thir)n(d)g(Confer)n(enc)n(e)e(on)168 1833 y(Hyp)n(er)n(cub)n(e)g(Concurr)n(ent)f(Computers)i(and)f(Applic)n (ations)p Fx(,)e(pages)h(384{390.)d(A)o(CM)j(Press,)f(1988.)75 1923 y([24])21 b(A.)14 b(Skjellum)i(and)e(A.)g(Leung.)19 b(Zip)q(co)q(de:)h(a)14 b(p)q(ortable)h(m)o(ulticomputer)f(comm)o (unication)h(library)168 1980 y(atop)g(the)f(reactiv)o(e)h(k)o(ernel.) 20 b(In)15 b(D.)f(W.)g(W)l(alk)o(er)h(and)g(Q.)g(F.)f(Stout,)g (editors,)g Fl(Pr)n(o)n(c)n(e)n(e)n(dings)g(of)i(the)168 2036 y(Fifth)21 b(Distribute)n(d)h(Memory)f(Concurr)n(ent)g(Computing)g (Confer)n(enc)n(e)p Fx(,)e(pages)i(767{776.)d(IEEE)168 2093 y(Press,)d(1990.)75 2184 y([25])21 b(A.)12 b(Skjellum,)i(S.)e (Smith,)h(C.)e(Still,)j(A.)d(Leung,)i(and)g(M.)e(Morari.)i(The)g(Zip)q (co)q(de)g(message)f(passing)168 2240 y(system.)20 b(T)l(ec)o(hnical)d (rep)q(ort,)d(La)o(wrence)i(Liv)o(ermore)f(National)h(Lab)q(oratory)l (,)e(Septem)o(b)q(er)i(1992.)75 2331 y([26])21 b(An)o(thon)o(y)12 b(Skjellum,)i(Nathan)e(E.)g(Doss,)g(and)g(Purushotham)g(V.)g (Bangalore.)j(W)l(riting)e(Libraries)168 2387 y(in)i(MPI.)j(In)d(An)o (thon)o(y)f(Skjellum)h(and)f(Donna)g(S.)g(Reese,)h(editors,)f Fl(Pr)n(o)n(c)n(e)n(e)n(dings)f(of)i(the)g(Sc)n(alable)168 2444 y(Par)n(al)r(lel)h(Libr)n(aries)f(Confer)n(enc)n(e)p Fx(,)d(pages)j(166{173.)e(IEEE)i(Computer)g(So)q(ciet)o(y)g(Press,)g (Octob)q(er)168 2500 y(1993.)75 2591 y([27])21 b(An)o(thon)o(y)c (Skjellum,)h(Stev)o(en)f(G.)e(Smith,)j(Nathan)e(E.)g(Doss,)g(Alvin)i(P) e(.)g(Leung,)h(and)g(Manfred)168 2647 y(Morari.)32 b(The)19 b(Design)h(and)f(Ev)o(olution)h(of)f(Zip)q(co)q(de.)33 b Fl(Par)n(al)r(lel)19 b(Computing)p Fx(,)h(1994.)31 b(\(In)o(vited)168 2704 y(P)o(ap)q(er,)15 b(to)g(app)q(ear)g(in)h(Sp)q (ecial)h(Issue)f(on)f(Message)g(P)o(assing\).)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 206 214 206 213 bop 75 -100 a Fx(206)1368 b Fm(BIBLIOGRAPHY)75 45 y Fx([28])21 b(An)o(thon)o(y)d(Skjellum,)j(Stev)o(en)d(G.)g(Smith,)h (Nathan)f(E.)g(Doss,)g(Charles)h(H.)f(Still,)i(Alvin)g(P)l(.)e(Le-)168 102 y(ung,)g(and)f(Manfred)g(Morari.)24 b(Zip)q(co)q(de:)h(A)17 b(Portable)g(Comm)o(unication)h(La)o(y)o(er)e(for)h(High)g(P)o(er-)168 158 y(formance)g(Multicomputing.)28 b(T)l(ec)o(hnical)20 b(Rep)q(ort)d(UCRL-JC-106725)g(\(revised)h(9/92,)f(12/93,)168 214 y(4/94\),)f(La)o(wrence)i(Liv)o(ermore)g(National)f(Lab)q(oratory)l (,)g(Marc)o(h)f(1991.)25 b(T)l(o)17 b(app)q(ear)h(in)g(Concur-)168 271 y(rency:)j(Practice)15 b(&)h(Exp)q(erience.)75 365 y([29])21 b(D.)12 b(W)l(alk)o(er.)k(Standards)c(for)g(message)g (passing)g(in)i(a)e(distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec) o(h-)168 421 y(nical)i(Rep)q(ort)f(TM-12147,)d(Oak)i(Ridge)h(National)g (Lab)q(oratory)l(,)e(August)h(1992.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 207 215 207 214 bop 75 356 a Fo(Annex)33 b(A)75 564 y Fr(Language)41 b(Binding)75 805 y Fn(A.1)59 b(Intro)r(duction)75 906 y Fx(In)15 b(this)f(section)h(w)o(e)f(summarize)g(the)h(sp)q(eci\014c)h (bindings)g(for)d(b)q(oth)h(F)l(ortran)f(and)h(C.)g(W)l(e)g(presen)o(t) g(\014rst)75 963 y(the)h(C)g(bindings,)i(then)e(the)h(F)l(ortran)e (bindings.)22 b(Listings)16 b(are)f(alphab)q(etical)i(within)f(c)o (hapter.)75 1106 y Fn(A.2)59 b(De\014ned)19 b(Constants)g(fo)n(r)i(C)f (and)f(F)n(o)n(rtran)75 1207 y Fx(These)e(are)g(required)h(de\014ned)h (constan)o(ts,)d(to)g(b)q(e)i(de\014ned)h(in)e(the)h(\014les)g Fu(mpi.h)e Fx(\(for)g(C\))h(and)g Fu(mpif.h)75 1264 y Fx(\(for)d(F)l(ortran\).)75 1370 y Fu(/*)24 b(return)f(codes)g(\(both)g (C)h(and)f(Fortran\))g(*/)75 1427 y(MPI_SUCCESS)75 1483 y(MPI_ERR_BUFFER)75 1539 y(MPI_ERR_COUNT)75 1596 y(MPI_ERR_TYPE)75 1652 y(MPI_ERR_TAG)75 1709 y(MPI_ERR_COMM)75 1765 y(MPI_ERR_RANK)75 1822 y(MPI_ERR_REQUEST)75 1878 y(MPI_ERR_ROOT)75 1935 y(MPI_ERR_GROUP)75 1991 y(MPI_ERR_OP)75 2048 y(MPI_ERR_TOPOLOGY)75 2104 y(MPI_ERR_DIMS)75 2160 y(MPI_ERR_ARG)75 2217 y(MPI_ERR_UNKNOWN)75 2273 y(MPI_ERR_TRUNCATE)75 2330 y(MPI_ERR_OTHER)75 2386 y(MPI_ERR_INTERN)75 2443 y(MPI_ERR_LASTCODE)75 2556 y(/*)h(assorted)e (constants)h(\(both)g(C)h(and)f(Fortran\))g(*/)75 2612 y(MPI_BOTTOM)75 2669 y(MPI_PROC_NULL)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 208 216 208 215 bop 75 -100 a Fx(208)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(MPI_ANY_SOURCE)75 102 y(MPI_ANY_TAG)75 158 y(MPI_UNDEFINED)75 214 y(MPI_UB)75 271 y(MPI_LB)75 384 y(/*)24 b(status)f(size)g(and)g(reserved)g(index)g (values)g(\(Fortran\))g(*/)75 440 y(MPI_STATUS_SIZE)75 497 y(MPI_SOURCE)75 553 y(MPI_TAG)75 666 y(/*)h(Error-handling)e (specifiers)g(\(C)i(and)f(Fortran\))g(*/)75 723 y(MPI_ERRORS_ARE_FATAL) 75 779 y(MPI_ERRORS_RETURN)75 892 y(/*)h(Maximum)e(sizes)i(for)f (strings)g(*/)75 948 y(MPI_MAX_PROCESSOR_NAME)75 1005 y(MPI_MAX_ERROR_STRING)75 1118 y(/*)h(elementary)e(datatypes)h(\(C\))g (*/)75 1174 y(MPI_CHAR)75 1231 y(MPI_SHORT)75 1287 y(MPI_INT)75 1344 y(MPI_LONG)75 1400 y(MPI_UNSIGNED_CHAR)75 1456 y (MPI_UNSIGNED_SHORT)75 1513 y(MPI_UNSIGNED)75 1569 y(MPI_UNSIGNED_LONG) 75 1626 y(MPI_FLOAT)75 1682 y(MPI_DOUBLE)75 1739 y(MPI_LONG_DOUBLE)75 1795 y(MPI_BYTE)75 1852 y(MPI_PACKED)75 2077 y(/*)h(elementary)e (datatypes)h(\(Fortran\))f(*/)75 2134 y(MPI_INTEGER)75 2190 y(MPI_REAL)75 2247 y(MPI_DOUBLE_PRECISION)75 2303 y(MPI_COMPLEX)75 2360 y(MPI_DOUBLE_COMPLEX)75 2416 y(MPI_LOGICAL)75 2473 y(MPI_CHARACTER)75 2529 y(MPI_BYTE)75 2586 y(MPI_PACKED)75 2698 y(/*)i(datatypes)e(for)i(reduction)e(functions)h(\(C\))g(*/)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 209 217 209 216 bop 75 -100 a Fm(A.2.)29 b(DEFINED)15 b(CONST)l(ANTS)h(F)o(OR)f (C)g(AND)g(F)o(OR)l(TRAN)590 b Fx(209)75 45 y Fu(MPI_FLOAT_INT)75 102 y(MPI_DOUBLE_INT)75 158 y(MPI_LONG_INT)75 214 y(MPI_2INT)75 271 y(MPI_SHORT_INT)75 327 y(MPI_LONG_DOUBLE_INT)75 440 y(/*)24 b(datatypes)e(for)i(reduction)e(functions)h(\(Fortran\))g(*/)75 497 y(MPI_2REAL)75 553 y(MPI_2DOUBLE_PRECISION)75 610 y(MPI_2INTEGER)75 666 y(MPI_2COMPLEX)75 779 y(/*)h(optional)e (datatypes)h(\(Fortran\))g(*/)75 892 y(MPI_INTEGER1)75 948 y(MPI_INTEGER2)75 1005 y(MPI_INTEGER4)75 1061 y(MPI_REAL2)75 1118 y(MPI_REAL4)75 1174 y(MPI_REAL8)75 1287 y(/*)h(optional)e (datatypes)h(\(C\))g(*/)75 1344 y(MPI_LONG_LONG_INT)75 1456 y(/*)h(reserved)e(communicators)g(\(C)i(and)f(Fortran\))g(*/)75 1513 y(MPI_COMM_WORLD)75 1569 y(MPI_COMM_SELF)75 1682 y(/*)h(results)e(of)i(communicator)e(and)i(group)f(comparisons)f(*/)75 1795 y(MPI_IDENT)75 1852 y(MPI_CONGRUENT)75 1908 y(MPI_SIMILAR)75 1965 y(MPI_UNEQUAL)75 2077 y(/*)i(environmental)e(inquiry)h(keys)g(\(C) g(and)h(Fortran\))f(*/)75 2134 y(MPI_TAG_UB)75 2190 y(MPI_IO)75 2247 y(MPI_HOST)75 2360 y(/*)h(collective)e(operations)h(\(C)g(and)h (Fortran\))e(*/)75 2416 y(MPI_MAX)75 2473 y(MPI_MIN)75 2529 y(MPI_SUM)75 2586 y(MPI_PROD)75 2642 y(MPI_MAXLOC)75 2698 y(MPI_MINLOC)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 210 218 210 217 bop 75 -100 a Fx(210)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(MPI_BAND)75 102 y(MPI_BOR)75 158 y(MPI_BXOR)75 214 y(MPI_LAND)75 271 y(MPI_LOR)75 327 y(MPI_LXOR)75 440 y(/*)24 b(Null)f(handles)g(*/)75 497 y(MPI_GROUP_NULL)75 553 y(MPI_COMM_NULL)75 610 y(MPI_DATATYPE_NULL) 75 666 y(MPI_REQUEST_NULL)75 723 y(MPI_OP_NULL)75 779 y(MPI_ERRHANDLER_NULL)75 892 y(/*)h(Empty)f(group)g(*/)75 948 y(MPI_GROUP_EMPTY)75 1061 y(/*)h(topologies)e(\(C)i(and)f (Fortran\))g(*/)75 1118 y(MPI_GRAPH)75 1174 y(MPI_CART)166 1337 y Fx(The)15 b(follo)o(wing)h(are)f(de\014ned)i(C)e(t)o(yp)q(e)g (de\014nitions,)i(also)e(included)j(in)e(the)f(\014le)h Fu(mpi.h)p Fx(.)75 1443 y Fu(/*)24 b(opaque)f(types)g(\(C\))g(*/)75 1500 y(MPI_Aint)75 1556 y(MPI_Status)75 1669 y(/*)h(handles)e(to)i (assorted)f(structures)f(\(C\))i(*/)75 1725 y(MPI_Group)75 1782 y(MPI_Comm)75 1838 y(MPI_Datatype)75 1895 y(MPI_Request)75 1951 y(MPI_Op)75 2064 y(/*)g(prototypes)e(for)i(user-defined)e (functions)g(\(C\))i(*/)75 2121 y(typedef)f(int)47 b (MPI_Copy_function\(MPI_Comm)21 b(*oldcomm,)h(*newcomm,)h(int)g (*keyval,)552 2177 y(void)h(*extra_state\))75 2234 y(typedef)f(int)47 b(MPI_Delete_function\(MPI_Comm)20 b(*comm,)j(int)h(*keyval,)839 2290 y(void)f(*extra_state\)})75 2346 y(typedef)g(void)g (MPI_Handler_function\(MPI_Com)o(m)e(*,)j(int)f(*,)h(...\);)75 2403 y(typedef)f(void)g(MPI_User_function\()f(void)h(*invec,)g(void)g (*inoutvec,)g(int)g(*len,)600 2459 y(MPI_Datatype)f(*datatype\);)166 2566 y Fx(F)l(or)d(F)l(ortran,)g(here)i(are)e(examples)i(of)e(ho)o(w)g (eac)o(h)h(of)g(the)f(user-de\014ned)j(functions)e(should)h(b)q(e)75 2622 y(declared.)166 2679 y(The)15 b(user-function)i(argumen)o(t)d(to)h Fv(MPI)p 878 2679 14 2 v 15 w(OP)p 955 2679 V 17 w(CREA)l(TE)i Fx(should)f(b)q(e)g(declared)g(lik)o(e)g(this:)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 211 219 211 218 bop 75 -100 a Fm(A.3.)34 b(C)15 b(BINDINGS)h(F)o(OR)g (POINT-TO-POINT)h(COMMUNICA)l(TION)380 b Fx(211)75 45 y Fu(FUNCTION)23 b(USER_FUNCTION\()f(INVEC\(*\),)g(INOUTVEC\(*\),)h (LEN,)g(TYPE\))75 102 y()g(INVEC\(LEN\),)f(INOUTVEC\(LEN\))99 158 y(INTEGER)h(LEN,)g(TYPE)166 252 y Fx(The)15 b(cop)o(y-function)h (argumen)o(t)f(to)f Fv(MPI)p 887 252 14 2 v 16 w(KEYV)l(AL)p 1073 252 V 17 w(CREA)l(TE)i Fx(should)h(b)q(e)e(declared)i(lik)o(e)f (this:)75 346 y Fu(FUNCTION)23 b(COPY_FUNCTION\(OLDCOMM,)e(KEYVAL,)i (EXTRA_STATE,)f(ATTRIBUTE_VAL_IN,)1244 402 y(ATTRIBUTE_VAL_OUT,)g (FLAG\))99 459 y(INTEGER)h(OLDCOMM,)g(KEYVAL,)f(EXTRA_STATE,)h (ATTRIBUTE_VAL_IN,)e(ATTRIBUTE_VAL_OUT)99 515 y(LOGICAL)i(FLAG)166 609 y Fx(The)15 b(delete-function)j(argumen)o(t)c(to)h Fv(MPI)p 913 609 V 15 w(KEYV)l(AL)p 1098 609 V 17 w(CREA)l(TE)i Fx(should)f(b)q(e)g(declared)g(lik)o(e)g(this:)75 703 y Fu(FUNCTION)23 b(DELETE_FUNCTION\(COMM,)e(KEYVAL,)i(ATTRIBUTE_VAL,)f (EXTRA_STATE\))99 759 y(INTEGER)h(COMM,)g(KEYVAL,)g(ATTRIBUTE_VAL,)f (EXTRA_STATE)75 902 y Fn(A.3)59 b(C)20 b(bindings)g(fo)n(r)h(P)n (oint-to-P)n(oint)e(Communication)75 1004 y Fx(These)d(are)e(presen)o (ted)i(here)g(in)g(the)f(order)g(of)g(their)g(app)q(earance)h(in)g(the) g(c)o(hapter.)75 1060 y Fu(int)23 b(MPI)p 245 1060 15 2 v 17 w(Send\(void*)g(buf,)g(int)g(count,)h(MPI)p 979 1060 V 16 w(Datatype)f(datatype,)g(int)g(dest,)393 1117 y(int)h(tag,)f(MPI)p 683 1117 V 17 w(Comm)g(comm\))75 1203 y(int)g(MPI)p 245 1203 V 17 w(Recv\(void*)g(buf,)g(int)g(count,)h (MPI)p 979 1203 V 16 w(Datatype)f(datatype,)g(int)g(source,)393 1260 y(int)h(tag,)f(MPI)p 683 1260 V 17 w(Comm)g(comm,)g(MPI)p 1034 1260 V 17 w(Status)g(*status\))75 1346 y(int)g(MPI)p 245 1346 V 17 w(Get)p 334 1346 V 17 w(count\(MPI)p 567 1346 V 16 w(Status)g(status,)g(MPI)p 1013 1346 V 17 w(Datatype)f (datatype,)h(int)h(*count\))75 1433 y(int)f(MPI)p 245 1433 V 17 w(Bsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1433 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 1489 y(int)h(tag,)f(MPI)p 683 1489 V 17 w(Comm)g(comm\))75 1575 y(int)g(MPI)p 245 1575 V 17 w(Ssend\(void*)g(buf,)g(int)g(count,)g (MPI)p 1002 1575 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 1632 y(int)h(tag,)f(MPI)p 683 1632 V 17 w(Comm)g(comm\))75 1718 y(int)g(MPI)p 245 1718 V 17 w(Rsend\(void*)g(buf,)g(int)g(count,)g (MPI)p 1002 1718 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 1775 y(int)h(tag,)f(MPI)p 683 1775 V 17 w(Comm)g(comm\))75 1861 y(int)g(MPI)p 245 1861 V 17 w(Buffer)p 406 1861 V 17 w(attach\()f(void*)i(buffer,)f(int)g(size\))75 1948 y(int)g(MPI)p 245 1948 V 17 w(Buffer)p 406 1948 V 17 w(detach\()f(void**)i(buffer,)e(int*)i(size\))75 2034 y(int)f(MPI)p 245 2034 V 17 w(Isend\(void*)g(buf,)g(int)g(count,)g(MPI) p 1002 2034 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2091 y(int)h(tag,)f(MPI)p 683 2091 V 17 w(Comm)g(comm,)g(MPI)p 1034 2091 V 17 w(Request)g(*request\))75 2177 y(int)g(MPI)p 245 2177 V 17 w(Ibsend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 2177 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 2234 y(int)g(tag,)f(MPI)p 683 2234 V 17 w(Comm)g(comm,)g(MPI)p 1034 2234 V 17 w(Request)g(*request\))75 2320 y(int)g(MPI)p 245 2320 V 17 w(Issend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 2320 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 2376 y(int)g(tag,)f(MPI)p 683 2376 V 17 w(Comm)g(comm,)g(MPI)p 1034 2376 V 17 w(Request)g(*request\))75 2463 y(int)g(MPI)p 245 2463 V 17 w(Irsend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 2463 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 2519 y(int)g(tag,)f(MPI)p 683 2519 V 17 w(Comm)g(comm,)g(MPI)p 1034 2519 V 17 w(Request)g(*request\))75 2606 y(int)g(MPI)p 245 2606 V 17 w(Irecv\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 2606 V 17 w(Datatype)g(datatype,)g(int)g(source,)393 2662 y(int)h(tag,)f(MPI)p 683 2662 V 17 w(Comm)g(comm,)g(MPI)p 1034 2662 V 17 w(Request)g(*request\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 212 220 212 219 bop 75 -100 a Fx(212)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(int)23 b(MPI)p 245 45 15 2 v 17 w(Wait\(MPI)p 454 45 V 16 w(Request)g(*request,)g(MPI) p 972 45 V 17 w(Status)g(*status\))75 132 y(int)g(MPI)p 245 132 V 17 w(Test\(MPI)p 454 132 V 16 w(Request)g(*request,)g(int)g (*flag,)g(MPI)p 1234 132 V 17 w(Status)g(*status\))75 218 y(int)g(MPI)p 245 218 V 17 w(Request)p 430 218 V 16 w(free\(MPI)p 638 218 V 17 w(Request)f(*request\))75 304 y(int)h(MPI)p 245 304 V 17 w(Waitany\(int)g(count,)g(MPI)p 788 304 V 16 w(Request)g(*array)p 1139 304 V 17 w(of)p 1204 304 V 17 w(requests,)f(int)i(*index,)393 361 y(MPI)p 468 361 V 17 w(Status)f(*status\))75 447 y(int)g(MPI)p 245 447 V 17 w(Testany\(int)g(count,)g(MPI)p 788 447 V 16 w(Request)g(*array)p 1139 447 V 17 w(of)p 1204 447 V 17 w(requests,)f(int)i(*index,)393 504 y(int)g(*flag,)f(MPI)p 731 504 V 16 w(Status)g(*status\))75 590 y(int)g(MPI)p 245 590 V 17 w(Waitall\(int)g(count,)g(MPI)p 788 590 V 16 w(Request)g(*array)p 1139 590 V 17 w(of)p 1204 590 V 17 w(requests,)393 647 y(MPI)p 468 647 V 17 w(Status)g(*array)p 796 647 V 16 w(of)p 860 647 V 17 w(statuses\))75 733 y(int)g(MPI)p 245 733 V 17 w(Testall\(int)g(count,)g(MPI)p 788 733 V 16 w(Request)g(*array)p 1139 733 V 17 w(of)p 1204 733 V 17 w(requests,)f(int)i(*flag,)393 790 y(MPI)p 468 790 V 17 w(Status)f(*array)p 796 790 V 16 w(of)p 860 790 V 17 w(statuses\))75 876 y(int)g(MPI)p 245 876 V 17 w(Waitsome\(int)f(incount,)h(MPI)p 859 876 V 17 w(Request)g(*array)p 1211 876 V 16 w(of)p 1275 876 V 17 w(requests,)g(int)g(*outcount,)393 933 y(int)h(*array)p 636 933 V 16 w(of)p 700 933 V 17 w(indices,)f(MPI)p 1004 933 V 16 w(Status)g(*array)p 1331 933 V 17 w(of)p 1396 933 V 17 w(statuses\))75 1019 y(int)g(MPI)p 245 1019 V 17 w(Testsome\(int)f(incount,)h(MPI)p 859 1019 V 17 w(Request)g(*array)p 1211 1019 V 16 w(of)p 1275 1019 V 17 w(requests,)g(int)g(*outcount,)393 1075 y(int)h(*array)p 636 1075 V 16 w(of)p 700 1075 V 17 w(indices,)f(MPI)p 1004 1075 V 16 w(Status)g(*array)p 1331 1075 V 17 w(of)p 1396 1075 V 17 w(statuses\))75 1162 y(int)g(MPI)p 245 1162 V 17 w(Iprobe\(int)g(source,)g(int)g(tag,)g(MPI)p 1002 1162 V 17 w(Comm)g(comm,)h(int)f(*flag,)393 1218 y(MPI)p 468 1218 V 17 w(Status)g(*status\))75 1305 y(int)g(MPI)p 245 1305 V 17 w(Probe\(int)g(source,)g(int)g(tag,)h(MPI)p 979 1305 V 16 w(Comm)g(comm,)f(MPI)p 1330 1305 V 17 w(Status)g (*status\))75 1391 y(int)g(MPI)p 245 1391 V 17 w(Cancel\(MPI)p 502 1391 V 16 w(Request)g(*request\))75 1478 y(int)g(MPI)p 245 1478 V 17 w(Test)p 358 1478 V 17 w(cancelled\(MPI)p 687 1478 V 15 w(Status)g(status,)g(int)h(*flag\))75 1564 y(int)f(MPI)p 245 1564 V 17 w(Send)p 358 1564 V 17 w(init\(void*)f (buf,)i(int)f(count,)g(MPI)p 1091 1564 V 17 w(Datatype)g(datatype,)f (int)i(dest,)393 1621 y(int)g(tag,)f(MPI)p 683 1621 V 17 w(Comm)g(comm,)g(MPI)p 1034 1621 V 17 w(Request)g(*request\))75 1707 y(int)g(MPI)p 245 1707 V 17 w(Bsend)p 382 1707 V 17 w(init\(void*)f(buf,)i(int)f(count,)g(MPI)p 1115 1707 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 1764 y(int)g(tag,)f(MPI)p 683 1764 V 17 w(Comm)g(comm,)g(MPI)p 1034 1764 V 17 w(Request)g (*request\))75 1850 y(int)g(MPI)p 245 1850 V 17 w(Ssend)p 382 1850 V 17 w(init\(void*)f(buf,)i(int)f(count,)g(MPI)p 1115 1850 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 1907 y(int)g(tag,)f(MPI)p 683 1907 V 17 w(Comm)g(comm,)g(MPI)p 1034 1907 V 17 w(Request)g(*request\))75 1993 y(int)g(MPI)p 245 1993 V 17 w(Rsend)p 382 1993 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 1993 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 2049 y(int)g(tag,)f(MPI)p 683 2049 V 17 w(Comm)g(comm,)g(MPI)p 1034 2049 V 17 w(Request)g(*request\))75 2136 y(int)g(MPI)p 245 2136 V 17 w(Recv)p 358 2136 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1091 2136 V 17 w(Datatype)g(datatype,)f(int)i(source,) 393 2192 y(int)g(tag,)f(MPI)p 683 2192 V 17 w(Comm)g(comm,)g(MPI)p 1034 2192 V 17 w(Request)g(*request\))75 2279 y(int)g(MPI)p 245 2279 V 17 w(Start\(MPI)p 478 2279 V 16 w(Request)g(*request\))75 2365 y(int)g(MPI)p 245 2365 V 17 w(Startall\(int)f(count,)h(MPI)p 811 2365 V 17 w(Request)g(*array)p 1163 2365 V 17 w(of)p 1228 2365 V 16 w(requests\))75 2452 y(int)g(MPI)p 245 2452 V 17 w(Sendrecv\(void)f(*sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 2452 V 17 w(Datatype)f(sendtype,)393 2508 y(int)i(dest,)f(int)g (sendtag,)g(void)g(*recvbuf,)g(int)g(recvcount,)393 2565 y(MPI)p 468 2565 V 17 w(Datatype)g(recvtype,)f(int)i(source,)f(MPI)p 1297 2565 V 16 w(Datatype)g(recvtag,)393 2621 y(MPI)p 468 2621 V 17 w(Comm)g(comm,)g(MPI)p 819 2621 V 17 w(Status)g (*status\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 213 221 213 220 bop 75 -100 a Fm(A.3.)34 b(C)15 b(BINDINGS)h(F)o(OR)g (POINT-TO-POINT)h(COMMUNICA)l(TION)380 b Fx(213)75 45 y Fu(int)23 b(MPI)p 245 45 15 2 v 17 w(Sendrecv)p 454 45 V 16 w(replace\(void*)f(buf,)i(int)f(count,)g(MPI)p 1258 45 V 17 w(Datatype)g(datatype,)393 102 y(int)h(dest,)f(int)g (sendtag,)g(int)g(source,)g(int)h(recvtag,)f(MPI)p 1614 102 V 16 w(Comm)h(comm,)393 158 y(MPI)p 468 158 V 17 w(Status)f(*status\))75 244 y(int)g(MPI)p 245 244 V 17 w(Type)p 358 244 V 17 w(contiguous\(int)f(count,)h(MPI)p 972 244 V 17 w(Datatype)f(oldtype,)393 301 y(MPI)p 468 301 V 17 w(Datatype)h(*newtype\))75 387 y(int)g(MPI)p 245 387 V 17 w(Type)p 358 387 V 17 w(vector\(int)f(count,)h(int)h (blocklength,)e(int)i(stride,)393 444 y(MPI)p 468 444 V 17 w(Datatype)f(oldtype,)f(MPI)p 986 444 V 17 w(Datatype)h (*newtype\))75 530 y(int)g(MPI)p 245 530 V 17 w(Type)p 358 530 V 17 w(hvector\(int)f(count,)h(int)h(blocklength,)e(MPI)p 1306 530 V 17 w(Aint)h(stride,)393 587 y(MPI)p 468 587 V 17 w(Datatype)g(oldtype,)f(MPI)p 986 587 V 17 w(Datatype)h (*newtype\))75 673 y(int)g(MPI)p 245 673 V 17 w(Type)p 358 673 V 17 w(indexed\(int)f(count,)h(int)h(*array)p 1068 673 V 16 w(of)p 1132 673 V 17 w(blocklengths,)393 730 y(int)g(*array)p 636 730 V 16 w(of)p 700 730 V 17 w(displacements,)e(MPI)p 1147 730 V 17 w(Datatype)g(oldtype,)393 786 y(MPI)p 468 786 V 17 w(Datatype)h(*newtype\))75 873 y(int)g(MPI)p 245 873 V 17 w(Type)p 358 873 V 17 w(hindexed\(int)f (count,)h(int)h(*array)p 1092 873 V 16 w(of)p 1156 873 V 17 w(blocklengths,)393 929 y(MPI)p 468 929 V 17 w(Aint)f(*array)p 748 929 V 17 w(of)p 813 929 V 17 w(displacements,)e(MPI)p 1259 929 V 17 w(Datatype)i(oldtype,)393 985 y(MPI)p 468 985 V 17 w(Datatype)g(*newtype\))75 1072 y(int)g(MPI)p 245 1072 V 17 w(Type)p 358 1072 V 17 w(struct\(int)f(count,)h(int)h (*array)p 1044 1072 V 16 w(of)p 1108 1072 V 17 w(blocklengths,)393 1128 y(MPI)p 468 1128 V 17 w(Aint)f(*array)p 748 1128 V 17 w(of)p 813 1128 V 17 w(displacements,)e(MPI)p 1259 1128 V 17 w(Datatype)i(*array)p 1635 1128 V 16 w(of)p 1699 1128 V 17 w(types,)393 1185 y(MPI)p 468 1185 V 17 w(Datatype)g(*newtype\))75 1271 y(int)g(MPI)p 245 1271 V 17 w(Address\(void*)f(location,)h(MPI)p 907 1271 V 17 w(Aint)g(*address\))75 1358 y(int)g(MPI)p 245 1358 V 17 w(Type)p 358 1358 V 17 w(extent\(MPI)p 615 1358 V 16 w(Datatype)f(datatype,)h(int)h(*extent\))75 1444 y(int)f(MPI)p 245 1444 V 17 w(Type)p 358 1444 V 17 w(size\(MPI)p 567 1444 V 16 w(Datatype)g(datatype,)f(int)i(*size\))75 1531 y(int)f(MPI)p 245 1531 V 17 w(Type)p 358 1531 V 17 w(count\(MPI)p 591 1531 V 16 w(Datatype)g(datatype,)f(int)i (*count\))75 1617 y(int)f(MPI)p 245 1617 V 17 w(Type)p 358 1617 V 17 w(lb\(MPI)p 519 1617 V 16 w(Datatype)g(datatype,)g(int*)g (displacement\))75 1704 y(int)g(MPI)p 245 1704 V 17 w(Type)p 358 1704 V 17 w(ub\(MPI)p 519 1704 V 16 w(Datatype)g(datatype,)g(int*)g (displacement\))75 1790 y(int)g(MPI)p 245 1790 V 17 w(Type)p 358 1790 V 17 w(commit\(MPI)p 615 1790 V 16 w(Datatype)f(*datatype\))75 1877 y(int)h(MPI)p 245 1877 V 17 w(Type)p 358 1877 V 17 w(free\(MPI)p 567 1877 V 16 w(Datatype)g(*datatype\))75 1963 y(int)g(MPI)p 245 1963 V 17 w(Get)p 334 1963 V 17 w(elements\(MPI)p 639 1963 V 16 w(Status)g(status,)f(MPI)p 1084 1963 V 17 w(Datatype)h(datatype,)g(int)g(*count\))75 2049 y(int)g(MPI)p 245 2049 V 17 w(Pack\(void*)g(inbuf,)g(int)g (incount,)g(MPI)p 1074 2049 V 17 w(Datatype)g(datatype,)f(void)h (*outbuf,)393 2106 y(int)h(outsize,)e(int)i(*position,)e(MPI)p 1136 2106 V 17 w(Comm)h(comm\))75 2192 y(int)g(MPI)p 245 2192 V 17 w(Unpack\(void*)f(inbuf,)h(int)h(insize,)f(int)g (*position,)g(void)g(*outbuf,)393 2249 y(int)h(outcount,)e(MPI)p 802 2249 V 17 w(Datatype)h(datatype,)f(MPI)p 1344 2249 V 17 w(Comm)i(comm\))75 2335 y(int)f(MPI)p 245 2335 V 17 w(Pack)p 358 2335 V 17 w(size\(int)g(incount,)f(MPI)p 876 2335 V 17 w(Datatype)h(datatype,)g(MPI)p 1419 2335 V 16 w(Comm)h(comm,)393 2392 y(int)g(*size\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 214 222 214 221 bop 75 -100 a Fx(214)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fn(A.4)59 b(C)20 b(Bindings)g(fo)n (r)h(Collective)e(Communication)75 147 y Fu(int)k(MPI)p 245 147 15 2 v 17 w(Barrier\(MPI)p 526 147 V 16 w(Comm)g(comm)h(\))75 233 y(int)f(MPI)p 245 233 V 17 w(Bcast\(void*)g(buffer,)f(int)i(count,) f(MPI)p 1074 233 V 17 w(Datatype)g(datatype,)f(int)i(root,)393 289 y(MPI)p 468 289 V 17 w(Comm)f(comm)h(\))75 376 y(int)f(MPI)p 245 376 V 17 w(Gather\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1217 376 V 17 w(Datatype)h(sendtype,)393 432 y(void*)g(recvbuf,)g(int)h (recvcount,)e(MPI)p 1184 432 V 17 w(Datatype)h(recvtype,)f(int)i(root,) 393 489 y(MPI)p 468 489 V 17 w(Comm)f(comm\))75 575 y(int)g(MPI)p 245 575 V 17 w(Gatherv\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 575 V 17 w(Datatype)h(sendtype,)393 632 y(void*)g(recvbuf,)g(int)h (*recvcounts,)e(int)h(*displs,)393 688 y(MPI)p 468 688 V 17 w(Datatype)g(recvtype,)f(int)i(root,)f(MPI)p 1249 688 V 17 w(Comm)g(comm\))75 775 y(int)g(MPI)p 245 775 V 17 w(Scatter\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 775 V 17 w(Datatype)h(sendtype,)393 831 y(void*)g(recvbuf,)g(int)h (recvcount,)e(MPI)p 1184 831 V 17 w(Datatype)h(recvtype,)f(int)i(root,) 393 888 y(MPI)p 468 888 V 17 w(Comm)f(comm\))75 974 y(int)g(MPI)p 245 974 V 17 w(Scatterv\(void*)f(sendbuf,)h(int)g(*sendcounts,)g(int)g (*displs,)393 1030 y(MPI)p 468 1030 V 17 w(Datatype)g(sendtype,)f (void*)i(recvbuf,)e(int)i(recvcount,)393 1087 y(MPI)p 468 1087 V 17 w(Datatype)f(recvtype,)f(int)i(root,)f(MPI)p 1249 1087 V 17 w(Comm)g(comm\))75 1173 y(int)g(MPI)p 245 1173 V 17 w(Allgather\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1289 1173 V 17 w(Datatype)f(sendtype,)393 1230 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1230 V 17 w(Datatype)h(recvtype,)393 1286 y(MPI)p 468 1286 V 17 w(Comm)g(comm\))75 1373 y(int)g(MPI)p 245 1373 V 17 w(Allgatherv\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 1373 V 16 w(Datatype)g(sendtype,)393 1429 y(void*)g(recvbuf,)g (int)h(*recvcounts,)e(int)h(*displs,)393 1486 y(MPI)p 468 1486 V 17 w(Datatype)g(recvtype,)f(MPI)p 1010 1486 V 17 w(Comm)h(comm\))75 1572 y(int)g(MPI)p 245 1572 V 17 w(Alltoall\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1572 V 17 w(Datatype)f(sendtype,)393 1629 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1629 V 17 w(Datatype)h(recvtype,)393 1685 y(MPI)p 468 1685 V 17 w(Comm)g(comm\))75 1772 y(int)g(MPI)p 245 1772 V 17 w(Alltoallv\(void*)f(sendbuf,)h(int)g(*sendcounts,)f(int) i(*sdispls,)393 1828 y(MPI)p 468 1828 V 17 w(Datatype)f(sendtype,)f (void*)i(recvbuf,)e(int)i(*recvcounts,)393 1884 y(int)g(*rdispls,)e (MPI)p 802 1884 V 17 w(Datatype)h(recvtype,)f(MPI)p 1344 1884 V 17 w(Comm)i(comm\))75 1971 y(int)f(MPI)p 245 1971 V 17 w(Reduce\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)h(count,)393 2027 y(MPI)p 468 2027 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2027 V 17 w(Op)i(op,)f(int)h(root,)f(MPI)p 1505 2027 V 17 w(Comm)g(comm\))75 2114 y(int)g(MPI)p 245 2114 V 17 w(Op)p 310 2114 V 17 w(create\(MPI)p 567 2114 V 16 w(User)p 679 2114 V 17 w(function)f(*function,)h(int)g(commute,)g (MPI)p 1555 2114 V 17 w(Op)g(*op\))75 2200 y(int)g(MPI)p 245 2200 V 17 w(Op)p 310 2200 V 17 w(free\()g(MPI)p 542 2200 V 17 w(Op)h(*op\))75 2287 y(int)f(MPI)p 245 2287 V 17 w(Allreduce\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)g(count,) 393 2343 y(MPI)p 468 2343 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 2343 V 17 w(Op)i(op,)f(MPI)p 1266 2343 V 17 w(Comm)g(comm\))75 2430 y(int)g(MPI)p 245 2430 V 17 w(Reduce)p 406 2430 V 17 w(scatter\(void*)f(sendbuf,)g(void*)i(recvbuf,)e(int)i (*recvcounts,)393 2486 y(MPI)p 468 2486 V 17 w(Datatype)f(datatype,)f (MPI)p 1010 2486 V 17 w(Op)i(op,)f(MPI)p 1266 2486 V 17 w(Comm)g(comm\))75 2573 y(int)g(MPI)p 245 2573 V 17 w(Scan\(void*)g(sendbuf,)f(void*)i(recvbuf,)e(int)i(count,)393 2629 y(MPI)p 468 2629 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2629 V 17 w(Op)i(op,)f(MPI)p 1266 2629 V 17 w(Comm)g(comm)h(\)) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 215 223 215 222 bop 75 -100 a Fm(A.5.)34 b(C)15 b(BINDINGS)h(F)o(OR)g(GR)o (OUPS,)f(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)160 b Fx(215)75 45 y Fn(A.5)59 b(C)20 b(Bindings)g(fo)n(r)h(Groups,)e(Contexts,)f(and)h (Communicato)n(rs)75 147 y Fu(int)k(MPI)p 245 147 15 2 v 17 w(Group)p 382 147 V 17 w(size\(MPI)p 591 147 V 16 w(Group)g(group,)g(int)g(*size\))75 233 y(int)g(MPI)p 245 233 V 17 w(Group)p 382 233 V 17 w(rank\(MPI)p 591 233 V 16 w(Group)g(group,)g(int)g(*rank\))75 319 y(int)g(MPI)p 245 319 V 17 w(Group)p 382 319 V 17 w(translate)p 615 319 V 16 w(ranks)g(\(MPI)p 870 319 V 16 w(Group)h(group1,)f(int)g(n,)h (int)f(*ranks1,)393 376 y(MPI)p 468 376 V 17 w(Group)g(group2,)g(int)g (*ranks2\))75 462 y(int)g(MPI)p 245 462 V 17 w(Group)p 382 462 V 17 w(compare\(MPI)p 663 462 V 15 w(Group)h(group1,MPI)p 1062 462 V 15 w(Group)g(group2,)e(int)i(*result\))75 549 y(int)f(MPI)p 245 549 V 17 w(Comm)p 358 549 V 17 w(group\(MPI)p 591 549 V 16 w(Comm)g(comm,)g(MPI)p 941 549 V 17 w(Group)g(*group\))75 635 y(int)g(MPI)p 245 635 V 17 w(Group)p 382 635 V 17 w(union\(MPI)p 615 635 V 16 w(Group)g(group1,)g(MPI)p 1037 635 V 17 w(Group)g(group2,)g(MPI)p 1460 635 V 16 w(Group)h(*newgroup\))75 722 y(int)f(MPI)p 245 722 V 17 w(Group)p 382 722 V 17 w(intersection\(MPI)p 783 722 V 15 w(Group)g(group1,)g(MPI)p 1204 722 V 17 w(Group)g(group2,)393 778 y(MPI)p 468 778 V 17 w(Group)g(*newgroup\))75 865 y(int)g(MPI)p 245 865 V 17 w(Group)p 382 865 V 17 w(difference\(MPI)p 735 865 V 15 w(Group)g(group1,)g(MPI)p 1156 865 V 17 w(Group)g(group2,)393 921 y(MPI)p 468 921 V 17 w(Group)g(*newgroup\))75 1008 y(int)g(MPI)p 245 1008 V 17 w(Group)p 382 1008 V 17 w(incl\(MPI)p 591 1008 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1008 V 17 w(Group)h(*newgroup\))75 1094 y(int)g(MPI)p 245 1094 V 17 w(Group)p 382 1094 V 17 w(excl\(MPI)p 591 1094 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1094 V 17 w(Group)h(*newgroup\))75 1181 y(int)g(MPI)p 245 1181 V 17 w(Group)p 382 1181 V 17 w(range)p 519 1181 V 16 w(incl\(MPI)p 727 1181 V 16 w(Group)g(group,)h(int)f(n,)h(int)f (ranges[][3],)393 1237 y(MPI)p 468 1237 V 17 w(Group)g(*newgroup\))75 1323 y(int)g(MPI)p 245 1323 V 17 w(Group)p 382 1323 V 17 w(range)p 519 1323 V 16 w(excl\(MPI)p 727 1323 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(ranges[][3],)393 1380 y(MPI)p 468 1380 V 17 w(Group)g(*newgroup\))75 1466 y(int)g(MPI)p 245 1466 V 17 w(Group)p 382 1466 V 17 w(free\(MPI)p 591 1466 V 16 w(Group)g(*group\))75 1553 y(int)g(MPI)p 245 1553 V 17 w(Comm)p 358 1553 V 17 w(size\(MPI)p 567 1553 V 16 w(Comm)g(comm,)g(int)h(*size\))75 1639 y(int)f(MPI)p 245 1639 V 17 w(Comm)p 358 1639 V 17 w(rank\(MPI)p 567 1639 V 16 w(Comm)g(comm,)g(int)h(*rank\))75 1726 y(int)f(MPI)p 245 1726 V 17 w(Comm)p 358 1726 V 17 w(compare\(MPI)p 639 1726 V 16 w(Comm)g(comm1,)g(comm2,)g(int)g(*result\))75 1812 y(int)g(MPI)p 245 1812 V 17 w(Comm)p 358 1812 V 17 w(dup\(MPI)p 543 1812 V 16 w(Comm)g(comm,)h(MPI)p 894 1812 V 16 w(Comm)g(*newcomm\))75 1899 y(int)f(MPI)p 245 1899 V 17 w(Comm)p 358 1899 V 17 w(create\(MPI)p 615 1899 V 16 w(Comm)g(comm,)g(MPI)p 965 1899 V 17 w(Group)g(group,)g (MPI)p 1364 1899 V 17 w(Comm)g(*newcomm\))75 1985 y(int)g(MPI)p 245 1985 V 17 w(Comm)p 358 1985 V 17 w(split\(MPI)p 591 1985 V 16 w(Comm)g(comm,)g(int)h(color,)f(int)g(key,)h(MPI)p 1419 1985 V 16 w(Comm)g(*newcomm\))75 2072 y(int)f(MPI)p 245 2072 V 17 w(Comm)p 358 2072 V 17 w(free\(MPI)p 567 2072 V 16 w(Comm)g(*comm\))75 2158 y(int)g(MPI)p 245 2158 V 17 w(Comm)p 358 2158 V 17 w(test)p 471 2158 V 16 w(inter\(MPI)p 703 2158 V 16 w(Comm)h(comm,)f(int)g(*flag\))75 2244 y(int)g(MPI)p 245 2244 V 17 w(Comm)p 358 2244 V 17 w(remote)p 519 2244 V 16 w(size\(MPI)p 727 2244 V 16 w(Comm)h(comm,)f(int)g(*size\))75 2331 y(int)g(MPI)p 245 2331 V 17 w(Comm)p 358 2331 V 17 w(remote)p 519 2331 V 16 w(group\(MPI)p 751 2331 V 16 w(Comm)g(comm,)h(MPI)p 1102 2331 V 16 w(Group)g(*group\))75 2417 y(int)f(MPI)p 245 2417 V 17 w(Intercomm)p 478 2417 V 16 w(create\(MPI)p 734 2417 V 16 w(Comm)g(local)p 989 2417 V 17 w(comm,)g(int)g(local)p 1364 2417 V 17 w(leader,)393 2474 y(MPI)p 468 2474 V 17 w(Comm)g(peer)p 700 2474 V 17 w(comm,)g(int)h(remote)p 1100 2474 V 16 w(leader,)f(int)g(tag,)393 2530 y(MPI)p 468 2530 V 17 w(Comm)g(*newintercomm\))75 2617 y(int)g(MPI)p 245 2617 V 17 w(Intercomm)p 478 2617 V 16 w(merge\(MPI)p 710 2617 V 16 w(Comm)g(intercomm,)g(int)g(high,)393 2673 y(MPI)p 468 2673 V 17 w(Comm)g(*newintracomm\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 216 224 216 223 bop 75 -100 a Fx(216)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(int)23 b(MPI)p 245 45 15 2 v 17 w(Keyval)p 406 45 V 17 w(create\(MPI)p 663 45 V 15 w(Copy)p 774 45 V 17 w(function)g(*copy)p 1126 45 V 16 w(fn,)h(MPI)p 1310 45 V 17 w(Delete)p 1471 45 V 16 w(function)393 102 y(*delete)p 564 102 V 16 w(fn,)g(int)f (*keyval,)g(void*)g(extra)p 1249 102 V 17 w(state\))75 188 y(int)g(MPI)p 245 188 V 17 w(Keyval)p 406 188 V 17 w(free\(int)f(*keyval\))75 274 y(int)h(MPI)p 245 274 V 17 w(Attr)p 358 274 V 17 w(put\(MPI)p 543 274 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void*)g(attribute)p 1467 274 V 16 w(val\))75 361 y(int)g(MPI)p 245 361 V 17 w(Attr)p 358 361 V 17 w(get\(MPI)p 543 361 V 16 w(Comm)g(comm,)h (int)f(keyval,)g(void)g(**attribute)p 1491 361 V 16 w(val,)g(int)h (*flag\))75 447 y(int)f(MPI)p 245 447 V 17 w(Attr)p 358 447 V 17 w(delete\(MPI)p 615 447 V 16 w(Comm)g(comm,)g(int)h(keyval\)) 75 621 y Fn(A.6)59 b(C)20 b(Bindings)g(fo)n(r)h(Pro)r(cess)e(T)-5 b(op)r(ologies)75 722 y Fu(int)23 b(MPI)p 245 722 V 17 w(Cart)p 358 722 V 17 w(create\(MPI)p 615 722 V 16 w(Comm)g(comm)p 846 722 V 17 w(old,)g(int)g(ndims,)g(int)h(*dims,)f(int)g(*periods,)393 779 y(int)h(reorder,)e(MPI)p 778 779 V 17 w(Comm)h(*comm)p 1034 779 V 17 w(cart\))75 865 y(int)g(MPI)p 245 865 V 17 w(Dims)p 358 865 V 17 w(create\(int)f(nnodes,)h(int)h(ndims,)f(int)g (*dims\))75 951 y(int)g(MPI)p 245 951 V 17 w(Graph)p 382 951 V 17 w(create\(MPI)p 639 951 V 16 w(Comm)g(comm)p 870 951 V 16 w(old,)h(int)f(nnodes,)g(int)h(*index,)f(int)g(*edges,)393 1008 y(int)h(reorder,)e(MPI)p 778 1008 V 17 w(Comm)h(*comm)p 1034 1008 V 17 w(graph\))75 1094 y(int)g(MPI)p 245 1094 V 17 w(Topo)p 358 1094 V 17 w(test\(MPI)p 567 1094 V 16 w(Comm)g(comm,)g(int)h(*status\))75 1181 y(int)f(MPI)p 245 1181 V 17 w(Graphdims)p 478 1181 V 16 w(get\(MPI)p 662 1181 V 16 w(Comm)h(comm,)f(int)g(*nnodes,)g(int)h(*nedges\))75 1267 y(int)f(MPI)p 245 1267 V 17 w(Graph)p 382 1267 V 17 w(get\(MPI)p 567 1267 V 16 w(Comm)g(comm,)g(int)h(maxindex,)f(int)g (maxedges,)g(int)g(*index,)393 1324 y(int)h(*edges\))75 1410 y(int)f(MPI)p 245 1410 V 17 w(Cartdim)p 430 1410 V 16 w(get\(MPI)p 614 1410 V 17 w(Comm)g(comm,)g(int)h(*ndims\))75 1497 y(int)f(MPI)p 245 1497 V 17 w(Cart)p 358 1497 V 17 w(get\(MPI)p 543 1497 V 16 w(Comm)g(comm,)h(int)f(maxdims,)g(int)g (*dims,)g(int)h(*periods,)393 1553 y(int)g(*coords\))75 1640 y(int)f(MPI)p 245 1640 V 17 w(Cart)p 358 1640 V 17 w(rank\(MPI)p 567 1640 V 16 w(Comm)g(comm,)g(int)h(*coords,)f(int)g (*rank\))75 1726 y(int)g(MPI)p 245 1726 V 17 w(Cart)p 358 1726 V 17 w(coords\(MPI)p 615 1726 V 16 w(Comm)g(comm,)g(int)h (rank,)f(int)g(maxdims,)g(int)g(*coords\))75 1812 y(int)g(MPI)p 245 1812 V 17 w(Graph)p 382 1812 V 17 w(neighbors)p 615 1812 V 16 w(count\(MPI)p 847 1812 V 16 w(Comm)g(comm,)g(int)h(rank,)f (int)g(*nneighbors\))75 1899 y(int)g(MPI)p 245 1899 V 17 w(Graph)p 382 1899 V 17 w(neighbors\(MPI)p 711 1899 V 15 w(Comm)g(comm,)h(int)f(rank,)g(int)h(maxneighbors,)393 1955 y(int)g(*neighbors\))75 2042 y(int)f(MPI)p 245 2042 V 17 w(Cart)p 358 2042 V 17 w(shift\(MPI)p 591 2042 V 16 w(Comm)g(comm,)g(int)h(direction,)e(int)i(disp,)f(int)g(*rank)p 1681 2042 V 17 w(source,)393 2098 y(int)h(*rank)p 612 2098 V 16 w(dest\))75 2185 y(int)f(MPI)p 245 2185 V 17 w(Cart)p 358 2185 V 17 w(sub\(MPI)p 543 2185 V 16 w(Comm)g(comm,)h(int) f(*remain)p 1085 2185 V 16 w(dims,)g(MPI)p 1316 2185 V 17 w(Comm)h(*newcomm\))75 2271 y(int)f(MPI)p 245 2271 V 17 w(Cart)p 358 2271 V 17 w(map\(MPI)p 543 2271 V 16 w(Comm)g(comm,)h(int)f(ndims,)g(int)h(*dims,)f(int)g(*periods,)393 2328 y(int)h(*newrank\))75 2414 y(int)f(MPI)p 245 2414 V 17 w(Graph)p 382 2414 V 17 w(map\(MPI)p 567 2414 V 16 w(Comm)g(comm,)g(int)h(nnodes,)f(int)g(*index,)g(int)h(*edges,)393 2471 y(int)g(*newrank\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 217 225 217 224 bop 75 -100 a Fm(A.7.)34 b(C)15 b(BINDINGS)h(F)o(OR)g(ENVIR)o (ONMENT)l(AL)g(INQUIR)l(Y)582 b Fx(217)75 45 y Fn(A.7)59 b(C)20 b(bindings)g(fo)n(r)h(Environmental)d(Inquiry)75 147 y Fu(int)23 b(MPI)p 245 147 15 2 v 17 w(Get)p 334 147 V 17 w(processor)p 567 147 V 16 w(name\(char)g(*name,)g(int)g (*resultlen\))75 233 y(int)g(MPI)p 245 233 V 17 w(Errhandler)p 502 233 V 16 w(create\(MPI)p 758 233 V 16 w(Handler)p 942 233 V 16 w(function)g(*function,)393 289 y(MPI)p 468 289 V 17 w(Errhandler)f(*errhandler\))75 376 y(int)h(MPI)p 245 376 V 17 w(Errhandler)p 502 376 V 16 w(set\(MPI)p 686 376 V 16 w(Comm)h(comm,)f(MPI)p 1037 376 V 17 w(Errhandler)f (errhandler\))75 462 y(int)h(MPI)p 245 462 V 17 w(Errhandler)p 502 462 V 16 w(get\(MPI)p 686 462 V 16 w(Comm)h(comm,)f(MPI)p 1037 462 V 17 w(Errhandler)f(*errhandler\))75 549 y(int)h(MPI)p 245 549 V 17 w(Errhandler)p 502 549 V 16 w(free\(MPI)p 710 549 V 16 w(Errhandler)g(*errhandler\))75 635 y(int)g(MPI)p 245 635 V 17 w(Error)p 382 635 V 17 w(string\(int)f(errorcode,)h(char)g (*string,)g(int)g(*resultlen\))75 722 y(int)g(MPI)p 245 722 V 17 w(Error)p 382 722 V 17 w(class\(int)f(errorcode,)h(int)g (*errorclass\))75 808 y(int)g(double)g(MPI)p 412 808 V 17 w(Wtime\(void\))75 895 y(int)g(double)g(MPI)p 412 895 V 17 w(Wtick\(void\))75 981 y(int)g(MPI)p 245 981 V 17 w(Init\(int)g(*argc,)g(char)g(***argv\))75 1068 y(int)g(MPI)p 245 1068 V 17 w(Finalize\(void\))75 1154 y(int)g(MPI)p 245 1154 V 17 w(Initialized\(int)f(*flag\))75 1241 y(int)h(MPI)p 245 1241 V 17 w(Abort\(MPI)p 478 1241 V 16 w(Comm)g(comm,)h(int)f(errorcode\))75 1414 y Fn(A.8)59 b(C)20 b(Bindings)g(fo)n(r)h(Pro\014ling)75 1515 y Fu(int)i(MPI)p 245 1515 V 17 w(Pcontrol\(const)f(int)i(level,)f Fp(:)8 b(:)g(:)d Fu(\))75 1688 y Fn(A.9)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(P)n(oint-to-P)n(oint)f(Communication)75 1790 y Fu(MPI)p 150 1790 V 17 w(SEND\(BUF,)i(COUNT,)h(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(IERROR\))170 1846 y()g(BUF\(*\))170 1903 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 1989 y(MPI)p 150 1989 V 17 w(RECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,) g(TAG,)g(COMM,)g(STATUS,)g(IERROR\))170 2046 y()g(BUF\(*\))170 2102 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS\(MPI)p 1463 2102 V 16 w(STATUS)p 1623 2102 V 17 w(SIZE\),)170 2159 y(IERROR)75 2245 y(MPI)p 150 2245 V 17 w(GET)p 239 2245 V 17 w(COUNT\(STATUS,)f(DATATYPE,)g(COUNT,)h (IERROR\))170 2302 y(INTEGER)g(STATUS\(MPI)p 604 2302 V 16 w(STATUS)p 764 2302 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h(IERROR) 75 2388 y(MPI)p 150 2388 V 17 w(BSEND\(BUF,)f(COUNT,)h(DATATYPE,)g (DEST,)g(TAG,)g(COMM,)h(IERROR\))170 2444 y()f(BUF\(*\))170 2501 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 2587 y(MPI)p 150 2587 V 17 w(SSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 2644 y()f(BUF\(*\))170 2700 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 218 226 218 225 bop 75 -100 a Fx(218)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(RSEND\(BUF,)22 b(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)h (IERROR\))170 102 y()f(BUF\(*\))170 158 y(INTEGER)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 244 y(MPI)p 150 244 V 17 w(BUFFER)p 311 244 V 16 w(ATTACH\()g(BUFFER,)g(SIZE,)g (IERROR\))170 301 y()g(BUFFER\(*\))170 357 y(INTEGER)g(SIZE,)h (IERROR)75 444 y(MPI)p 150 444 V 17 w(BUFFER)p 311 444 V 16 w(DETACH\()f(BUFFER,)g(SIZE,)g(IERROR\))170 500 y()g(BUFFER\(*\))170 557 y(INTEGER)g(SIZE,)h(IERROR)75 643 y(MPI)p 150 643 V 17 w(ISEND\(BUF,)e(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(REQUEST,)e(IERROR\))170 700 y()h(BUF\(*\))170 756 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)75 843 y(MPI)p 150 843 V 17 w(IBSEND\(BUF,)g(COUNT,)h (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 899 y()g(BUF\(*\))170 955 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,) g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 1042 y(MPI)p 150 1042 V 17 w(ISSEND\(BUF,)g(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR\))170 1098 y()g(BUF\(*\))170 1155 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)75 1241 y(MPI)p 150 1241 V 17 w(IRSEND\(BUF,)g(COUNT,)h (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1298 y()g(BUF\(*\))170 1354 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 1441 y(MPI)p 150 1441 V 17 w(IRECV\(BUF,)g(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g (COMM,)g(REQUEST,)g(IERROR\))170 1497 y()g(BUF\(*\))170 1554 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR)75 1640 y(MPI)p 150 1640 V 17 w(WAIT\(REQUEST,)f (STATUS,)h(IERROR\))170 1696 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 1696 V 16 w(STATUS)p 979 1696 V 16 w(SIZE\),)g(IERROR)75 1783 y(MPI)p 150 1783 V 17 w(TEST\(REQUEST,)f(FLAG,)h(STATUS,)g (IERROR\))170 1839 y(LOGICAL)g(FLAG)170 1896 y(INTEGER)g(REQUEST,)g (STATUS\(MPI)p 819 1896 V 16 w(STATUS)p 979 1896 V 16 w(SIZE\),)g(IERROR)75 1982 y(MPI)p 150 1982 V 17 w(REQUEST)p 335 1982 V 16 w(FREE\(REQUEST,)f(IERROR\))170 2039 y(INTEGER)h (REQUEST,)g(IERROR)75 2125 y(MPI)p 150 2125 V 17 w(WAITANY\(COUNT,)f (ARRAY)p 645 2125 V 16 w(OF)p 709 2125 V 17 w(REQUESTS,)h(INDEX,)g (STATUS,)g(IERROR\))170 2182 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2182 V 17 w(OF)p 716 2182 V 17 w(REQUESTS\(*\),)f(INDEX,)h (STATUS\(MPI)p 1450 2182 V 16 w(STATUS)p 1610 2182 V 16 w(SIZE\),)170 2238 y(IERROR)75 2325 y(MPI)p 150 2325 V 17 w(TESTANY\(COUNT,)f(ARRAY)p 645 2325 V 16 w(OF)p 709 2325 V 17 w(REQUESTS,)h(INDEX,)g(FLAG,)g(STATUS,)g(IERROR\))170 2381 y(LOGICAL)g(FLAG)170 2438 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2438 V 17 w(OF)p 716 2438 V 17 w(REQUESTS\(*\),)f(INDEX,)h (STATUS\(MPI)p 1450 2438 V 16 w(STATUS)p 1610 2438 V 16 w(SIZE\),)170 2494 y(IERROR)75 2580 y(MPI)p 150 2580 V 17 w(WAITALL\(COUNT,)f(ARRAY)p 645 2580 V 16 w(OF)p 709 2580 V 17 w(REQUESTS,)h(ARRAY)p 1085 2580 V 16 w(OF)p 1149 2580 V 17 w(STATUSES,)g(IERROR\))170 2637 y(INTEGER)g(COUNT,)g (ARRAY)p 651 2637 V 17 w(OF)p 716 2637 V 17 w(REQUESTS\(*\),)170 2693 y(ARRAY)p 293 2693 V 17 w(OF)p 358 2693 V 17 w(STATUSES\(MPI)p 663 2693 V 15 w(STATUS)p 822 2693 V 17 w(SIZE,*\),)g(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 219 227 219 226 bop 75 -100 a Fm(A.9.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o (OR)g(POINT-TO-POINT)h(COMMUNICA)l(TION)185 b Fx(219)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(TESTALL\(COUNT,)22 b(ARRAY)p 645 45 V 16 w(OF)p 709 45 V 17 w(REQUESTS,)h(FLAG,)g(ARRAY)p 1228 45 V 16 w(OF)p 1292 45 V 17 w(STATUSES,)g(IERROR\))170 102 y(LOGICAL)g(FLAG)170 158 y(INTEGER)g(COUNT,)g(ARRAY)p 651 158 V 17 w(OF)p 716 158 V 17 w(REQUESTS\(*\),)170 214 y(ARRAY)p 293 214 V 17 w(OF)p 358 214 V 17 w(STATUSES\(MPI)p 663 214 V 15 w(STATUS)p 822 214 V 17 w(SIZE,*\),)g(IERROR)75 301 y(MPI)p 150 301 V 17 w(WAITSOME\(INCOUNT,)e(ARRAY)p 716 301 V 17 w(OF)p 781 301 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 301 V 16 w(OF)p 1459 301 V 17 w(INDICES,)393 357 y(ARRAY)p 516 357 V 17 w(OF)p 581 357 V 17 w(STATUSES,)f(IERROR\))170 414 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 414 V 17 w(OF)p 764 414 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 414 V 17 w(OF)p 1514 414 V 17 w(INDICES\(*\),)170 470 y(ARRAY)p 293 470 V 17 w(OF)p 358 470 V 17 w(STATUSES\(MPI)p 663 470 V 15 w(STATUS)p 822 470 V 17 w(SIZE,*\),)h(IERROR)75 557 y(MPI)p 150 557 V 17 w(TESTSOME\(INCOUNT,)e(ARRAY)p 716 557 V 17 w(OF)p 781 557 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 557 V 16 w(OF)p 1459 557 V 17 w(INDICES,)393 613 y(ARRAY)p 516 613 V 17 w(OF)p 581 613 V 17 w(STATUSES,)f(IERROR\))170 670 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 670 V 17 w(OF)p 764 670 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 670 V 17 w(OF)p 1514 670 V 17 w(INDICES\(*\),)170 726 y(ARRAY)p 293 726 V 17 w(OF)p 358 726 V 17 w(STATUSES\(MPI)p 663 726 V 15 w(STATUS)p 822 726 V 17 w(SIZE,*\),)h(IERROR)75 813 y(MPI)p 150 813 V 17 w(IPROBE\(SOURCE,)f(TAG,)h(COMM,)g(FLAG,)g (STATUS,)g(IERROR\))170 869 y(LOGICAL)g(FLAG)170 925 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 925 V 16 w(STATUS)p 1218 925 V 16 w(SIZE\),)g(IERROR)75 1012 y(MPI)p 150 1012 V 17 w(PROBE\(SOURCE,)f(TAG,)h(COMM,)g(STATUS,)g (IERROR\))170 1068 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 1068 V 16 w(STATUS)p 1218 1068 V 16 w(SIZE\),)g(IERROR)75 1155 y(MPI)p 150 1155 V 17 w(CANCEL\(REQUEST,)f(IERROR\))170 1211 y(INTEGER)h(REQUEST,)g(IERROR)75 1298 y(MPI)p 150 1298 V 17 w(TEST)p 263 1298 V 16 w(CANCELLED\(STATUS,)f(FLAG,)h (IERROR\))170 1354 y(LOGICAL)g(FLAG)170 1411 y(INTEGER)g(STATUS\(MPI)p 604 1411 V 16 w(STATUS)p 764 1411 V 17 w(SIZE\),)g(IERROR)75 1497 y(MPI)p 150 1497 V 17 w(SEND)p 263 1497 V 16 w(INIT\(BUF,)g (COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1554 y()g(BUF\(*\))170 1610 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 1696 y(MPI)p 150 1696 V 17 w(BSEND)p 287 1696 V 16 w(INIT\(BUF,)g (COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1753 y()g(BUF\(*\))170 1809 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 1896 y(MPI)p 150 1896 V 17 w(SSEND)p 287 1896 V 16 w(INIT\(BUF,)g (COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1952 y()g(BUF\(*\))170 2009 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 2095 y(MPI)p 150 2095 V 17 w(RSEND)p 287 2095 V 16 w(INIT\(BUF,)h(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2152 y()g(BUF\(*\))170 2208 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 2295 y(MPI)p 150 2295 V 17 w(RECV)p 263 2295 V 16 w(INIT\(BUF,)h(COUNT,)g(DATATYPE,) g(SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2351 y()g(BUF\(*\))170 2408 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 2494 y(MPI)p 150 2494 V 17 w(START\(REQUEST,)f(IERROR\))170 2550 y(INTEGER)h(REQUEST,)g (IERROR)75 2637 y(MPI)p 150 2637 V 17 w(STARTALL\(COUNT,)f(ARRAY)p 669 2637 V 16 w(OF)p 733 2637 V 17 w(REQUESTS,)h(IERROR\))170 2693 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2693 V 17 w(OF)p 716 2693 V 17 w(REQUESTS\(*\),)f(IERROR)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 220 228 220 227 bop 75 -100 a Fx(220)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(SENDRECV\(SENDBUF,)21 b(SENDCOUNT,)i(SENDTYPE,)f(DEST,)i (SENDTAG,)e(RECVBUF,)393 102 y(RECVCOUNT,)h(RECVTYPE,)f(SOURCE,)h (RECVTAG,)g(COMM,)g(STATUS,)g(IERROR\))170 158 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 214 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVCOUNT,)h(RECVTYPE,)170 271 y(SOURCE,)g(RECVTAG,)g(COMM,)g(STATUS\(MPI)p 962 271 V 16 w(STATUS)p 1122 271 V 16 w(SIZE\),)h(IERROR)75 357 y(MPI)p 150 357 V 17 w(SENDRECV)p 359 357 V 16 w(REPLACE\(BUF,)e (COUNT,)h(DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)393 414 y(COMM,)g(STATUS,)g(IERROR\))170 470 y()g(BUF\(*\))170 527 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g (RECVTAG,)g(COMM,)170 583 y(STATUS\(MPI)p 413 583 V 16 w(STATUS)p 573 583 V 17 w(SIZE\),)g(IERROR)75 670 y(MPI)p 150 670 V 17 w(TYPE)p 263 670 V 16 w(CONTIGUOUS\(COUNT,)f(OLDTYPE,)h (NEWTYPE,)f(IERROR\))170 726 y(INTEGER)h(COUNT,)g(OLDTYPE,)g(NEWTYPE,)g (IERROR)75 813 y(MPI)p 150 813 V 17 w(TYPE)p 263 813 V 16 w(VECTOR\(COUNT,)f(BLOCKLENGTH,)h(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h (IERROR\))170 869 y(INTEGER)g(COUNT,)g(BLOCKLENGTH,)g(STRIDE,)g (OLDTYPE,)f(NEWTYPE,)h(IERROR)75 955 y(MPI)p 150 955 V 17 w(TYPE)p 263 955 V 16 w(HVECTOR\(COUNT,)f(BLOCKLENGTH,)h(STRIDE,)f (OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1012 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)75 1098 y(MPI)p 150 1098 V 17 w(TYPE)p 263 1098 V 16 w(INDEXED\(COUNT,)f (ARRAY)p 757 1098 V 17 w(OF)p 822 1098 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1293 1098 V 16 w(OF)p 1357 1098 V 17 w(DISPLACEMENTS,)393 1155 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1211 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1211 V 17 w(OF)p 716 1211 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1211 V 17 w(OF)p 1323 1211 V 17 w(DISPLACEMENTS\(*\),)170 1268 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 1354 y(MPI)p 150 1354 V 17 w(TYPE)p 263 1354 V 16 w(HINDEXED\(COUNT,)f(ARRAY)p 781 1354 V 17 w(OF)p 846 1354 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1317 1354 V 16 w(OF)p 1381 1354 V 17 w(DISPLACEMENTS,)393 1411 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1467 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1467 V 17 w(OF)p 716 1467 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1467 V 17 w(OF)p 1323 1467 V 17 w(DISPLACEMENTS\(*\),)170 1524 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 1610 y(MPI)p 150 1610 V 17 w(TYPE)p 263 1610 V 16 w(STRUCT\(COUNT,)f(ARRAY)p 733 1610 V 17 w(OF)p 798 1610 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1269 1610 V 16 w(OF)p 1333 1610 V 17 w(DISPLACEMENTS,)393 1666 y(ARRAY)p 516 1666 V 17 w(OF)p 581 1666 V 17 w(TYPES,)h(NEWTYPE,)f (IERROR\))170 1723 y(INTEGER)h(COUNT,)g(ARRAY)p 651 1723 V 17 w(OF)p 716 1723 V 17 w(BLOCKLENGTHS\(*\),)e(ARRAY)p 1258 1723 V 17 w(OF)p 1323 1723 V 17 w(DISPLACEMENTS\(*\),)170 1779 y(ARRAY)p 293 1779 V 17 w(OF)p 358 1779 V 17 w(TYPES\(*\),)h (NEWTYPE,)h(IERROR)75 1866 y(MPI)p 150 1866 V 17 w(ADDRESS\(LOCATION,)e (ADDRESS,)i(IERROR\))170 1922 y()g(LOCATION\(*\))170 1979 y(INTEGER)g(ADDRESS,)g(IERROR)75 2065 y(MPI)p 150 2065 V 17 w(TYPE)p 263 2065 V 16 w(EXTENT\(DATATYPE,)f(EXTENT,)h (IERROR\))170 2122 y(INTEGER)g(DATATYPE,)g(EXTENT,)g(IERROR)75 2208 y(MPI)p 150 2208 V 17 w(TYPE)p 263 2208 V 16 w(SIZE\(DATATYPE,)f (SIZE,)i(IERROR\))170 2265 y(INTEGER)f(DATATYPE,)g(SIZE,)g(IERROR)75 2351 y(MPI)p 150 2351 V 17 w(TYPE)p 263 2351 V 16 w(COUNT\(DATATYPE,)f (COUNT,)h(IERROR\))170 2408 y(INTEGER)g(DATATYPE,)g(COUNT,)g(IERROR)75 2494 y(MPI)p 150 2494 V 17 w(TYPE)p 263 2494 V 16 w(LB\()h(DATATYPE,)e (DISPLACEMENT,)h(IERROR\))170 2550 y(INTEGER)g(DATATYPE,)g (DISPLACEMENT,)f(IERROR)75 2637 y(MPI)p 150 2637 V 17 w(TYPE)p 263 2637 V 16 w(UB\()i(DATATYPE,)e(DISPLACEMENT,)h(IERROR\)) 170 2693 y(INTEGER)g(DATATYPE,)g(DISPLACEMENT,)f(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 221 229 221 228 bop 75 -100 a Fm(A.10.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o(OR)g(COLLECTIVE)g(COMMUNICA)l(TION)257 b Fx(221)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(TYPE)p 263 45 V 16 w(COMMIT\(DATATYPE,)22 b(IERROR\))170 102 y(INTEGER)h (DATATYPE,)g(IERROR)75 188 y(MPI)p 150 188 V 17 w(TYPE)p 263 188 V 16 w(FREE\(DATATYPE,)f(IERROR\))170 244 y(INTEGER)h (DATATYPE,)g(IERROR)75 331 y(MPI)p 150 331 V 17 w(GET)p 239 331 V 17 w(ELEMENTS\(STATUS,)e(DATATYPE,)i(COUNT,)g(IERROR\))170 387 y(INTEGER)g(STATUS\(MPI)p 604 387 V 16 w(STATUS)p 764 387 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h(IERROR)75 474 y(MPI)p 150 474 V 17 w(PACK\(INBUF,)f(INCOUNT,)h(DATATYPE,)g (OUTBUF,)f(OUTCOUNT,)h(POSITION,)g(COMM,)393 530 y(IERROR\))170 587 y()g(INBUF\(*\),)g(OUTBUF\(*\))170 643 y(INTEGER)g(INCOUNT,)g (DATATYPE,)g(OUTCOUNT,)f(POSITION,)h(COMM,)g(IERROR)75 730 y(MPI)p 150 730 V 17 w(UNPACK\(INBUF,)f(INSIZE,)h(POSITION,)f (OUTBUF,)h(OUTCOUNT,)g(DATATYPE,)g(COMM,)393 786 y(IERROR\))170 843 y()g(INBUF\(*\),)g(OUTBUF\(*\))170 899 y(INTEGER)g(INSIZE,)g (POSITION,)g(OUTCOUNT,)g(DATATYPE,)f(COMM,)h(IERROR)75 985 y(MPI)p 150 985 V 17 w(PACK)p 263 985 V 16 w(SIZE\(INCOUNT,)f (DATATYPE,)h(COMM,)g(SIZE,)g(IERROR\))170 1042 y(INTEGER)g(INCOUNT,)g (DATATYPE,)g(COMM,)g(SIZE,)g(IERROR)75 1215 y Fn(A.10)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Collective)e(Communication)75 1317 y Fu(MPI)p 150 1317 V 17 w(BARRIER\(COMM,)j(IERROR\))170 1373 y(INTEGER)h(COMM,)h(IERROR)75 1460 y(MPI)p 150 1460 V 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR\)) 170 1516 y()g(BUFFER\(*\))170 1572 y(INTEGER)g(COUNT,)g (DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)75 1659 y(MPI)p 150 1659 V 17 w(GATHER\(SENDBUF,)f(SENDCOUNT,)g(SENDTYPE,)h(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 1715 y(ROOT,)g(COMM,)h(IERROR\))170 1772 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 1828 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g (COMM,)g(IERROR)75 1915 y(MPI)p 150 1915 V 17 w(GATHERV\(SENDBUF,)e (SENDCOUNT,)i(SENDTYPE,)g(RECVBUF,)f(RECVCOUNTS,)h(DISPLS,)393 1971 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\))170 2028 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2084 y(INTEGER)g (SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g(DISPLS\(*\),)h(RECVTYPE,)g (ROOT,)170 2141 y(COMM,)h(IERROR)75 2227 y(MPI)p 150 2227 V 17 w(SCATTER\(SENDBUF,)d(SENDCOUNT,)i(SENDTYPE,)g(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 2283 y(ROOT,)g(COMM,)h(IERROR\))170 2340 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 2396 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g (COMM,)g(IERROR)75 2483 y(MPI)p 150 2483 V 17 w(SCATTERV\(SENDBUF,)e (SENDCOUNTS,)i(DISPLS,)g(SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)393 2539 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\))170 2596 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 222 230 222 229 bop 75 -100 a Fx(222)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y Fu(INTEGER)23 b(SENDCOUNTS\(*\),)f (DISPLS\(*\),)h(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)170 102 y(COMM,)h(IERROR)75 188 y(MPI)p 150 188 15 2 v 17 w(ALLGATHER\(SENDBUF,)d(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,) g(RECVTYPE,)393 244 y(COMM,)g(IERROR\))170 301 y()g (SENDBUF\(*\),)g(RECVBUF\(*\))170 357 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g(IERROR)75 444 y(MPI)p 150 444 V 17 w(ALLGATHERV\(SENDBUF,)e(SENDCOUNT,)i (SENDTYPE,)f(RECVBUF,)h(RECVCOUNTS,)f(DISPLS,)393 500 y(RECVTYPE,)h(COMM,)g(IERROR\))170 557 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 613 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f (RECVCOUNTS\(*\),)g(DISPLS\(*\),)h(RECVTYPE,)g(COMM,)170 670 y(IERROR)75 756 y(MPI)p 150 756 V 17 w(ALLTOALL\(SENDBUF,)e (SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)g(RECVTYPE,)393 813 y(COMM,)g(IERROR\))170 869 y()g(SENDBUF\(*\),)g(RECVBUF\(*\)) 170 925 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g (COMM,)g(IERROR)75 1012 y(MPI)p 150 1012 V 17 w(ALLTOALLV\(SENDBUF,)e (SENDCOUNTS,)i(SDISPLS,)f(SENDTYPE,)h(RECVBUF,)g(RECVCOUNTS,)393 1068 y(RDISPLS,)g(RECVTYPE,)g(COMM,)g(IERROR\))170 1125 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1181 y(INTEGER)g (SENDCOUNTS\(*\),)f(SDISPLS\(*\),)h(SENDTYPE,)f(RECVCOUNTS\(*\),)g (RDISPLS\(*\),)170 1238 y(RECVTYPE,)h(COMM,)g(IERROR)75 1324 y(MPI)p 150 1324 V 17 w(REDUCE\(SENDBUF,)f(RECVBUF,)g(COUNT,)h (DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR\))170 1381 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1437 y(INTEGER)g(COUNT,)g (DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR)75 1524 y(MPI)p 150 1524 V 17 w(OP)p 215 1524 V 17 w(CREATE\()g(FUNCTION,)f(COMMUTE,)h (OP,)g(IERROR\))170 1580 y(EXTERNAL)g(FUNCTION)170 1636 y(LOGICAL)g(COMMUTE)170 1693 y(INTEGER)g(OP,)h(IERROR)75 1779 y(MPI)p 150 1779 V 17 w(OP)p 215 1779 V 17 w(FREE\()f(OP,)g (IERROR\))170 1836 y(INTEGER)g(OP,)h(IERROR)75 1922 y(MPI)p 150 1922 V 17 w(ALLREDUCE\(SENDBUF,)d(RECVBUF,)i(COUNT,)g(DATATYPE,)g (OP,)g(COMM,)g(IERROR\))170 1979 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 2035 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)75 2122 y(MPI)p 150 2122 V 17 w(REDUCE)p 311 2122 V 16 w(SCATTER\(SENDBUF,)e(RECVBUF,)h(RECVCOUNTS,)f(DATATYPE,)h (OP,)g(COMM,)393 2178 y(IERROR\))170 2235 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 2291 y(INTEGER)g(RECVCOUNTS\(*\),)f(DATATYPE,)h(OP,)g (COMM,)h(IERROR)75 2377 y(MPI)p 150 2377 V 17 w(SCAN\(SENDBUF,)e (RECVBUF,)h(COUNT,)g(DATATYPE,)f(OP,)i(COMM,)f(IERROR\))170 2434 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2490 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 223 231 223 230 bop 75 -100 a Fm(A.11.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o(OR)g(GR)o(OUPS,)f(CONTEXTS,)g(ETC.)375 b Fx(223)75 45 y Fn(A.11)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i (Groups,)f(Contexts,)d(etc.)75 147 y Fu(MPI)p 150 147 15 2 v 17 w(GROUP)p 287 147 V 16 w(SIZE\(GROUP,)23 b(SIZE,)g(IERROR\)) 170 203 y(INTEGER)g(GROUP,)g(SIZE,)h(IERROR)75 289 y(MPI)p 150 289 V 17 w(GROUP)p 287 289 V 16 w(RANK\(GROUP,)f(RANK,)g(IERROR\)) 170 346 y(INTEGER)g(GROUP,)g(RANK,)h(IERROR)75 432 y(MPI)p 150 432 V 17 w(GROUP)p 287 432 V 16 w(TRANSLATE)p 519 432 V 16 w(RANKS\(GROUP1,)e(N,)i(RANKS1,)f(GROUP2,)g(RANKS2,)g (IERROR\))170 489 y(INTEGER)g(GROUP1,)g(N,)h(RANKS1\(*\),)e(GROUP2,)h (RANKS2\(*\),)g(IERROR)75 575 y(MPI)p 150 575 V 17 w(GROUP)p 287 575 V 16 w(COMPARE\(GROUP1,)f(GROUP2,)h(RESULT,)g(IERROR\))170 632 y(INTEGER)g(GROUP1,)g(GROUP2,)g(RESULT,)g(IERROR)75 718 y(MPI)p 150 718 V 17 w(COMM)p 263 718 V 16 w(GROUP\(COMM,)g(GROUP,) g(IERROR\))170 775 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)75 861 y(MPI)p 150 861 V 17 w(GROUP)p 287 861 V 16 w(UNION\(GROUP1,)f (GROUP2,)h(NEWGROUP,)g(IERROR\))170 918 y(INTEGER)g(GROUP1,)g(GROUP2,)g (NEWGROUP,)g(IERROR)75 1004 y(MPI)p 150 1004 V 17 w(GROUP)p 287 1004 V 16 w(INTERSECTION\(GROUP1,)e(GROUP2,)i(NEWGROUP,)g(IERROR\)) 170 1060 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 1147 y(MPI)p 150 1147 V 17 w(GROUP)p 287 1147 V 16 w (DIFFERENCE\(GROUP1,)f(GROUP2,)h(NEWGROUP,)f(IERROR\))170 1203 y(INTEGER)h(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 1290 y(MPI)p 150 1290 V 17 w(GROUP)p 287 1290 V 16 w(INCL\(GROUP,)g(N,) g(RANKS,)g(NEWGROUP,)g(IERROR\))170 1346 y(INTEGER)g(GROUP,)g(N,)h (RANKS\(*\),)f(NEWGROUP,)f(IERROR)75 1433 y(MPI)p 150 1433 V 17 w(GROUP)p 287 1433 V 16 w(EXCL\(GROUP,)h(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 1489 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)75 1576 y(MPI)p 150 1576 V 17 w(GROUP)p 287 1576 V 16 w(RANGE)p 423 1576 V 17 w(INCL\(GROUP,)g(N,)i(RANGES,)f (NEWGROUP,)f(IERROR\))170 1632 y(INTEGER)h(GROUP,)g(N,)h (RANGES\(3,*\),)e(NEWGROUP,)h(IERROR)75 1719 y(MPI)p 150 1719 V 17 w(GROUP)p 287 1719 V 16 w(RANGE)p 423 1719 V 17 w(EXCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1775 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(3,*\),)e(NEWGROUP,)h(IERROR)75 1862 y(MPI)p 150 1862 V 17 w(GROUP)p 287 1862 V 16 w(FREE\(GROUP,)g (IERROR\))170 1918 y(INTEGER)g(GROUP,)g(IERROR)75 2004 y(MPI)p 150 2004 V 17 w(COMM)p 263 2004 V 16 w(SIZE\(COMM,)g(SIZE,)g (IERROR\))170 2061 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 2147 y(MPI)p 150 2147 V 17 w(COMM)p 263 2147 V 16 w(RANK\(COMM,)g (RANK,)g(IERROR\))170 2204 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)75 2290 y(MPI)p 150 2290 V 17 w(COMM)p 263 2290 V 16 w(COMPARE\(COMM1,)f (COMM2,)h(RESULT,)g(IERROR\))170 2347 y(INTEGER)g(COMM1,)g(COMM2,)g (RESULT,)g(IERROR)75 2433 y(MPI)p 150 2433 V 17 w(COMM)p 263 2433 V 16 w(DUP\(COMM,)g(NEWCOMM,)g(IERROR\))170 2490 y(INTEGER)g(COMM,)h(NEWCOMM,)e(IERROR)75 2576 y(MPI)p 150 2576 V 17 w(COMM)p 263 2576 V 16 w(CREATE\(COMM,)h(GROUP,)g (NEWCOMM,)f(IERROR\))170 2633 y(INTEGER)h(COMM,)h(GROUP,)f(NEWCOMM,)f (IERROR)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 224 232 224 231 bop 75 -100 a Fx(224)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Fu(MPI)p 150 45 15 2 v 17 w(COMM)p 263 45 V 16 w(SPLIT\(COMM,)23 b(COLOR,)g(KEY,)g (NEWCOMM,)g(IERROR\))170 102 y(INTEGER)g(COMM,)h(COLOR,)f(KEY,)g (NEWCOMM,)g(IERROR)75 188 y(MPI)p 150 188 V 17 w(COMM)p 263 188 V 16 w(FREE\(COMM,)g(IERROR\))170 244 y(INTEGER)g(COMM,)h (IERROR)75 331 y(MPI)p 150 331 V 17 w(COMM)p 263 331 V 16 w(TEST)p 375 331 V 17 w(INTER\(COMM,)e(FLAG,)i(IERROR\))170 387 y(INTEGER)f(COMM,)h(IERROR)170 444 y(LOGICAL)f(FLAG)75 530 y(MPI)p 150 530 V 17 w(COMM)p 263 530 V 16 w(REMOTE)p 423 530 V 17 w(SIZE\(COMM,)f(SIZE,)h(IERROR\))170 587 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 673 y(MPI)p 150 673 V 17 w(COMM)p 263 673 V 16 w(REMOTE)p 423 673 V 17 w(GROUP\(COMM,)f(GROUP,)h(IERROR\))170 730 y(INTEGER)g(COMM,)h(GROUP,)f (IERROR)75 816 y(MPI)p 150 816 V 17 w(INTERCOMM)p 383 816 V 16 w(CREATE\(LOCAL)p 687 816 V 15 w(COMM,)g(LOCAL)p 965 816 V 17 w(LEADER,)g(PEER)p 1269 816 V 16 w(COMM,)h(REMOTE)p 1573 816 V 16 w(LEADER,)f(TAG,)393 873 y(NEWINTERCOMM,)f(IERROR\))170 929 y(INTEGER)h(LOCAL)p 484 929 V 17 w(COMM,)g(LOCAL)p 764 929 V 17 w(LEADER,)f(PEER)p 1067 929 V 17 w(COMM,)h(REMOTE)p 1371 929 V 17 w(LEADER,)g(TAG,)170 985 y(NEWINTERCOMM,)g(IERROR)75 1072 y(MPI)p 150 1072 V 17 w(INTERCOMM)p 383 1072 V 16 w(MERGE\(INTERCOMM,)e(HIGH,)j(INTRACOMM,)e(IERROR\))170 1128 y(INTEGER)h(INTERCOMM,)g(INTRACOMM,)f(IERROR)170 1185 y(LOGICAL)h(HIGH)75 1271 y(MPI)p 150 1271 V 17 w(KEYVAL)p 311 1271 V 16 w(CREATE\(COPY)p 591 1271 V 16 w(FN,)g(DELETE)p 846 1271 V 17 w(FN,)g(KEYVAL,)g(EXTRA)p 1269 1271 V 16 w(STATE,)h(IERROR\))170 1328 y(EXTERNAL)f(COPY)p 484 1328 V 17 w(FN,)g(DELETE)p 740 1328 V 17 w(FN)170 1384 y(INTEGER)g(KEYVAL,)g(EXTRA)p 675 1384 V 17 w(STATE,)g(IERROR)75 1471 y(MPI)p 150 1471 V 17 w(KEYVAL)p 311 1471 V 16 w(FREE\(KEYVAL,)f (IERROR\))170 1527 y(INTEGER)h(KEYVAL,)g(IERROR)75 1614 y(MPI)p 150 1614 V 17 w(ATTR)p 263 1614 V 16 w(PUT\(COMM,)g(KEYVAL,)g (ATTRIBUTE)p 925 1614 V 16 w(VAL,)g(IERROR\))170 1670 y(INTEGER)g(COMM,)h(KEYVAL,)e(ATTRIBUTE)p 914 1670 V 16 w(VAL,)i(IERROR)75 1756 y(MPI)p 150 1756 V 17 w(ATTR)p 263 1756 V 16 w(GET\(COMM,)f(KEYVAL,)g(ATTRIBUTE)p 925 1756 V 16 w(VAL,)g(FLAG,)g(IERROR\))170 1813 y(INTEGER)g(COMM,)h (KEYVAL,)e(ATTRIBUTE)p 914 1813 V 16 w(VAL,)i(IERROR)170 1869 y(LOGICAL)f(FLAG)75 1956 y(MPI)p 150 1956 V 17 w(ATTR)p 263 1956 V 16 w(DELETE\(COMM,)g(KEYVAL,)g(IERROR\))170 2012 y(INTEGER)g(COMM,)h(KEYVAL,)e(IERROR)75 2186 y Fn(A.12)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Pro)r(cess)e(T)-5 b(op)r(ologies)75 2287 y Fu(MPI)p 150 2287 V 17 w(CART)p 263 2287 V 16 w(CREATE\(COMM)p 543 2287 V 16 w(OLD,)23 b(NDIMS,)g(DIMS,)h(PERIODS,)e(REORDER,)h(COMM)p 1514 2287 V 17 w(CART,)g(IERROR\))170 2343 y(INTEGER)g(COMM)p 460 2343 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g(COMM)p 1074 2343 V 17 w(CART,)g(IERROR)170 2400 y(LOGICAL)g(PERIODS\(*\),)g (REORDER)75 2486 y(MPI)p 150 2486 V 17 w(DIMS)p 263 2486 V 16 w(CREATE\(NNODES,)f(NDIMS,)h(DIMS,)h(IERROR\))170 2543 y(INTEGER)f(NNODES,)g(NDIMS,)g(DIMS\(*\),)g(IERROR)75 2629 y(MPI)p 150 2629 V 17 w(GRAPH)p 287 2629 V 16 w(CREATE\(COMM)p 567 2629 V 16 w(OLD,)g(NNODES,)g(INDEX,)g(EDGES,)g(REORDER,)g(COMM)p 1538 2629 V 17 w(GRAPH,)393 2686 y(IERROR\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 225 233 225 232 bop 75 -100 a Fm(A.13.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o(OR)g(ENVIR)o(ONMENT)l(AL)g(INQUIR)l(Y)364 b Fx(225)170 45 y Fu(INTEGER)23 b(COMM)p 460 45 15 2 v 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g(EDGES\(*\),)f(COMM)p 1360 45 V 17 w(GRAPH,)h(IERROR)170 102 y(LOGICAL)g(REORDER)75 188 y(MPI)p 150 188 V 17 w(TOPO)p 263 188 V 16 w(TEST\(COMM,)g(STATUS,) g(IERROR\))170 244 y(INTEGER)g(COMM,)h(STATUS,)e(IERROR)75 331 y(MPI)p 150 331 V 17 w(GRAPHDIMS)p 383 331 V 16 w(GET\(COMM,)g (NNODES,)h(NEDGES,)g(IERROR\))170 387 y(INTEGER)g(COMM,)h(NNODES,)e (NEDGES,)h(IERROR)75 474 y(MPI)p 150 474 V 17 w(GRAPH)p 287 474 V 16 w(GET\(COMM,)g(MAXINDEX,)g(MAXEDGES,)f(INDEX,)h(EDGES,)g (IERROR\))170 530 y(INTEGER)g(COMM,)h(MAXINDEX,)e(MAXEDGES,)h (INDEX\(*\),)f(EDGES\(*\),)h(IERROR)75 617 y(MPI)p 150 617 V 17 w(CARTDIM)p 335 617 V 16 w(GET\(COMM,)g(NDIMS,)g(IERROR\))170 673 y(INTEGER)g(COMM,)h(NDIMS,)f(IERROR)75 760 y(MPI)p 150 760 V 17 w(CART)p 263 760 V 16 w(GET\(COMM,)g(MAXDIMS,)g(DIMS,)g (PERIODS,)g(COORDS,)g(IERROR\))170 816 y(INTEGER)g(COMM,)h(MAXDIMS,)e (DIMS\(*\),)h(COORDS\(*\),)g(IERROR)170 873 y(LOGICAL)g(PERIODS\(*\))75 959 y(MPI)p 150 959 V 17 w(CART)p 263 959 V 16 w(RANK\(COMM,)g(COORDS,) g(RANK,)g(IERROR\))170 1015 y(INTEGER)g(COMM,)h(COORDS\(*\),)e(RANK,)h (IERROR)75 1102 y(MPI)p 150 1102 V 17 w(CART)p 263 1102 V 16 w(COORDS\(COMM,)g(RANK,)g(MAXDIMS,)g(COORDS,)f(IERROR\))170 1158 y(INTEGER)h(COMM,)h(RANK,)f(MAXDIMS,)f(COORDS\(*\),)h(IERROR)75 1245 y(MPI)p 150 1245 V 17 w(GRAPH)p 287 1245 V 16 w(NEIGHBORS)p 519 1245 V 16 w(COUNT\(COMM,)g(RANK,)g(NNEIGHBORS,)f(IERROR\))170 1301 y(INTEGER)h(COMM,)h(RANK,)f(NNEIGHBORS,)f(IERROR)75 1388 y(MPI)p 150 1388 V 17 w(GRAPH)p 287 1388 V 16 w(NEIGHBORS\(COMM,)g (RANK,)h(MAXNEIGHBORS,)f(NEIGHBORS,)h(IERROR\))170 1444 y(INTEGER)g(COMM,)h(RANK,)f(MAXNEIGHBORS,)f(NEIGHBORS\(*\),)g(IERROR)75 1531 y(MPI)p 150 1531 V 17 w(CART)p 263 1531 V 16 w(SHIFT\(COMM,)h (DIRECTION,)f(DISP,)h(RANK)p 1067 1531 V 17 w(SOURCE,)g(RANK)p 1371 1531 V 17 w(DEST,)g(IERROR\))170 1587 y(INTEGER)g(COMM,)h (DIRECTION,)e(DISP,)h(RANK)p 1009 1587 V 17 w(SOURCE,)g(RANK)p 1313 1587 V 16 w(DEST,)h(IERROR)75 1674 y(MPI)p 150 1674 V 17 w(CART)p 263 1674 V 16 w(SUB\(COMM,)f(REMAIN)p 662 1674 V 16 w(DIMS,)h(NEWCOMM,)e(IERROR\))170 1730 y(INTEGER)h(COMM,)h (NEWCOMM,)e(IERROR)170 1786 y(LOGICAL)h(REMAIN)p 508 1786 V 17 w(DIMS\(*\))75 1873 y(MPI)p 150 1873 V 17 w(CART)p 263 1873 V 16 w(MAP\(COMM,)g(NDIMS,)g(DIMS,)g(PERIODS,)g(NEWRANK,)g (IERROR\))170 1929 y(INTEGER)g(COMM,)h(NDIMS,)f(DIMS\(*\),)f(NEWRANK,)h (IERROR)170 1986 y(LOGICAL)g(PERIODS\(*\))75 2072 y(MPI)p 150 2072 V 17 w(GRAPH)p 287 2072 V 16 w(MAP\(COMM,)g(NNODES,)g(INDEX,)g (EDGES,)g(NEWRANK,)g(IERROR\))170 2129 y(INTEGER)g(COMM,)h(NNODES,)e (INDEX\(*\),)h(EDGES\(*\),)g(NEWRANK,)f(IERROR)75 2302 y Fn(A.13)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Environmental)d (Inquiry)75 2403 y Fu(MPI)p 150 2403 V 17 w(GET)p 239 2403 V 17 w(PROCESSOR)p 472 2403 V 15 w(NAME\(NAME,)23 b(RESULTLEN,)f(IERROR\))170 2460 y(CHARACTER*\(*\))h(NAME)170 2516 y(INTEGER)g(RESULTLEN,)g(IERROR)75 2603 y(MPI)p 150 2603 V 17 w(ERRHANDLER)p 407 2603 V 16 w(CREATE\(FUNCTION,)e (HANDLER,)i(IERROR\))170 2659 y(EXTERNAL)g(FUNCTION)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 226 234 226 233 bop 75 -100 a Fx(226)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y Fu(INTEGER)23 b(ERRHANDLER,)g (IERROR)75 132 y(MPI)p 150 132 15 2 v 17 w(ERRHANDLER)p 407 132 V 16 w(SET\(COMM,)f(ERRHANDLER,)h(IERROR\))170 188 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)75 274 y(MPI)p 150 274 V 17 w(ERRHANDLER)p 407 274 V 16 w(GET\(COMM,)g(ERRHANDLER,)h (IERROR\))170 331 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)75 417 y(MPI)p 150 417 V 17 w(ERRHANDLER)p 407 417 V 16 w(FREE\(ERRHANDLER,)f(IERROR\))170 474 y(INTEGER)i(ERRHANDLER,)g (IERROR)75 560 y(MPI)p 150 560 V 17 w(ERROR)p 287 560 V 16 w(STRING\(ERRORCODE,)f(STRING,)h(RESULTLEN,)f(IERROR\))170 617 y(INTEGER)h(ERRORCODE,)g(RESULTLEN,)f(IERROR)170 673 y(CHARACTER*\(*\))h(STRING)75 760 y(MPI)p 150 760 V 17 w(ERROR)p 287 760 V 16 w(CLASS\(ERRORCODE,)f(ERRORCLASS,)g (IERROR\))170 816 y(INTEGER)h(ERRORCODE,)g(ERRORCLASS,)f(IERROR)75 903 y(DOUBLE)h(PRECISION)g(MPI)p 556 903 V 16 w(WTIME\(\))75 989 y(DOUBLE)g(PRECISION)g(MPI)p 556 989 V 16 w(WTICK\(\))75 1075 y(MPI)p 150 1075 V 17 w(INIT\(IERROR\))170 1132 y(INTEGER)g(IERROR)75 1218 y(MPI)p 150 1218 V 17 w(FINALIZE\(IERROR\)) 170 1275 y(INTEGER)g(IERROR)75 1361 y(MPI)p 150 1361 V 17 w(INITIALIZED\(FLAG,)e(IERROR\))170 1418 y(LOGICAL)i(FLAG)170 1474 y(INTEGER)g(IERROR)75 1561 y(MPI)p 150 1561 V 17 w(ABORT\(COMM,)f(ERRORCODE,)h(IERROR\))170 1617 y(INTEGER)g(COMM,)h (ERRORCODE,)e(IERROR)75 1790 y Fn(A.14)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Pro\014ling)75 1892 y Fu(MPI)p 150 1892 V 17 w(PCONTROL\(level\))170 1948 y(INTEGER)i(LEVEL,)g Fp(:)8 b(:)g(:)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 227 235 227 234 bop 75 283 a Fr(MPI)40 b(F)-10 b(unction)40 b(Index)75 490 y Fx(MPI)p 167 490 14 2 v 16 w(ABOR)l(T,)16 b(197)75 547 y(MPI)p 167 547 V 16 w(ADDRESS,)g(67)75 603 y(MPI)p 167 603 V 16 w(ALLGA)l(THER,)g(107)75 660 y(MPI)p 167 660 V 16 w(ALLGA)l(THER)-5 b(V,)16 b(108)75 716 y(MPI)p 167 716 V 16 w(ALLREDUCE,)g(122)75 773 y(MPI)p 167 773 V 16 w(ALL)l(TO)o(ALL,)h(109)75 829 y(MPI)p 167 829 V 16 w(ALL)l(TO)o(ALL)-5 b(V,)17 b(110)75 886 y(MPI)p 167 886 V 16 w(A)l(TTR)p 312 886 V 17 w(DELETE,)e(169)75 942 y(MPI)p 167 942 V 16 w(A)l(TTR)p 312 942 V 17 w(GET,)f(169)75 999 y(MPI)p 167 999 V 16 w(A)l(TTR)p 312 999 V 17 w(PUT,)h(168)75 1096 y(MPI)p 167 1096 V 16 w(BARRIER,)i(93)75 1153 y(MPI)p 167 1153 V 16 w(BCAST,)e(93)75 1209 y(MPI)p 167 1209 V 16 w(BSEND,)g(27)75 1266 y(MPI)p 167 1266 V 16 w(BSEND)p 340 1266 V 17 w(INIT,)g(53)75 1322 y(MPI)p 167 1322 V 16 w(BUFFER)p 373 1322 V 17 w(A)l(TT)l(A)o(CH,)f(33)75 1379 y(MPI)p 167 1379 V 16 w(BUFFER)p 373 1379 V 17 w(DET)l(A)o(CH,)g (33)75 1477 y(MPI)p 167 1477 V 16 w(CANCEL,)h(51)75 1533 y(MPI)p 167 1533 V 16 w(CAR)l(T)p 312 1533 V 17 w(COORDS,)h(182)75 1590 y(MPI)p 167 1590 V 16 w(CAR)l(T)p 312 1590 V 17 w(CREA)l(TE,)f(177)75 1646 y(MPI)p 167 1646 V 16 w(CAR)l(T)p 312 1646 V 17 w(GET,)f(181)75 1703 y(MPI)p 167 1703 V 16 w(CAR)l(T)p 312 1703 V 17 w(MAP)l(,)h(186)75 1759 y(MPI)p 167 1759 V 16 w(CAR)l(T)p 312 1759 V 17 w(RANK,)h(182)75 1815 y(MPI)p 167 1815 V 16 w(CAR)l(T)p 312 1815 V 17 w(SHIFT,)f(184)75 1872 y(MPI)p 167 1872 V 16 w(CAR)l(T)p 312 1872 V 17 w(SUB,)g(185)75 1928 y(MPI)p 167 1928 V 16 w(CAR)l(TDIM)p 405 1928 V 17 w(GET,)f(181)75 1985 y(MPI)p 167 1985 V 16 w(COMM)p 335 1985 V 16 w(COMP)l(ARE,)i(142)75 2041 y(MPI)p 167 2041 V 16 w(COMM)p 335 2041 V 16 w(CREA)l(TE,)f(143)75 2098 y(MPI)p 167 2098 V 16 w(COMM)p 335 2098 V 16 w(DUP)l(,)g(143)75 2154 y(MPI)p 167 2154 V 16 w(COMM)p 335 2154 V 16 w(FREE,)g(145)75 2211 y(MPI)p 167 2211 V 16 w(COMM)p 335 2211 V 16 w(GR)o(OUP)l(,)g(137) 75 2267 y(MPI)p 167 2267 V 16 w(COMM)p 335 2267 V 16 w(RANK,)h(142)75 2324 y(MPI)p 167 2324 V 16 w(COMM)p 335 2324 V 16 w(REMOTE)p 556 2324 V 17 w(GR)o(OUP)l(,)f(155)75 2380 y(MPI)p 167 2380 V 16 w(COMM)p 335 2380 V 16 w(REMOTE)p 556 2380 V 17 w(SIZE,)g(155)75 2436 y(MPI)p 167 2436 V 16 w(COMM)p 335 2436 V 16 w(SIZE,)g(141)75 2493 y(MPI)p 167 2493 V 16 w(COMM)p 335 2493 V 16 w(SPLIT,)h(144)75 2549 y(MPI)p 167 2549 V 16 w(COMM)p 335 2549 V 16 w(TEST)p 473 2549 V 16 w(INTER,)g(154)75 2647 y(MPI)p 167 2647 V 16 w(DIMS)p 301 2647 V 17 w(CREA)l(TE,)f(177)1048 490 y(MPI)p 1140 490 V 16 w(ERRHANDLER)p 1482 490 V 18 w(CREA)l(TE,)g(192) 1048 547 y(MPI)p 1140 547 V 16 w(ERRHANDLER)p 1482 547 V 18 w(FREE,)g(193)1048 603 y(MPI)p 1140 603 V 16 w(ERRHANDLER)p 1482 603 V 18 w(GET,)f(193)1048 660 y(MPI)p 1140 660 V 16 w(ERRHANDLER)p 1482 660 V 18 w(SET,)h(192)1048 716 y(MPI)p 1140 716 V 16 w(ERR)o(OR)p 1320 716 V 18 w(CLASS,)g(195)1048 773 y(MPI)p 1140 773 V 16 w(ERR)o(OR)p 1320 773 V 18 w(STRING,)g(193)1048 871 y(MPI)p 1140 871 V 16 w(FINALIZE,)h(196)1048 968 y(MPI)p 1140 968 V 16 w(GA)l(THER,)f(94)1048 1025 y(MPI)p 1140 1025 V 16 w(GA)l(THER)-5 b(V,)15 b(95)1048 1081 y(MPI)p 1140 1081 V 16 w(GET)p 1256 1081 V 16 w(COUNT,)g(21)1048 1138 y(MPI)p 1140 1138 V 16 w(GET)p 1256 1138 V 16 w(ELEMENTS,)g(73) 1048 1194 y(MPI)p 1140 1194 V 16 w(GET)p 1256 1194 V 16 w(PR)o(OCESSOR)p 1552 1194 V 18 w(NAME,)f(190)1048 1251 y(MPI)p 1140 1251 V 16 w(GRAPH)p 1324 1251 V 17 w(CREA)l(TE,)h(178)1048 1307 y(MPI)p 1140 1307 V 16 w(GRAPH)p 1324 1307 V 17 w(GET,)f(181)1048 1364 y(MPI)p 1140 1364 V 16 w(GRAPH)p 1324 1364 V 17 w(MAP)l(,)g(187)1048 1420 y(MPI)p 1140 1420 V 16 w(GRAPH)p 1324 1420 V 17 w(NEIGHBORS,)i(183)1048 1477 y(MPI)p 1140 1477 V 16 w(GRAPH)p 1324 1477 V 17 w(NEIGHBORS)p 1617 1477 V 17 w(COUNT,)10 b(183)1048 1533 y(MPI)p 1140 1533 V 16 w(GRAPHDIMS)p 1442 1533 V 17 w(GET,)k(180)1048 1590 y(MPI)p 1140 1590 V 16 w(GR)o(OUP)p 1324 1590 V 17 w(COMP)l(ARE,)h(136)1048 1646 y(MPI)p 1140 1646 V 16 w(GR)o(OUP)p 1324 1646 V 17 w(DIFFERENCE,)f(138)1048 1702 y(MPI)p 1140 1702 V 16 w(GR)o(OUP)p 1324 1702 V 17 w(EX)o(CL,)g(139)1048 1759 y(MPI)p 1140 1759 V 16 w(GR)o(OUP)p 1324 1759 V 17 w(FREE,)g(140)1048 1815 y(MPI)p 1140 1815 V 16 w(GR)o(OUP)p 1324 1815 V 17 w(INCL,)h(138)1048 1872 y(MPI)p 1140 1872 V 16 w(GR)o(OUP)p 1324 1872 V 17 w(INTERSECTION,)h(137)1048 1928 y(MPI)p 1140 1928 V 16 w(GR)o(OUP)p 1324 1928 V 17 w(RANGE)p 1509 1928 V 16 w(EX)o(CL,)f(140)1048 1985 y(MPI)p 1140 1985 V 16 w(GR)o(OUP)p 1324 1985 V 17 w(RANGE)p 1509 1985 V 16 w(INCL,)h(139)1048 2041 y(MPI)p 1140 2041 V 16 w(GR)o(OUP)p 1324 2041 V 17 w(RANK,)f(135)1048 2098 y(MPI)p 1140 2098 V 16 w(GR)o(OUP)p 1324 2098 V 17 w(SIZE,)g(135)1048 2154 y(MPI)p 1140 2154 V 16 w(GR)o(OUP)p 1324 2154 V 17 w(TRANSLA)l(TE)p 1622 2154 V 17 w(RANKS,)c(136)1048 2211 y(MPI)p 1140 2211 V 16 w(GR)o(OUP)p 1324 2211 V 17 w(UNION,)k(137)1048 2308 y(MPI)p 1140 2308 V 16 w(IBSEND,)g(37)1048 2365 y(MPI)p 1140 2365 V 16 w(INIT,)h(196)1048 2421 y(MPI)p 1140 2421 V 16 w(INITIALIZED,)g(197)1048 2478 y(MPI)p 1140 2478 V 16 w(INTER)o(COMM)p 1454 2478 V 16 w(CREA)l(TE,)f(157)1048 2534 y(MPI)p 1140 2534 V 16 w(INTER)o(COMM)p 1454 2534 V 16 w(MER)o(GE,)g(157)1048 2591 y(MPI)p 1140 2591 V 16 w(IPR)o(OBE,)h(48)1048 2647 y(MPI)p 1140 2647 V 16 w(IRECV,)g(38)1048 2704 y(MPI)p 1140 2704 V 16 w(IRSEND,)g(38)941 2828 y(227)p eop %%Page: 228 236 228 235 bop 75 -100 a Fx(228)1326 b Fm(MPI)15 b(F)l(unction)h(Index)75 45 y Fx(MPI)p 167 45 14 2 v 16 w(ISEND,)g(36)75 102 y(MPI)p 167 102 V 16 w(ISSEND,)g(37)75 196 y(MPI)p 167 196 V 16 w(KEYV)-5 b(AL)p 374 196 V 18 w(CREA)l(TE,)15 b(166)75 253 y(MPI)p 167 253 V 16 w(KEYV)-5 b(AL)p 374 253 V 18 w(FREE,)14 b(168)75 348 y(MPI)p 167 348 V 16 w(OP)p 249 348 V 17 w(CREA)l(TE,)h(118)75 404 y(MPI)p 167 404 V 16 w(OP)p 249 404 V 17 w(FREE,)g(120)75 499 y(MPI)p 167 499 V 16 w(P)l(A)o(CK,)g(83)75 556 y(MPI)p 167 556 V 16 w(P)l(A)o(CK)p 311 556 V 17 w(SIZE,)g(86)75 612 y(MPI)p 167 612 V 16 w(PCONTR)o(OL,)h(199)75 669 y(MPI)p 167 669 V 16 w(PR)o(OBE,)g(49)75 764 y(MPI)p 167 764 V 16 w(RECV,)f(19)75 820 y(MPI)p 167 820 V 16 w(RECV)p 314 820 V 17 w(INIT,)h(55)75 877 y(MPI)p 167 877 V 16 w(REDUCE,)f(111)75 933 y(MPI)p 167 933 V 16 w(REDUCE)p 380 933 V 17 w(SCA)l(TTER,)g(123)75 989 y(MPI)p 167 989 V 16 w(REQUEST)p 405 989 V 18 w(FREE,)f(41)75 1046 y(MPI)p 167 1046 V 16 w(RSEND,)i(28)75 1102 y(MPI)p 167 1102 V 16 w(RSEND)p 341 1102 V 17 w(INIT,)g(54)75 1197 y(MPI)p 167 1197 V 16 w(SCAN,)f(124)75 1254 y(MPI)p 167 1254 V 16 w(SCA)l(TTER,)h(103)75 1310 y(MPI)p 167 1310 V 16 w(SCA)l(TTER)-5 b(V,)16 b(104)75 1367 y(MPI)p 167 1367 V 16 w(SEND,)f(16)75 1423 y(MPI)p 167 1423 V 16 w(SEND)p 308 1423 V 17 w(INIT,)g(53)75 1480 y(MPI)p 167 1480 V 16 w(SENDRECV,)g(57)75 1536 y(MPI)p 167 1536 V 16 w(SENDRECV)p 439 1536 V 17 w(REPLA)o(CE,)g(58)75 1592 y(MPI)p 167 1592 V 16 w(SSEND,)g(27)75 1649 y(MPI)p 167 1649 V 16 w(SSEND)p 333 1649 V 17 w(INIT,)h(54)75 1705 y(MPI)p 167 1705 V 16 w(ST)l(AR)l(T,)g(55)75 1762 y(MPI)p 167 1762 V 16 w(ST)l(AR)l(T)l(ALL,)h(56)75 1857 y(MPI)p 167 1857 V 16 w(TEST,)e(40)75 1913 y(MPI)p 167 1913 V 16 w(TEST)p 305 1913 V 17 w(CANCELLED,)g(52)75 1970 y(MPI)p 167 1970 V 16 w(TEST)l(ALL,)h(45)75 2026 y(MPI)p 167 2026 V 16 w(TEST)l(ANY,)f(44)75 2083 y(MPI)p 167 2083 V 16 w(TESTSOME,)g(46)75 2139 y(MPI)p 167 2139 V 16 w(TOPO)p 317 2139 V 17 w(TEST,)g(180)75 2195 y(MPI)p 167 2195 V 16 w(TYPE)p 312 2195 V 17 w(COMMIT,)f(70)75 2252 y(MPI)p 167 2252 V 16 w(TYPE)p 312 2252 V 17 w(CONTIGUOUS,)h(60)75 2308 y(MPI)p 167 2308 V 16 w(TYPE)p 312 2308 V 17 w(COUNT,)g(69)75 2365 y(MPI)p 167 2365 V 16 w(TYPE)p 312 2365 V 17 w(EXTENT,)f(68)75 2421 y(MPI)p 167 2421 V 16 w(TYPE)p 312 2421 V 17 w(FREE,)g(71)75 2478 y(MPI)p 167 2478 V 16 w(TYPE)p 312 2478 V 17 w(HINDEXED,)g(65)75 2534 y(MPI)p 167 2534 V 16 w(TYPE)p 312 2534 V 17 w(HVECTOR,)h(63)75 2591 y(MPI)p 167 2591 V 16 w(TYPE)p 312 2591 V 17 w(INDEXED,)f(64)75 2647 y(MPI)p 167 2647 V 16 w(TYPE)p 312 2647 V 17 w(LB,)h(70)75 2704 y(MPI)p 167 2704 V 16 w(TYPE)p 312 2704 V 17 w(SIZE,)g(68)1048 45 y(MPI)p 1140 45 V 16 w(TYPE)p 1285 45 V 16 w(STR)o(UCT,)g(66)1048 102 y(MPI)p 1140 102 V 16 w(TYPE)p 1285 102 V 16 w(UB,)g(70)1048 158 y(MPI)p 1140 158 V 16 w(TYPE)p 1285 158 V 16 w(VECTOR,)g(61)1048 256 y(MPI)p 1140 256 V 16 w(UNP)l(A)o(CK,)g(84)1048 354 y(MPI)p 1140 354 V 16 w(W)-5 b(AIT,)15 b(39)1048 410 y(MPI)p 1140 410 V 16 w(W)-5 b(AIT)l(ALL,)16 b(44)1048 467 y(MPI)p 1140 467 V 16 w(W)-5 b(AIT)l(ANY,)15 b(43)1048 523 y(MPI)p 1140 523 V 16 w(W)-5 b(AITSOME,)15 b(46)1048 580 y(MPI)p 1140 580 V 16 w(WTICK,)g(195)1048 636 y(MPI)p 1140 636 V 16 w(WTIME,)f(195)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF .