%!PS-Adobe-2.0 %%Creator: dvips 5.528 Copyright 1986, 1994 Radical Eye Software %%Title: mpi-report.dvi %%CreationDate: Mon Jun 12 12:53:26 1995 %%Pages: 240 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips -o mpi-report.ps mpi-report %DVIPSParameters: dpi=300, comments removed %DVIPSSource: TeX output 1995.06.12:1251 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /hsize 8.5 72 mul N /landplus90{false}def /@rigin{isls{[0 landplus90{1 -1}{-1 1} ifelse 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{landplus90{VResolution 72 div vsize mul 0 exch}{Resolution -72 div hsize mul 0}ifelse TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{ statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{/nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{/sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0]N df-tail}B /E{ pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get} B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]}if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{cc 1 add D }B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet %%BeginProcSet: special.pro TeXDict begin /SDict 200 dict N SDict begin /@SpecialDefaults{/hs 612 N /vs 792 N /ho 0 N /vo 0 N /hsc 1 N /vsc 1 N /ang 0 N /CLIP 0 N /rwiSeen false N /rhiSeen false N /letter{}N /note{}N /a4{}N /legal{}N}B /@scaleunit 100 N /@hscale{@scaleunit div /hsc X}B /@vscale{@scaleunit div /vsc X}B /@hsize{/hs X /CLIP 1 N}B /@vsize{/vs X /CLIP 1 N}B /@clip{ /CLIP 2 N}B /@hoffset{/ho X}B /@voffset{/vo X}B /@angle{/ang X}B /@rwi{ 10 div /rwi X /rwiSeen true N}B /@rhi{10 div /rhi X /rhiSeen true N}B /@llx{/llx X}B /@lly{/lly X}B /@urx{/urx X}B /@ury{/ury X}B /magscale true def end /@MacSetUp{userdict /md known{userdict /md get type /dicttype eq{userdict begin md length 10 add md maxlength ge{/md md dup length 20 add dict copy def}if end md begin /letter{}N /note{}N /legal{} N /od{txpose 1 0 mtx defaultmatrix dtransform S atan/pa X newpath clippath mark{transform{itransform moveto}}{transform{itransform lineto} }{6 -2 roll transform 6 -2 roll transform 6 -2 roll transform{ itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll curveto}}{{ closepath}}pathforall newpath counttomark array astore /gc xdf pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack}if}N /txpose{pxs pys scale ppr aload pop por{noflips{pop S neg S TR pop 1 -1 scale}if xflip yflip and{pop S neg S TR 180 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{pop S neg S TR pop 180 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{ppr 1 get neg ppr 0 get neg TR}if}{noflips{TR pop pop 270 rotate 1 -1 scale}if xflip yflip and{TR pop pop 90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{TR pop pop 90 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{TR pop pop 270 rotate ppr 2 get ppr 0 get neg sub neg 0 S TR}if}ifelse scaleby96{ppr aload pop 4 -1 roll add 2 div 3 1 roll add 2 div 2 copy TR .96 dup scale neg S neg S TR}if}N /cp {pop pop showpage pm restore}N end}if}if}N /normalscale{Resolution 72 div VResolution 72 div neg scale magscale{DVImag dup scale}if 0 setgray} N /psfts{S 65781.76 div N}N /startTexFig{/psf$SavedState save N userdict maxlength dict begin /magscale false def normalscale currentpoint TR /psf$ury psfts /psf$urx psfts /psf$lly psfts /psf$llx psfts /psf$y psfts /psf$x psfts currentpoint /psf$cy X /psf$cx X /psf$sx psf$x psf$urx psf$llx sub div N /psf$sy psf$y psf$ury psf$lly sub div N psf$sx psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub TR /showpage{}N /erasepage{}N /copypage{}N /p 3 def @MacSetUp}N /doclip{ psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll S lineto S lineto S lineto closepath clip newpath moveto}N /endTexFig{end psf$SavedState restore}N /@beginspecial{SDict begin /SpecialSave save N gsave normalscale currentpoint TR @SpecialDefaults count /ocount X /dcount countdictstack N}N /@setspecial {CLIP 1 eq{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto closepath clip}if ho vo TR hsc vsc scale ang rotate rwiSeen{rwi urx llx sub div rhiSeen{rhi ury lly sub div}{dup}ifelse scale llx neg lly neg TR }{rhiSeen{rhi ury lly sub div dup scale llx neg lly neg TR}if}ifelse CLIP 2 eq{newpath llx lly moveto urx lly lineto urx ury lineto llx ury lineto closepath clip}if /showpage{}N /erasepage{}N /copypage{}N newpath }N /@endspecial{count ocount sub{pop}repeat countdictstack dcount sub{ end}repeat grestore SpecialSave restore end}N /@defspecial{SDict begin} N /@fedspecial{end}B /li{lineto}B /rl{rlineto}B /rc{rcurveto}B /np{ /SaveX currentpoint /SaveY X N 1 setlinecap newpath}N /st{stroke SaveX SaveY moveto}N /fil{fill SaveX SaveY moveto}N /ellipse{/endangle X /startangle X /yrad X /xrad X /savematrix matrix currentmatrix N TR xrad yrad scale 0 0 1 startangle endangle arc savematrix setmatrix}N end %%EndProcSet TeXDict begin 40258431 52099146 1000 300 300 (/tmp_mnt/Net/charlotte/ftp/pub/mpi/mpi-1.jun95/mpi-report.dvi) @start /Fa 2 63 df<0000038000000F0000003C000000F0000003C000000F0000003C 000000F0000003C000000F0000003C000000F0000000F00000003C0000000F00000003C0 000000F00000003C0000000F00000003C0000000F00000003C0000000F00000003801918 7D9520>60 D 62 D E /Fb 17 118 df<387C7E7E3E0E1E1C78F060070B798416>44 D<1FE0003FF0007FF800783C00300E00000E00000E0003FE001FFE003E0E00700E00E00E 00E00E00E00E00783E007FFFE03FE7E00F83E013127E9116>97 D<03F80FFC1FFE3C1E78 0C7000E000E000E000E000E000F000700778073E0E1FFC0FF803F010127D9116>99 D<003F00007F00003F0000070000070000070000070003C7000FF7001FFF003C1F00780F 00700700E00700E00700E00700E00700E00700E00700700F00700F003C1F001FFFE00FE7 F007C7E014197F9816>I<03E00FF81FFC3C1E780E7007E007FFFFFFFFFFFFE000E00070 0778073C0F1FFE0FFC03F010127D9116>I<001F00007F8000FF8001E78001C30001C000 01C0007FFF00FFFF00FFFF0001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C0003FFE007FFF003FFE0011197F9816>I<018003C003C00180 00000000000000007FC07FC07FC001C001C001C001C001C001C001C001C001C001C001C0 01C07FFFFFFF7FFF101A7D9916>105 D<003000780078003000000000000000001FF81F F81FF8003800380038003800380038003800380038003800380038003800380038003800 38003800386070F0F0FFE07FC03F800D237E9916>I108 DI<7E3C00FEFE007FFF000F 87800F03800E03800E03800E03800E03800E03800E03800E03800E03800E03800E03807F C7F0FFE7F87FC7F01512809116>I<03E0000FF8001FFC003C1E00780F00700700E00380 E00380E00380E00380E00380F00780700700780F003C1E001FFC000FF80003E00011127E 9116>I<7E3E00FEFF007FFF800F83C00F00E00E00E00E00700E00700E00700E00700E00 700E00700E00E00F01E00F83C00FFF800EFF000E3C000E00000E00000E00000E00000E00 000E00007FC000FFE0007FC000141B809116>I114 D<0FEC3FFC7FFCF03CE01CE01C70007F801FF007F8003C600EE0 0EF00EF81EFFFCFFF8C7E00F127D9116>I<0300000700000700000700000700007FFF00 FFFF00FFFF00070000070000070000070000070000070000070000070100070380070380 07038007870003FE0001FC0000F80011177F9616>I<7E1F80FE3F807E1F800E03800E03 800E03800E03800E03800E03800E03800E03800E03800E03800E03800E0F800FFFF007FB F803E3F01512809116>I E /Fc 1 1 df0 D E /Fd 1 106 df<040C0000000000705898983030606464683006127E910B>105 D E /Fe 11 90 df22 D<0038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 380038003800380038003800380038003800380038003800380038003800380038003800 38003800380038003800380038FFF8FFF8FFF80D63808118>I<00001800003000006000 00E00000C0000180000380000700000600000E00000C00001C0000380000380000700000 700000E00000E00001E00001C00001C0000380000380000380000780000700000700000F 00000E00000E00001E00001E00001E00001C00001C00003C00003C00003C00003C000038 0000780000780000780000780000780000780000780000780000700000F00000F00000F0 0000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0 0000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F0000070 00007800007800007800007800007800007800007800007800003800003C00003C00003C 00003C00001C00001C00001E00001E00001E00000E00000E00000F000007000007000007 800003800003800003800001C00001C00001E00000E00000E00000700000700000380000 3800001C00000C00000E000006000007000003800001800000C00000E000006000003000 0018157C768121>32 DI<00001C00003C00 00F80001E00003C0000780000F00000E00001E00003C00003C00003C0000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 F00000F00000F00001E00001E00003C0000380000700000E00001C0000780000E00000E0 00007800001C00000E000007000003800003C00001E00001E00000F00000F00000F00000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 7800007800003C00003C00003C00001E00000E00000F000007800003C00001E00000F800 003C00001C167C7B8121>40 D<0018007800F001E003C007800F001F001E003E003C007C 007C007800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F8 00F800F800F800F800F800F800F8000D25707E25>56 D58 D<007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C007C00F800F800F800F001F001E003E003C0078007000E001C 003800F000C000F00038001C000E000700078003C003E001E001F000F000F800F800F800 7C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C00 7C007C007C007C007C007C0E4D798025>60 D62 D80 D89 D E /Ff 7 116 df<60F0F070101020204040040A7D830A>59 D<003E000C000C000C000C0018001800180018073018F0307060706060C060C060C06080 C080C480C4C1C446C838700F177E9612>100 D<03000380030000000000000000000000 00001C002400460046008C000C0018001800180031003100320032001C0009177F960C> 105 D<00180038001000000000000000000000000001C002200430043008600060006000 6000C000C000C000C001800180018001806300E300C60078000D1D80960E>I<383C1E00 44C6630047028100460301008E0703000C0603000C0603000C060600180C0600180C0620 180C0C20180C0C4030180440301807801B0E7F8D1F>109 D<383C0044C6004702004602 008E06000C06000C06000C0C00180C00180C40181840181880300880300F00120E7F8D15 >I<07C00C201870187038001E000FC003E000606060E060C0C0C1803F000C0E7E8D10> 115 D E /Fg 12 94 df<07C018303018701C600C600CE00EE00EE00EE00EE00EE00EE0 0EE00EE00E600C600C701C30181C7007C00F157F9412>48 D<03000700FF000700070007 00070007000700070007000700070007000700070007000700070007007FF00C157E9412 >I<0F8030E040708030C038E0384038003800700070006000C00180030006000C080808 10183FF07FF0FFF00D157E9412>I<0FE030306018701C701C001C00180038006007E000 300018000C000E000EE00EE00EC00C401830300FE00F157F9412>I<00300030007000F0 01F001700270047008701870107020704070C070FFFE0070007000700070007003FE0F15 7F9412>I<20303FE03FC0240020002000200020002F8030E020700030003800384038E0 38E0388030406020C01F000D157E9412>I<01F00608080C181C301C70006000E000E3E0 EC30F018F00CE00EE00EE00E600E600E300C3018183007C00F157F9412>I<40007FFE7F FC7FF8C008801080200040008000800100010003000200060006000E000E000E000E000E 0004000F167E9512>I<07E018302018600C600C700C78183E101F600FC00FF018F8607C 601EC00EC006C006C004600C38300FE00F157F9412>I61 D91 D93 D E /Fh 5 82 df0 D<020002000200C218F2 783AE00F800F803AE0F278C2180200020002000D0E7E8E12>3 D<000300030006000600 0C000C00180018003000300060006000C000C00180018003000300060006000C000C0018 0018003000300060006000C0004000101E7B9600>54 D<01FFE00FFFF818E0FC20E03E60 E01EC0E00E00E00E00E00C00C00C00C01801C01001C02001804001818003BE0003780003 00000700000600000600000E00000C00000C000018000017187F9619>80 D<001FC0007FE00181F00200F804007808003C18003C30001C30001C60001C60001CE000 18E00018E00030E00030F00060F808407C30803FE1001F02000004000038003FE0067FE0 0C0FFC0800FFF0001FC0171B7C961D>I E /Fi 63 122 df<01F1C003F1C007F1C00701 C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFF1C0FFF1C00E01C00E01C00E01 C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01 C00E01C0121D809C16>13 D<00F0000001F8000003FC0000039C0000070E0000070E0000 070E0000070E0000070E0000071E0000073C0000073800000778000003F0000003E00300 07C003000F8006001FC006003DC0060078E00C0070F00C00F0701800E0783800E03C3000 E01C6000E00EC000700F800070078040381FE1C01FF8FFC007E03F001A1F7E9D1F>38 D<00E001C0038007000E000E001C001C003800380038007000700070007000E000E000E0 00E000E000E000E000E000E000E000E000E00070007000700070003800380038001C001C 000E000E000700038001C000E00B2A7E9E10>40 DI<018001C0018001806186F99F7DBE1FF807E007E01FF87DBEF99F618601800180 01C0018010127E9E15>I<00060000000600000006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000FFFFFFE0FFFFFFE000 060000000600000006000000060000000600000006000000060000000600000006000000 0600000006000000060000000600001B1C7E9720>II< FFE0FFE00B0280890E>I<03C00FF01FF83C3C381C700E700E700EE007E007E007E007E0 07E007E007E007E007E007E007E007E007700E700E700E381C3C3C1FF80FF007E0101D7E 9B15>48 D<010007003F00FF00C700070007000700070007000700070007000700070007 000700070007000700070007000700070007000700FFF8FFF80D1C7C9B15>I<07C01FF0 3878701C601EE00EC00F400F400700070007000F000E001E001C003C007800F001E001C0 038007000E001C0038007000FFFFFFFF101C7E9B15>I58 D<7FFFFFC0FFFFFFE0000000000000000000000000000000 0000000000000000000000000000000000FFFFFFE07FFFFFC01B0C7E8F20>61 D<001C0000003E0000003E0000002E0000006700000067000000E7800000C7800000C380 0001C3C0000183C0000181C0000381E0000381E0000700F0000700F0000600F0000E0078 000FFFF8000FFFF8001C003C001C003C0018003C0038001E0038001E0070001F0070000F 0070000F00E0000780191D7F9C1C>65 DI<003FC000FFF003C0F00780300F00001E00003C00003C0000780000780000780000 F00000F00000F00000F00000F00000F00000F00000F00000F00000780000780000780000 3C00003C00001E00000F000807801803C07800FFF0003F80151F7D9D1B>IIII<003F8001FFF003C0F80780 380F00181E00003C00003C0000780000780000780000F00000F00000F00000F00000F000 00F00000F007F8F007F8F000387800387800387800383C00383C00381E00380F00380780 3803C0F801FFF0003F80151F7D9D1C>III75 DIII<003F000001FFE00003FFF00007C0F800 0F807C001E001E003E001F003C000F00780007807800078078000780F00003C0F00003C0 F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F80007C07800078078000780 7C000F803C000F003E001F001F003E000F807C0007C0F80003FFF00001FFE000003F0000 1A1F7E9D1F>II<003F000001FFE0 0003FFF00007C0F8000F807C001F003E003E001F003C000F007800078078000780780007 80F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003C0F00003 C07800078078000780780E07803C0F0F003E079F001E03DE000F83FC0007C1F80003FFF0 0001FFF800003F780000003C0000003E0000001F0000000F801A237E9D1F>II<03F8000FFE001C0F00380700700300600000 E00000E00000E00000E00000F000007800007F00003FE0001FFC0007FE0001FF00001F80 0007800003C00003C00001C00001C00001C00001C0C00180E00380F007007C0E001FFC00 07F000121F7E9D17>IIII I<78000E007C001E003C003C001E0038000F0070000F00F0000781E00003C1C00001C3C0 0001E7800000F70000007E0000003E0000003C0000003C0000007E00000077000000E780 0001E3800003C1C0000381E0000700F0000F00F8000E0078001C003C003C003E0078001F 0070000F00F0000F80191D7F9C1C>II<7FFFF07FFF F00001E00003E00003C00007C0000780000F00001F00001E00003E00003C0000780000F8 0000F00001F00001E00003C00007C0000780000F80000F00001E00003E00003C00007C00 00780000FFFFF0FFFFF0141D7E9C19>II93 D<0FC03FF07FF87038401C001C001C00FC0FFC3FFC781CE01CE01CE01CF07C 7FFC7FDC3F1C0E127E9114>97 DI<07E00FF81FFC3C1C70047000E000E000E000E000E000E000700070043C 1C1FFC0FF807E00E127E9112>I<000E000E000E000E000E000E000E000E000E000E000E 0F8E1FEE3FFE7C3E700E700EE00EE00EE00EE00EE00EE00EF00E701E7C3E3FFE1FEE0F8E 0F1D7E9C15>I<07C01FE03FF078787018601CFFFCFFFCFFFCE000E000E000700070043C 1C3FFC1FF807E00E127E9112>I<00FC01FC03FC07000E000E000E000E000E000E000E00 FFE0FFE00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E1D809C0D>I<03C3C00FFFC01FFFC01C3800381C00381C00381C00381C00381C001C38 001FF8001FF0003BC0003800003800001FFC001FFF003FFF80700780E001C0E001C0E001 C0F003C07C0F803FFF001FFE0007F800121B7F9115>III107 DIII<03F0000FFC001FFE003C0F 00780780700380E001C0E001C0E001C0E001C0E001C0F003C07003807807803C0F001FFE 000FFC0003F00012127F9115>I I114 D<1FC03FF07FF0F030E000E000F0007F003FC01FE000F000380038 8038F078FFF07FE01FC00D127F9110>I<1C001C001C001C001C001C00FFE0FFE01C001C 001C001C001C001C001C001C001C001C001C001C001C201FF00FF007C00C187F970F>I< E01CE01CE01CE01CE01CE01CE01CE01CE01CE01CE01CE01CE01CE01CE07CFFFC7FDC3F1C 0E127D9115>III<7003807807003C0E001C1C000E1C00 07380003F00001E00001C00001E00003F0000738000E18000E1C001C0E00380700700380 F003C01212809113>II E /Fj 64 122 df<003F83F800FF8FF801FF9FF803E03E00038038000780780007807800 078078000780780007807800078078000780780007807800FFFC7FC0FFFC7FC0FFFC7FC0 078078000780780007807800078078000780780007807800078078000780780007807800 078078000780780007807800078078000780780007807800078078000780780007807800 078078001D2380A21C>11 D<000078007C7800FC7801FC7803C000038000078000078000 078000078000078000078000078000078000FFFC78FFFC78FFFC78078078078078078078 078078078078078078078078078078078078078078078078078078078078078078078078 078078078078078078078078152480A31A>I<0000C018000000C018000000C018000001 803000000180300000018030000001803000000300600000030060000003006000000300 60000003006000000600C000000600C000000600C000000600C000000C018000FFFFFFFF C0FFFFFFFFC0001803000000180300000018030000001803000000300600000030060000 00300600000030060000FFFFFFFFC0FFFFFFFFC000600C000000C018000000C018000000 C018000000C0180000018030000001803000000180300000018030000003006000000300 600000030060000003006000000600C000000600C000000600C00000222D7DA229>35 D<787878781830306060E0050A7CA20E>39 D<007000E001C003C0078007000F000E001E 001C003C003C00380078007800780078007000F000F000F000F000F000F000F000F000F0 00F000F000F000F000F0007000780078007800780038003C003C001C001E000E000F0007 00078003C001C000E000700C327DA413>II<787878781830306060E0050A7C830E>44 DII<00C001C007C0FFC0FFC0FBC0 03C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C0 03C003C003C003C003C003C003C0FFFFFFFFFFFF10227CA118>49 D<03F0000FFC001FFE003C1F003007807007C06003C0E003E0C001E04001E04001E00001 E00001E00001E00003C00003C0000780000780000F00001E00003C0000780000F00001E0 0001C0000380000700000E00001C0000380000700000FFFFE0FFFFE0FFFFE013227EA118 >I<01F00007FC001FFF003E0F003807807003C02003C02003C00003C00003C00003C000 0780000780000F00001E0003FC0003F80003FE00000F000007800003C00003C00001E000 01E00001E00001E00001E08001E0C003C0E003C07007803C0F801FFF000FFC0003F00013 237EA118>I<001F00001F00002F00002F00006F0000EF0000CF0001CF0001CF00038F00 038F00078F00070F000F0F000E0F001E0F003C0F003C0F00780F00780F00F00F00FFFFF8 FFFFF8FFFFF8000F00000F00000F00000F00000F00000F00000F00000F00000F0015217F A018>I<3FFF803FFF803FFF803C00003C00003C00003C00003C00003C00003C00003C00 003C00003CF8003FFE003FFF003F0F803E07803C03C03803C00001E00001E00001E00001 E00001E00001E00001E04003C04003C0E003C07007807C1F003FFE000FFC0003F0001322 7EA018>I<007E0001FF0003FF0007C1000F00001E00001E00003C00003C000078000078 000078F800F1FE00F7FF00FF0F80FC0780F803C0F803C0F801C0F001E0F001E0F001E0F0 01E0F001E07001E07001E07801E07803C03803C03C03801C07801F0F000FFE0007FC0001 F00013237EA118>II<01F00007FC000FFE001E0F003C07803C07807803C07803C07803C07803 C07803C03803803C07801E0F000F1E0007FC0003F8000FFE001E0F003C07807803C07803 C0F001E0F001E0F001E0F001E0F001E0F001E07803C07803C03C07803E0F801FFF0007FC 0001F00013237EA118>I<01F00007FC000FFE001E0F003C07003807807803807803C0F0 03C0F001C0F001E0F001E0F001E0F001E0F001E0F001E07003E07803E07803E03C07E03E 1FE01FFDE00FF1E003E3C00003C00003C0000380000780000700000F00101E00183C003F F8001FF00007C00013237EA118>II<001F0000001F0000003F8000003F8000003B8000007BC0000073C000 0071C00000F1E00000F1E00000E0E00001E0F00001E0F00001C0F00003C0780003C07800 0380780007803C0007803C0007003C000F001E000F001E000FFFFE001FFFFF001FFFFF00 1C000F003C0007803C00078038000780780003C0780003C0700003C0F00001E0F00001E0 E00001E01B237EA220>65 DI<000FF000007FFC0000FFFF0001F01F0003C00700 078000000F0000001E0000003E0000003C0000003C000000780000007800000078000000 F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000 7800000078000000780000003C0000003C0000003E0000001E0000000F00000007800080 03C0038001F00F8000FFFF00007FFC00000FF00019257DA31F>IIII<000FF000003FFE0000FFFF8001F80F8003E00380078000000F0000001E0000001E00 00003C0000003C000000780000007800000078000000F0000000F0000000F0000000F000 0000F0000000F0000000F000FFC0F000FFC0F000FFC0780003C0780003C0780003C03C00 03C03C0003C01E0003C01E0003C00F0003C0078003C003E003C001F807C000FFFFC0003F FF00000FF8001A257DA321>III76 DII<001FC000007FF00001FFFC0003F07E0007C01F000F800F801F0007C0 1E0003C03C0001E03C0001E0780000F0780000F0780000F070000070F0000078F0000078 F0000078F0000078F0000078F0000078F0000078F0000078F0000078780000F0780000F0 780000F07C0001F03C0001E03E0003E01E0003C01F0007C00F800F8007C01F0003F07E00 01FFFC00007FF000001FC0001D257DA324>II82 D<00FE0003FFC007FFE00F81E01E00603C00003C00007800007800007800007800007800 007C00003C00003F00001FC0000FFC0007FF0001FF80003FC00007E00001F00000F00000 F8000078000078000078000078000078000078C000F0E000F0F801E07E07C03FFF800FFF 0001FC0015257EA31B>IIII<780003E07C0003C03E0007C0 1E000F800F000F000F801F0007C03E0003C03C0003E07C0001F0780000F0F0000079F000 007DE000003FC000001FC000001F8000000F0000001F8000001FC000003FC000007DE000 0079F00000F0F00001F0780001E03C0003C03E0007C01E0007800F000F000F801F000780 1E0003C03C0003E07C0001F0780000F0F80000F81D237FA220>88 D<07E01FF83FFC381E201E000F000F000F000F00FF07FF1FFF3E0F780FF00FF00FF00FF0 0FF83F7FFF3FEF1F8F10167E9517>97 DI<01FC0007FF000FFF801F03803C0180 780000780000700000F00000F00000F00000F00000F00000F00000780000780000780000 3C00401F03C00FFFC007FF8001FC0012167E9516>I<0003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C003E3C00FFBC01FFFC03F0F C03C07C07803C07803C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07803 C07803C03C07C03E0FC01FFFC00FFBC003E3C012237EA219>I<03F00007FC001FFE003E 0F003C0780780380780380F001C0FFFFC0FFFFC0FFFFC0F00000F00000F0000070000078 00007800003C00801F07800FFF8007FF0001F80012167E9516>I<003F00FF01FF03C003 8007800780078007800780078007800780FFF8FFF8FFF807800780078007800780078007 80078007800780078007800780078007800780078007800780102380A20F>I<01F07807 FFF80FFFF81F1F001E0F003C07803C07803C07803C07803C07801E0F001F1F000FFE001F FC0019F0003800003800003C00001FFE001FFFC01FFFE03FFFF07801F07800F8F00078F0 0078F00078F000787800F03E03E01FFFC00FFF8001FC0015217F9518>III<00F000F000F000F000000000000000000000000000000000000000F000F000 F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000F000 F000F000F000F000F000F000F080F0E3E0FFE07FC01F000C2D83A20D>IIIII<01FC0007FF000FFF801F07C03C01E07800F07800F0700070F00078F00078 F00078F00078F00078F000787800F07800F07C01F03E03E01F07C00FFF8007FF0001FC00 15167F9518>II<03E3C007FBC00FFFC01F0FC03C07C07C03C07803C0F803C0F003C0F003C0F003C0F0 03C0F003C0F003C0F803C07803C07803C03C07C03F0FC01FFFC00FFBC003E3C00003C000 03C00003C00003C00003C00003C00003C00003C00003C00003C012207E9519>II<07F01FFC3FFE3C0E7806780078007C003F003FF01FF80FFC 01FE001F000F000F000FC00FF81EFFFE3FFC0FF010167F9513>I<0F000F000F000F000F 000F00FFF8FFF8FFF80F000F000F000F000F000F000F000F000F000F000F000F000F000F 000F080F1C07FC07F803E00E1C7F9B12>IIII<7801F07C01E03E03C01E07C00F0780 078F0007DE0003FC0001FC0000F80000700000F80001FC0003DC00039E00078F000F0780 1E07801E03C03C01E07800F0F800F81516809516>II E /Fk 60 124 df<007E1F0001C1B1800303E3 C00703C3C00E03C1800E01C0000E01C0000E01C0000E01C0000E01C0000E01C000FFFFFC 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0 000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0007F87FC001A1D80 9C18>11 D<007E0001C1800301800703C00E03C00E01800E00000E00000E00000E00000E 0000FFFFC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C07F87F8151D809C17>I<6060F0F0F8F8686808 0808080808101010102020404080800D0C7F9C15>34 D<004000800100020006000C000C 0018001800300030007000600060006000E000E000E000E000E000E000E000E000E000E0 00E000E000600060006000700030003000180018000C000C00060002000100008000400A 2A7D9E10>40 D<800040002000100018000C000C00060006000300030003800180018001 8001C001C001C001C001C001C001C001C001C001C001C001C00180018001800380030003 00060006000C000C00180010002000400080000A2A7E9E10>I<60F0F070101010102020 4080040C7C830C>44 DI<60F0F06004047C830C>I<0001000300 0600060006000C000C000C0018001800180030003000300060006000C000C000C0018001 800180030003000300060006000C000C000C001800180018003000300030006000600060 00C000C00010297E9E15>I<03C00C301818300C300C700E60066006E007E007E007E007 E007E007E007E007E007E007E007E007E00760066006700E300C300C18180C3007E0101D 7E9B15>I<030007003F00C7000700070007000700070007000700070007000700070007 0007000700070007000700070007000700070007000F80FFF80D1C7C9B15>I<07C01830 201C400C400EF00FF80FF807F8077007000F000E000E001C001C00380070006000C00180 030006010C01180110023FFE7FFEFFFE101C7E9B15>I<60F0F060000000000000000000 0060F0F06004127C910C>58 D<60F0F0600000000000000000000060F0F0701010101020 204080041A7C910C>I<7FFFFFC0FFFFFFE0000000000000000000000000000000000000 0000000000000000000000000000FFFFFFE07FFFFFC01B0C7E8F20>61 D<000600000006000000060000000F0000000F0000000F00000017800000178000001780 000023C0000023C0000023C0000041E0000041E0000041E0000080F0000080F0000180F8 000100780001FFF80003007C0002003C0002003C0006003E0004001E0004001E000C001F 001E001F00FF80FFF01C1D7F9C1F>65 D<001F808000E0618001801980070007800E0003 801C0003801C00018038000180780000807800008070000080F0000000F0000000F00000 00F0000000F0000000F0000000F0000000F0000000700000807800008078000080380000 801C0001001C0001000E000200070004000180080000E03000001FC000191E7E9C1E>67 DIII<001F808000E0618001801980070007800E0003801C00 03801C00018038000180780000807800008070000080F0000000F0000000F0000000F000 0000F0000000F0000000F000FFF0F0000F80700007807800078078000780380007801C00 07801C0007800E00078007000B800180118000E06080001F80001C1E7E9C21>I73 D76 DII<003F800000E0E0000380380007 001C000E000E001C0007003C00078038000380780003C0780003C0700001C0F00001E0F0 0001E0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0700001C0780003C078 0003C0380003803C0007801C0007000E000E0007001C000380380000E0E000003F80001B 1E7E9C20>II82 D<07E0801C1980300580700380600180E00180E00080E00080E00080F00000F8 00007C00007FC0003FF8001FFE0007FF0000FF80000F800007C00003C00001C08001C080 01C08001C0C00180C00180E00300D00200CC0C0083F800121E7E9C17>I<7FFFFFC0700F 01C0600F00C0400F0040400F0040C00F0020800F0020800F0020800F0020000F0000000F 0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F 0000000F0000000F0000000F0000000F0000000F0000000F0000001F800003FFFC001B1C 7F9B1E>II89 D<08081010202040404040808080808080B0B0F8F8787830300D0C7A9C15>92 D<1FC000307000783800781C00301C00001C00001C0001FC000F1C00381C00701C00601C 00E01C40E01C40E01C40603C40304E801F870012127E9115>97 DI<07E00C301878307870306000E000E000E000E000E000E0 0060007004300418080C3007C00E127E9112>I<003F0000070000070000070000070000 070000070000070000070000070000070003E7000C1700180F00300700700700600700E0 0700E00700E00700E00700E00700E00700600700700700300700180F000C370007C7E013 1D7E9C17>I<03E00C301818300C700E6006E006FFFEE000E000E000E000600070023002 18040C1803E00F127F9112>I<00F8018C071E061E0E0C0E000E000E000E000E000E00FF E00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E007FE00F 1D809C0D>I<00038003C4C00C38C01C3880181800381C00381C00381C00381C00181800 1C38000C300013C0001000003000001800001FF8001FFF001FFF803003806001C0C000C0 C000C0C000C06001803003001C0E0007F800121C7F9215>II<18003C003C0018000000000000000000000000000000FC001C 001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00FF80091D7F 9C0C>I<00C001E001E000C000000000000000000000000000000FE000E000E000E000E0 00E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E060E0F0C0 F1C061803E000B25839C0D>IIIII<03F0000E1C00180600300300700380600180E001C0E001C0E001C0E001C0 E001C0E001C06001807003803003001806000E1C0003F00012127F9115>II<03C1000C3300180B00300F00700700700700E00700E00700E00700E0 0700E00700E00700600700700700300F00180F000C370007C70000070000070000070000 0700000700000700000700003FE0131A7E9116>II<1F9030704030C010 C010E010F8007F803FE00FF000F880388018C018C018E010D0608FC00D127F9110>I<04 000400040004000C000C001C003C00FFE01C001C001C001C001C001C001C001C001C001C 101C101C101C101C100C100E2003C00C1A7F9910>III< FF3FCFE03C0F03801C0701801C0701001C0B01000E0B82000E0B82000E1182000711C400 0711C4000720C40003A0E80003A0E80003C0680001C0700001C070000180300000802000 1B127F911E>I<7F8FF00F03800F030007020003840001C80001D80000F0000070000078 0000F800009C00010E00020E000607000403801E07C0FF0FF81512809116>II<7FFC70386038407040F040E041C003C0038007000F040E041C043C 0C380870087038FFF80E127F9112>II E /Fl 58 124 df<00003FE00000E0100001803800038078000300780007003000070000000700 0000070000000E0000000E0000000E000000FFFFE0000E00E0001C01C0001C01C0001C01 C0001C01C0001C0380003803800038038000380380003807000038070000700700007007 1000700E2000700E2000700E2000E00E2000E0064000E0038000E0000000C0000001C000 0001C000003180000079800000F3000000620000003C0000001D29829F1A>12 D<0E1F3F3F1D0102020404081020C0080E779F0E>39 D<00010002000400080010002000 6000C0018001800300070006000E000C001C0018003800380030007000700060006000E0 00E000C000C000C000C000C000C000C000C000C000C000C000C000C00040006000600020 00100010000800102E79A113>I<00100000080000040000060000020000030000030000 030000010000018000018000018000018000018000018000018000038000038000038000 0300000300000300000700000700000600000600000E00000C00000C00001C0000180000 380000300000700000600000E00000C0000180000100000300000600000C000018000030 0000600000800000112E80A113>I<1C3C3C3C3C040408081020204080060E7D840E>44 D<7FF0FFE07FE00C037D8A10>I<70F8F8F0E005057B840E>I<000F800030E000E07001C0 700380300380380700380F00780F00780E00781E00781E00703C00F03C00F03C00F03C00 F07801E07801E07801E07801C07003C0F003C0F00380F00780F00700700700700E00701C 003038001870000FC000151F7C9D17>48 D<000200020006000E003C00DC031C001C0038 003800380038007000700070007000E000E000E000E001C001C001C001C0038003800380 03800780FFF80F1E7B9D17>I<001F000061800080E00100E00200700220700420700410 700820F00820F00820F00840E00881E00703C0000380000700000C000018000060000080 000300000400000800401000401000802001807E030047FF0041FE0080FC00807800141F 7C9D17>I<0000600000E00000E00000E00001C00001C00001C000038000038000030000 0700000700000600000E00000C0000180000180000300000300000630000C70000870001 0700030700060E00040E00080E003F8E00607C00801FC0001C00001C0000380000380000 380000380000700000700000600013277E9D17>52 D<001F0000718000C0C00180C00380 E00700E00F00E00F01E01E01E01E01E01E01E01E01C01C03C01C03C01C03C01C07C01C0F 800C0F8006378003C700000F00000E00000E00001C00601C00F03800F07000E0600080C0 004380003E0000131F7B9D17>57 D<00000200000006000000060000000E0000001E0000 001E0000003F0000002F0000004F0000004F0000008F0000010F0000010F0000020F0000 020F0000040F00000C0F0000080F0000100F0000100F0000200F80003FFF800040078000 C007800080078001000780010007800200078002000780060007801E000F80FF807FF81D 207E9F22>65 D<01FFFFC0001E00F0001E0078001E0038001E003C003C003C003C003C00 3C003C003C003C0078007800780078007800F0007801E000F0078000FFFE0000F00F8000 F003C001E001C001E001E001E001E001E001E003C001E003C001E003C001E003C001C007 8003C00780078007800F0007801E000F007800FFFFE0001E1F7D9E20>I<0000FE020007 8186001C004C0038003C0060003C00C0001C01C0001803800018070000180F0000181E00 00101E0000103C0000003C00000078000000780000007800000078000000F0000000F000 0000F0000000F0000000F000008070000080700000807000010038000100380002001800 04000C001800060020000381C00000FE00001F217A9F21>I<01FFFF80001E00E0001E00 70001E0038001E001C003C001C003C000E003C000E003C000E0078000E0078000E007800 0E0078000E00F0001E00F0001E00F0001E00F0001E01E0003C01E0003C01E0003C01E000 7803C0007003C0007003C000E003C001C0078001C00780038007800E0007801C000F0070 00FFFFC0001F1F7D9E22>I<01FFFFFE001E001C001E000C001E0004001E0004003C0004 003C0004003C0004003C00040078080800780800007808000078180000F0300000FFF000 00F0300000F0300001E0200001E0200001E0200001E0001003C0002003C0002003C00040 03C00040078000800780018007800100078007000F001F00FFFFFE001F1F7D9E1F>I<01 FFFFFC001E0038001E0018001E0008001E0008003C0008003C0008003C0008003C000800 78001000780800007808000078080000F0100000F0300000FFF00000F0300001E0200001 E0200001E0200001E0200003C0000003C0000003C0000003C00000078000000780000007 800000078000000F800000FFF800001E1F7D9E1E>I<0000FC040007030C001C00980030 007800E0007801C000380380003003800030070000300E0000301E0000201E0000203C00 00003C00000078000000780000007800000078000000F0000000F000FFF0F0000780F000 0780F0000F0070000F0070000F0070000F0070001E0038001E0018003E001C002E000E00 CC000383040000FC00001E217A9F23>I<01FFF3FFE0001F003E00001E003C00001E003C 00001E003C00003C007800003C007800003C007800003C007800007800F000007800F000 007800F000007800F00000F001E00000FFFFE00000F001E00000F001E00001E003C00001 E003C00001E003C00001E003C00003C007800003C007800003C007800003C00780000780 0F000007800F000007800F000007800F00000F801F0000FFF1FFE000231F7D9E22>I<01 FFF0001F00001E00001E00001E00003C00003C00003C00003C0000780000780000780000 780000F00000F00000F00000F00001E00001E00001E00001E00003C00003C00003C00003 C0000780000780000780000780000F8000FFF800141F7D9E12>I<001FFF0000F80000F0 0000F00000F00001E00001E00001E00001E00003C00003C00003C00003C0000780000780 000780000780000F00000F00000F00000F00001E00001E00301E00781E00F83C00F83C00 F0780080700040E00021C0001F000018207D9E18>I<01FFF03FE0001F000F80001E000E 00001E000800001E001000003C002000003C004000003C010000003C0200000078040000 00780800000078100000007830000000F0F0000000F1F8000000F278000000F478000001 E83C000001F03C000001E03C000001E01E000003C01E000003C01E000003C00F000003C0 0F000007800F00000780078000078007800007800780000F8007C000FFF03FF800231F7D 9E23>I<01FFF800001F0000001E0000001E0000001E0000003C0000003C0000003C0000 003C00000078000000780000007800000078000000F0000000F0000000F0000000F00000 01E0000001E0000001E0000001E0008003C0010003C0010003C0030003C0020007800600 0780060007800C0007801C000F007800FFFFF800191F7D9E1D>I<01FE00007FC0001E00 00FC00001E0000F80000170001780000170001780000270002F00000270004F000002700 04F00000270008F00000470009E00000470011E00000470021E00000470021E000008700 43C00000838043C00000838083C00000838083C000010381078000010382078000010382 0780000103840780000203840F00000203880F00000203900F00000203900F00000401E0 1E00000401E01E00000401C01E00000C01801E00001C01803E0000FF8103FFC0002A1F7D 9E29>I<01FF007FE0001F000F00001F0004000017800400001780040000278008000023 C008000023C008000023C008000041E010000041E010000041F010000040F010000080F0 200000807820000080782000008078200001003C400001003C400001003C400001001E40 0002001E800002001E800002000F800002000F800004000F000004000700000400070000 0C000700001C00020000FF80020000231F7D9E22>I<0001FC0000070700001C01C00030 00E000E0006001C000700380007007800038070000380E0000381E0000381C0000383C00 00383C00003878000078780000787800007878000078F00000F0F00000F0F00000E0F000 01E0F00001C0F00003C0700003807000070078000F0038001E0038003C001C0070000E00 E0000783800001FC00001D217A9F23>I<01FFFF80001E00E0001E0070001E0038001E00 3C003C003C003C003C003C003C003C003C0078007800780078007800F0007800E000F003 C000F00F0000FFFC0000F0000001E0000001E0000001E0000001E0000003C0000003C000 0003C0000003C00000078000000780000007800000078000000F800000FFF000001E1F7D 9E1F>I<01FFFF00001E03C0001E00E0001E0070001E0078003C0078003C0078003C0078 003C0078007800F0007800F0007801E0007801C000F0070000F01E0000FFF00000F03800 01E01C0001E01E0001E00E0001E00F0003C01E0003C01E0003C01E0003C01E0007803C00 07803C0807803C0807803C100F801C10FFF00C20000007C01D207D9E21>82 D<0007E040001C18C0003005800060038000C0038001C001800180010003800100038001 00038001000380000003C0000003C0000003F8000001FF800001FFE000007FF000001FF0 000001F80000007800000078000000380000003800200038002000380020003000600070 00600060006000E0007000C000E8038000C606000081F800001A217D9F1A>I<0FFFFFF0 1E0780E0180780201007802020078020200F0020600F0020400F0020400F0020801E0040 001E0000001E0000001E0000003C0000003C0000003C0000003C00000078000000780000 007800000078000000F0000000F0000000F0000000F0000001E0000001E0000001E00000 01E0000003E00000FFFF00001C1F789E21>I<7FFC1FF807C003C0078001000780010007 8001000F0002000F0002000F0002000F0002001E0004001E0004001E0004001E0004003C 0008003C0008003C0008003C00080078001000780010007800100078001000F0002000F0 002000F0002000F0004000F0004000700080007001000030020000380400000C18000007 E000001D20779E22>I I<00F1800389C00707800E03801C03803C0380380700780700780700780700F00E00F00E 00F00E00F00E20F01C40F01C40703C40705C40308C800F070013147C9317>97 D<07803F8007000700070007000E000E000E000E001C001C001CF01D0C3A0E3C0E380F38 0F700F700F700F700FE01EE01EE01EE01CE03CE038607060E031C01F0010207B9F15>I< 007E0001C1000300800E07801E07801C07003C0200780000780000780000F00000F00000 F00000F00000F0000070010070020030040018380007C00011147C9315>I<0000780003 F80000700000700000700000700000E00000E00000E00000E00001C00001C000F1C00389 C00707800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E 20F01C40F01C40703C40705C40308C800F070015207C9F17>I<007C01C207010E011C01 3C013802780C7BF07C00F000F000F000F0007000700170023804183807C010147C9315> I<00007800019C00033C00033C000718000700000700000E00000E00000E00000E00000E 0001FFE0001C00001C00001C00001C000038000038000038000038000038000070000070 0000700000700000700000700000E00000E00000E00000E00000C00001C00001C0000180 003180007B0000F300006600003C00001629829F0E>I<003C6000E27001C1E00380E007 00E00F00E00E01C01E01C01E01C01E01C03C03803C03803C03803C03803C07003C07001C 0F001C17000C2E0003CE00000E00000E00001C00001C00301C00783800F0700060E0003F 8000141D7E9315>I<01E0000FE00001C00001C00001C00001C000038000038000038000 038000070000070000071E000763000E81800F01C00E01C00E01C01C03801C03801C0380 1C0380380700380700380700380E10700E20700C20701C20700C40E00CC060070014207D 9F17>I<00C001E001E001C000000000000000000000000000000E003300230043804300 470087000E000E000E001C001C001C003840388030807080310033001C000B1F7C9E0E> I<01E0000FE00001C00001C00001C00001C0000380000380000380000380000700000700 000703C00704200E08E00E11E00E21E00E40C01C80001D00001E00001FC00038E0003870 00387000383840707080707080707080703100E03100601E0013207D9F15>107 D<03C01FC0038003800380038007000700070007000E000E000E000E001C001C001C001C 0038003800380038007000700070007100E200E200E200E200640038000A207C9F0C>I< 1C0F80F0002630C318004740640C004780680E004700700E004700700E008E00E01C000E 00E01C000E00E01C000E00E01C001C01C038001C01C038001C01C038001C01C070803803 8071003803806100380380E10038038062007007006600300300380021147C9325>I<1C 0F802630C04740604780604700704700708E00E00E00E00E00E00E00E01C01C01C01C01C 01C01C03843803883803083807083803107003303001C016147C931A>I<007C0001C300 0301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F003C0F003C0F00780 F00700700F00700E0030180018700007C00013147C9317>I<01C1E002621804741C0478 1C04701E04701E08E01E00E01E00E01E00E01E01C03C01C03C01C03C01C0380380780380 700380E003C1C0072380071E000700000700000E00000E00000E00000E00001C00001C00 00FFC000171D809317>I<00F0400388C00705800E03801C03803C038038070078070078 0700780700F00E00F00E00F00E00F00E00F01C00F01C00703C00705C0030B8000F380000 380000380000700000700000700000700000E00000E0000FFE00121D7C9315>I<1C1E00 2661004783804787804707804703008E00000E00000E00000E00001C00001C00001C0000 1C000038000038000038000038000070000030000011147C9313>I<00FC030206010C03 0C070C060C000F800FF007F803FC003E000E700EF00CF00CE008401020601F8010147D93 13>I<018001C0038003800380038007000700FFF007000E000E000E000E001C001C001C 001C003800380038003820704070407080708031001E000C1C7C9B0F>I<0E00C03300E0 2301C04381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C0710 1C0E20180E20180E201C1E200C264007C38014147C9318>I<0E03803307802307C04383 C04301C04700C08700800E00800E00800E00801C01001C01001C01001C02001C02001C04 001C04001C08000E300003C00012147C9315>I<0E00C1C03300E3C02301C3E04381C1E0 4301C0E04701C060870380400E0380400E0380400E0380401C0700801C0700801C070080 1C0701001C0701001C0602001C0F02000C0F04000E13080003E1F0001B147C931E>I<03 83800CC4401068E01071E02071E02070C040E00000E00000E00000E00001C00001C00001 C00001C040638080F38080F38100E5810084C60078780013147D9315>I<0E00C03300E0 2301C04381C04301C04701C08703800E03800E03800E03801C07001C07001C07001C0700 1C0E00180E00180E001C1E000C3C0007DC00001C00001C00003800F03800F07000E06000 C0C0004380003E0000131D7C9316>I123 D E /Fm 49 121 df<1C3E7E7E3A0202040408081020C0070E7D840D>44 DI<3078F8787005057C840D>I<00000400000C00000C00001800 00180000300000300000600000600000C00000C000018000018000030000030000060000 0600000C00000C0000180000180000300000300000600000600000C00000C00001800001 80000300000300000600000600000600000C00000C000018000018000030000030000060 0000600000C00000C00000800000162D7EA117>I<007E0001C3000301800701C00E00C0 0E00E01C00E01C00E03C01E03801E07801E07801E07801E07801E07801E0F003C0F003C0 F003C0F003C0F003C0F00380F00780E00780E00700E00700E00E00600E00701C00303800 3870000FC000131F7C9D17>I<000C001C00FC0F38003800380038003800380070007000 7000700070007000E000E000E000E000E000E001C001C001C001C001C001C0038003C0FF FE0F1E7C9D17>I<003F8000C1E00100F00200780400780400780F007C0F807C0F807C0F 00780600780000F80000F00001E00001C0000380000700000E00001C0000380000600000 C0000180000300200600200800401000403FFFC07FFF80FFFF80161E7E9D17>I<007F00 0183C00201E00400F00700F00F00F00F01F00F01F00001E00001E00003C0000380000700 000E0000F800000E000007000007800007C00003C00007C03007C07807C0F807C0F807C0 F00780800F00400E00201C0018780007E000141F7D9D17>I<0000600000600000E00001 C00003C00005C0000DC00009C00011C000238000438000C3800083800103800203800407 00080700180700100700200700400700FFFFF0000E00000E00000E00000E00000E00001C 00001E0001FFE0141E7E9D17>I<01803001FFE003FFC003FF0003FC0002000002000002 0000040000040000040000047C000587000603800C01800801C00001C00001E00001E000 01E00001E07003C0F803C0F003C0E00380800780400700400E00201C0018700007C00014 1F7D9D17>I<000F8000704000C0200180E00301E00701E00E00C01E00001C00003C0000 38000078F800790E007A07007C0300F80380F80380F003C0F003C0F003C0F003C0F00780 E00780E00780E00700E00F00600E00701C0030180018700007C000131F7C9D17>I<2000 003FFFE07FFFC07FFF80400100C002008002008004000008000010000020000040000040 000080000180000300000300000700000600000E00000E00001E00001C00001C00003C00 003C00003C0000780000780000780000300000131F799D17>I<003F0000C1C001006002 00600400300C00300C00300C00300C00600E00600F80C00FC18007F60003FC0001FC0001 FF00063F800C0F801007C03003C06001C06000C0C000C0C000C0C000C0C00080C0010060 030030040018180007E000141F7D9D17>I<007E0001C3000301800601C00E01C01C00C0 3C00E03C00E03C01E07801E07801E07801E07801E07803E07803E03803C03807C01C0BC0 0C13C003E380000780000780000700000E00600E00F01C00F01800E0300080600041C000 3F0000131F7C9D17>I<07E01838201C401C701CF03CF03C603C0038007000E001C00180 0300020006000400040008000800080008000000000000000000000030007800F8007800 70000E20799F15>63 D<0000100000001800000038000000380000007800000078000000 FC000001BC0000013C0000033C0000023C0000063C0000043E0000081E0000081E000010 1E0000101E0000201E0000200F0000400F0000400F0000FFFF0000800F0001000F800100 0780020007800200078004000780040007800C0007C03E0007C0FF807FFC1E207E9F22> 65 D<07FFFF00007C01C0003C01E0003C00F0007800F8007800F8007800F8007800F800 7800F8007800F000F001F000F001E000F003C000F00F8000FFFE0000F00F0001E007C001 E003C001E003E001E001E001E001E001E001E003C001E003C003E003C003E003C003C003 C007C003C00F8007800F0007803E00FFFFF0001D1F7E9E20>I<0001F808000E06180038 0138007000F801E0007803C0007007800030078000300F0000301F0000301E0000303E00 00203C0000007C0000007C0000007C0000007C000000F8000000F8000000F8000000F800 0000F80000007800004078000080780000803C0000803C0001001C0002000E0002000600 0C000300100001C0E000003F00001D217B9F21>I<07FFFF00007C01E0003C00F0003C00 780078003C0078003C0078001E0078001E0078001E0078001F00F0001F00F0001F00F000 1F00F0001F00F0001F00F0001F01E0001E01E0003E01E0003E01E0003E01E0003C01E000 7C03C0007803C000F003C000F003C001E003C003C003C0078007800F0007803C00FFFFE0 00201F7E9E23>I<07FFFFF8007C0078003C0038003C0018007800180078000800780008 00780008007800080078080800F0100000F0100000F0100000F0300000FFF00000F07000 01E0200001E0200001E0200001E0200001E0000801E0001003C0001003C0001003C00020 03C0002003C0006003C000C0078001C0078007C0FFFFFF801D1F7E9E1F>I<07FFFFF800 7C0078003C0038003C001800780018007800080078000800780008007800080078000800 F0100000F0100000F0100000F0300000F0700000FFF00001E0600001E0200001E0200001 E0200001E0200001E0000003C0000003C0000003C0000003C0000003C0000003C0000007 80000007C00000FFFE00001D1F7E9E1E>I<0001FC04000F030C003C009C0070007C00E0 003C01C0003803800018078000180F0000181F0000181E0000183E0000103C0000007C00 00007C0000007C0000007C000000F8000000F8000000F8007FFCF80003E0780001E07800 01E0780003C0780003C03C0003C03C0003C01C0003C00E0007C007000B800380118001E0 6080003F80001E217B9F24>I<07FFC7FFC0007C00F800003C007800003C007800007800 F000007800F000007800F000007800F000007800F000007800F00000F001E00000F001E0 0000F001E00000F001E00000FFFFE00000F001E00001E003C00001E003C00001E003C000 01E003C00001E003C00001E003C00003C007800003C007800003C007800003C007800003 C007800003C007800007800F000007C00F8000FFF8FFF800221F7E9E22>I<07FFE0007C 00003C00003C0000780000780000780000780000780000780000F00000F00000F00000F0 0000F00000F00001E00001E00001E00001E00001E00001E00003C00003C00003C00003C0 0003C00003C00007800007C000FFFC00131F7F9E10>I<07FFE07FE0007C001F00003C00 0C00003C0018000078001000007800400000780080000078010000007802000000780400 0000F008000000F010000000F060000000F0F0000000F1F0000000F278000001E4780000 01E878000001F03C000001E03C000001E01E000001E01E000003C00F000003C00F000003 C00F000003C007800003C007800003C003C000078003C00007C007E000FFFC3FFC00231F 7E9E23>75 D<07FFF000007E0000003C0000003C00000078000000780000007800000078 0000007800000078000000F0000000F0000000F0000000F0000000F0000000F0000001E0 000001E0000001E0000001E0000001E0008001E0010003C0010003C0010003C0030003C0 020003C0060003C0060007801E0007807C00FFFFFC00191F7E9E1C>I<07FC0000FFC000 7C0000F800003C00017800003C00017800004E0002F000004E0002F000004E0004F00000 4E0004F000004E0008F000004E0008F00000870011E00000870011E00000870021E00000 870021E00000870041E00000838041E00001038083C00001038083C00001038103C00001 038203C0000101C203C0000101C403C0000201C40780000201C80780000201C807800002 01D00780000200F00780000600E00780000600E00F00000F00C00F8000FFE0C1FFF8002A 1F7E9E2A>I<07FC01FFC0003E003E00003E001800003E001800004F001000004F001000 004780100000478010000043C010000043C010000083C020000081E020000081E0200000 80F020000080F020000080782000010078400001007C400001003C400001003C40000100 1E400001001E400002000F800002000F800002000F800002000780000200078000060003 800006000300000F00010000FFE0010000221F7E9E22>I<0003F800001E0E0000380700 00E0038001C001C003C001E0078000E00F0000F00F0000F01E0000F01E0000F83E0000F8 3C0000F87C0000F87C0000F87C0000F87C0000F8F80001F0F80001F0F80001F0F80001F0 F80003E0780003E0780003C0780007C07C0007803C000F003C001E001E001C000E003800 0700F00003C3C00000FE00001D217B9F23>I<07FFFF00007C03C0003C01E0003C00F000 7800F0007800F8007800F8007800F8007800F8007800F000F001F000F001E000F003C000 F0078000F00F0000FFF80001E0000001E0000001E0000001E0000001E0000001E0000003 C0000003C0000003C0000003C0000003C0000003C000000780000007C00000FFFC00001D 1F7E9E1F>I<0003F800001E0E000038070000F0038001E001C003C001E0078001E00F00 00F00F0000F01F0000F01E0000F83E0000F83C0000F87C0000F87C0000F87C0000F87C00 00F8F80001F0F80001F0F80001F0F80001F0F80003E0780003E0780003C0780007C0781E 07803C210F003C409E001E409C000E80F8000740F00003C1C04000FEC0400000E0400000 E0800000E1800000FF800000FF000000FF0000007E0000003C001D297B9F23>I<07FFFC 00007C0700003C03C0003C01E0007801E0007801F0007801F0007801F0007801F0007801 E000F003E000F003C000F0078000F00F0000F03C0000FFF00001E0300001E0380001E01C 0001E01C0001E01C0001E01E0003C03E0003C03E0003C03E0003C03E0003C03E0003C03E 0207803E0407C01F04FFFC0F18000003E01F207E9E21>I<003F040060CC01803C03801C 03001C0700180600080E00080E00080E00080E00000F00000F80000FE00007FE0003FF80 01FFC0007FE00007E00001E00000E00000F00000F04000E04000E04000E04000E06000C0 600180E00380F80300C60C0081F80016217D9F19>I<3FFFFFF03C0780F0300780306007 8030400F0010400F0010C00F0010800F0010800F0010800F0010001E0000001E0000001E 0000001E0000001E0000001E0000003C0000003C0000003C0000003C0000003C0000003C 000000780000007800000078000000780000007800000078000000F0000001F800007FFF E0001C1F7A9E21>II< FFF003FE1F8000F80F0000600F0000400F0000400F800080078001800780010007800200 0780020007C0040003C0040003C0080003C0080003C0100003E0100001E0200001E02000 01E0400001E0400001F0800000F1000000F1000000F2000000F2000000FC0000007C0000 00780000007800000070000000700000002000001F207A9E22>II<03FFC0FFC0007F007E00003E003800001E003000001E00200000 0F004000000F008000000F81000000078200000007C600000003C400000003E800000001 F000000001F000000000F000000000F800000000F8000000017C000000023C000000043C 0000000C1E000000081E000000101F000000200F000000400F800000C0078000008007C0 00010003C000070003E0001F8007E000FFE01FFE00221F7F9E22>II<03FFFFC003E007800380078007000F0006001E0004 003C000C003C00080078000800F0000801F0000001E0000003C000000780000007800000 0F0000001E0000003C0000003C00000078000000F0010001E0010001E0020003C0020007 8002000F8006000F0004001E000C003C001C007C003C007800F800FFFFF8001A1F7D9E1C >I<01FC07060E0F1C0F380E78007000F000F000F000F000E000E000E000E000F0027004 300818300FC010147C9314>99 D<0000700003F00000F00000700000700000E00000E000 00E00000E00000E00000E00001C000F9C00305C00E03C01C03C03801C0780380700380F0 0380F00380F00380F00380E00700E00700E00700E00700E00700700F00301E00186F000F 8FE014207C9F19>I<00F800070E000E07001C0700380380780380700380F00380F00380 FFFF80F00000E00000E00000E00000E00000F001007002003004001C180007E00011147D 9314>I<01C003E003E003C0018000000000000000000000000003801F80078003800380 0700070007000700070007000E000E000E000E000E000E001C001E00FF800B1F7F9E0C> 105 D<038F801F90E007A0E003C0600380600780E00700E00700E00700E00700E00700E0 0E01C00E01C00E01C00E01C00E01C00E01C01C03801E03C0FFCFF815147E9319>110 D<00FC000387000E01801C00C03800E03800E07000F0F000F0F000F0F000F0F000F0E001 E0E001E0E001C0E003C0F00380700700380E001C1C0007E00014147D9317>I<00800100 01000100030007000F001E00FFF80E000E000E000E001C001C001C001C001C001C003800 38203820382038203840384018800F000D1C7C9B12>116 D<1C0380FC1F803C07801C03 801C0380380700380700380700380700380700380700700E00700E00700E00700E00701E 00701E00703C00305E001F9FC012147B9319>I<1FF0FF03C07801C06001C04000E08000 E180007300007600003C00003C00001C00002E00004E000087000107000203800603800C 01C03E03E0FF07FC18147F9318>120 D E /Fn 62 123 df<000FF01FE0003FF07FE000 FFF0FFE001FFF1FFE001F003F00003E003E00003C007C00007C007C00007C007C00007C0 07C00007C007C00007C007C00007C007C00007C007C00007C007C000FFFF07FF00FFFF07 FF00FFFF07FF0007C007C00007C007C00007C007C00007C007C00007C007C00007C007C0 0007C007C00007C007C00007C007C00007C007C00007C007C00007C007C00007C007C000 07C007C00007C007C00007C007C00007C007C00007C007C00007C007C00007C007C00007 C007C00007C007C00007C007C00007C007C000232A7FA923>11 D<00000F80003F8F8000 7F8F8000FF8F8001FF8F8003E0000003C0000007C0000007C0000007C0000007C0000007 C0000007C0000007C0000007C0000007C00000FFFF8F80FFFF8F80FFFF8F8007C00F8007 C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007 C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007C00F8007 C00F8007C00F8007C00F8007C00F8007C00F80192B7FAA20>I44 DII<0000 0600000E00000E00001C00001C00001C0000380000380000380000700000700000E00000 E00000E00001C00001C00001C0000380000380000380000700000700000700000E00000E 00000E00001C00001C00001C0000380000380000700000700000700000E00000E00000E0 0001C00001C00001C0000380000380000380000700000700000700000E00000E00000E00 001C00001C0000380000380000380000700000700000700000E00000E00000C00000173C 7DAC1E>I<007F000001FFC00007FFF0000FFFF8000FC1F8001F007C003F007E003E003E 003C001E007C001F007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F 80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F 80F8000F80F8000F8078000F007C001F007C001F007C001F003E003E003E003E003F007E 001F80FC000FC1F8000FFFF80007FFF00001FFC000007F000019297EA71E>I<00180000 380000F80007F800FFF800FFF800F8F80000F80000F80000F80000F80000F80000F80000 F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000 F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F8007F FFF07FFFF07FFFF014287BA71E>I<00FE0003FFC007FFE00FFFF01F03F83C00FC38007E 78003E70003EF0001FF0001F60001F20001F00001F00001F00001F00003E00003E00007C 00007C0000F80001F00001E00003C0000780000F00001E00003C0000780000F00001E000 03C0000780000F00001E00003C00007FFFFF7FFFFF7FFFFF7FFFFF18287EA71E>I<007F 000001FFC00007FFF0000FFFF8001FC1F8003E007C003C003E0078003E0038003E001000 3E0000003E0000003E0000003C0000007C000000FC000001F8000007F00000FFE00000FF C00000FFE00000FFF0000001FC0000007C0000003E0000001F0000001F0000000F800000 0F8000000F8000000F8000000F8040000F8060001F00F0001F00F8003F007E007E003F81 FC001FFFF8000FFFF00003FFE000007F000019297EA71E>I<0003F0000007F0000005F0 00000DF000000DF000001DF0000039F0000039F0000079F0000079F00000F1F00000F1F0 0001E1F00003E1F00003E1F00007C1F00007C1F0000F81F0000F81F0001F01F0001F01F0 003E01F0007C01F0007C01F000F801F000FFFFFF80FFFFFF80FFFFFF80FFFFFF800001F0 000001F0000001F0000001F0000001F0000001F0000001F0000001F0000001F0000001F0 0019277EA61E>I<3FFFFC3FFFFC3FFFFC3FFFFC3E00003E00003E00003E00003E00003E 00003E00003E00003E00003E00003E3F003EFFC03FFFE03FFFF03FE1F83F807C3F003E3E 003E00003E00001F00001F00001F00001F00001F00001F00001F20001F60003E70003EF8 007C7C00FC3F03F81FFFF00FFFE007FF8000FE0018287EA61E>I<000FF000003FFC0000 FFFC0001FFFC0003F80C0007E000000FC000000F8000001F0000001E0000003E0000003C 0000007C0000007C0000007C3FE000F8FFF000F9FFF800FBFFFC00FF807E00FF003E00FE 003F00FC001F00FC001F00FC000F80F8000F80F8000F80F8000F80F8000F8078000F807C 000F807C000F807C000F003E001F003E001F001F003E001F807C000FC1FC0007FFF80003 FFF00001FFC000007F000019297EA71E>I I<007F000001FFC00007FFF0000FFFF8001FC1FC003F007E003E003E007E003F007C001F 007C001F007C001F007C001F007C001F003E003E003E003E001F007C000FC1F80007FFF0 0003FFE00003FFE0000FFFF8001FC1FC003F007E003E003E007C001F007C001F00F8000F 80F8000F80F8000F80F8000F80F8000F80F8000F807C001F007C001F007E003F003F007E 001FC1FC000FFFF80007FFF00003FFE000007F000019297EA71E>I<007F000001FFC000 03FFE0000FFFF0000FC1F8001F007C003E007C007C003E007C001E007C001F00F8001F00 F8001F00F8000F00F8000F80F8000F80F8000F80F8000F80F8001F807C001F807C001F80 7E003F803E007F803F00FF801FFFEF800FFFCF8007FF8F8003FE1F0000001F0000001F00 00001E0000003E0000003E0000007C0000007C000000F8001801F0001E07E0003FFFC000 1FFF80000FFE000003F8000019297EA71E>I<03FC001FFF807FFFC0FFFFE0FC07F07001 F04000F80000F80000F80000F80000F80001F80001F00007F0000FE0001FC0001F80003E 00003E00007C0000780000F80000F00000F00000F00000F00000F00001F00001F00001F0 0000000000000000000000000000000000000000000001F00001F00001F00001F00001F0 00152A7DA91C>63 D<0001F000000003F800000003F800000007FC00000007BC00000007 BC0000000F3E0000000F1E0000000F1E0000001F1F0000001E1F0000001E0F0000003E0F 8000003C0F8000003C078000007C07C000007807C00000F803E00000F803E00000F003E0 0001F001F00001F001F00001E001F00003E000F80003E000F80003C000F80007FFFFFC00 07FFFFFC000FFFFFFE000F80003E000F80003E001F00003F001F00001F001E00001F003E 00000F803E00000F803C00000F807C000007C07C000007C078000007C0F8000003E0F800 0003E0232A7EA928>65 DI<0001FF00000FFFE0003FFFF8007FFFF800FE01F801F8003003F0001007C0 00000F8000001F8000001F0000003E0000003E0000007E0000007C0000007C0000007C00 0000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F800 0000F80000007C0000007C0000007C0000007E0000003E0000003E0000001F0000001F80 00000F80000007C0000003F0000401F8001C00FE00FC007FFFFC003FFFF8000FFFE00001 FF001E2C7CAA26>IIII< 0001FF00000FFFE0003FFFFC007FFFFE00FF01FE01F8003E03F0000C07C000000FC00000 1F8000001F0000003F0000003E0000007E0000007C0000007C0000007C000000F8000000 F8000000F8000000F8000000F8000000F8000000F8000000F8001FFEF8001FFEF8001FFE 7C001FFE7C00003E7C00003E7E00003E3E00003E3F00003E1F00003E1F80003E0FC0003E 07C0003E03F0003E01F8003E00FF00FE007FFFFE003FFFFC000FFFE00001FF001F2C7CAA 28>III76 DII<0001FC0000000FFF8000003FFFE000007FFFF00001FE03FC0003F800FE0007E000 3F0007C0001F000F80000F801F000007C01F000007C03E000003E03E000003E07C000001 F07C000001F07C000001F078000000F0F8000000F8F8000000F8F8000000F8F8000000F8 F8000000F8F8000000F8F8000000F8F8000000F8F8000000F8F8000000F87C000001F07C 000001F07C000001F07E000003F03E000003E03F000007E01F000007C01F80000FC00FC0 001F8007E0003F0007F0007F0003F800FE0001FE03FC0000FFFFF800003FFFE000000FFF 80000001FC0000252C7DAA2C>II82 D<007FC00001FFF80007FFFE000FFFFF001FC07F003F000F007E0006007C0000 007C000000F8000000F8000000F8000000F8000000F8000000FC0000007E0000007F0000 003F8000001FF800000FFF800007FFE00003FFF80000FFFC00000FFE000000FF0000003F 0000001F8000000F8000000FC0000007C0000007C0000007C0000007C0000007C0000007 C000000F8060000F80F0001F00FC003F00FF80FE007FFFFC001FFFF80007FFE00000FF80 001A2C7DAA21>IIIII<01FE000FFF803FFFC03FFFE03C03 F03001F00001F80000F80000F80000F80000F80000F8007FF807FFF81FFFF83FE0F87F00 F8FC00F8F800F8F800F8F800F8FC01F87E07F87FFFF83FFFF81FFCF80FE0F8151B7E9A1D >97 DI<007FC001FFF007FF FC0FFFFC1FC07C1F00083E00007C00007C00007C0000F80000F80000F80000F80000F800 00F80000F800007C00007C00007E00003E00001F000C1FC07C0FFFFC07FFFC01FFF0007F 80161B7E9A1B>I<00003E00003E00003E00003E00003E00003E00003E00003E00003E00 003E00003E00003E00003E00003E00003E00FC3E03FF3E07FFFE0FFFFE1FC1FE3F007E3E 003E7C003E7C003EFC003EF8003EF8003EF8003EF8003EF8003EF8003EF8003EFC003E7C 003E7C003E3E007E3F00FE1FC1FE0FFFFE07FFBE03FF3E00FC3E172A7EA91F>I<007E00 03FF8007FFC00FFFE01F83F03F00F03E00787C00787C003878003CFFFFFCFFFFFCFFFFFC FFFFFCF80000F80000F800007800007C00007C00003E00003F000C1FC07C0FFFFC07FFFC 01FFF0007F80161B7E9A1B>I<001FC0007FC000FFC001FFC003F00003E00007C00007C0 0007C00007C00007C00007C00007C00007C00007C000FFFE00FFFE00FFFE0007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0 0007C00007C00007C00007C00007C00007C00007C00007C00007C00007C000122A7FA912 >I<00F8078003FE7FC00FFFFFC01FFFFFC01F07C0003E03E0003E03E0007C01F0007C01 F0007C01F0007C01F0007C01F0007C01F0003E03E0003E03E0001F07C0001FFFC0003FFF 80003BFE000038F8000078000000780000003C0000003FFFC0003FFFF8001FFFFC001FFF FE003FFFFF007C007F00F8001F80F8000F80F8000F80F8000F80FC001F807E003F003F80 FE003FFFFE000FFFF80007FFF00000FF80001A287E9A1E>III107 DIII<007F000001FFC00007FFF0000FFFF8001FC1FC003F007E003E003E 007C001F007C001F0078000F00F8000F80F8000F80F8000F80F8000F80F8000F80F8000F 80F8000F807C001F007C001F007E003F003E003E003F007E001FC1FC000FFFF80007FFF0 0001FFC000007F0000191B7E9A1E>II<00FC3E03 FF3E07FFBE0FFFFE1FC1FE3F80FE3E007E7E003E7C003E7C003EF8003EF8003EF8003EF8 003EF8003EF8003EF8003EFC003E7C003E7E003E3E007E3F00FE1FC1FE1FFFFE0FFFBE03 FF3E01F83E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00 003E00003E17277E9A1F>I I<03FC001FFF803FFFC07FFFC07C07C0F80080F80000F80000F80000FC00007F80007FF8 003FFE001FFF0007FF8000FFC0000FE00007E00003E00003E04003E0E007E0FC0FC0FFFF C07FFF801FFE0003F800131B7E9A17>I<07C00007C00007C00007C00007C00007C00007 C000FFFFC0FFFFC0FFFFC007C00007C00007C00007C00007C00007C00007C00007C00007 C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C04007E1C003 FFE003FFE001FF8000FC0013227FA116>IIII<7C000FC03E001F803F001F001F803E000F807C0007C0FC0003E0F80001F1F00001 FBE00000FFC000007FC000003F8000001F0000001F0000003F8000007FC00000FBC00000 F3E00001F1F00003E0F80007C07C000F807C000F803E001F001F003E000F807E000FC0FC 0007E01B1B809A1C>III E /Fo 18 121 df<00001E000000003E00000000FE00000003FE00 00003FFE0000FFFFFE0000FFFFFE0000FFFFFE0000FFCFFE0000000FFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE000000 0FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE 0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE00007FFFFFFFC07FFFFFFFC07F FFFFFFC07FFFFFFFC0223879B731>49 D<0007FE0000007FFFE00001FFFFF80003FFFFFE 0007F01FFF000F8007FF801F0001FFC03E0000FFE07F8000FFF07FC0007FF0FFE0007FF8 FFF0003FF8FFF0003FFCFFF0003FFCFFF0003FFCFFF0003FFC7FE0003FFC3FC0003FFC1F 80003FFC0000003FFC0000003FF80000007FF80000007FF00000007FF0000000FFE00000 00FFC0000001FF80000001FF00000003FE00000007FC00000007F80000000FF00000001F C00000003F800000007F00000000FC00000001F800000001F0003C0003E0003C0007C000 3C000F000078001E000078003C00007800780000F800F00000F801FFFFFFF803FFFFFFF0 07FFFFFFF00FFFFFFFF01FFFFFFFF03FFFFFFFF07FFFFFFFF0FFFFFFFFF0FFFFFFFFE0FF FFFFFFE0FFFFFFFFE026387BB731>I<0003FF8000001FFFF000007FFFFE0000FE03FF00 01F000FF8003C000FFC00780007FE00FF0007FF00FF8007FF01FFC007FF81FFE007FF81F FE007FF81FFE007FF81FFE007FF81FFE007FF80FFC007FF007F8007FF003F0007FF00000 00FFE0000000FFC0000001FF80000001FF00000003FE00000007FC0000001FF000000FFF C000000FFF8000000FFFF800000003FE00000000FF800000007FE00000003FF00000003F F80000003FFC0000001FFC0000001FFE0000001FFE0200001FFF1FC0001FFF3FE0001FFF 7FF0001FFF7FF0001FFFFFF8001FFFFFF8001FFFFFF8001FFEFFF8001FFEFFF0001FFE7F F0003FFC7FE0003FFC3FC0003FF81F80007FF01FE000FFE007FC03FFC003FFFFFF0001FF FFFE00003FFFF0000007FF800028397CB731>I<00000007C0000000000FC0000000000F C0000000001FC0000000003FC0000000007FC000000000FFC000000000FFC000000001FF C000000003FFC000000007FFC00000000FFFC00000000FFFC00000001EFFC00000003CFF C00000007CFFC0000000F8FFC0000000F0FFC0000001E0FFC0000003C0FFC0000007C0FF C000000F80FFC000000F00FFC000001E00FFC000003C00FFC000007C00FFC00000F800FF C00000F000FFC00001E000FFC00003C000FFC00007C000FFC0000F8000FFC0000F0000FF C0001E0000FFC0003C0000FFC0007C0000FFC000F80000FFC000FFFFFFFFFFC0FFFFFFFF FFC0FFFFFFFFFFC0FFFFFFFFFFC0000001FFC000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FF C0000007FFFFFFC00007FFFFFFC00007FFFFFFC00007FFFFFFC02A377DB631>I<040000 00C00F800007C00FF8007FC00FFFFFFF800FFFFFFF000FFFFFFE000FFFFFFC000FFFFFF8 000FFFFFF0000FFFFFE0000FFFFF80000FFFFE00000FFFF800000F800000000F80000000 0F800000000F800000000F800000000F800000000F800000000F800000000F81FF00000F 8FFFE0000FBFFFF8000FFE03FE000FF000FF000FC000FF800F80007FC00F00007FE00700 007FF00000003FF00000003FF80000003FF80000003FF80000003FFC0000003FFC060000 3FFC1F80003FFC3FC0003FFC7FE0003FFCFFE0003FFCFFF0003FFCFFF0003FFCFFF0003F F8FFE0003FF8FFE0003FF87FC0007FF07F00007FF03C00007FE03E0000FFC01F0000FF80 0FC003FF0007F00FFE0003FFFFFC0001FFFFF000007FFFC000000FFC000026397BB731> I<00000FF80000007FFF000003FFFF80000FFC07C0001FE001E0003FC001F0007F0007F0 00FF000FF001FE001FF803FC003FF807FC003FF80FFC003FF80FF8003FF81FF8001FF01F F8000FE03FF80007C03FF00000003FF00000007FF00000007FF00000007FF00000007FF0 7FF000FFF0FFFE00FFF1F7FF00FFF3807F80FFF6003FE0FFFE001FF0FFFC001FF0FFFC00 0FF8FFF8000FFCFFF8000FFCFFF8000FFEFFF8000FFEFFF0000FFEFFF0000FFFFFF0000F FFFFF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF7FF0000FFF3FF0000FFF 3FF0000FFE3FF0000FFE1FF0000FFE1FF8000FFC0FF8000FFC0FF8001FF807FC001FF003 FC001FF001FE003FE000FF80FFC0007FFFFF00001FFFFE000007FFF8000000FFC0002839 7CB731>I<1E00000000001F00000000001FF0000000001FFFFFFFFFC01FFFFFFFFFC01F FFFFFFFFC03FFFFFFFFFC03FFFFFFFFF803FFFFFFFFF003FFFFFFFFE003FFFFFFFFC003F FFFFFFF8003FFFFFFFF0007FFFFFFFF0007C000007E0007C00000FC0007800001F800078 00001F00007800003E0000F000007E0000F00000FC0000F00001F80000000003F0000000 0003E00000000007E0000000000FC0000000000F80000000001F80000000003F80000000 003F00000000007F00000000007F0000000000FF0000000000FE0000000001FE00000000 01FE0000000003FE0000000003FE0000000003FE0000000007FC0000000007FC00000000 07FC000000000FFC000000000FFC000000000FFC000000000FFC000000000FFC00000000 1FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC00000000 1FFC000000001FFC000000001FFC000000000FF80000000007F00000000003E00000002A 3B7BB931>I<0001FF8000000FFFF800003FFFFE00007F00FF0000F8003F8001F0000FC0 03E0000FE007C00007F007C00007F00FC00003F80FC00003F81FC00003F81FC00003F81F E00003F81FF00003F81FF80003F81FFC0007F01FFF0007F01FFFC007E00FFFF00FE00FFF FC1FC007FFFE3F8007FFFFFF0003FFFFFC0001FFFFF80000FFFFFE00007FFFFF00003FFF FF80003FFFFFE000FFFFFFF001FDFFFFF007F07FFFF80FE01FFFFC1FC007FFFC3F8001FF FE3F8000FFFE7F00003FFF7F00000FFFFE000003FFFE000001FFFE000000FFFE000000FF FE0000007FFE0000007FFE0000007EFE0000007E7F0000007E7F000000FC3F800000FC3F C00001F81FE00003F00FF00007E007FE007FC003FFFFFF8000FFFFFE00003FFFF8000003 FF800028397CB731>I<00000001F80000000000000001F80000000000000003FC000000 0000000003FC0000000000000007FE0000000000000007FE0000000000000007FE000000 000000000FFF000000000000000FFF000000000000001FFF800000000000001FFF800000 000000001FFF800000000000003FFFC00000000000003FFFC00000000000007FFFE00000 000000007DFFE00000000000007DFFE0000000000000FDFFF0000000000000F8FFF00000 00000001F8FFF8000000000001F07FF8000000000001F07FF8000000000003F07FFC0000 00000003E03FFC000000000007E03FFE000000000007C01FFE00000000000FC01FFF0000 0000000F801FFF00000000000F800FFF00000000001F800FFF80000000001F0007FF8000 0000003F0007FFC0000000003E0003FFC0000000003E0003FFC0000000007E0003FFE000 0000007C0001FFE000000000FC0001FFF000000000F80000FFF000000000F80000FFF000 000001FFFFFFFFF800000001FFFFFFFFF800000003FFFFFFFFFC00000003FFFFFFFFFC00 000003E000003FFC00000007E000003FFE00000007C000001FFE0000000FC000001FFF00 00000F8000000FFF0000001F8000000FFF8000001F0000000FFF8000001F00000007FF80 00003F00000007FFC000003E00000003FFC000007E00000003FFE00000FF00000001FFE0 00FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0FFFFFC0001FFFFFFF0FFFFFC0001FFFFFF F0443B7DBA4B>65 D<0000001FFF000030000001FFFFE000F000000FFFFFFC01F000007F FFFFFE03F00001FFFE007F87F00003FFE0000FCFF0000FFF000003FFF0001FFC000001FF F0003FF80000007FF0007FF00000003FF000FFC00000003FF001FFC00000001FF003FF80 0000000FF007FF000000000FF00FFF0000000007F00FFE0000000007F01FFE0000000003 F01FFE0000000003F03FFC0000000003F03FFC0000000001F03FFC0000000001F07FFC00 00000001F07FF80000000001F07FF80000000000007FF8000000000000FFF80000000000 00FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF800 0000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF80000000000 00FFF80000000000007FF80000000000007FF80000000000007FF80000000000007FFC00 00000000F03FFC0000000000F03FFC0000000000F03FFC0000000000F01FFE0000000000 F01FFE0000000001E00FFE0000000001E00FFF0000000001E007FF0000000003C003FF80 00000003C001FFC0000000078000FFE00000000F00007FF00000001F00003FF80000003E 00001FFC0000007C00000FFF000001F8000003FFE00007F0000001FFFE003FC00000007F FFFFFF000000000FFFFFFC0000000001FFFFF000000000001FFF0000003C3D7BBB47>67 D<001FFF00000001FFFFF0000003FFFFFC000007F007FE00000FF801FF00001FFC00FF80 001FFC007FC0001FFC007FE0001FFC003FE0000FF8003FF0000FF8003FF00007F0003FF0 0001C0003FF0000000003FF0000000003FF0000000003FF0000000FFFFF000000FFFFFF0 00007FF83FF00001FF803FF00007FE003FF0000FF8003FF0001FF0003FF0003FE0003FF0 007FE0003FF0007FE0003FF000FFC0003FF000FFC0003FF000FFC0003FF000FFC0003FF0 00FFC0007FF0007FE0007FF0007FE000DFF0003FF0039FF8001FFC0F0FFFF007FFFE0FFF F001FFFC07FFF0003FE000FFF02C267DA530>97 D<0001FFC000000FFFF800003FFFFE00 00FF80FF0001FE003F8007FC001FC00FF8000FE00FF8000FF01FF00007F03FF00007F83F F00007F87FE00007F87FE00003FC7FE00003FC7FE00003FCFFE00003FCFFFFFFFFFCFFFF FFFFFCFFFFFFFFFCFFE0000000FFE0000000FFE0000000FFE00000007FE00000007FE000 00007FE00000003FE00000003FF000003C1FF000003C1FF000003C0FF800007807FC0000 F803FE0001F001FF0007E000FFC03FC0003FFFFF000007FFFC000000FFE00026267DA52D >101 D<00FF00000000FFFF00000000FFFF00000000FFFF00000000FFFF0000000007FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF007FC00003FF 01FFF80003FF07FFFC0003FF0F03FE0003FF1C01FF0003FF3001FF8003FF6000FF8003FF E000FFC003FFC000FFC003FF8000FFC003FF8000FFC003FF8000FFC003FF0000FFC003FF 0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF 0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF 0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF 0000FFC003FF0000FFC003FF0000FFC0FFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFFFFFF FC3FFFFF303C7CBB37>104 D<00FE007FC000FFFE01FFF800FFFE07FFFC00FFFE0F03FE 00FFFE1C01FF0007FE3001FF8003FE6000FF8003FEE000FFC003FEC000FFC003FF8000FF C003FF8000FFC003FF8000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FF C0FFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFF30267CA537>110 D<00FF01FF8000FFFF0FFFF000FFFF3FFFFC00FFFFFE03FF00FFFFF000FF8003FFC0007F C003FF80003FE003FF00003FF003FF00001FF803FF00001FFC03FF00000FFC03FF00000F FE03FF00000FFE03FF000007FE03FF000007FF03FF000007FF03FF000007FF03FF000007 FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007 FE03FF000007FE03FF00000FFE03FF00000FFC03FF00000FFC03FF00001FF803FF00001F F803FF00003FF003FF80003FE003FFC0007FC003FFF001FF8003FFFC07FF0003FF3FFFFC 0003FF0FFFF00003FF01FF000003FF0000000003FF0000000003FF0000000003FF000000 0003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF000000 0003FF0000000003FF0000000003FF00000000FFFFFC000000FFFFFC000000FFFFFC0000 00FFFFFC00000030377DA537>112 D<00FE03F000FFFE0FFE00FFFE1FFF00FFFE3C3F80 FFFE707FC007FE60FFE003FEE0FFE003FEC0FFE003FFC0FFE003FF807FC003FF807FC003 FF803F8003FF800E0003FF00000003FF00000003FF00000003FF00000003FF00000003FF 00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00 000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF0000 0003FF000000FFFFFE0000FFFFFE0000FFFFFE0000FFFFFE000023267DA529>114 D<0007800000078000000780000007800000078000000F8000000F8000000F8000000F80 00001F8000001F8000003F8000003F8000007F800000FF800001FF800007FF80001FFFFF F0FFFFFFF0FFFFFFF0FFFFFFF001FF800001FF800001FF800001FF800001FF800001FF80 0001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF80 0001FF800001FF800001FF800001FF800001FF803C01FF803C01FF803C01FF803C01FF80 3C01FF803C01FF803C01FF803C00FF807800FFC078007FC070003FE0E0001FFFC00007FF 800001FF001E377EB626>116 D120 D E /Fp 31 123 df<007E01C007000E001C003C003800780078007FF0F000F000F00070 0070007000300018000C1807E00F147E9312>15 D<70F8F8F87005057C840D>58 D<70F8FCFC74040404080810102040060E7C840D>I<000001C00000078000001E000000 78000001E00000078000000E00000038000000F0000003C000000F0000003C000000F000 0000F00000003C0000000F00000003C0000000F0000000380000000E0000000780000001 E0000000780000001E0000000780000001C01A1A7C9723>I62 D<000002000000060000000E0000000E0000001E0000001F0000002F0000002F0000004F 0000008F0000008F0000010F0000010F0000020F0000040F0000040F0000080F80000807 80001007800020078000200780007FFF8000400780008007800180078001000780020007 80020007C0040003C00C0003C01E0007C0FF807FFC1E207E9F22>65 D<0FFFFFFC1E03C0381803C0181003C0082003C008200780086007800840078008400780 08800F0010000F0000000F0000000F0000001E0000001E0000001E0000001E0000003C00 00003C0000003C0000003C00000078000000780000007800000078000000F0000000F000 0000F0000000F0000001F000007FFFC0001E1F7F9E1B>84 D<00F1800389C00707800E03 801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E10F01C20F01C 20703C20705C40308C400F078014147E9318>97 D<07803F8007000700070007000E000E 000E000E001C001C001CF01D0C3A0E3C0E380F380F700F700F700F700FE01EE01EE01EE0 1CE03CE038607060E031C01F0010207E9F14>I<007C01C207010E0F1E0F1C0E3C047800 78007800F000F000F000F000F00070017002300418380FC010147E9314>I<0000780003 F80000700000700000700000700000E00000E00000E00000E00001C00001C000F1C00389 C00707800E03801C03803C0380380700780700780700780700F00E00F00E00F00E00F00E 10F01C20F01C20703C20705C40308C400F078015207E9F18>I<007C01C207010E011C01 3C013802780C7BF07C00F000F000F000F0007000700170023004183807C010147E9315> I<00007C0000CE00019E00039E00030C000700000700000700000700000E00000E00000E 0000FFF0000E00000E00001C00001C00001C00001C00001C000038000038000038000038 0000380000700000700000700000700000700000E00000E00000E00000E00000C00001C0 00318000798000F300006200003C000017297E9F16>I<001E3000713800E0F001C07003 80700780700700E00F00E00F00E00F00E01E01C01E01C01E01C01E01C01E03801E03800E 07800E0B8006170001E700000700000700000E00000E00300E00781C00F038006070003F C000151D809316>I<00E001E001E000C000000000000000000000000000000E00130023 804380438043808700070007000E000E001C001C001C20384038403840388019000E000B 1F7E9E10>105 D<0000C00001E00001E00001C000000000000000000000000000000000 0000000000001E0000630000438000838001038001038002070000070000070000070000 0E00000E00000E00000E00001C00001C00001C00001C0000380000380000380000380000 700000700030700078E000F1C0006380003E00001328819E13>I<01E0000FE00001C000 01C00001C00001C0000380000380000380000380000700000700000701E00706100E0870 0E10F00E20F00E40601C80001D00001E00001FC000387000383800383800381C20703840 703840703840701880E01880600F0014207E9F18>I<03C01FC003800380038003800700 0700070007000E000E000E000E001C001C001C001C003800380038003800700070007000 7100E200E200E200E200640038000A207E9F0E>I<1E07C07C00231861860023A0320300 43C03403004380380380438038038087007007000700700700070070070007007007000E 00E00E000E00E00E000E00E00E000E00E01C101C01C01C201C01C038201C01C038401C01 C0184038038018801801800F0024147E9328>I<1E07802318C023A06043C07043807043 80708700E00700E00700E00700E00E01C00E01C00E01C00E03821C03841C07041C07081C 03083803101801E017147E931B>I<007C0001C3000301800E01C01E01C01C01E03C01E0 7801E07801E07801E0F003C0F003C0F003C0F00780F00700700F00700E00301800187000 07C00013147E9316>I<03C1E004621804741C08781C08701E08701E10E01E00E01E00E0 1E00E01E01C03C01C03C01C03C01C0380380780380700380E003C1C0072380071E000700 000700000E00000E00000E00000E00001C00001C0000FFC000171D819317>I<1E1E0023 210023C38043C7804387804383008700000700000700000700000E00000E00000E00000E 00001C00001C00001C00001C000038000018000011147E9315>114 D<007C018203010603060706060E00078007F803FC01FE001F00077007F006F006E00440 0820301FC010147E9315>I<00C000E001C001C001C001C003800380FFF8038007000700 070007000E000E000E000E001C001C001C001C10382038203820384018800F000D1C7F9B 10>I<0F00601180702180E021C0E041C0E04380E08381C00701C00701C00701C00E0380 0E03800E03800E03840E07080C07080C07080E0F1006131003E1E016147E931A>I<0F01 801183C02183E021C1E041C0E04380608380400700400700400700400E00800E00800E00 800E01000E01000C02000E04000E040006180001E00013147E9316>I<0F006060118070 F02180E0F821C0E07841C0E0384380E0188381C0100701C0100701C0100701C0100E0380 200E0380200E0380200E0380400E0380400E0380800E078080060781000709860001F078 001D147E9321>I<03C1C00C62201034701038F02038F020386040700000700000700000 700000E00000E00000E00000E02061C040F1C040F1C080E2C080446300383C0014147E93 1A>I<0F00601180702180E021C0E041C0E04380E08381C00701C00701C00701C00E0380 0E03800E03800E03800E07000C07000C07000E0F00061E0003EE00000E00000E00001C00 78180078380070700060600021C0001F0000141D7E9316>I<01E02003F04007F8C00C1F 8008010000020000040000080000100000600000C0000100000200000400800801001003 003F060061FC0040F80080700013147E9315>I E /Fq 56 122 df<000FF07F00007FF9 FF8000F83FC7C001E07F8FC003E07F0FC007C07F0FC007C03F078007C01F000007C01F00 0007C01F000007C01F000007C01F0000FFFFFFF800FFFFFFF80007C01F000007C01F0000 07C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007 C01F000007C01F000007C01F000007C01F000007C01F000007C01F000007C01F00003FF8 FFF0003FF8FFF0002220809F1F>11 D<000FE000007FF80000F81C0001E07C0003E07C00 07C07C0007C07C0007C0380007C0000007C0000007C0000007C1FE00FFFFFE00FFFFFE00 07C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E00 07C03E0007C03E0007C03E0007C03E0007C03E0007C03E0007C03E003FF9FFC03FF9FFC0 1A20809F1D>I<000FF03FFC00007FFDFFFC0000F83FE0FC0001E03F81FC0003E07F81FC 0007C07F00FC0007C03F00FC0007C01F007C0007C01F007C0007C01F007C0007C01F007C 0007C01F007C00FFFFFFFFFC00FFFFFFFFFC0007C01F007C0007C01F007C0007C01F007C 0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C 0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C0007C01F007C 0007C01F007C003FF8FFE3FF803FF8FFE3FF802920809F2C>15 D<387CFEFFFF7F3B0303 06060E0C18702008107C860F>44 DI<387CFEFEFE7C 3807077C860F>I<01FC0007FF001F07C01E03C03E03E07C01F07C01F07C01F0FC01F8FC 01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F87C 01F07C01F07C01F03E03E01E03C01F8FC007FF0001FC00151D7E9C1A>48 D<00E00001E0000FE000FFE000F3E00003E00003E00003E00003E00003E00003E00003E0 0003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E0 0003E00003E00003E000FFFF80FFFF80111D7C9C1A>I<07F0001FFE00383F007C1F80FE 0FC0FE0FC0FE0FE0FE07E07C07E03807E0000FE0000FC0000FC0001F80001F00003E0000 780000F00000E00001C0000380600700600E00601C00E01FFFC03FFFC07FFFC0FFFFC0FF FFC0131D7D9C1A>I<01FC0007FF000E0F801E0FC03F07E03F07E03F07E03F07E01E0FC0 000FC0000F80001F0001FC0001FC00000F800007C00003E00003F00003F83803F87C03F8 FE03F8FE03F8FE03F0FC03F07807E03C0FC01FFF8003FC00151D7E9C1A>I<0001C00003 C00007C00007C0000FC0001FC0003BC00073C00063C000C3C00183C00383C00703C00E03 C00C03C01803C03803C07003C0E003C0FFFFFEFFFFFE0007C00007C00007C00007C00007 C00007C000FFFE00FFFE171D7F9C1A>I<3803803FFF803FFF003FFE003FFC003FF0003F 800030000030000030000030000033F80037FE003C1F00380F801007C00007C00007E000 07E07807E0FC07E0FC07E0FC07E0FC07C0780FC0600F80381F001FFC0007F000131D7D9C 1A>I<003F0001FFC007E0E00F81E01F03F01E03F03E03F07C03F07C01E07C0000FC1000 FCFF00FDFFC0FD03E0FE01F0FE01F0FC01F8FC01F8FC01F8FC01F87C01F87C01F87C01F8 3C01F03E01F01E03E00F07C007FF8001FE00151D7E9C1A>I<6000007FFFF87FFFF87FFF F07FFFE07FFFC0E00180C00300C00300C00600000C000018000038000038000078000070 0000F00000F00001F00001F00001F00001F00003F00003F00003F00003F00003F00003F0 0001E00000C000151E7D9D1A>I<01FC0007FF000F07801E03C01C01E03C01E03C01E03E 01E03F01E03FC3C01FE3801FFF000FFE0007FF8007FFC01FFFE03C3FF0780FF07803F8F0 01F8F000F8F00078F00078F000707800707C00E03E03C00FFF8003FC00151D7E9C1A>I< 01FC000FFF001F07803E03C07C03E07C01E0FC01F0FC01F0FC01F0FC01F8FC01F8FC01F8 FC01F87C03F87C03F83E05F81FFDF807F9F80041F80001F03C01F07E01F07E03E07E03E0 7E07C03C0780381F001FFC0007F000151D7E9C1A>I<7FFFFFFF80FFFFFFFFC0FFFFFFFF C00000000000000000000000000000000000000000000000000000000000000000000000 00000000000000000000FFFFFFFFC0FFFFFFFFC07FFFFFFF80220F7D9229>61 D<0000E000000000E000000001F000000001F000000001F000000003F800000003F80000 0006FC00000006FC0000000EFE0000000C7E0000000C7E000000183F000000183F000000 303F800000301F800000701FC00000600FC00000600FC00000C007E00000FFFFE00001FF FFF000018003F000018003F000030001F800030001F800060001FC00060000FC000E0000 FE00FFE00FFFE0FFE00FFFE0231F7E9E28>65 DI<0007FC02003FFF0E00FE03DE03F000FE07E0003E0FC0001E1F80001E3F 00000E3F00000E7F0000067E0000067E000006FE000000FE000000FE000000FE000000FE 000000FE000000FE0000007E0000007E0000067F0000063F0000063F00000C1F80000C0F C0001807E0003803F0007000FE01C0003FFF800007FC001F1F7D9E26>I69 DI<0007FC0200003FFF0E0000FE03DE0003F000FE0007E0003E000FC0001E001F 80001E003F00000E003F00000E007F000006007E000006007E00000600FE00000000FE00 000000FE00000000FE00000000FE00000000FE003FFFE0FE003FFFE07E00007E007E0000 7E007F00007E003F00007E003F00007E001F80007E000FC0007E0007E0007E0003F000FE 0000FE01FE00003FFF8E000007FC0600231F7D9E29>I73 D76 DI80 D<001FF80000FFFF0001F81F8007E007E00FC003F01F8001F81F80 01F83F0000FC7F0000FE7F0000FE7E00007EFE00007FFE00007FFE00007FFE00007FFE00 007FFE00007FFE00007FFE00007FFE00007F7E00007E7E00007E7F0000FE3F0000FC3F87 C1FC1F8FE1F80FD833F007F83FE001F81F8000FFFF00001FFE0300001E0300001F070000 0FFF00000FFF000007FE000007FE000003FC000001F8000000F020287D9E27>I<03FC08 0FFF381E03F83800F8700078700038F00038F00018F00018F80000FC00007FC0007FFE00 3FFF801FFFE00FFFF007FFF000FFF80007F80000FC00007C00003CC0003CC0003CC0003C E00038E00078F80070FE01E0E7FFC081FF00161F7D9E1D>83 D<7FFFFFFC7FFFFFFC7C07 E07C7007E01C6007E00C6007E00CE007E00EC007E006C007E006C007E006C007E0060007 E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007E0000007 E0000007E0000007E0000007E0000007E0000007E0000007E0000007E00003FFFFC003FF FFC01F1E7E9D24>I86 D<07FC001FFF003F0F803F07C03F03E03F03E00C03E00003E0007FE007FBE01F03E03C03 E07C03E0F803E0F803E0F803E0FC05E07E0DE03FF8FE0FE07E17147F9319>97 DI<01FE0007 FF801F0FC03E0FC03E0FC07C0FC07C0300FC0000FC0000FC0000FC0000FC0000FC00007C 00007E00003E00603F00C01F81C007FF0001FC0013147E9317>I<0007F80007F80000F8 0000F80000F80000F80000F80000F80000F80000F80000F80000F801F8F80FFEF81F83F8 3E01F87E00F87C00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00F8FC00F87C00F87C00F8 7E00F83E01F81F07F80FFEFF03F8FF18207E9F1D>I<01FE0007FF800F83C01E01E03E00 F07C00F07C00F8FC00F8FFFFF8FFFFF8FC0000FC0000FC00007C00007C00003E00181E00 180F807007FFE000FF8015147F9318>I<001F8000FFC001F3E003E7E003C7E007C7E007 C3C007C00007C00007C00007C00007C000FFFC00FFFC0007C00007C00007C00007C00007 C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C0003F FC003FFC0013207F9F10>I<01FC3C07FFFE0F079E1E03DE3E03E03E03E03E03E03E03E0 3E03E01E03C00F07800FFF0009FC001800001800001C00001FFF800FFFF007FFF81FFFFC 3C007C70003EF0001EF0001EF0001E78003C78003C3F01F80FFFE001FF00171E7F931A> II<1C003E00 7F007F007F003E001C00000000000000000000000000FF00FF001F001F001F001F001F00 1F001F001F001F001F001F001F001F001F001F001F00FFE0FFE00B217EA00E>I<003800 7C00FE00FE00FE007C003800000000000000000000000001FE01FE003E003E003E003E00 3E003E003E003E003E003E003E003E003E003E003E003E003E003E003E003E303E783EFC 3CFC7C78783FF01FC00F2A83A010>IIIII<01FF0007FFC01F83F03E00F83E00F87C007C7C007CFC007EFC007EFC007EFC00 7EFC007EFC007E7C007C7C007C3E00F83E00F81F83F007FFC001FF0017147F931A>II<01F81807FE381F87783F01F83E01F8 7E00F87C00F8FC00F8FC00F8FC00F8FC00F8FC00F8FC00F87C00F87E00F87E00F83F01F8 1F87F80FFEF803F8F80000F80000F80000F80000F80000F80000F80000F80007FF0007FF 181D7E931C>II<0FE63FFE701E600EE006E006F800FFC07FF83FFC1FFE03FE001FC007C007E007F006 F81EFFFCC7F010147E9315>I<01800180018003800380038007800F803F80FFFCFFFC0F 800F800F800F800F800F800F800F800F800F800F860F860F860F860F8607CC03F801F00F 1D7F9C14>I IIIII E /Fr 37 122 df<00000007FF800000000001FFFFF0000000000F FFFFFC000000003FFFFFFE00000000FFFC00FF00000001FFC0003F80000007FF00007FC0 00000FFE0001FFC000001FFC0001FFE000001FF80003FFE000003FF00003FFE000003FF0 0003FFE000007FE00003FFE000007FE00003FFE000007FE00003FFE000007FE00003FFE0 00007FE00001FFC000007FE00000FF8000007FE000003E0000007FE00000000000007FE0 0000000000007FE00000000000007FE00000000000007FE00000000000007FE000000000 00007FE0003FFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFFFFFFFFF000FFFFFF FFFFFFF000FFFFFFFFFFFFF000007FF00000FFF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF000007FF0 00007FF000007FF000007FF0003FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFFE03FFFFF E03FFFFFE03FFFFFE03FFFFFE03FFFFFE03B487EC742>12 D<07C0001FF0003FFC007FFC 007FFE00FFFE00FFFF00FFFF00FFFF80FFFF80FFFF807FFF807FFF803FFF801FFF8007E7 80000780000780000F00000F00000F00000F00001E00001E00003C00003C0000780000F8 0000F00001F00003E00007C0000F80001F00001E00000C00001124788F21>44 DI<0000000003E000000000000000000007F000000000000000 000007F00000000000000000000FF80000000000000000000FF80000000000000000000F F80000000000000000001FFC0000000000000000001FFC0000000000000000003FFE0000 000000000000003FFE0000000000000000007FFF0000000000000000007FFF0000000000 000000007FFF000000000000000000FFFF800000000000000000FFFF8000000000000000 01FFFFC00000000000000001FFFFC00000000000000001FFFFC00000000000000003FFFF E00000000000000003EFFFE00000000000000007EFFFF00000000000000007CFFFF00000 000000000007C7FFF0000000000000000FC7FFF8000000000000000F83FFF80000000000 00001F83FFFC000000000000001F03FFFC000000000000001F01FFFC000000000000003F 01FFFE000000000000003E00FFFE000000000000007E00FFFF000000000000007C007FFF 00000000000000FC007FFF80000000000000F8007FFF80000000000000F8003FFF800000 00000001F8003FFFC0000000000001F0001FFFC0000000000003F0001FFFE00000000000 03E0001FFFE0000000000003E0000FFFE0000000000007E0000FFFF0000000000007C000 07FFF000000000000FC00007FFF800000000000F800007FFF800000000000F800003FFF8 00000000001F800003FFFC00000000001F000001FFFC00000000003FFFFFFFFFFE000000 00003FFFFFFFFFFE00000000007FFFFFFFFFFF00000000007FFFFFFFFFFF00000000007F FFFFFFFFFF0000000000FC0000007FFF8000000000F80000003FFF8000000001F8000000 3FFFC000000001F00000003FFFC000000001F00000001FFFC000000003F00000001FFFE0 00000003E00000000FFFE000000007E00000000FFFF000000007C00000000FFFF0000000 07C000000007FFF00000000FC000000007FFF80000000F8000000003FFF80000001F8000 000003FFFC0000001F0000000003FFFC0000007FC000000001FFFC0000FFFFFFC00007FF FFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF80FFFFFFC00007FFFFFFFF 80FFFFFFC00007FFFFFFFF8051487CC75A>65 DI<000000003FFE00000E0000000FFFFFC0001E 0000007FFFFFF8003E000003FFFFFFFE00FE00000FFFFFFFFF81FE00003FFFF800FFC3FE 0000FFFF80000FF7FE0001FFFC000003FFFE0007FFF0000001FFFE000FFFC00000007FFE 001FFF800000003FFE003FFF000000001FFE007FFE000000000FFE00FFFC0000000007FE 01FFF80000000007FE03FFF00000000003FE03FFF00000000001FE07FFE00000000001FE 07FFE00000000000FE0FFFC00000000000FE0FFFC000000000007E1FFFC000000000007E 1FFF8000000000007E3FFF8000000000007E3FFF8000000000003E3FFF8000000000003E 7FFF8000000000003E7FFF0000000000003E7FFF000000000000007FFF00000000000000 FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000 FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000 FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000 FFFF000000000000007FFF000000000000007FFF000000000000007FFF00000000000000 7FFF8000000000003E3FFF8000000000003E3FFF8000000000003E3FFF8000000000003E 1FFF8000000000003E1FFFC000000000003E0FFFC000000000007C0FFFC000000000007C 07FFE000000000007C07FFE00000000000F803FFF00000000000F803FFF00000000001F8 01FFF80000000001F000FFFC0000000003E0007FFE0000000007E0003FFF000000000FC0 001FFF800000001F80000FFFC00000003F000007FFF0000000FE000001FFFC000001FC00 0000FFFF80000FF80000003FFFF8007FF00000000FFFFFFFFFC000000003FFFFFFFF0000 0000007FFFFFFC00000000000FFFFFE00000000000003FFE000000474979C756>I69 DI<000000003FFE00000E000000000FFFFFC0001E00000000 7FFFFFF8003E00000003FFFFFFFE00FE0000000FFFFFFFFF81FE0000003FFFF800FFC3FE 000000FFFF80000FF7FE000001FFFC000003FFFE000007FFF0000001FFFE00000FFFC000 00007FFE00001FFF800000003FFE00003FFF000000001FFE00007FFE000000000FFE0000 FFFC0000000007FE0001FFF80000000007FE0003FFF00000000003FE0003FFF000000000 01FE0007FFE00000000001FE0007FFE00000000000FE000FFFC00000000000FE000FFFC0 00000000007E001FFFC000000000007E001FFF8000000000007E003FFF8000000000007E 003FFF8000000000003E003FFF8000000000003E007FFF8000000000003E007FFF000000 0000003E007FFF00000000000000007FFF0000000000000000FFFF0000000000000000FF FF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000 000000FFFF0000000000000000FFFF0000000000000000FFFF0000000000000000FFFF00 00000000000000FFFF0000000000000000FFFF0000000000000000FFFF00000000000000 00FFFF0000007FFFFFFFFE7FFF0000007FFFFFFFFE7FFF0000007FFFFFFFFE7FFF800000 7FFFFFFFFE7FFF8000007FFFFFFFFE3FFF8000000000FFFE003FFF8000000000FFFE003F FF8000000000FFFE001FFF8000000000FFFE001FFFC000000000FFFE000FFFC000000000 FFFE000FFFC000000000FFFE0007FFE000000000FFFE0007FFE000000000FFFE0003FFF0 00000000FFFE0003FFF800000000FFFE0001FFF800000000FFFE0000FFFC00000000FFFE 00007FFE00000000FFFE00003FFF00000000FFFE00001FFF80000000FFFE00000FFFE000 0001FFFE000007FFF0000003FFFE000001FFFC000007FFFE000000FFFF80001FFFFE0000 003FFFFC00FFC7FE0000000FFFFFFFFF83FE00000003FFFFFFFE00FE000000007FFFFFF8 003E000000000FFFFFE0000E00000000003FFE000000004F4979C75D>I73 D76 DI80 D<3FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFFFFFFFFFF003FFFFFFFFF FFFFFFFF003FFFFFFFFFFFFFFFFF003FFF0003FFF8003FFF007FF80003FFF80007FF807F E00003FFF80001FF807FC00003FFF80000FF807F800003FFF800007F807F000003FFF800 003F807F000003FFF800003F807E000003FFF800001F807E000003FFF800001F807E0000 03FFF800000F807C000003FFF800000F807C000003FFF800000F807C000003FFF800000F 807C000003FFF800000F80FC000003FFF800000FC0F8000003FFF8000007C0F8000003FF F8000007C0F8000003FFF8000007C0F8000003FFF8000007C0F8000003FFF8000007C000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 00000000000003FFF80000000000000003FFF80000000000000003FFF800000000000000 03FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8000000 0000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FF F80000000000000003FFF80000000000000003FFF80000000000000003FFF80000000000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 00000000000003FFF80000000000000003FFF80000000000000003FFF800000000000000 03FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF8000000 0000000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FF F80000000000000003FFF80000000000000003FFF80000000000000003FFF80000000000 000003FFF80000000000000003FFF80000000000000003FFF80000000000000003FFF800 0000000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FFFFFFFFFFF800000003FF FFFFFFFFF800000003FFFFFFFFFFF800004A467CC553>84 D<0007FFFC000000007FFFFF C0000001FFFFFFF8000003FFFFFFFE000007FE001FFF000007FF0003FFC0000FFF8001FF E0000FFF8000FFF0000FFF80007FF0000FFF80007FF8000FFF80007FF80007FF00003FFC 0007FF00003FFC0003FE00003FFC0000F800003FFC00000000003FFC00000000003FFC00 000000003FFC00000000003FFC00000007FFFFFC000000FFFFFFFC000007FFFFFFFC0000 3FFFE03FFC0000FFFE003FFC0003FFF0003FFC0007FFC0003FFC000FFF00003FFC001FFE 00003FFC003FFC00003FFC007FF800003FFC007FF800003FFC00FFF000003FFC00FFF000 003FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FFF000007FFC007FF80000 FFFC007FF80001EFFC003FFC0003EFFC003FFF0007CFFF000FFFC03F8FFFF807FFFFFF07 FFFC01FFFFFC03FFFC007FFFF001FFFC0003FF80007FF8362E7DAD3A>97 D<007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC0 0000000000FFFFC0000000000003FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC000000000 0001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0 000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00FFC0000 0001FFC07FFFC0000001FFC3FFFFF0000001FFCFFFFFFC000001FFDFF00FFF000001FFFF 8003FF800001FFFE0001FFC00001FFF800007FE00001FFF000007FF00001FFE000003FF8 0001FFE000001FFC0001FFE000001FFC0001FFE000001FFE0001FFE000001FFE0001FFE0 00000FFF0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001FFE000000FFF 8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE0 00000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF 8001FFE000000FFF0001FFE000000FFF0001FFE000000FFF0001FFE000001FFE0001FFE0 00001FFE0001FFE000001FFC0001FFE000001FFC0001FFE000003FF80001FFF000003FF8 0001FFF800007FF00001FFFC0000FFE00001FFFE0001FFC00001FFBF0007FF800001FF1F E01FFE000001FE0FFFFFFC000001FC03FFFFF0000001F800FFFF80000001F0001FF80000 0039487CC742>I<00001FFFC0000000FFFFF8000007FFFFFE00001FFFFFFF80007FFC00 FFC000FFE001FFC001FFC003FFE003FF8003FFE007FF0003FFE00FFE0003FFE00FFE0003 FFE01FFC0001FFC01FFC0001FFC03FFC0000FF803FFC00003E007FF8000000007FF80000 00007FF800000000FFF800000000FFF800000000FFF800000000FFF800000000FFF80000 0000FFF800000000FFF800000000FFF800000000FFF800000000FFF8000000007FF80000 00007FF8000000007FFC000000003FFC000000003FFC000000001FFC000000F81FFE0000 00F80FFE000000F80FFF000001F007FF800003F003FFC00007E001FFE0000FC000FFF000 1F80007FFE00FF00001FFFFFFE000007FFFFF8000000FFFFE00000001FFE00002D2E7CAD 35>I<00000000007FC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000 000000FFFFC00000000000FFFFC0000000000003FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000000FFC01 FFC0000000FFFF81FFC0000007FFFFE1FFC000001FFFFFF9FFC000007FFC03FFFFC00000 FFF0007FFFC00001FFC0001FFFC00003FF80000FFFC00007FF000007FFC0000FFE000003 FFC0000FFE000003FFC0001FFC000003FFC0001FFC000003FFC0003FFC000003FFC0003F FC000003FFC0007FF8000003FFC0007FF8000003FFC0007FF8000003FFC000FFF8000003 FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FF F8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003FFC000FFF8000003 FFC0007FF8000003FFC0007FF8000003FFC0007FF8000003FFC0003FF8000003FFC0003F FC000003FFC0003FFC000003FFC0001FFC000003FFC0001FFC000003FFC0000FFE000007 FFC00007FF00000FFFC00003FF00001FFFC00001FFC0003FFFC00000FFE000FFFFE00000 7FF807FBFFFF80001FFFFFF3FFFF800007FFFFC3FFFF800001FFFF03FFFF8000001FF803 FFFF8039487CC742>I<00001FFE00000001FFFFE0000007FFFFF800001FFFFFFE00007F FC07FF0000FFE001FF8001FFC0007FC003FF80003FE007FF00003FF00FFE00001FF01FFE 00000FF81FFC00000FF83FFC00000FFC3FFC000007FC7FFC000007FC7FF8000007FC7FF8 000007FE7FF8000007FEFFF8000007FEFFF8000007FEFFFFFFFFFFFEFFFFFFFFFFFEFFFF FFFFFFFEFFFFFFFFFFFCFFF800000000FFF800000000FFF800000000FFF8000000007FF8 000000007FF8000000007FFC000000003FFC000000003FFC000000003FFC0000001C1FFE 0000003E0FFE0000003E07FF0000007E07FF000000FC03FF800001F801FFC00003F0007F F0001FE0003FFE00FFC0001FFFFFFF800007FFFFFE000000FFFFF80000000FFF80002F2E 7DAD36>I<000000FFC000000007FFF80000003FFFFC000000FFFFFF000001FFC1FF0000 07FF03FF80000FFC03FF80000FF807FFC0001FF807FFC0003FF007FFC0003FF007FFC000 3FE003FF80007FE003FF80007FE001FF00007FE000FE00007FE0003800007FE000000000 7FE0000000007FE0000000007FE0000000007FE0000000007FE0000000007FE000000000 7FE0000000007FE0000000007FE0000000FFFFFFFE0000FFFFFFFE0000FFFFFFFE0000FF FFFFFE0000FFFFFFFE0000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF0000000007FF0000000007FF0000000007FF0000000007FF000000000 7FF0000000007FF00000003FFFFFF800003FFFFFF800003FFFFFF800003FFFFFF800003F FFFFF800002A487DC724>I<00000000001F8000007FF000FFE00007FFFF03FFF0001FFF FFC7FFF0007FFFFFFFC7F800FFE03FFE0FF801FF800FFC0FF803FF0007FE0FF807FE0003 FF07F007FE0003FF07F00FFC0001FF81C00FFC0001FF80000FFC0001FF80001FFC0001FF C0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0001FFC0 000FFC0001FF80000FFC0001FF80000FFC0001FF800007FE0003FF000007FE0003FF0000 03FF0007FE000001FF800FFC000000FFE03FF8000001FFFFFFF0000001DFFFFFC0000003 C7FFFF00000003C07FF000000007C0000000000007C0000000000007C0000000000007C0 000000000007E0000000000007F0000000000007F8000000000007FFFFFFF0000007FFFF FFFF000003FFFFFFFFE00003FFFFFFFFF80001FFFFFFFFFE0001FFFFFFFFFF0000FFFFFF FFFF80007FFFFFFFFF8003FFFFFFFFFFC00FFFFFFFFFFFC01FF800001FFFE03FE0000001 FFE07FC00000007FF07FC00000003FF0FF800000001FF0FF800000001FF0FF800000001F F0FF800000001FF0FF800000001FF07FC00000003FE07FC00000003FE03FE00000007FC0 3FF0000000FFC01FFC000003FF800FFF00000FFF0003FFF000FFFC0000FFFFFFFFF00000 3FFFFFFFC0000007FFFFFE000000003FFFC0000035447DAE3B>I<007FC00000000000FF FFC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000000000FFFFC00000 00000003FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001 FFC0000000000001FFC0000000000001FFC0000000000001FFC0000000000001FFC00000 00000001FFC0000000000001FFC0000000000001FFC001FFC0000001FFC00FFFF8000001 FFC03FFFFE000001FFC0FFFFFF000001FFC1FC07FF800001FFC3E003FFC00001FFC7C001 FFC00001FFCF0001FFE00001FFDE0000FFE00001FFDC0000FFE00001FFFC0000FFF00001 FFF80000FFF00001FFF00000FFF00001FFF00000FFF00001FFF00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF000FFFFFFC07FFFFFE0FFFFFFC07F FFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B487CC742>I<00FC 0001FE0003FF0007FF800FFFC01FFFE01FFFE01FFFE01FFFE01FFFE01FFFE00FFFC007FF 8003FF0001FE0000FC000000000000000000000000000000000000000000000000000000 00000000000000007FC0FFFFC0FFFFC0FFFFC0FFFFC0FFFFC003FFC001FFC001FFC001FF C001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FF C001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FF C001FFC001FFC001FFC001FFC001FFC001FFC001FFC0FFFFFFFFFFFFFFFFFFFFFFFFFFFF FF18497CC820>I<007FC000000000FFFFC000000000FFFFC000000000FFFFC000000000 FFFFC000000000FFFFC00000000003FFC00000000001FFC00000000001FFC00000000001 FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FF C00000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC0 0000000001FFC00000000001FFC00000000001FFC00000000001FFC00000000001FFC000 00000001FFC00000000001FFC0007FFFF801FFC0007FFFF801FFC0007FFFF801FFC0007F FFF801FFC0007FFFF801FFC0000FFE0001FFC00007F00001FFC0000FE00001FFC0003FC0 0001FFC0007F800001FFC000FE000001FFC001FC000001FFC007F8000001FFC00FF00000 01FFC01FC0000001FFC03F80000001FFC0FF00000001FFC1FE00000001FFC3FF00000001 FFCFFF80000001FFDFFFC0000001FFFFFFC0000001FFFFFFE0000001FFFFFFF0000001FF FCFFF8000001FFF87FFC000001FFE03FFC000001FFC01FFE000001FFC01FFF000001FFC0 0FFF800001FFC007FFC00001FFC003FFC00001FFC001FFE00001FFC001FFF00001FFC000 FFF80001FFC0007FFC0001FFC0003FFC0001FFC0001FFE0001FFC0000FFF0001FFC0000F FF8001FFC0000FFFC0FFFFFF807FFFFFFFFFFF807FFFFFFFFFFF807FFFFFFFFFFF807FFF FFFFFFFF807FFFFF38487CC73F>107 D<007FC000FFFFC000FFFFC000FFFFC000FFFFC0 00FFFFC00003FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000FFFFFF 80FFFFFF80FFFFFF80FFFFFF80FFFFFF8019487CC720>I<007FC001FFC00000FFE00000 FFFFC00FFFF80007FFFC0000FFFFC03FFFFE001FFFFF0000FFFFC0FFFFFF007FFFFF8000 FFFFC1FC07FF80FE03FFC000FFFFC3E003FFC1F001FFE00003FFC7C001FFC3E000FFE000 01FFCF0001FFE78000FFF00001FFDE0000FFEF00007FF00001FFDC0000FFEE00007FF000 01FFFC0000FFFE00007FF80001FFF80000FFFC00007FF80001FFF00000FFF800007FF800 01FFF00000FFF800007FF80001FFF00000FFF800007FF80001FFE00000FFF000007FF800 01FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF800 01FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF800 01FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF800 01FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF800 01FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF800 01FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF800 01FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF800 01FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007FF800 01FFE00000FFF000007FF800FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0 FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0 5C2E7CAD63>I<007FC001FFC00000FFFFC00FFFF80000FFFFC03FFFFE0000FFFFC0FFFF FF0000FFFFC1FC07FF8000FFFFC3E003FFC00003FFC7C001FFC00001FFCF0001FFE00001 FFDE0000FFE00001FFDC0000FFE00001FFFC0000FFF00001FFF80000FFF00001FFF00000 FFF00001FFF00000FFF00001FFF00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001 FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000 FFF00001FFE00000FFF000FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FF FFFFC07FFFFFE0FFFFFFC07FFFFFE03B2E7CAD42>I<00000FFF0000000000FFFFF00000 0007FFFFFE0000001FFFFFFF8000003FFC03FFC00000FFE0007FF00001FF80001FF80003 FF00000FFC0007FE000007FE000FFE000007FF000FFC000003FF001FFC000003FF803FFC 000003FFC03FF8000001FFC03FF8000001FFC07FF8000001FFE07FF8000001FFE07FF800 0001FFE0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF80000 01FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001 FFF07FF8000001FFE07FF8000001FFE07FF8000001FFE07FF8000001FFE03FFC000003FF C03FFC000003FFC01FFC000003FF801FFE000007FF800FFE000007FF0007FF00000FFE00 03FF80001FFC0001FFC0003FF80000FFE0007FF000007FFC03FFE000001FFFFFFF800000 07FFFFFE00000000FFFFF0000000000FFF000000342E7DAD3B>I<007FC00FFC000000FF FFC07FFFC00000FFFFC3FFFFF00000FFFFCFFFFFFC0000FFFFDFF01FFF0000FFFFFF8007 FF800003FFFE0001FFC00001FFF80000FFE00001FFF00000FFF00001FFE000007FF80001 FFE000003FFC0001FFE000003FFC0001FFE000003FFE0001FFE000001FFE0001FFE00000 1FFF0001FFE000001FFF0001FFE000001FFF0001FFE000000FFF0001FFE000000FFF8001 FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE00000 0FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001FFE000000FFF8001 FFE000000FFF0001FFE000001FFF0001FFE000001FFF0001FFE000001FFE0001FFE00000 1FFE0001FFE000003FFC0001FFE000003FFC0001FFE000007FF80001FFF000007FF80001 FFF80000FFF00001FFFC0001FFE00001FFFE0003FFC00001FFFF0007FF800001FFFFE03F FE000001FFEFFFFFFC000001FFE3FFFFF0000001FFE0FFFF80000001FFE01FF800000001 FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE00000 00000001FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001 FFE0000000000001FFE0000000000001FFE0000000000001FFE0000000000001FFE00000 00000001FFE00000000000FFFFFFC000000000FFFFFFC000000000FFFFFFC000000000FF FFFFC000000000FFFFFFC00000000039427CAD42>I<00FF803F8000FFFF80FFF000FFFF 83FFFC00FFFF87FFFE00FFFF8FC3FF00FFFF8F07FF0003FF9E0FFF8001FFBC0FFF8001FF B80FFF8001FFF80FFF8001FFF00FFF8001FFF007FF0001FFF007FF0001FFE003FE0001FF E000F80001FFE000000001FFE000000001FFC000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC000000001FFC000000001FFC000000001FFC000000001FF C000000001FFC000000001FFC0000000FFFFFFE00000FFFFFFE00000FFFFFFE00000FFFF FFE00000FFFFFFE00000292E7CAD31>114 D<000FFF00E0007FFFF3E001FFFFFFE007FF FFFFE00FF800FFE01FC0001FE03F80000FE03F000007E07F000003E07F000003E0FF0000 03E0FF000003E0FF800003E0FFC0000000FFF0000000FFFE000000FFFFF800007FFFFFC0 007FFFFFF0003FFFFFFC001FFFFFFF000FFFFFFF8007FFFFFFC003FFFFFFE000FFFFFFF0 003FFFFFF00003FFFFF800001FFFF8000000FFFC0000001FFC7800000FFCF8000007FCF8 000003FCFC000003FCFC000003FCFE000003F8FE000003F8FF000003F8FF800007F0FFC0 000FF0FFF0001FE0FFFC00FFC0FFFFFFFF80FC7FFFFE00F81FFFF800E003FF8000262E7C AD2F>I<0001F000000001F000000001F000000001F000000001F000000001F000000003 F000000003F000000003F000000007F000000007F000000007F00000000FF00000000FF0 0000001FF00000003FF00000003FF00000007FF0000001FFF0000003FFF000000FFFFFFF C0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC0FFFFFFFFC000FFF0000000FFF0000000FFF00000 00FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000 FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FF F0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF001F000FFF0 01F000FFF001F000FFF001F000FFF001F000FFF001F000FFF001F000FFF001F000FFF001 F0007FF001E0007FF803E0003FF803E0003FFC07C0001FFE0F80000FFFFF800007FFFE00 0001FFFC0000001FF00024427EC12E>I<007FE000003FF000FFFFE0007FFFF000FFFFE0 007FFFF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE0007FFFF00003FFE00001FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00001FFF0 0001FFE00001FFF00001FFE00001FFF00001FFE00003FFF00000FFE00007FFF00000FFE0 000F7FF000007FE0001F7FF000007FF0003E7FF800003FFC00FC7FFFE0001FFFFFF87FFF E00007FFFFE07FFFE00001FFFF807FFFE000003FFE007FFFE03B2E7CAD42>I II< 7FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFFFF007FFFFF801FFFFF007FFFFF80 1FFFFF00007FF80001FF0000007FFC0001FC0000003FFE0003F80000001FFF0007F00000 000FFF0007E000000007FF800FC000000003FFC01F8000000003FFE03F8000000001FFF0 7F0000000000FFF8FE00000000007FF9FC00000000003FFFF800000000003FFFF0000000 00001FFFE000000000000FFFC0000000000007FFC0000000000003FFC0000000000001FF E0000000000001FFF0000000000001FFF8000000000003FFFC000000000003FFFE000000 000007FFFE00000000000FEFFF00000000001FCFFF80000000003F87FFC0000000007F03 FFE000000000FE01FFE000000001FC00FFF000000001F8007FF800000003F0007FFC0000 0007F0003FFE0000000FE0001FFF0000001FC0000FFF0000003F800007FF800000FF8000 07FFC000FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC0FFFFF8003FFFFFC0 FFFFF8003FFFFFC03A2E7EAD3F>I<7FFFFFC000FFFF807FFFFFC000FFFF807FFFFFC000 FFFF807FFFFFC000FFFF807FFFFFC000FFFF8000FFF000000FE00000FFF800000FC00000 FFF800000FC000007FFC00000F8000007FFC00001F8000003FFC00001F0000003FFE0000 3F0000001FFE00003E0000001FFF00007E0000000FFF00007C0000000FFF8000FC000000 07FF8000F800000007FFC001F800000003FFC001F000000003FFE003F000000003FFE003 F000000001FFF003E000000001FFF007E000000000FFF007C000000000FFF80FC0000000 007FF80F80000000007FFC1F80000000003FFC1F00000000003FFE3F00000000001FFE3E 00000000001FFF7E00000000000FFF7C00000000000FFFFC00000000000FFFFC00000000 0007FFF8000000000007FFF8000000000003FFF0000000000003FFF0000000000001FFE0 000000000001FFE0000000000000FFC0000000000000FFC00000000000007F8000000000 00007F800000000000003F000000000000003F000000000000003F000000000000003E00 0000000000007E000000000000007C00000000000000FC000000001F8000F8000000003F C001F8000000007FE001F000000000FFF003F000000000FFF003E000000000FFF007E000 000000FFF00FC000000000FFF01F8000000000FFF03F80000000007FE07F00000000007F 43FE00000000003FFFF800000000001FFFF0000000000007FFC0000000000001FE000000 00000039427EAD3F>I E /Fs 10 58 df<1F00318060C04040C060C060C060C060C060C0 60C060C060404060C031801F000B107F8F0F>48 D<0C003C00CC000C000C000C000C000C 000C000C000C000C000C000C000C00FF8009107E8F0F>I<1F00618040C08060C0600060 006000C00180030006000C00102020207FC0FFC00B107F8F0F>I<1F00218060C060C000 C0008001800F00008000400060C060C060804060801F000B107F8F0F>I<030003000700 0F000B001300330023004300C300FFE003000300030003001FE00B107F8F0F>I<20803F 002C002000200020002F0030802040006000600060C06080C061801F000B107F8F0F>I< 0780184030C060C06000C000CF00F080E040C060C060C060406060C030801F000B107F8F 0F>I<40007FE07FC08080808001000200040004000C0008000800180018001800180018 000B117E900F>I<1F00318060C060C060C071803F000F00338061C0C060C060C0604040 60801F000B107F8F0F>I<1F00318060C0C040C060C060C06040E021E01E600060004060 C0608043003E000B107F8F0F>I E /Ft 89 126 df<70F8F8F8F8F8F8F8F8F8F8F8F8F8 F8F8F870000000000070F8F8F870051C779B18>33 D<4010E038F078E038E038E038E038 E038E038E038E038E038E03860300D0E7B9C18>I<030600078F00078F00078F00078F00 078F00078F007FFFC0FFFFE0FFFFE07FFFC00F1E000F1E000F1E000F1E000F1E000F1E00 7FFFC0FFFFE0FFFFE07FFFC01E3C001E3C001E3C001E3C001E3C001E3C000C1800131C7E 9B18>I<3803007C07807C0780EE0F80EE0F00EE0F00EE1F00EE1E00EE1E00EE3E007C3C 007C3C00387C0000780000780000F80000F00001F00001E00001E00003E00003C00003C0 0007C0000783800787C00F87C00F0EE00F0EE01F0EE01E0EE01E0EE03E0EE03C07C03C07 C018038013247E9F18>37 D<01C00007E0000FF0000E70001C38001C38001C38001C3800 1C73F01C73F01CE3F00FE3800FC7000F87000F07001F0E003F0E007B8E0073DC00E1DC00 E0F800E0F800E07070E0787070FC707FFFE03FCFE00F03C0141C7F9B18>I<387C7C7E3E 0E0E0E1C1C38F8F0C0070E789B18>I<007000F001E003C007800F001E001C0038003800 7000700070007000E000E000E000E000E000E000E000E000700070007000700038003800 1C001E000F00078003C001F000F000700C24799F18>I<6000F00078003C001E000F0007 80038001C001C000E000E000E000E00070007000700070007000700070007000E000E000 E000E001C001C0038007800F001E003C007800F00060000C247C9F18>I<01C00001C000 01C00001C000C1C180F1C780F9CF807FFF001FFC0007F00007F0001FFC007FFF00F9CF80 F1C780C1C18001C00001C00001C00001C00011147D9718>I<00600000F00000F00000F0 0000F00000F00000F00000F0007FFFC0FFFFE0FFFFE07FFFC000F00000F00000F00000F0 0000F00000F00000F00000600013147E9718>I<1C3E7E7F3F1F070E1E7CF860080C7885 18>I<7FFF00FFFF80FFFF807FFF0011047D8F18>I<3078FCFC78300606778518>I<0003 00000780000780000F80000F00001F00001E00001E00003E00003C00007C000078000078 0000F80000F00001F00001E00003E00003C00003C00007C0000780000F80000F00000F00 001F00001E00003E00003C00003C00007C0000780000F80000F00000F000006000001124 7D9F18>I<01F00007FC000FFE001F1F001C07003803807803C07001C07001C0E000E0E0 00E0E000E0E000E0E000E0E000E0E000E0E000E0E000E0F001E07001C07001C07803C038 03801C07001F1F000FFE0007FC0001F000131C7E9B18>I<01800380038007800F803F80 FF80FB804380038003800380038003800380038003800380038003800380038003800380 03807FFCFFFE7FFC0F1C7B9B18>I<03F0000FFE003FFF007C0F807003C0E001C0F000E0 F000E06000E00000E00000E00001C00001C00003C0000780000F00001E00003C00007800 00F00001E00007C0000F80001E00E03C00E07FFFE0FFFFE07FFFE0131C7E9B18>I<07F8 001FFE003FFF007807807803C07801C03001C00001C00003C0000380000F0003FF0003FE 0003FF000007800003C00001C00000E00000E00000E0F000E0F000E0F001C0F003C07C07 803FFF001FFE0003F800131C7E9B18>I<001F00003F0000770000770000E70001E70001 C7000387000787000707000E07001E07003C0700380700780700F00700FFFFF8FFFFF8FF FFF8000700000700000700000700000700000700007FF000FFF8007FF0151C7F9B18>I< 1FFF803FFF803FFF803800003800003800003800003800003800003800003800003BF800 3FFE003FFF003C07801803C00001C00000E00000E06000E0F000E0F000E0E001C07003C0 7C0F803FFF001FFC0003F000131C7E9B18>I<007E0001FF0007FF800F83C01E03C01C03 C0380180380000700000700000E1F800E7FE00FFFF00FE0780F803C0F001C0F000E0E000 E0F000E07000E07000E07000E03801C03C03C01E07800FFF0007FE0001F800131C7E9B18 >II<03F8000FFE001FFF003E0F80 3803807001C07001C07001C07001C03803803C07801FFF0007FC000FFE001F1F003C0780 7001C0F001E0E000E0E000E0E000E0E000E07001C07803C03E0F801FFF000FFE0003F800 131C7E9B18>I<03F0000FFC001FFE003C0F00780780700380E001C0E001C0E001C0E001 E0E001E07001E07803E03C0FE01FFFE00FFEE003F0E00000E00001C00001C00001C03003 80780780780F00783E003FFC001FF00007C000131C7E9B18>I<3078FCFC783000000000 000000003078FCFC78300614779318>I<183C7E7E3C180000000000000000183C7E7E3E 1E0E1C3C78F060071A789318>I<000300000780001F80003F00007E0001FC0003F00007 E0001FC0003F00007E0000FC0000FC00007E00003F00001FC00007E00003F00001FC0000 7E00003F00001F8000078000030011187D9918>I<7FFFC0FFFFE0FFFFE0FFFFE0000000 000000000000000000FFFFE0FFFFE0FFFFE07FFFC0130C7E9318>I<600000F00000FC00 007E00003F00001FC00007E00003F00001FC00007E00003F00001F80001F80003F00007E 0001FC0003F00007E0001FC0003F00007E0000FC0000F0000060000011187D9918>I<00 7C0001FE0007FF000F87801E03C03C1DC0387FC070FFE071E3E071C1E0E1C1E0E380E0E3 80E0E380E0E380E0E380E0E380E0E1C1C071C1C071E3C070FF80387F003C1C001E00E00F 83E007FFC001FF80007E00131C7E9B18>64 D<00700000F80000F80000D80000D80001DC 0001DC0001DC00018C00038E00038E00038E00038E000306000707000707000707000707 000FFF800FFF800FFF800E03800E03801C01C01C01C07F07F0FF8FF87F07F0151C7F9B18 >II<00F8E003FEE007FFE00F07E01E03E0 3C01E03800E07000E07000E0700000E00000E00000E00000E00000E00000E00000E00000 E000007000007000E07000E03800E03C00E01E01C00F07C007FF8003FE0000F800131C7E 9B18>I<7FF800FFFE007FFF001C0F801C03C01C03C01C01E01C00E01C00E01C00F01C00 701C00701C00701C00701C00701C00701C00701C00701C00F01C00E01C00E01C01E01C01 C01C03C01C0F807FFF00FFFE007FF800141C7F9B18>III<01F1C003FDC00FFFC01F0F C01C03C03803C03801C07001C07001C0700000E00000E00000E00000E00000E00000E00F F0E01FF0E00FF07001C07001C07003C03803C03803C01C07C01F0FC00FFFC003FDC001F1 C0141C7E9B18>I<7F07F0FF8FF87F07F01C01C01C01C01C01C01C01C01C01C01C01C01C 01C01C01C01C01C01FFFC01FFFC01FFFC01C01C01C01C01C01C01C01C01C01C01C01C01C 01C01C01C01C01C01C01C07F07F0FF8FF87F07F0151C7F9B18>I<7FFF00FFFF807FFF00 01C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C000 01C00001C00001C00001C00001C00001C00001C00001C00001C00001C0007FFF00FFFF80 7FFF00111C7D9B18>I<7F07F0FF87F87F07F01C03C01C07801C07001C0E001C1E001C3C 001C38001C70001CF0001DF0001DF0001FB8001FB8001F1C001E1C001C0E001C0E001C07 001C07001C03801C03801C01C07F03F0FF87F87F03F0151C7F9B18>75 D<7FE000FFE0007FE0000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00000E00700E00700E00700E00 700E00707FFFF0FFFFF07FFFF0141C7F9B18>II<7E07F0FF0FF87F07F01D81C01D81C01D81C01DC1C01CC1C01CC1C01CE1C01CE1C0 1CE1C01C61C01C71C01C71C01C31C01C39C01C39C01C39C01C19C01C19C01C1DC01C0DC0 1C0DC01C0DC07F07C0FF87C07F03C0151C7F9B18>I<0FF8003FFE007FFF00780F007007 00F00780E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E003 80E00380E00380E00380E00380E00380F00780700700780F007FFF003FFE000FF800111C 7D9B18>II<0FF8003FFE007FFF00780F00 700700F00780E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380 E00380E00380E00380E00380E1E380E1E380F0E78070F700787F007FFF003FFE000FFC00 001C00001E00000E00000F0000070000070011227D9B18>I<7FF800FFFE007FFF001C0F 801C03801C03C01C01C01C01C01C01C01C03C01C03801C0F801FFF001FFE001FFE001C0F 001C07001C03801C03801C03801C03801C03801C039C1C039C1C039C7F01F8FF81F87F00 F0161C7F9B18>I<03F3801FFF803FFF807C0F80700780E00380E00380E00380E0000070 00007800003F00001FF00007FE0000FF00000F800003C00001C00000E00000E06000E0E0 00E0E001E0F001C0F80780FFFF80FFFE00E7F800131C7E9B18>I<7FFFF8FFFFF8FFFFF8 E07038E07038E07038E07038007000007000007000007000007000007000007000007000 00700000700000700000700000700000700000700000700000700000700007FF0007FF00 07FF00151C7F9B18>IIII<7F8FE07F9F E07F8FE00E07000F0700070E00078E00039C0003DC0001F80001F80000F00000F0000070 0000F00000F80001F80001DC00039E00038E00070F000707000E07800E03801E03C07F07 F0FF8FF87F07F0151C7F9B18>II<3FFFE0 7FFFE07FFFE07001C07003C0700780700700000F00001E00001C00003C00007800007000 00F00001E00001C00003C0000780000700000F00001E00E01C00E03C00E07800E07000E0 FFFFE0FFFFE0FFFFE0131C7E9B18>II<600000F00000F00000F8000078 00007C00003C00003C00003E00001E00001F00000F00000F00000F800007800007C00003 C00003C00003E00001E00001F00000F00000F800007800007800007C00003C00003E0000 1E00001E00001F00000F00000F8000078000078000030011247D9F18>I I<7FFF00FFFF80FFFF807FFF0011047D7F18>95 D<061E3E387070E0E0E0F8FC7C7C3807 0E789E18>I<1FE0003FF8007FFC00781E00300E0000070000070000FF0007FF001FFF00 7F0700780700E00700E00700E00700F00F00781F003FFFF01FFBF007E1F014147D9318> I<7E0000FE00007E00000E00000E00000E00000E00000E00000E3E000EFF800FFFC00FC1 E00F80E00F00700E00700E00380E00380E00380E00380E00380E00380F00700F00700F80 E00FC1E00FFFC00EFF80063E00151C809B18>I<01FE0007FF001FFF803E078038030070 0000700000E00000E00000E00000E00000E00000E000007000007001C03801C03E03C01F FF8007FF0001FC0012147D9318>I<001F80003F80001F80000380000380000380000380 00038003E3800FFB801FFF803C1F80380F80700780700380E00380E00380E00380E00380 E00380E00380700780700780380F803C1F801FFFF00FFBF803E3F0151C7E9B18>I<01F0 0007FC001FFE003E0F00380780700380700380E001C0E001C0FFFFC0FFFFC0FFFFC0E000 007000007001C03801C03E03C01FFF8007FF0001FC0012147D9318>I<001F80007FC000 FFE000E1E001C0C001C00001C00001C0007FFFC0FFFFC0FFFFC001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0007FFF007F FF007FFF00131C7F9B18>I<01E1F007FFF80FFFF81E1E301C0E00380700380700380700 3807003807001C0E001E1E001FFC001FF80039E0003800001C00001FFE001FFFC03FFFE0 7801F0700070E00038E00038E00038E000387800F07E03F01FFFC00FFF8001FC00151F7F 9318>I<7E0000FE00007E00000E00000E00000E00000E00000E00000E3E000EFF800FFF C00FC1C00F80E00F00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00 E00E00E00E00E07FC3FCFFE7FE7FC3FC171C809B18>I<03800007C00007C00007C00003 80000000000000000000000000007FC000FFC0007FC00001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C000FFFF00FFFF80FF FF00111D7C9C18>I<0038007C007C007C003800000000000000000FFC1FFC0FFC001C00 1C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C6038F078FFF07FE03F800E277E9C18>II<7FE000FFE0007FE00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E0007FFFC0FFFFE07FFFC0131C7E9B18>I<7CE0E000FFFBF8007FFFF800 1F1F1C001E1E1C001E1E1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C00 1C1C1C001C1C1C001C1C1C001C1C1C001C1C1C007F1F1F00FFBFBF807F1F1F0019148193 18>I<7E3E00FEFF807FFFC00FC1C00F80E00F00E00E00E00E00E00E00E00E00E00E00E0 0E00E00E00E00E00E00E00E00E00E00E00E07FC3FCFFE7FE7FC3FC1714809318>I<01F0 000FFE001FFF003E0F803803807001C07001C0E000E0E000E0E000E0E000E0E000E0F001 E07001C07803C03C07803E0F801FFF000FFE0001F00013147E9318>I<7E3E00FEFF807F FFC00FC1E00F80E00F00700E00700E00380E00380E00380E00380E00380E00380F00700F 00700F80E00FC1E00FFFC00EFF800E3E000E00000E00000E00000E00000E00000E00000E 00007FC000FFE0007FC000151E809318>I<01E38007FB801FFF803E1F80380F80700780 700780E00380E00380E00380E00380E00380E00380700780700780380F803C1F801FFF80 0FFB8003E380000380000380000380000380000380000380000380003FF8003FF8003FF8 151E7E9318>I<7F87E0FF9FF07FBFF803F87803F03003E00003C00003C0000380000380 000380000380000380000380000380000380000380007FFE00FFFF007FFE0015147F9318 >I<07F7003FFF007FFF00780F00E00700E00700E007007C00007FE0001FFC0003FE0000 1F00600780E00380E00380F00380F80F00FFFF00FFFC00E7F00011147D9318>I<018000 0380000380000380000380007FFFC0FFFFC0FFFFC0038000038000038000038000038000 0380000380000380000380000380400380E00380E00380E001C1C001FFC000FF80003E00 13197F9818>I<7E07E0FE0FE07E07E00E00E00E00E00E00E00E00E00E00E00E00E00E00 E00E00E00E00E00E00E00E00E00E00E00E01E00F03E007FFFC03FFFE01FCFC1714809318 >I<7F8FF0FF8FF87F8FF01E03C00E03800E03800E0380070700070700070700038E0003 8E00038E00038E0001DC0001DC0001DC0000F80000F80000700015147F9318>II<7F8FF07F9FF07F8F F0070700078E00039E0001DC0001F80000F80000700000F00000F80001DC00039E00038E 000707000F07807F8FF0FF8FF87F8FF015147F9318>I<7F8FF0FF8FF87F8FF00E01C00E 03800E0380070380070700070700038700038600038E0001CE0001CE0000CC0000CC0000 DC0000780000780000780000700000700000700000F00000E00079E0007BC0007F80003F 00001E0000151E7F9318>I<3FFFF07FFFF07FFFF07001E07003C0700780000F00001E00 003C0000F80001F00003C0000780000F00701E00703C0070780070FFFFF0FFFFF0FFFFF0 14147F9318>I<0007E0001FE0007FE000780000E00000E00000E00000E00000E00000E0 0000E00000E00000E00000E00000E00001E0007FC000FF8000FF80007FC00001E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000007800007F E0001FE00007E013247E9F18>I<60F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0F0 F0F0F0F0F0F0F0F0F0F0F0F0F0600424769F18>I<7C0000FF0000FFC00003C00000E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000F000007FC0 003FE0003FE0007FC000F00000E00000E00000E00000E00000E00000E00000E00000E000 00E00000E00000E00003C000FFC000FF00007C000013247E9F18>I E /Fu 1 98 df<001800001800001800003C00003C00004E00004E00004E000087000087 000187800103800103800201C00201C003FFC00400E00400E00800700800701800703C00 78FE01FF18177F961C>97 D E /Fv 91 128 df<001F83E000F06E3001C078780380F878 0300F03007007000070070000700700007007000070070000700700007007000FFFFFF80 070070000700700007007000070070000700700007007000070070000700700007007000 070070000700700007007000070070000700700007007000070070000700700007007000 7FE3FF001D20809F1B>11 D<003F0000E0C001C0C00381E00701E00701E0070000070000 070000070000070000070000FFFFE00700E00700E00700E00700E00700E00700E00700E0 0700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E07FC3FE 1720809F19>I<003FE000E0E001C1E00381E00700E00700E00700E00700E00700E00700 E00700E00700E0FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700 E00700E00700E00700E00700E00700E00700E00700E00700E00700E07FE7FE1720809F19 >I<001F81F80000F04F040001C07C06000380F80F000300F00F000700F00F0007007000 0007007000000700700000070070000007007000000700700000FFFFFFFF000700700700 070070070007007007000700700700070070070007007007000700700700070070070007 007007000700700700070070070007007007000700700700070070070007007007000700 700700070070070007007007007FE3FE3FF02420809F26>I<001F81FF0000F06F070001 C07C0F000380F80F000300F0070007007007000700700700070070070007007007000700 70070007007007000700700700FFFFFFFF00070070070007007007000700700700070070 070007007007000700700700070070070007007007000700700700070070070007007007 000700700700070070070007007007000700700700070070070007007007000700700700 7FE3FE3FF02420809F26>I22 D<70F8F8F8F8F8F8F8707070707070 707070702020202020000000000070F8F8F87005217CA00D>33 D<7038F87CFC7EFC7E74 3A0402040204020804080410081008201040200F0E7E9F17>I<000300C0000300C00003 00C0000300C00006018000060180000601800006018000060180000C0300000C0300000C 0300000C0300000C0300001806007FFFFFFCFFFFFFFE00300C0000300C0000300C000030 0C0000601800006018000060180000601800FFFFFFFE7FFFFFFC00C03000018060000180 60000180600001806000018060000300C0000300C0000300C0000300C0000300C0000601 800006018000060180001F297D9F26>I<00780000008400000184000003020000070200 00070200000702000007020000070400000704000007080000070800000310000003A00F FC03C003E0038001C001C0008001C0010003E0010004E0020008F0020018700400307808 0070380800701C1000F01E1000F00E2000F0074000F003C0087003C0087801C010380670 301C18386007E00F801E227EA023>38 D<70F8FCFC74040404080810102040060E7C9F0D >I<0020004000800100020006000C000C00180018003000300030007000600060006000 E000E000E000E000E000E000E000E000E000E000E000E000600060006000700030003000 3000180018000C000C000600020001000080004000200B2E7DA112>I<80004000200010 0008000C00060006000300030001800180018001C000C000C000C000E000E000E000E000 E000E000E000E000E000E000E000E000C000C000C001C001800180018003000300060006 000C00080010002000400080000B2E7DA112>I<00060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 0006000000060000FFFFFFF0FFFFFFF00006000000060000000600000006000000060000 000600000006000000060000000600000006000000060000000600000006000000060000 000600001C207D9A23>43 D<70F8FCFC74040404080810102040060E7C840D>II<70F8F8F87005057C840D>I<000100030003000600060006000C000C00 0C00180018001800300030003000600060006000C000C000C00180018001800300030003 000600060006000C000C000C00180018001800300030003000600060006000C000C000C0 00102D7DA117>I<03F0000E1C001C0E00180600380700700380700380700380700380F0 03C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F0 03C07003807003807003807807803807001806001C0E000E1C0003F000121F7E9D17>I< 018003800F80F38003800380038003800380038003800380038003800380038003800380 038003800380038003800380038003800380038007C0FFFE0F1E7C9D17>I<03F0000C1C 00100E00200700400780800780F007C0F803C0F803C0F803C02007C00007C00007800007 80000F00000E00001C0000380000700000600000C0000180000300000600400C00401800 401000803FFF807FFF80FFFF80121E7E9D17>I<03F0000C1C00100E00200F00780F8078 0780780780380F80000F80000F00000F00000E00001C0000380003F000003C00000E0000 0F000007800007800007C02007C0F807C0F807C0F807C0F00780400780400F00200E001C 3C0003F000121F7E9D17>I<000600000600000E00000E00001E00002E00002E00004E00 008E00008E00010E00020E00020E00040E00080E00080E00100E00200E00200E00400E00 C00E00FFFFF0000E00000E00000E00000E00000E00000E00000E0000FFE0141E7F9D17> I<1803001FFE001FFC001FF8001FE00010000010000010000010000010000010000011F0 00161C00180E001007001007800003800003800003C00003C00003C07003C0F003C0F003 C0E00380400380400700200600100E000C380003E000121F7E9D17>I<007C0001820007 01000E03800C07801C0780380300380000780000700000700000F1F000F21C00F40600F8 0700F80380F80380F003C0F003C0F003C0F003C0F003C07003C07003C070038038038038 07001807000C0E00061C0001F000121F7E9D17>I<4000007FFFC07FFF807FFF80400100 80020080020080040000080000080000100000200000200000400000400000C00000C000 01C000018000038000038000038000038000078000078000078000078000078000078000 078000030000121F7D9D17>I<03F0000C0C001006003003002001806001806001806001 807001807803003E03003F06001FC8000FF00003F80007FC000C7E00103F00300F806003 804001C0C001C0C000C0C000C0C000C0C000806001802001001002000C0C0003F000121F 7E9D17>I<03F0000E18001C0C00380600380700700700700380F00380F00380F003C0F0 03C0F003C0F003C0F003C07007C07007C03807C0180BC00E13C003E3C000038000038000 0380000700300700780600780E00700C002018001070000FC000121F7E9D17>I<70F8F8 F8700000000000000000000070F8F8F87005147C930D>I<70F8F8F87000000000000000 00000070F0F8F878080808101010202040051D7C930D>I<7FFFFFE0FFFFFFF000000000 00000000000000000000000000000000000000000000000000000000FFFFFFF07FFFFFE0 1C0C7D9023>61 D<0FC0307040384038E03CF03CF03C603C0038007000E000C001800180 010003000200020002000200020002000000000000000000000007000F800F800F800700 0E207D9F15>63 D<000100000003800000038000000380000007C0000007C0000007C000 0009E0000009E0000009E0000010F0000010F0000010F000002078000020780000207800 00403C0000403C0000403C0000801E0000801E0000FFFE0001000F0001000F0001000F00 020007800200078002000780040003C00E0003C01F0007E0FFC03FFE1F207F9F22>65 DI<000FC040007030C001C009C0 038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00004078000040 F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003800200 01C0040000703800000FC0001A217D9F21>IIII<000FE0200078186000E004E0038002E0070001E0 0F0000E01E0000601E0000603C0000603C0000207C00002078000020F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F8007FFCF80003E0780001E07C0001E0 3C0001E03C0001E01E0001E01E0001E00F0001E0070001E0038002E000E0046000781820 000FE0001E217D9F24>III<0FFFC0007C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C 00003C00003C00003C00003C00003C00003C00003C00003C00003C00203C00F83C00F83C 00F83C00F0380040780040700030E0000F800012207E9E17>IIIII<001F800000F0F00001C0380007801E000F000F000E0007001E00 07803C0003C03C0003C07C0003E0780001E0780001E0F80001F0F80001F0F80001F0F800 01F0F80001F0F80001F0F80001F0F80001F0F80001F0780001E07C0003E07C0003E03C00 03C03C0003C01E0007800E0007000F000F0007801E0001C0380000F0F000001F80001C21 7D9F23>II<001F800000F0F000 01C0380007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E07C0003E0 780001E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0 F80001F0780001E0780001E07C0003E03C0003C03C0F03C01E1087800E2047000F204F00 07A03E0001E0380000F0F010001FB01000003010000038300000387000003FF000001FE0 00001FE000000FC0000007801C297D9F23>II<07E0800C1980100780300380600180600180E00180E00080E000 80E00080F00000F000007800007F00003FF0001FFC000FFE0003FF00001F800007800003 C00003C00001C08001C08001C08001C08001C0C00180C00380E00300F00600CE0C0081F8 0012217D9F19>I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F0030800F0010 800F0010800F0010800F0010000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000001F800007FFFE001C1F7E9E21>IIII<7FF83FF8 0FE00FC007C0070003C0020001E0040001F00C0000F0080000781000007C1000003C2000 003E4000001E4000000F8000000F8000000780000003C0000007E0000005E0000009F000 0018F8000010780000207C0000603C0000401E0000801F0001800F0001000780020007C0 070003C01F8007E0FFE01FFE1F1F7F9E22>II<7FFFF87C00F87000F06001E04001E0C003C0C003C0800780800F80800F00 001E00001E00003C00003C0000780000F80000F00001E00001E00003C00403C004078004 0F80040F000C1E000C1E00083C00183C0018780038F801F8FFFFF8161F7D9E1C>II<080410082010201040204020804080408040B85CFC7E FC7E7C3E381C0F0E7B9F17>II<08102020404080 8080B8FCFC7C38060E7D9F0D>96 D<1FE000303000781800781C00300E00000E00000E00 000E0000FE00078E001E0E00380E00780E00F00E10F00E10F00E10F01E10781E10386720 0F83C014147E9317>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E3E000EC3800F01C00F00E00E00E00E00700E00700E00780E00 780E00780E00780E00780E00780E00700E00700E00E00F00E00D01C00CC300083E001520 7F9F19>I<03F80E0C1C1E381E380C70007000F000F000F000F000F000F0007000700038 0138011C020E0C03F010147E9314>I<000380003F800003800003800003800003800003 8000038000038000038000038000038003E380061B801C07803803803803807003807003 80F00380F00380F00380F00380F00380F003807003807003803803803807801C07800E1B 8003E3F815207E9F19>I<03F0000E1C001C0E00380700380700700700700380F00380F0 0380FFFF80F00000F00000F000007000007000003800801800800C010007060001F80011 147F9314>I<007C00C6018F038F07060700070007000700070007000700FFF007000700 07000700070007000700070007000700070007000700070007000700070007007FF01020 809F0E>I<0000E003E3300E3C301C1C30380E00780F00780F00780F00780F00780F0038 0E001C1C001E380033E0002000002000003000003000003FFE001FFF800FFFC03001E060 0070C00030C00030C00030C000306000603000C01C038003FC00141F7F9417>I<0E0000 FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E3E00 0E43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16207F9F19>I<1C001E003E001E00 1C000000000000000000000000000E007E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E00FFC00A1F809E0C>I<00E001F001F001F000E000 0000000000000000000000007007F000F000700070007000700070007000700070007000 70007000700070007000700070007000700070007000706070F060F0C061803F000C2882 9E0E>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E0FF00E03C00E03000E02000E04000E08000E10000E30000E70000EF8000F38 000E1C000E1E000E0E000E07000E07800E03800E03C00E03E0FFCFF815207F9F18>I<0E 00FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B20809F0C>I<0E1F 01F000FE618618000E81C81C000F00F00E000F00F00E000E00E00E000E00E00E000E00E0 0E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E 000E00E00E000E00E00E000E00E00E000E00E00E00FFE7FE7FE023147F9326>I<0E3E00 FE43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16147F9319>I<01F800070E001C03 803801C03801C07000E07000E0F000F0F000F0F000F0F000F0F000F0F000F07000E07000 E03801C03801C01C0380070E0001F80014147F9317>I<0E3E00FEC3800F01C00F00E00E 00E00E00F00E00700E00780E00780E00780E00780E00780E00780E00700E00F00E00E00F 01E00F01C00EC3000E3E000E00000E00000E00000E00000E00000E00000E00000E0000FF E000151D7F9319>I<03E0800619801C05803C0780380380780380700380F00380F00380 F00380F00380F00380F003807003807803803803803807801C0B800E138003E380000380 000380000380000380000380000380000380000380003FF8151D7E9318>I<0E78FE8C0F 1E0F1E0F0C0E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00F 147F9312>I<1F9030704030C010C010C010E00078007F803FE00FF00070803880188018 C018C018E030D0608F800D147E9312>I<020002000200060006000E000E003E00FFF80E 000E000E000E000E000E000E000E000E000E000E000E080E080E080E080E080610031001 E00D1C7F9B12>I<0E01C0FE1FC00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E03C00603C0030DC001F1FC16147F93 19>III<7FC3FC0F01E00701C007018003810001C20000E40000EC 00007800003800003C00007C00004E000087000107000303800201C00601E01E01E0FF07 FE1714809318>II<3FFF380E200E 201C40384078407000E001E001C00380078007010E011E011C0338027006700EFFFE1014 7F9314>III<30307878F87C787830 300E057C9E17>127 D E /Fw 79 123 df<007F07F001FF1FF003FF3FF0078078000700 70000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F000FFF8FF80FFF8 FF80FFF8FF800F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00 F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00 F0001C20809F1B>11 D<0000F000F8F001F8F003F8F00780000700000F00000F00000F00 000F00000F00000F00000F0000FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00 F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00 F0142180A018>I<00F8F001F8F003F8F00780F00700F00F00F00F00F00F00F00F00F00F 00F00F00F00F00F0FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00F00F00F00F 00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F01420809F 18>I<783C783C783C783C180C3018301860306030E0700E0A7F9F17>34 D<007C000000FE000001FF000003C78000038780000383C0000783C0000703C0000703C0 000783C00007878000078F8000078F0000079E000003BC008003F800E003F001E003E001 C007E001C00FF003C01EF003803CF8078078780700787C0F00F03E0E00F01F1E00F01F3C 00F00FF800F807F0007803E0007C0FF8303FFFFFF01FFE3FF007F00FC01C227EA022>38 D<007000E001C00380078007000E001E001E003C003C003C0078007800780078007000F0 00F000F000F000F000F000F000F000F000F000F000F000700078007800780078003C003C 003C001E001E000E0007000780038001C000E000700C2E7EA112>40 DI<018001C001800180 C183E187F99F7DBE1FF807E007E01FF87DBEF99FE187C1830180018001C0018010147DA1 17>I<000600000006000000060000000600000006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000FFFFFFF0FFFFFFF000 060000000600000006000000060000000600000006000000060000000600000006000000 06000000060000000600000006000000060000000600001C207D9A23>I<787878781830 306060E0050A7D830D>III<03F0 0007F8001FFE001E1E003C0F00380700780780780780700380F003C0F003C0F003C0F003 C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07807807807 807807803807003C0F001E1E001FFE000FFC0003F00012207E9E17>48 D<00C001C00FC0FFC0FFC0F3C003C003C003C003C003C003C003C003C003C003C003C003 C003C003C003C003C003C003C003C003C003C003C0FFFEFFFEFFFE0F1F7C9E17>I<07F0 000FFC001FFE00383F00700F00600780E00780E003C04003C04003C00003C00003C00003 C0000780000780000F00000E00001C00003C0000780000E00001C0000380000700000E00 001C0000380000700000FFFFC0FFFFC0FFFFC0121F7E9E17>I<03F0000FFC001FFE003C 1F00780F00300780200780000780000780000780000F00000F00003E0003FC0003F80003 FC00001E00000F000007800003800003C00003C00003C00003C08003C0C003C0C0078070 0F807C1F003FFE000FFC0003F00012207E9E17>I<003E00003E00005E00005E0000DE00 01DE00019E00039E00039E00079E00071E000F1E000E1E001E1E003C1E003C1E00781E00 781E00F01E00FFFFF0FFFFF0FFFFF0001E00001E00001E00001E00001E00001E00001E00 001E00141E7F9D17>I<7FFF007FFF007FFF007800007800007800007800007800007800 0078000079F0007FFC007FFE007F1F007C07007C07807807800003C00003C00003C00003 C00003C00003C00003C0400780600780F00F007C1E003FFC001FF80007E000121F7E9D17 >I<007C0001FE0003FE000782000F00001E00003C00003C000078000078000078F800F3 FC00F7FE00FE1F00FC0F80F80780F80780F003C0F003C0F003C0F003C0F003C07003C078 03C07803C07807803807803C0F001E1E000FFE0007F80001F00012207E9E17>II<03F0000FFC001FFE003E1F003C0F 007807807807807807807807807807807807803C0F001E1E000FFC0007F8000FFC001F3E 003C0F00780780780780F003C0F003C0F003C0F003C0F003C0F003C07807807C0F803E1F 001FFE000FFC0003F00012207E9E17>I<03F00007F8000FFC001E1E003C0F0078070078 0780F00780F00380F003C0F003C0F003C0F003C0F003C0F003C07807C07807C07C0FC03E 1FC01FFBC00FF3C007C780000780000780000700000F00001E00201E00307C007FF8003F F0000FC00012207E9E17>II61 D<001F0000001F0000003F800000 3B8000003B8000007BC0000073C0000071C00000F1E00000E1E00000E0E00001E0F00001 E0F00001C0F00003C0780003C078000380780007803C0007803C0007003C000FFFFE000F FFFE000FFFFE001E000F001E000F003C000F803C0007803C000780780007C0780003C078 0003C0F00003E01B207F9F1E>65 DI<001FC000FFF801FFFC03E03C07800C0F00001E00003E00003C00 007C0000780000780000780000F00000F00000F00000F00000F00000F00000F00000F000 007800007800007800007C00003C00003E00001E00000F000207800E03E03E01FFFC00FF F0001FC017227DA01D>II II<001FE000 FFF801FFFE03E03E07800E0F00001E00003E00003C00007C0000780000780000780000F0 0000F00000F00000F00000F00000F00000F003FEF003FE7803FE78001E78001E7C001E3C 001E3E001E1E001E0F001E07801E03E03E01FFFE00FFF8001FC017227DA01E>III<003C003C00 3C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C00 3C003C003C003C003C003C003C003CC078E078FFF07FE00FC00E217E9F15>IIIII<003F000000FFC00003FFF00007E1F8000F807C001F003E001E001E00 3C000F003C000F00780007807800078078000780F00003C0F00003C0F00003C0F00003C0 F00003C0F00003C0F00003C0F00003C0F00003C0F80007C0780007807800078078000780 3C000F003C000F001E001E001F003E000F807C0007E1F80003FFF00000FFC000003F0000 1A227DA021>II<003F000000FFC00003FFF00007E1F8000F807C001F003E001E001E003C000F003C00 0F00780007807800078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F000 03C0F00003C0F00003C0F00003C0F00003C07800078078000780781E07803C0F0F003C0F 8F001E079E001F03FE000F83FC0007E1F80003FFF00000FFF800003F780000007C000000 3E0000001E0000001F0000000F801A277DA021>II<01FC0007FF800FFFC01F03C03C00C03C00007800 007800007800007800007800007C00003C00003F00001FE0000FFC0007FE0001FF00003F 800007C00003C00003E00001E00001E00001E00001E00001E00001C0C003C0F007C0FC0F 807FFF001FFE0003F80013227EA019>IIIII<780007807C000F003E001F001E001E00 0F003C000F807C000780780003C0F00003E1F00001F1E00000F3C000007FC000007F8000 003F0000001F0000001E0000003F0000007F8000007FC00000F3C00001F1E00001E0F000 03C0F80007C0780007803C000F003E001F001E001E000F003C000F807C0007C0780003C0 F00003E01B207F9F1E>IIII<381C3018301860306030C060F078F078F078F0780E0A79 9F17>II<07E03FF87FFC701E401F000F000F000F 003F07FF1FFF7E0FF80FF00FF00FF00FF83F7FFF3FEF1F8F10147E9316>97 DI<03F00FFC 1FFE3E0E3C0278007800F000F000F000F000F000F000780078003C013E0F1FFF0FFE03F0 10147E9314>I<0007800007800007800007800007800007800007800007800007800007 8000078000078007C7800FF7801FFF803E1F807C0780780780F80780F00780F00780F007 80F00780F00780F00780F00780780780780F803E1F801FFF800FF78007C78011207E9F17 >I<03F0000FFC001FFE003E1F003C0700780700700380FFFF80FFFF80FFFF80F00000F0 0000F000007000007800003C01003E07001FFF0007FE0001F80011147F9314>I<007E01 FE03FE078007000F000F000F000F000F000F000F00FFF0FFF0FFF00F000F000F000F000F 000F000F000F000F000F000F000F000F000F000F000F000F000F20809F0E>I<03E0F00F FFF01FFFF03E3E003C1E00780F00780F00780F00780F00780F003C1E003E3E001FFC003F F80033E0003000003800003FFE003FFF801FFFC03FFFE07803F0F000F0F000F0F000F0F8 01F07E07E03FFFC00FFF0003FC00141E7F9317>III<01E001E001E001E00000000000 000000000000000000000001E001E001E001E001E001E001E001E001E001E001E001E001 E001E001E001E001E001E001E001E001E001E001E001E001E0C3C0FFC0FF803F000B2983 9F0C>IIIII<01F80007FE001FFF 803F0FC03C03C07801E07801E0F000F0F000F0F000F0F000F0F000F0F000F07801E07801 E03C03C03F0FC01FFF8007FE0001F80014147F9317>II<03C7800FF7801FFF803E1F807C0F80780780780780F00780F00780 F00780F00780F00780F00780F807807807807C0F803E1F801FFF800FF78007C780000780 000780000780000780000780000780000780000780000780111D7E9317>II<07F01FFC3FFC780C7800780078007C003FC01FF00FF803F8007C003C003C C03CF07CFFF87FF00FC00E147F9311>I<1E001E001E001E001E001E00FFF0FFF0FFF01E 001E001E001E001E001E001E001E001E001E001E001E001E001E201FF00FF007C00C1A7F 9910>IIII<7801E07C03C03E07801E0F 000F0F00079E0003FC0003F80001F80000F00001F00001F80003FC00079E000F0F000E0F 001E07803C03C07801E0F801F01414809315>II<7FFF7FFF7FFF003E003C007800F800F001E003E007C007800F001F001E00 3C007C00FFFFFFFFFFFF10147F9314>I E /Fx 18 104 df0 D<70F8F8F87005057C8D0D>I<400004C0000C6000183000301800600C00C00601 8003030001860000CC0000780000300000300000780000CC000186000303000601800C00 C0180060300030600018C0000C40000416187A9623>I<01800180018001800180C183F1 8F399C0FF003C003C00FF0399CF18FC1830180018001800180018010147D9417>I<0001 FE00000007FF8000001E01E000007800780000E0001C0001800006000300000300060000 01800C000000C00C000000C0180000006030000000303000000030300000003060000000 1860000000186000000018C00000000CC00000000CC00000000CC00000000CC00000000C C00000000CC00000000CC00000000CC00000000C60000000186000000018600000001830 000000303000000030300000003018000000600C000000C00C000000C006000001800300 000300018000060000E0001C000078007800001E01E0000007FF80000001FE0000262B7D A02D>13 D<03C00FF01C38300C60066006C003C003C003C003C003C00360066006300C1C 380FF003C010127D9317>I<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFFFFFFFFFF 7FFE7FFE3FFC1FF80FF003C010127D9317>I<000000C0000003C000000F0000003C0000 00F0000003C00000070000001C00000078000001E00000078000001E00000078000000E0 000000780000001E0000000780000001E0000000780000001C0000000700000003C00000 00F00000003C0000000F00000003C0000000C00000000000000000000000000000000000 00000000000000000000007FFFFF80FFFFFFC01A247C9C23>20 DI<003F F800FFF803C0000700000C0000180000300000300000600000600000C00000C00000C000 00FFFFF8FFFFF8C00000C00000C000006000006000003000003000001800000C00000700 0003C00000FFF8003FF8151C7C981E>50 D<00000C00000C000018000018000030000030 0000600000600000C00000C0000180000180000180000300000300000600000600000C00 000C0000180000180000300000300000600000600000C00000C000018000018000030000 0300000600000600000600000C00000C0000180000180000300000300000600000600000 C00000400000162C7AA000>54 D62 D<00040000000C0000000C 0000000C0000000C0000000C0000000C0000000C0000000C0000000C0000000C0000000C 0000000C0000000C0000000C0000000C0000000C0000000C0000000C0000000C0000000C 0000000C0000000C0000000C0000000C0000000C0000FFFFFFE0FFFFFFE01B1C7C9B23> I<003FFFC001FFFFF0071E03FC081E00FE181E003E701E001F701E001FC01E000F001C00 0F001C000F003C000E003C000E003C001C0038001C003800380078003000780040007000 800070030000F01C0000F3F00000E7800000E0000001E0000001C0000001C0000003C000 00038000000380000007000000070000000E0000000C00000020217F9E20>80 D<0000FE000007FF80001C1FC0002007E000C003F0018001F0030000F0060000F80E0000 F81C0000781C00007838000078380000787000007870000078F0000070F0000070F00000 F0F00000E0F00000E0F80001C0F80001C07C0003807E0083003F0707001FFC0E000FE00C 0000001000000020000000C000000F00001FFC000C7FF8001807FF0030007FF060000FFF C00000FF001E257B9F25>I<400002C00006C00006C00006C00006C00006C00006C00006 C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006C00006 C0000660000C60000C3000181C00700F01E003FF8000FE00171C7D9A1E>91 D<000F0038006000E001C001C001C001C001C001C001C001C001C001C001C001C001C001 C001C0038007001E00F8001E000700038001C001C001C001C001C001C001C001C001C001 C001C001C001C001C001C000E000600038000F102D7DA117>102 DI E /Fy 47 122 df<70F8FCFC7404040404080810102040060F7C840E>44 DI<70F8F8F87005057C840E>I<01F000071C000C060018030038 03803803807001C07001C07001C07001C0F001E0F001E0F001E0F001E0F001E0F001E0F0 01E0F001E0F001E0F001E0F001E0F001E0F001E0F001E07001C07001C07001C07803C038 03803803801C07000C0600071C0001F00013227EA018>48 D<008003800F80F380038003 800380038003800380038003800380038003800380038003800380038003800380038003 80038003800380038003800380038007C0FFFE0F217CA018>I<03F0000C1C0010070020 07804003C04003C08003E0F003E0F801E0F801E0F801E02003E00003E00003C00003C000 0780000700000E00001C0000180000300000600000C00001800001000002002004002008 00201800603000403FFFC07FFFC0FFFFC013217EA018>I<03F8000C1E00100700200780 4007C07807C07803C07807C03807C0000780000780000700000F00000E0000380003F000 001C00000F000007800007800003C00003C00003E02003E07003E0F803E0F803E0F003C0 4003C0400780200780100F000C1C0003F00013227EA018>I<000200000600000E00000E 00001E00001E00002E00004E00004E00008E00008E00010E00020E00020E00040E00040E 00080E00100E00100E00200E00200E00400E00800E00FFFFF8000E00000E00000E00000E 00000E00000E00000E00001F0001FFF015217FA018>I<1000801E07001FFF001FFE001F F80013E00010000010000010000010000010000010000010F800130E0014070018038010 03800001C00001C00001E00001E00001E00001E07001E0F001E0F001E0E001C08001C040 03C04003802007001006000C1C0003F00013227EA018>I<007E0001C1000300800601C0 0E03C01C03C0180180380000380000780000700000700000F0F800F30C00F40600F40300 F80380F801C0F001C0F001E0F001E0F001E0F001E0F001E07001E07001E07001E03801C0 3801C01803801C03000C0600070C0001F00013227EA018>I<01F800060E000803001001 802001802000C06000C06000C06000C07000C07801803E01003F02001FC4000FF80003F8 0003FC00067F00083F80100F803007C06001C06000E0C000E0C00060C00060C00060C000 606000406000C03000801803000E0E0003F00013227EA018>56 D<01F000060C000C0600 180700380380700380700380F001C0F001C0F001C0F001E0F001E0F001E0F001E0F001E0 7001E07003E03803E01805E00C05E00619E003E1E00001C00001C00001C0000380000380 300300780700780600700C002018001030000FC00013227EA018>I<0001800000018000 000180000003C0000003C0000003C0000005E0000005E000000DF0000008F0000008F000 0010F800001078000010780000203C0000203C0000203C0000401E0000401E0000401E00 00800F0000800F0000FFFF000100078001000780030007C0020003C0020003C0040003E0 040001E0040001E00C0000F00C0000F03E0001F8FF800FFF20237EA225>65 D<0007E0100038183000E0063001C00170038000F0070000F00E0000701E0000701C0000 303C0000303C0000307C0000107800001078000010F8000000F8000000F8000000F80000 00F8000000F8000000F8000000F800000078000000780000107C0000103C0000103C0000 101C0000201E0000200E000040070000400380008001C0010000E0020000381C000007E0 001C247DA223>67 D69 DI73 D<03FFF0001F00000F00000F00000F00000F00000F00000F00 000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00 000F00000F00000F00000F00000F00000F00700F00F80F00F80F00F80E00F01E00401C00 20380018700007C00014237EA119>I77 DI80 D82 D<03F0200C0C601802603001E07000E0600060E000 60E00060E00020E00020E00020F00000F000007800007F00003FF0001FFE000FFF0003FF 80003FC00007E00001E00000F00000F0000070800070800070800070800070C00060C000 60E000C0F000C0C80180C6070081FC0014247DA21B>I<7FFFFFF8780780786007801840 0780084007800840078008C007800C800780048007800480078004800780040007800000 078000000780000007800000078000000780000007800000078000000780000007800000 078000000780000007800000078000000780000007800000078000000780000007800000 07800000078000000FC00003FFFF001E227EA123>I<0FE0001838003C0C003C0E001807 0000070000070000070000FF0007C7001E07003C0700780700700700F00708F00708F007 08F00F087817083C23900FC1E015157E9418>97 D<0E0000FE00001E00000E00000E0000 0E00000E00000E00000E00000E00000E00000E00000E00000E00000E1F000E61C00E8060 0F00300E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E001E0E001E0E001C 0E003C0E00380F00700C80600C41C0083F0017237FA21B>I<01FE000703000C07801C07 80380300780000700000F00000F00000F00000F00000F00000F00000F000007000007800 403800401C00800C010007060001F80012157E9416>I<0000E0000FE00001E00000E000 00E00000E00000E00000E00000E00000E00000E00000E00000E00000E001F8E00704E00C 02E01C01E03800E07800E07000E0F000E0F000E0F000E0F000E0F000E0F000E0F000E070 00E07800E03800E01801E00C02E0070CF001F0FE17237EA21B>I<01FC000707000C0380 1C01C03801C07801E07000E0F000E0FFFFE0F00000F00000F00000F00000F00000700000 7800203800201C00400E008007030000FC0013157F9416>I<003C00C6018F038F030F07 0007000700070007000700070007000700FFF80700070007000700070007000700070007 0007000700070007000700070007000700070007807FF8102380A20F>I<00007001F198 071E180E0E181C07001C07003C07803C07803C07803C07801C07001C07000E0E000F1C00 19F0001000001000001800001800001FFE000FFFC00FFFE03800F0600030400018C00018 C00018C000186000306000303800E00E038003FE0015217F9518>I<0E0000FE00001E00 000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E1F 800E60C00E80E00F00700F00700E00700E00700E00700E00700E00700E00700E00700E00 700E00700E00700E00700E00700E00700E00700E0070FFE7FF18237FA21B>I<1C001E00 3E001E001C00000000000000000000000000000000000E00FE001E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E00FFC00A227FA10E>I<01 C003E003E003E001C00000000000000000000000000000000001E00FE001E000E000E000 E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000 E000E000E060E0F0C0F18061803E000B2C82A10F>I<0E0000FE00001E00000E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00000E00000E03FC0E01F00E01 C00E01800E02000E04000E08000E10000E38000EF8000F1C000E1E000E0E000E07000E07 800E03C00E01C00E01E00E00F00E00F8FFE3FE17237FA21A>I<0E00FE001E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E00FFE00B237FA20E>I<0E1FC07F00 FE60E183801E807201C00F003C00E00F003C00E00E003800E00E003800E00E003800E00E 003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E00 3800E00E003800E00E003800E00E003800E00E003800E0FFE3FF8FFE27157F942A>I<0E 1F80FE60C01E80E00F00700F00700E00700E00700E00700E00700E00700E00700E00700E 00700E00700E00700E00700E00700E00700E00700E0070FFE7FF18157F941B>I<01FC00 0707000C01801800C03800E0700070700070F00078F00078F00078F00078F00078F00078 F000787000707800F03800E01C01C00E038007070001FC0015157F9418>I<0E1F00FE61 C00E80600F00700E00380E003C0E001C0E001E0E001E0E001E0E001E0E001E0E001E0E00 1E0E003C0E003C0E00380F00700E80E00E41C00E3F000E00000E00000E00000E00000E00 000E00000E00000E00000E0000FFE000171F7F941B>I<0E3CFE461E8F0F0F0F060F000E 000E000E000E000E000E000E000E000E000E000E000E000E000F00FFF010157F9413> 114 D<0F8830786018C018C008C008E008F0007F803FE00FF001F8003C801C800C800CC0 0CC008E018D0308FC00E157E9413>I<02000200020002000600060006000E001E003E00 FFF80E000E000E000E000E000E000E000E000E000E000E000E040E040E040E040E040E04 0708030801F00E1F7F9E13>I<0E0070FE07F01E00F00E00700E00700E00700E00700E00 700E00700E00700E00700E00700E00700E00700E00700E00700E00F00E00F00601700382 7800FC7F18157F941B>III121 D E /Fz 18 117 df45 D<78FCFCFCFC780000000000000000000000 000000000000000078FCFCFCFC78061F7A9E12>58 D<0000030000000000030000000000 0300000000000780000000000780000000000FC0000000000FC0000000000FC000000000 17E00000000013E00000000013E00000000023F00000000021F00000000021F000000000 40F80000000040F80000000040F800000000807C00000000807C00000001807E00000001 003E00000001003E00000002003F00000002001F00000002001F00000004000F80000004 000F80000004000F800000080007C00000080007C00000180007E000001FFFFFE000001F FFFFE00000200003F00000200001F00000200001F00000400001F80000400000F8000040 0000F800008000007C00008000007C00008000007C00010000003E00010000003E000300 00003F00030000001F00070000001F001F8000003F80FFE00003FFFCFFE00003FFFC2E32 7EB132>65 D73 D77 D80 D<007F802001FFE02007C078600F001C601E0006E03C0003E0380001 E0780000E0700000E070000060F0000060F0000060F0000020F0000020F0000020F80000 20F80000007C0000007E0000003F0000003FC000001FF800000FFF800007FFF80003FFFC 0000FFFF00000FFF800000FFC000001FE0000007E0000003F0000001F0000000F0000000 F8000000F88000007880000078800000788000007880000078C0000078C0000070E00000 F0E00000E0F00000E0F80001C0EC000380C7000700C1F01E00807FFC00800FF0001D337C B125>83 D<00FE00000303C0000C00E00010007000100038003C003C003E001C003E001E 003E001E0008001E0000001E0000001E0000001E00000FFE0000FC1E0003E01E000F801E 001F001E003E001E003C001E007C001E00F8001E04F8001E04F8001E04F8003E04F8003E 0478003E047C005E043E008F080F0307F003FC03E01E1F7D9E21>97 D<003F8000E0600380180700040F00041E001E1C003E3C003E7C003E7C0008780000F800 00F80000F80000F80000F80000F80000F80000F80000F800007800007C00007C00003C00 011E00011E00020F000207000403801800E060003F80181F7D9E1D>99 D<000001E000003FE000003FE0000003E0000001E0000001E0000001E0000001E0000001 E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001 E0000001E0001F81E000F061E001C019E0078005E00F0003E00E0003E01E0001E03C0001 E03C0001E07C0001E0780001E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001 E0F80001E0F80001E0F80001E0780001E0780001E03C0001E03C0001E01C0001E01E0003 E00E0005E0070009E0038011F000E061FF003F81FF20327DB125>I<003F800000E0E000 0380380007003C000E001E001E001E001C000F003C000F007C000F0078000F8078000780 F8000780F8000780FFFFFF80F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000003C0000003C0000801E0000800E0001000F0002000780020001C00C00 00F03000001FC000191F7E9E1D>I<0007E0001C1000383800707C00E07C01E07C01C038 03C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C000 FFFFC0FFFFC003C00003C00003C00003C00003C00003C00003C00003C00003C00003C000 03C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C000 03C00003C00003C00003C00007E0007FFF007FFF0016327FB114>I<000000F0007F0308 01C1C41C0380E81C070070080F0078001E003C001E003C003E003E003E003E003E003E00 3E003E003E003E003E003E001E003C001E003C000F007800070070000780E00009C1C000 087F000018000000180000001800000018000000180000001C0000000E0000000FFFF800 07FFFF0003FFFF800E000FC0180001E0300000F070000070E0000038E0000038E0000038 E0000038E00000387000007070000070380000E01C0001C00700070001C01C00003FE000 1E2F7E9F21>I<07000F801F801F800F8007000000000000000000000000000000000000 00000000000780FF80FF800F800780078007800780078007800780078007800780078007 800780078007800780078007800780078007800780078007800FC0FFF8FFF80D307EAF12 >105 D<0780FE0000FF83078000FF8C03C0000F9001E00007A001E00007A000F00007C0 00F00007C000F000078000F000078000F000078000F000078000F000078000F000078000 F000078000F000078000F000078000F000078000F000078000F000078000F000078000F0 00078000F000078000F000078000F000078000F000078000F000078000F000078000F000 0FC001F800FFFC1FFF80FFFC1FFF80211F7E9E25>110 D<0783E0FF8C18FF907C0F907C 07A07C07C03807C00007C00007C000078000078000078000078000078000078000078000 078000078000078000078000078000078000078000078000078000078000078000078000 0FC000FFFE00FFFE00161F7E9E19>114 D<01FC100E03301800F0300070600030E00030 E00010E00010E00010F00010F800007E00003FF0001FFF000FFFC003FFE0003FF00001F8 0000F880003C80003C80001CC0001CC0001CE0001CE00018F00038F00030CC0060C301C0 80FE00161F7E9E1A>I<00400000400000400000400000400000C00000C00000C00001C0 0001C00003C00007C0000FC0001FFFE0FFFFE003C00003C00003C00003C00003C00003C0 0003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C01003C0 1003C01003C01003C01003C01003C01003C01001C02001E02000E0400078C0001F00142C 7FAB19>I E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 0 1 0 0 bop 340 938 a Fz(MPI:)22 b(A)g(Message-P)n(assing)e(In)n(terface)j (Standard)621 1133 y Fy(Message)16 b(P)o(assing)h(In)o(terface)e(F)l (orum)828 1259 y(June)i(12,)f(1995)77 1317 y(This)h(w)o(ork)f(w)o(as)h (supp)q(orted)g(in)f(part)h(b)o(y)e(ARP)l(A)h(and)h(NSF)e(under)h(gran) o(t)h(ASC-9310330,)i(the)192 1375 y(National)d(Science)f(F)l(oundation) i(Science)e(and)i(T)l(ec)o(hnology)f(Cen)o(ter)f(Co)q(op)q(erativ)o(e) 76 1433 y(Agreemen)o(t)e(No.)22 b(CCR-8809615,)d(and)e(b)o(y)e(the)h (Commission)e(of)j(the)f(Europ)q(ean)i(Comm)o(unit)n(y)654 1492 y(through)f(Esprit)f(pro)s(ject)g(P6643.)p eop %%Page: 1 2 1 1 bop 1937 77 a Fx(>)75 165 y Fw(V)o(ersion)16 b(1.1:)21 b(June,)c(1995.)44 b Fv(Beginning)18 b(in)f(Marc)o(h,)e(1995,)g(the)h (Message)f(P)o(assing)h(In)o(terface)g(F)l(orum)75 222 y(recon)o(v)o(ened)g(to)f(correct)g(errors)g(and)g(mak)o(e)g (clari\014cations)i(in)f(the)g(MPI)f(do)q(cumen)o(t)h(of)f(Ma)o(y)g(5,) f(1994,)75 278 y(referred)19 b(to)f(b)q(elo)o(w)h(as)f(V)l(ersion)h (1.0.)29 b(These)19 b(discussions)h(resulted)f(in)h(V)l(ersion)f(1.1,)f (whic)o(h)h(is)h(this)75 335 y(do)q(cumen)o(t.)27 b(The)17 b(c)o(hanges)h(from)e(V)l(ersion)i(1.0)f(are)g(minor.)26 b(A)17 b(v)o(ersion)h(of)f(this)h(do)q(cumen)o(t)f(with)h(all)75 391 y(c)o(hanges)d(mark)o(ed)g(is)h(a)o(v)m(ailable.)21 b(This)16 b(paragraph)e(is)i(an)f(example)h(of)f(a)g(c)o(hange.)436 b Fx(?)75 511 y Fw(V)o(ersion)13 b(1.0:)18 b(June,)d(1994.)44 b Fv(The)13 b(Message)f(P)o(assing)h(In)o(terface)g(F)l(orum)g (\(MPIF\),)e(with)j(participation)75 567 y(from)e(o)o(v)o(er)h(40)f (organizations,)i(has)f(b)q(een)h(meeting)g(since)g(Jan)o(uary)f(1993)f (to)g(discuss)j(and)e(de\014ne)h(a)f(set)75 624 y(of)g(library)i(in)o (terface)f(standards)g(for)f(message)g(passing.)20 b(MPIF)14 b(is)g(not)g(sanctioned)g(or)g(supp)q(orted)g(b)o(y)75 680 y(an)o(y)h(o\016cial)h(standards)f(organization.)166 737 y(The)i(goal)f(of)g(the)g(Message)g(P)o(assing)g(In)o(terface,)g (simply)i(stated,)e(is)g(to)g(dev)o(elop)h(a)g(widely)g(used)75 793 y(standard)h(for)g(writing)h(message-passing)g(programs.)29 b(As)19 b(suc)o(h)g(the)g(in)o(terface)g(should)g(establish)h(a)75 850 y(practical,)c(p)q(ortable,)f(e\016cien)o(t,)g(and)h(\015exible)h (standard)e(for)g(message)f(passing.)166 906 y(This)20 b(is)g(the)g(\014nal)h(rep)q(ort,)f(V)l(ersion)g(1.0,)g(of)f(the)h (Message)f(P)o(assing)h(In)o(terface)f(F)l(orum.)33 b(This)75 963 y(do)q(cumen)o(t)18 b(con)o(tains)g(all)h(the)f(tec)o(hnical)i (features)d(prop)q(osed)h(for)g(the)g(in)o(terface.)28 b(This)19 b(cop)o(y)e(of)h(the)75 1019 y(draft)d(w)o(as)f(pro)q(cessed) i(b)o(y)f(L)554 1013 y Fu(a)575 1019 y Fv(T)600 1033 y(E)626 1019 y(X)g(on)g(June)h(12,)e(1995.)166 1076 y(Please)j(send)g (commen)o(ts)f(on)g Fw(MPI)g Fv(to)g Ft(mpi-comments@cs.utk.edu)p Fv(.)21 b(Y)l(our)16 b(commen)o(t)g(will)i(b)q(e)75 1132 y(forw)o(arded)c(to)h(MPIF)g(committee)g(mem)o(b)q(ers)g(who)g(will)i (attempt)d(to)h(resp)q(ond.)179 2388 y(c)166 2389 y Fx(\015)p Fv(1993,)f(1994,)h(1995)f(Univ)o(ersit)o(y)i(of)f(T)l(ennessee,)i(Kno)o (xville,)g(T)l(ennessee.)22 b(P)o(ermission)16 b(to)f(cop)o(y)75 2445 y(without)20 b(fee)g(all)h(or)f(part)f(of)h(this)g(material)g(is)h (gran)o(ted,)f(pro)o(vided)h(the)f(Univ)o(ersit)o(y)g(of)g(T)l (ennessee)75 2501 y(cop)o(yrigh)o(t)15 b(notice)i(and)f(the)f(title)i (of)e(this)h(do)q(cumen)o(t)g(app)q(ear,)g(and)g(notice)g(is)g(giv)o (en)g(that)f(cop)o(ying)h(is)75 2558 y(b)o(y)f(p)q(ermission)i(of)e (the)g(Univ)o(ersit)o(y)h(of)e(T)l(ennessee.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 3 2 2 bop 75 377 a Fr(Con)m(ten)m(ts)75 645 y Fq(Ac)o(kno)o(wledgmen)o (ts)1346 b(vi)75 747 y(1)42 b(In)o(tro)q(duction)19 b(to)f(MPI)1230 b(1)143 804 y Fv(1.1)46 b(Ov)o(erview)16 b(and)f(Goals)38 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fv(1)143 860 y(1.2)46 b(Who)15 b(Should)h(Use)f(This)h(Standard?)32 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fv(2)143 916 y(1.3)46 b(What)14 b(Platforms)h(Are)g(T)l(argets)f(F)l(or)h(Implemen)o(tation?) 38 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fv(3)143 973 y(1.4)46 b(What)14 b(Is)i(Included)h(In)f(The)g (Standard?)42 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fv(3)143 1029 y(1.5)46 b(What)14 b(Is)i(Not)e(Included)k(In)e(The)f(Standard?)23 b Fp(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)91 b Fv(3)143 1086 y(1.6)46 b(Organization)16 b(of)e(this)i(Do)q(cumen)o(t)34 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fv(4)75 1188 y Fq(2)42 b(MPI)17 b(T)l(erms)f(and)i(Con)o(v)o(en)o (tions)1042 b(6)143 1244 y Fv(2.1)46 b(Do)q(cumen)o(t)15 b(Notation)44 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)91 b Fv(6)143 1301 y(2.2)46 b(Pro)q(cedure)16 b(Sp)q(eci\014cation) 44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fv(6)143 1357 y(2.3)46 b(Seman)o(tic)15 b(T)l(erms)g Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 91 b Fv(7)143 1414 y(2.4)46 b(Data)14 b(T)o(yp)q(es)30 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)91 b Fv(8)248 1470 y(2.4.1)50 b(Opaque)16 b(ob)s(jects)21 b Fp(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fv(8)248 1526 y(2.4.2)50 b(Arra)o(y)14 b(argumen)o(ts)28 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fv(9)248 1583 y(2.4.3)50 b(State)14 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fv(9)248 1639 y(2.4.4)50 b(Named)15 b(constan)o(ts)23 b Fp(:)f(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)91 b Fv(9)248 1696 y(2.4.5)50 b(Choice)20 b Fp(:)i(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(10)248 1752 y(2.4.6)50 b(Addresses)28 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(10)143 1809 y(2.5)46 b(Language)15 b(Binding)45 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b Fv(10)248 1865 y(2.5.1)50 b(F)l(ortran)14 b(77)h(Binding)i(Issues)30 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(10)248 1922 y(2.5.2)50 b(C)15 b(Binding)i(Issues)31 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(11)143 1978 y(2.6)46 b(Pro)q(cesses)40 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b Fv(12)143 2035 y(2.7)46 b(Error)14 b(Handling)32 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fv(12)143 2091 y(2.8)46 b(Implemen)o(tation)16 b(issues)36 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(14)248 2147 y(2.8.1)50 b(Indep)q(endence)19 b(of)14 b(Basic)i(Run)o(time)g(Routines)42 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(14)248 2204 y(2.8.2)50 b(In)o(teraction)16 b(with)f(signals)h(in)g(POSIX)42 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)69 b Fv(14)143 2260 y(2.9)46 b(Examples)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)69 b Fv(15)75 2362 y Fq(3)42 b(P)o(oin)o(t-to-P)o(oin)o(t) 19 b(Comm)o(unication)975 b(16)143 2419 y Fv(3.1)46 b(In)o(tro)q (duction)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) h(:)f(:)g(:)h(:)f(:)69 b Fv(16)143 2475 y(3.2)46 b(Blo)q(c)o(king)16 b(Send)g(and)g(Receiv)o(e)g(Op)q(erations)24 b Fp(:)e(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(17)248 2532 y(3.2.1)50 b(Blo)q(c)o(king)17 b(send)h Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(17)248 2588 y(3.2.2)50 b(Message)15 b(data)26 b Fp(:)d(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(17)248 2645 y(3.2.3)50 b(Message)15 b(en)o(v)o(elop)q(e)j Fp(:)k(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(19)248 2701 y(3.2.4)50 b(Blo)q(c)o(king)17 b(receiv)o(e)44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(20)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 4 3 3 bop 248 45 a Fv(3.2.5)50 b(Return)16 b(status)j Fp(:)k(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(21)143 102 y(3.3)46 b(Data)14 b(t)o(yp)q(e)h(matc)o(hing)g(and)h(data)e(con)o(v)o (ersion)j Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)69 b Fv(23)248 158 y(3.3.1)50 b(T)o(yp)q(e)16 b(matc)o(hing)f(rules)27 b Fp(:)22 b(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)69 b Fv(23)248 214 y(3.3.2)50 b(Data)14 b(con)o(v)o(ersion)46 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b Fv(25)143 271 y(3.4)46 b(Comm)o(unication)15 b(Mo)q(des)i Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(27)143 327 y(3.5)46 b(Seman)o(tics)15 b(of)g(p)q(oin)o(t-to-p)q (oin)o(t)h(comm)o(unication)40 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(30)143 384 y(3.6)46 b(Bu\013er)15 b(allo)q(cation)h(and)g(usage)38 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(34)248 440 y(3.6.1)50 b(Mo)q(del)16 b(implemen)o(tation)h(of)d(bu\013ered)i (mo)q(de)27 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)69 b Fv(36)143 497 y(3.7)46 b(Non)o(blo)q(c)o(king)16 b(comm)o(unication)j Fp(:)j(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(36)248 553 y(3.7.1)50 b(Comm)o(unication)16 b(Ob)s(jects)23 b Fp(:)g(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(38)248 610 y(3.7.2)50 b(Comm)o(unication)16 b(initiation)32 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 69 b Fv(38)248 666 y(3.7.3)50 b(Comm)o(unication)16 b(Completion)i Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(41)248 723 y(3.7.4)50 b(Seman)o(tics)16 b(of)f(Non)o(blo)q(c)o(king)h(Comm)o(unications)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(44)248 779 y(3.7.5)50 b(Multiple)17 b(Completions)41 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(45)143 835 y(3.8)46 b(Prob)q(e)15 b(and)g(Cancel)g Fp(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(51)143 892 y(3.9)46 b(P)o(ersisten)o(t)15 b(comm)o(unication)h(requests)33 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(55)143 948 y(3.10)23 b(Send-receiv)o(e)d Fp(:)i(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(59)143 1005 y(3.11)23 b(Null)16 b(pro)q(cesses)i Fp(:)k(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(60)143 1061 y(3.12)23 b(Deriv)o(ed)15 b(datat)o(yp)q(es)42 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(61)248 1118 y(3.12.1)27 b(Datat)o(yp)q(e)14 b(constructors)27 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)69 b Fv(62)248 1174 y(3.12.2)27 b(Address)16 b(and)f(exten)o(t)g(functions)32 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(69)248 1231 y(3.12.3)27 b(Lo)o(w)o(er-b)q(ound)16 b(and)f(upp)q(er-b)q(ound)j(mark)o(ers)33 b Fp(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(70)248 1287 y(3.12.4)27 b(Commit)15 b(and)g(free)28 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(72)248 1344 y(3.12.5)27 b(Use)16 b(of)e(general)i(datat)o(yp)q(es)f(in)h(comm) o(unication)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)69 b Fv(73)248 1400 y(3.12.6)27 b(Correct)15 b(use)g(of)g (addresses)26 b Fp(:)d(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(76)248 1456 y(3.12.7)27 b(Examples)33 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(77)143 1513 y(3.13)23 b(P)o(ac)o(k)14 b(and)i(unpac)o(k)29 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(85)75 1615 y Fq(4)42 b(Collectiv)o(e)19 b(Comm)o(unication)1077 b(92)143 1671 y Fv(4.1)46 b(In)o(tro)q(duction)16 b(and)f(Ov)o(erview) 44 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(92)143 1728 y(4.2)46 b(Comm)o(unicator)14 b(argumen)o(t)e Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(95)143 1784 y(4.3)46 b(Barrier)15 b(sync)o(hronization)44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(95)143 1841 y(4.4)46 b(Broadcast)27 b Fp(:)c(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(95)248 1897 y(4.4.1)50 b(Example)16 b(using)g Fw(MPI)p 786 1897 14 2 v 16 w(BCAST)26 b Fp(:)d(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(96)143 1954 y(4.5)46 b(Gather)17 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)69 b Fv(96)248 2010 y(4.5.1)50 b(Examples)16 b(using)g Fw(MPI)p 804 2010 V 16 w(GA)l(THER)p Fv(,)g Fw(MPI)p 1106 2010 V 16 w(GA)l(THERV)35 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)69 b Fv(99)143 2066 y(4.6)46 b(Scatter)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b Fv(105)248 2123 y(4.6.1)k(Examples)16 b(using)g Fw(MPI)p 804 2123 V 16 w(SCA)l(TTER)p Fv(,)g Fw(MPI)p 1129 2123 V 15 w(SCA)l(TTERV)25 b Fp(:)e(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b Fv(107)143 2179 y(4.7)g(Gather-to-all)40 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b Fv(110)248 2236 y(4.7.1)k(Examples)16 b(using)g Fw(MPI)p 804 2236 V 16 w(ALLGA)l(THER)p Fv(,)f Fw(MPI)p 1185 2236 V 16 w(ALLGA)l(THERV)i Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(111)143 2292 y(4.8)g(All-to-All)17 b(Scatter/Gather)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(112)143 2349 y(4.9)g(Global)16 b(Reduction)g(Op)q(erations)34 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(114)248 2405 y(4.9.1)k(Reduce)45 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(114)248 2462 y(4.9.2)k(Prede\014ned) 17 b(reduce)f(op)q(erations)37 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(115)248 2518 y(4.9.3)k(MINLOC)16 b(and)f(MAXLOC)44 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(117)248 2575 y(4.9.4)k (User-De\014ned)16 b(Op)q(erations)h Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fv(121)248 2631 y(4.9.5)k(All-Reduce)43 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(125)143 2687 y(4.10)23 b(Reduce-Scatter)33 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(126)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 5 4 4 bop 143 45 a Fv(4.11)23 b(Scan)28 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(127)248 102 y(4.11.1)27 b(Example)16 b(using)g Fw(MPI)p 786 102 14 2 v 16 w(SCAN)k Fp(:)i(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(127)143 158 y(4.12)23 b(Correctness)32 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fv(129)75 260 y Fq(5)c(Groups,)17 b(Con)o(texts,)f(and)i(Comm)o (unicators)757 b(133)143 316 y Fv(5.1)46 b(In)o(tro)q(duction)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b Fv(133)248 373 y(5.1.1)k(F)l(eatures)15 b(Needed)h(to)f(Supp)q(ort)h(Libraries)40 b Fp(:)23 b(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(133)248 429 y(5.1.2)k Fw(MPI)p Fv('s)14 b(Supp)q(ort)i(for)f(Libraries)30 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(134)143 486 y(5.2)g(Basic)16 b(Concepts)30 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b Fv(136)248 542 y(5.2.1)k(Groups)43 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fv(136)248 599 y(5.2.2)k(Con)o(texts)c Fp(:)22 b(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(136)248 655 y(5.2.3)k(In)o(tra-Comm)o(unicators)44 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(137)248 712 y(5.2.4)k (Prede\014ned)17 b(In)o(tra-Comm)o(unicators)30 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b Fv(137)143 768 y(5.3)g(Group)15 b(Managemen)o(t)41 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(138)248 824 y(5.3.1)k(Group)15 b(Accessors)34 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(138)248 881 y(5.3.2)k(Group)15 b(Constructors)37 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(139)248 937 y(5.3.3)k(Group)15 b(Destructors)27 b Fp(:)c(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b Fv(143)143 994 y(5.4)g(Comm)o(unicator)14 b(Managemen)o(t)i Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(144)248 1050 y(5.4.1)k(Comm)o(unicator)15 b(Accessors)43 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(144)248 1107 y(5.4.2)k(Comm)o (unicator)15 b(Constructors)10 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(146)248 1163 y(5.4.3)k(Comm)o(unicator)15 b(Destructors)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(148)143 1220 y(5.5)g(Motiv)m(ating) 15 b(Examples)h Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:) f(:)46 b Fv(149)248 1276 y(5.5.1)k(Curren)o(t)15 b(Practice)g(#1)27 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(149)248 1333 y(5.5.2)k(Curren)o(t)15 b(Practice)g(#2)27 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(150)248 1389 y(5.5.3)k(\(Appro)o(ximate\))15 b(Curren)o(t)f(Practice)i(#3)35 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b Fv(150)248 1445 y(5.5.4)k(Example)16 b(#4)44 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(151)248 1502 y(5.5.5)k(Library)16 b(Example)g(#1)22 b Fp(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(152)248 1558 y(5.5.6)k(Library)16 b(Example)g(#2)22 b Fp(:)g(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b Fv(154)143 1615 y(5.6)g(In)o(ter-Comm)o(unication) 14 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(156)248 1671 y(5.6.1)k(In)o(ter-comm)o(unicator)15 b(Accessors)h Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(158)248 1728 y(5.6.2)k(In)o(ter-comm)o(unicator)15 b(Op)q(erations)22 b Fp(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b Fv(159)248 1784 y(5.6.3)k(In)o(ter-Comm)o (unication)16 b(Examples)g Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(161)143 1841 y(5.7)g(Cac)o(hing)32 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(168)248 1897 y(5.7.1)k(F)l(unctionalit)o(y)32 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(168)248 1954 y(5.7.2)k(A)o (ttributes)15 b(Example)43 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fv(173)143 2010 y(5.8)g(F)l(ormalizing)16 b(the)f(Lo)q(osely)h (Sync)o(hronous)g(Mo)q(del)j Fp(:)j(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(175)248 2066 y(5.8.1)k(Basic)16 b(Statemen)o(ts)22 b Fp(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(175)248 2123 y(5.8.2)k(Mo)q(dels)16 b(of)f(Execution)29 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(175)75 2225 y Fq(6)c(Pro)q(cess)17 b(T)l(op)q(ologies)1224 b(177)143 2281 y Fv(6.1)46 b(In)o(tro)q(duction)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(177)143 2338 y(6.2)g(Virtual)16 b(T)l(op)q(ologies)37 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(178)143 2394 y(6.3)g(Em)o(b)q(edding)16 b(in)g Fw(MPI)29 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(178)143 2451 y(6.4)g(Ov)o(erview)16 b(of)e(the)i(F)l(unctions)22 b Fp(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(179)143 2507 y(6.5)g(T)l(op)q(ology)15 b(Constructors)20 b Fp(:)i(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(180)248 2564 y(6.5.1)k(Cartesian)15 b(Constructor)27 b Fp(:)22 b(:)h(:)f(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:) h(:)f(:)46 b Fv(180)248 2620 y(6.5.2)k(Cartesian)15 b(Con)o(v)o (enience)h(F)l(unction:)21 b Fw(MPI)p 1152 2620 V 16 w(DIMS)p 1279 2620 V 16 w(CREA)l(TE)39 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b Fv(180)248 2676 y(6.5.3)k(General)16 b(\(Graph\))e (Constructor)27 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(181)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 6 5 5 bop 248 45 a Fv(6.5.4)50 b(T)l(op)q(ology)15 b(inquiry)i(functions) 42 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(183)248 102 y(6.5.5)k(Cartesian) 15 b(Shift)h(Co)q(ordinates)23 b Fp(:)g(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(187)248 158 y(6.5.6)k(P)o(artitioning)16 b(of)e(Cartesian)h (structures)45 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b Fv(188)248 214 y(6.5.7)k(Lo)o(w-lev)o(el)17 b(top)q(ology)d(functions)45 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(189)143 271 y(6.6)g(An)15 b(Application)i(Example)g Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(190)75 373 y Fq(7)c(MPI)17 b(En)o(vironmen)o(tal)g(Managemen)o(t) 883 b(192)143 429 y Fv(7.1)46 b(Implemen)o(tation)16 b(information)22 b Fp(:)g(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(192)248 486 y(7.1.1)k(En)o(vironmen)o(tal)16 b(Inquiries)28 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(192)143 542 y(7.2)g(Error)14 b(handling)41 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(194)143 599 y(7.3)g(Error)14 b(co)q(des)i(and)f(classes)e Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b Fv(197)143 655 y(7.4)g(Timers)15 b(and)h(sync)o (hronization)28 b Fp(:)22 b(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(199)143 712 y(7.5)g(Startup)40 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:) h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(199)75 813 y Fq(8)c(Pro\014ling)18 b(In)o(terface)1238 b(202)143 870 y Fv(8.1)46 b(Requiremen)o(ts)28 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(202)143 926 y(8.2)g(Discussion)21 b Fp(:)i(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(202)143 983 y(8.3)g(Logic)16 b(of)e(the)i(design)29 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(203)248 1039 y(8.3.1)k(Miscellaneous)17 b(con)o(trol)e(of)g (pro\014ling)28 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(203)143 1096 y(8.4)g(Examples)36 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:) g(:)h(:)f(:)46 b Fv(204)248 1152 y(8.4.1)k(Pro\014ler)16 b(implemen)o(tation)35 b Fp(:)23 b(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(204)248 1209 y(8.4.2)k(MPI)15 b(library)h(implemen)o(tation)h Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(205)248 1265 y(8.4.3)k (Complications)e Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:) 46 b Fv(206)143 1322 y(8.5)g(Multiple)17 b(lev)o(els)f(of)f(in)o (terception)25 b Fp(:)d(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(207)75 1423 y Fq(Bibliograph)o(y)1430 b(208)75 1525 y(A)28 b(Language)19 b(Binding)1242 b(211)143 1582 y Fv(A.1)35 b(In)o(tro)q(duction)15 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:) g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(211)143 1638 y(A.2)35 b(De\014ned)16 b(Constan)o(ts)e(for)g(C)h(and)h(F)l (ortran)21 b Fp(:)h(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g (:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(211)143 1695 y(A.3)35 b(C)15 b(bindings)i(for)d(P)o(oin)o(t-to-P)o(oin)o(t)h(Comm)o (unication)21 b Fp(:)i(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f (:)g(:)h(:)f(:)46 b Fv(215)143 1751 y(A.4)35 b(C)15 b(Bindings)i(for)d (Collectiv)o(e)j(Comm)o(unication)34 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(218)143 1808 y(A.5)35 b(C)15 b(Bindings)i(for)d(Groups,)h(Con)o (texts,)f(and)h(Comm)o(unicators)26 b Fp(:)c(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b Fv(219)143 1864 y(A.6)35 b(C)15 b(Bindings)i(for)d(Pro)q(cess)h(T)l(op)q(ologies)43 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(220)143 1921 y(A.7)35 b(C)15 b(bindings)i(for)d(En)o(vironmen)o(tal)i(Inquiry)42 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)46 b Fv(221)143 1977 y(A.8)35 b(C)15 b(Bindings)i(for)d(Pro\014ling)35 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:) f(:)g(:)h(:)f(:)46 b Fv(221)143 2033 y(A.9)35 b(F)l(ortran)14 b(Bindings)j(for)d(P)o(oin)o(t-to-P)o(oin)o(t)h(Comm)o(unication)39 b Fp(:)22 b(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(221)143 2090 y(A.10)12 b(F)l(ortran)i(Bindings)j(for)d(Collectiv)o (e)j(Comm)o(unication)23 b Fp(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)h(:)f(:)g(:)h(:)f(:)46 b Fv(225)143 2146 y(A.11)12 b(F)l(ortran)i(Bindings)j(for)d(Groups,)h(Con)o(texts,)f(etc.)27 b Fp(:)22 b(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h (:)f(:)46 b Fv(227)143 2203 y(A.12)12 b(F)l(ortran)i(Bindings)j(for)d (Pro)q(cess)h(T)l(op)q(ologies)32 b Fp(:)23 b(:)f(:)g(:)h(:)f(:)g(:)h (:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(228)143 2259 y(A.13)12 b(F)l(ortran)i(Bindings)j(for)d(En)o (vironmen)o(tal)i(Inquiry)24 b Fp(:)e(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f (:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(229)143 2316 y(A.14)12 b(F)l(ortran)i(Bindings)j(for)d(Pro\014ling)24 b Fp(:)e(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:)f(:)g(:)h(:) f(:)g(:)h(:)f(:)h(:)f(:)g(:)h(:)f(:)46 b Fv(230)75 2418 y Fq(MPI)17 b(F)l(unction)h(Index)1252 b(231)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 7 6 6 bop 75 75 a Fr(Ac)m(kno)m(wledgmen)m(ts)166 282 y Fv(The)19 b(tec)o(hnical)i(dev)o(elopmen)o(t)f(w)o(as)e(carried)i(out)f (b)o(y)g(subgroups,)h(whose)f(w)o(ork)g(w)o(as)f(review)o(ed)75 338 y(b)o(y)d(the)h(full)h(committee.)k(During)15 b(the)h(p)q(erio)q(d) h(of)e(dev)o(elopmen)o(t)h(of)f(the)h(Message)f(P)o(assing)g(In)o (terface)75 395 y(\()p Fw(MPI)p Fv(\),)f(man)o(y)g(p)q(eople)j(serv)o (ed)e(in)h(p)q(ositions)g(of)f(resp)q(onsibilit)o(y)j(and)d(are)g (listed)i(b)q(elo)o(w.)143 497 y Fx(\017)23 b Fv(Jac)o(k)15 b(Dongarra,)e(Da)o(vid)i(W)l(alk)o(er,)g(Con)o(v)o(eners)g(and)g (Meeting)h(Chairs)143 589 y Fx(\017)23 b Fv(Ewing)15 b(Lusk,)h(Bob)f(Knigh)o(ten,)h(Min)o(utes)143 682 y Fx(\017)23 b Fv(Marc)14 b(Snir,)i(William)h(Gropp,)d(Ewing)i(Lusk,)f(P)o(oin)o (t-to-P)o(oin)o(t)f(Comm)o(unications)143 774 y Fx(\017)23 b Fv(Al)16 b(Geist,)e(Marc)h(Snir,)h(Stev)o(e)f(Otto,)f(Collectiv)o(e)j (Comm)o(unications)143 867 y Fx(\017)23 b Fv(Stev)o(e)15 b(Otto,)f(Editor)143 959 y Fx(\017)23 b Fv(Rolf)15 b(Hemp)q(el,)i(Pro)q (cess)e(T)l(op)q(ologies)143 1052 y Fx(\017)23 b Fv(Ewing)15 b(Lusk,)h(Language)f(Binding)143 1144 y Fx(\017)23 b Fv(William)17 b(Gropp,)d(En)o(vironmen)o(tal)i(Managemen)o(t)143 1237 y Fx(\017)23 b Fv(James)15 b(Co)o(wnie,)g(Pro\014ling)143 1330 y Fx(\017)23 b Fv(T)l(on)o(y)16 b(Skjellum,)j(Lyndon)f(Clark)o(e,) f(Marc)g(Snir,)h(Ric)o(hard)g(Little\014eld,)h(Mark)e(Sears,)g(Groups,) 189 1386 y(Con)o(texts,)c(and)j(Comm)o(unicators)143 1479 y Fx(\017)23 b Fv(Stev)o(en)15 b(Huss-Lederman,)h(Initial)h (Implemen)o(tation)f(Subset)166 1580 y(The)23 b(follo)o(wing)g(list)g (includes)i(some)d(of)g(the)g(activ)o(e)h(participan)o(ts)g(in)g(the)g Fw(MPI)f Fv(pro)q(cess)g(not)75 1637 y(men)o(tioned)16 b(ab)q(o)o(v)o(e.)100 1771 y(Ed)f(Anderson)114 b(Rob)q(ert)15 b(Babb)135 b(Jo)q(e)16 b(Baron)186 b(Eric)16 b(Barszcz)100 1827 y(Scott)f(Berryman)56 b(Rob)15 b(Bjornson)119 b(Nathan)15 b(Doss)133 b(Anne)16 b(Elster)100 1883 y(Jim)g(F)l(eeney)149 b(Vince)16 b(F)l(ernando)81 b(Sam)15 b(Fineb)q(erg)115 b(Jon)16 b(Flo)o(w)o(er)100 1940 y(Daniel)g(F)l(ry)o(e)143 b(Ian)15 b(Glendinning)74 b(Adam)15 b(Green)o(b)q(erg)50 b(Rob)q(ert)16 b(Harrison)100 1996 y(Leslie)h(Hart)153 b(T)l(om)15 b(Haupt)163 b(Don)15 b(Heller)176 b(T)l(om)15 b(Henderson)100 2053 y(Alex)h(Ho)210 b(C.T.)14 b(Ho)o(w)o(ard)g(Ho)62 b(Gary)14 b(Ho)o(w)o(ell)143 b(John)16 b(Kap)q(enga)100 2109 y(James)f(Kohl)141 b(Susan)15 b(Krauss)125 b(Bob)15 b(Leary)182 b(Arth)o(ur)15 b(Maccab)q(e)100 2166 y(P)o(eter)g(Madams)83 b(Alan)16 b(Main)o(w)o(aring)49 b(Oliv)o(er)17 b(McBry)o(an)70 b(Phil)17 b(McKinley)100 2222 y(Charles)e(Mosher)63 b(Dan)15 b(Nessett)149 b(P)o(eter)15 b(P)o(ac)o(heco)109 b(Ho)o(w)o(ard)14 b(P)o(almer)100 2279 y(P)o(aul)h(Pierce)147 b(Sanja)o(y)14 b(Rank)m(a)121 b(P)o(eter)15 b(Rigsb)q(ee)117 b(Arc)o(h)15 b(Robison)100 2335 y(Eric)o(h)h(Sc)o(hikuta)81 b(Am)o(buj)15 b(Singh)133 b(Alan)16 b(Sussman)108 b(Rob)q(ert)16 b(T)l(omlinson)100 2392 y(Rob)q(ert)f(G.)g(V)l(oigt)50 b(Dennis)16 b(W)l(eeks)117 b(Stephen)16 b(Wheat)83 b(Stev)o(e)15 b(Zenith)166 2534 y(The)e(Univ)o(ersit)o(y)h(of)f(T)l(ennessee)i(and)e(Oak)g(Ridge)i (National)e(Lab)q(oratory)g(made)g(the)g(draft)g(a)o(v)m(ail-)75 2591 y(able)j(b)o(y)f(anon)o(ymous)f(FTP)h(mail)g(serv)o(ers)g(and)g(w) o(ere)g(instrumen)o(tal)g(in)h(distributing)h(the)e(do)q(cumen)o(t.)166 2647 y Fw(MPI)c Fv(op)q(erated)h(on)f(a)h(v)o(ery)f(tigh)o(t)h(budget)g (\(in)g(realit)o(y)l(,)g(it)g(had)g(no)g(budget)g(when)g(the)g(\014rst) f(meeting)75 2704 y(w)o(as)i(announced\).)20 b(ARP)l(A)14 b(and)f(NSF)h(ha)o(v)o(e)e(supp)q(orted)i(researc)o(h)g(at)e(v)m (arious)i(institutions)h(that)d(ha)o(v)o(e)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 8 7 7 bop 75 45 a Fv(made)17 b(a)f(con)o(tribution)h(to)o(w)o(ards)e(tra) o(v)o(el)h(for)g(the)g(U.S.)g(academics.)25 b(Supp)q(ort)17 b(for)f(sev)o(eral)h(Europ)q(ean)75 102 y(participan)o(ts)f(w)o(as)e (pro)o(vided)i(b)o(y)f(ESPRIT.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 1 9 1 8 bop 75 356 a Fo(Chapter)34 b(1)75 564 y Fr(In)m(tro)s(duction)41 b(to)g(MPI)75 805 y Fn(1.1)59 b(Overview)19 b(and)g(Goals)75 906 y Fv(Message)11 b(passing)i(is)f(a)g(paradigm)g(used)g(widely)i(on) d(certain)i(classes)f(of)g(parallel)h(mac)o(hines,)g(esp)q(ecially)75 963 y(those)j(with)i(distributed)g(memory)l(.)24 b(Although)17 b(there)g(are)f(man)o(y)h(v)m(ariations,)g(the)g(basic)g(concept)g(of) 75 1019 y(pro)q(cesses)j(comm)o(unicating)h(through)e(messages)h(is)g (w)o(ell)h(understo)q(o)q(d.)34 b(Ov)o(er)20 b(the)g(last)g(ten)g(y)o (ears,)75 1076 y(substan)o(tial)11 b(progress)f(has)g(b)q(een)h(made)g (in)g(casting)f(signi\014can)o(t)i(applications)g(in)f(this)g (paradigm.)18 b(Eac)o(h)75 1132 y(v)o(endor)13 b(has)g(implemen)o(ted)i (its)e(o)o(wn)g(v)m(arian)o(t.)19 b(More)13 b(recen)o(tly)l(,)h(sev)o (eral)f(systems)g(ha)o(v)o(e)f(demonstrated)75 1188 y(that)k(a)h (message)f(passing)h(system)g(can)g(b)q(e)g(e\016cien)o(tly)h(and)f(p)q (ortably)h(implemen)o(ted.)26 b(It)17 b(is)g(th)o(us)g(an)75 1245 y(appropriate)c(time)g(to)f(try)h(to)f(de\014ne)i(b)q(oth)f(the)g (syn)o(tax)f(and)h(seman)o(tics)g(of)g(a)g(core)f(of)h(library)h (routines)75 1301 y(that)i(will)i(b)q(e)f(useful)h(to)e(a)g(wide)h (range)f(of)h(users)f(and)h(e\016cien)o(tly)h(implemen)o(table)g(on)f (a)f(wide)h(range)75 1358 y(of)e(computers.)166 1414 y(In)c(designing)h Fw(MPI)e Fv(w)o(e)h(ha)o(v)o(e)f(sough)o(t)g(to)g (mak)o(e)g(use)h(of)f(the)h(most)e(attractiv)o(e)h(features)g(of)g(a)h (n)o(um)o(b)q(er)75 1471 y(of)16 b(existing)i(message)e(passing)h (systems,)g(rather)f(than)g(selecting)j(one)d(of)h(them)f(and)h (adopting)g(it)g(as)75 1527 y(the)g(standard.)26 b(Th)o(us,)18 b Fw(MPI)e Fv(has)h(b)q(een)i(strongly)e(in\015uenced)j(b)o(y)d(w)o (ork)f(at)h(the)g(IBM)h(T.)f(J.)g(W)l(atson)75 1584 y(Researc)o(h)e (Cen)o(ter)g([1)o(,)g(2)o(],)f(In)o(tel's)i(NX/2)e([23)o(],)g(Express)h ([22)o(],)f(nCUBE's)h(V)l(ertex)g([21)o(],)f(p4)h([7)o(,)g(6],)f(and)75 1640 y(P)l(ARMA)o(CS)h([5,)f(8].)19 b(Other)c(imp)q(ortan)o(t)g(con)o (tributions)h(ha)o(v)o(e)e(come)h(from)f(Zip)q(co)q(de)j([24)o(,)d (25],)g(Chimp)75 1697 y([14)o(,)h(15)o(],)g(PVM)g([4)o(,)g(11)o(],)g (Chameleon)g([19)o(],)g(and)g(PICL)h([18)o(].)166 1753 y(The)e Fw(MPI)f Fv(standardization)h(e\013ort)e(in)o(v)o(olv)o(ed)j (ab)q(out)e(60)g(p)q(eople)i(from)e(40)g(organizations)h(mainly)75 1810 y(from)j(the)g(United)i(States)e(and)h(Europ)q(e.)27 b(Most)16 b(of)h(the)h(ma)s(jor)e(v)o(endors)h(of)g(concurren)o(t)h (computers)75 1866 y(w)o(ere)11 b(in)o(v)o(olv)o(ed)h(in)h Fw(MPI)p Fv(,)d(along)h(with)h(researc)o(hers)f(from)g(univ)o (ersities,)j(go)o(v)o(ernmen)o(t)c(lab)q(oratories,)i(and)75 1922 y(industry)l(.)21 b(The)14 b(standardization)h(pro)q(cess)f(b)q (egan)h(with)f(the)h(W)l(orkshop)f(on)g(Standards)g(for)g(Message)75 1979 y(P)o(assing)j(in)i(a)e(Distributed)h(Memory)f(En)o(vironmen)o(t,) h(sp)q(onsored)g(b)o(y)f(the)h(Cen)o(ter)f(for)g(Researc)o(h)h(on)75 2035 y(P)o(arallel)11 b(Computing,)g(held)h(April)f(29-30,)f(1992,)g (in)h(Williamsburg,)h(Virginia)g([29)o(].)18 b(A)o(t)9 b(this)i(w)o(orkshop)75 2092 y(the)17 b(basic)h(features)f(essen)o (tial)h(to)f(a)g(standard)g(message)g(passing)g(in)o(terface)h(w)o(ere) f(discussed,)i(and)e(a)75 2148 y(w)o(orking)e(group)g(established)i(to) d(con)o(tin)o(ue)i(the)f(standardization)h(pro)q(cess.)166 2205 y(A)e(preliminary)i(draft)e(prop)q(osal,)g(kno)o(wn)g(as)g(MPI1,)g (w)o(as)f(put)h(forw)o(ard)f(b)o(y)h(Dongarra,)f(Hemp)q(el,)75 2261 y(Hey)l(,)24 b(and)f(W)l(alk)o(er)g(in)g(No)o(v)o(em)o(b)q(er)f (1992,)h(and)f(a)h(revised)g(v)o(ersion)g(w)o(as)e(completed)j(in)f(F)l (ebruary)75 2318 y(1993)e([12)o(].)39 b(MPI1)22 b(em)o(b)q(o)q(died)h (the)f(main)g(features)g(that)f(w)o(ere)g(iden)o(ti\014ed)j(at)d(the)h (Williamsburg)75 2374 y(w)o(orkshop)d(as)h(b)q(eing)h(necessary)g(in)g (a)f(message)f(passing)i(standard.)34 b(Since)22 b(MPI1)d(w)o(as)h (primarily)75 2431 y(in)o(tended)12 b(to)d(promote)h(discussion)i(and)e (\\get)g(the)h(ball)g(rolling,")h(it)f(fo)q(cused)g(mainly)g(on)f(p)q (oin)o(t-to-p)q(oin)o(t)75 2487 y(comm)o(unications.)33 b(MPI1)20 b(brough)o(t)f(to)g(the)g(forefron)o(t)f(a)h(n)o(um)o(b)q(er) h(of)f(imp)q(ortan)o(t)h(standardization)75 2543 y(issues,)c(but)f(did) h(not)f(include)i(an)o(y)e(collectiv)o(e)i(comm)o(unication)f(routines) f(and)h(w)o(as)e(not)h(thread-safe.)166 2600 y(In)j(No)o(v)o(em)o(b)q (er)f(1992,)g(a)g(meeting)h(of)f(the)h Fw(MPI)f Fv(w)o(orking)g(group)h (w)o(as)e(held)j(in)g(Minneap)q(olis,)h(at)75 2656 y(whic)o(h)15 b(it)g(w)o(as)e(decided)k(to)c(place)j(the)e(standardization)h(pro)q (cess)g(on)f(a)g(more)g(formal)g(fo)q(oting,)g(and)h(to)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 10 2 9 bop 75 -100 a Fv(2)903 b Fm(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION)17 b(TO)e(MPI)75 45 y Fv(generally)i(adopt)f(the)g(pro)q(cedures)i(and)e (organization)g(of)g(the)g(High)h(P)o(erformance)f(F)l(ortran)f(F)l (orum.)75 102 y(Sub)q(committees)i(w)o(ere)e(formed)g(for)g(the)h(ma)s (jor)e(comp)q(onen)o(t)i(areas)f(of)g(the)h(standard,)f(and)g(an)h (email)75 158 y(discussion)23 b(service)g(established)g(for)e(eac)o(h.) 39 b(In)22 b(addition,)i(the)d(goal)h(of)f(pro)q(ducing)i(a)e(draft)g Fw(MPI)75 214 y Fv(standard)13 b(b)o(y)g(the)g(F)l(all)h(of)e(1993)g(w) o(as)g(set.)19 b(T)l(o)13 b(ac)o(hiev)o(e)h(this)f(goal)g(the)g Fw(MPI)g Fv(w)o(orking)g(group)f(met)h(ev)o(ery)75 271 y(6)k(w)o(eeks)g(for)f(t)o(w)o(o)g(da)o(ys)h(throughout)f(the)h (\014rst)g(9)g(mon)o(ths)f(of)h(1993,)f(and)h(presen)o(ted)h(the)f (draft)g Fw(MPI)75 327 y Fv(standard)d(at)g(the)g(Sup)q(ercomputing)i (93)e(conference)h(in)g(No)o(v)o(em)o(b)q(er)f(1993.)19 b(These)14 b(meetings)h(and)g(the)75 384 y(email)h(discussion)g (together)f(constituted)g(the)g Fw(MPI)f Fv(F)l(orum,)g(mem)o(b)q (ership)i(of)f(whic)o(h)h(has)e(b)q(een)i(op)q(en)75 440 y(to)f(all)h(mem)o(b)q(ers)f(of)g(the)g(high)h(p)q(erformance)f (computing)h(comm)o(unit)o(y)l(.)166 497 y(The)j(main)f(adv)m(an)o (tages)g(of)g(establishing)j(a)d(message-passing)g(standard)g(are)g(p)q (ortabilit)o(y)i(and)75 553 y(ease-of-use.)g(In)14 b(a)g(distributed)h (memory)e(comm)o(unication)i(en)o(vironmen)o(t)f(in)g(whic)o(h)h(the)f (higher)h(lev)o(el)75 610 y(routines)c(and/or)g(abstractions)f(are)h (build)i(up)q(on)e(lo)o(w)o(er)g(lev)o(el)h(message)e(passing)i (routines)f(the)g(b)q(ene\014ts)75 666 y(of)22 b(standardization)h(are) f(particularly)h(apparen)o(t.)41 b(F)l(urthermore,)24 b(the)e(de\014nition)i(of)e(a)g(message)75 723 y(passing)16 b(standard,)e(suc)o(h)i(as)e(that)h(prop)q(osed)g(here,)h(pro)o(vides)f (v)o(endors)h(with)f(a)g(clearly)h(de\014ned)h(base)75 779 y(set)10 b(of)g(routines)g(that)g(they)g(can)h(implemen)o(t)g (e\016cien)o(tly)l(,)i(or)c(in)i(some)f(cases)h(pro)o(vide)f(hardw)o (are)g(supp)q(ort)75 835 y(for,)k(thereb)o(y)h(enhancing)i(scalabilit)o (y)l(.)166 892 y(The)h(goal)g(of)g(the)g(Message)g(P)o(assing)g(In)o (terface)g(simply)i(stated)d(is)i(to)f(dev)o(elop)h(a)f(widely)h(used) 75 948 y(standard)f(for)g(writing)h(message-passing)g(programs.)29 b(As)19 b(suc)o(h)g(the)g(in)o(terface)g(should)g(establish)h(a)75 1005 y(practical,)c(p)q(ortable,)f(e\016cien)o(t,)g(and)h(\015exible)h (standard)e(for)g(message)f(passing.)166 1061 y(A)h(complete)h(list)g (of)f(goals)g(follo)o(ws.)143 1144 y Fx(\017)23 b Fv(Design)11 b(an)f(application)i(programming)e(in)o(terface)h(\(not)f(necessarily)i (for)d(compilers)j(or)e(a)g(system)189 1201 y(implemen)o(tation)16 b(library\).)143 1288 y Fx(\017)23 b Fv(Allo)o(w)15 b(e\016cien)o(t)h (comm)o(unication:)21 b(Av)o(oid)16 b(memory-to-memory)e(cop)o(ying)h (and)h(allo)o(w)f(o)o(v)o(erlap)189 1344 y(of)c(computation)h(and)g (comm)o(unication)h(and)f(o\017oad)f(to)g(comm)o(unication)i(co-pro)q (cessor,)f(where)189 1401 y(a)o(v)m(ailable.)143 1488 y Fx(\017)23 b Fv(Allo)o(w)15 b(for)g(implemen)o(tations)h(that)f(can)g (b)q(e)h(used)g(in)g(a)f(heterogeneous)g(en)o(vironmen)o(t.)143 1575 y Fx(\017)23 b Fv(Allo)o(w)15 b(con)o(v)o(enien)o(t)h(C)f(and)h(F) l(ortran)e(77)g(bindings)j(for)e(the)g(in)o(terface.)143 1662 y Fx(\017)23 b Fv(Assume)14 b(a)h(reliable)h(comm)o(unication)g (in)o(terface:)j(the)c(user)g(need)g(not)f(cop)q(e)h(with)g(comm)o (unica-)189 1718 y(tion)c(failures.)19 b(Suc)o(h)12 b(failures)f(are)g (dealt)g(with)g(b)o(y)g(the)g(underlying)i(comm)o(unication)e (subsystem.)143 1806 y Fx(\017)23 b Fv(De\014ne)15 b(an)f(in)o(terface) h(that)f(is)h(not)f(to)q(o)g(di\013eren)o(t)g(from)g(curren)o(t)h (practice,)f(suc)o(h)h(as)f(PVM,)g(NX,)189 1862 y(Express,)g(p4,)h (etc.,)g(and)g(pro)o(vides)h(extensions)g(that)e(allo)o(w)i(greater)e (\015exibilit)o(y)l(.)143 1949 y Fx(\017)23 b Fv(De\014ne)d(an)f(in)o (terface)h(that)e(can)i(b)q(e)g(implemen)o(ted)h(on)e(man)o(y)g(v)o (endor's)g(platforms,)g(with)h(no)189 2006 y(signi\014can)o(t)c(c)o (hanges)f(in)h(the)f(underlying)i(comm)o(unication)f(and)g(system)e (soft)o(w)o(are.)143 2093 y Fx(\017)23 b Fv(Seman)o(tics)15 b(of)g(the)g(in)o(terface)h(should)g(b)q(e)g(language)f(indep)q(enden)o (t.)143 2180 y Fx(\017)23 b Fv(The)15 b(in)o(terface)h(should)g(b)q(e)g (designed)g(to)f(allo)o(w)g(for)g(thread-safet)o(y)l(.)75 2320 y Fn(1.2)59 b(Who)20 b(Should)g(Use)g(This)f(Standa)n(rd?)75 2421 y Fv(This)d(standard)f(is)h(in)o(tended)h(for)e(use)g(b)o(y)h(all) g(those)f(who)g(w)o(an)o(t)g(to)f(write)i(p)q(ortable)g (message-passing)75 2478 y(programs)9 b(in)j(F)l(ortran)d(77)i(and)f (C.)h(This)g(includes)i(individual)h(application)e(programmers,)e(dev)o (elop)q(ers)75 2534 y(of)17 b(soft)o(w)o(are)e(designed)j(to)f(run)g (on)g(parallel)i(mac)o(hines,)f(and)f(creators)f(of)h(en)o(vironmen)o (ts)g(and)g(to)q(ols.)75 2591 y(In)g(order)f(to)f(b)q(e)i(attractiv)o (e)e(to)h(this)g(wide)i(audience,)f(the)g(standard)e(m)o(ust)h(pro)o (vide)h(a)e(simple,)j(easy-)75 2647 y(to-use)f(in)o(terface)g(for)f (the)g(basic)i(user)f(while)h(not)e(seman)o(tically)i(precluding)h(the) e(high-p)q(erformance)75 2704 y(message-passing)e(op)q(erations)h(a)o (v)m(ailable)h(on)e(adv)m(anced)h(mac)o(hines.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 11 3 10 bop 75 -100 a Fm(1.3.)34 b(WHA)l(T)15 b(PLA)l(TF)o(ORMS)h(ARE)g(T) l(AR)o(GETS)f(F)o(OR)g(IMPLEMENT)l(A)l(TION?)296 b Fv(3)75 45 y Fn(1.3)59 b(What)20 b(Platfo)n(rms)g(Are)g(T)-5 b(a)n(rgets)19 b(F)n(o)n(r)i(Implementat)o(ion?)75 147 y Fv(The)c(attractiv)o(eness)f(of)g(the)h(message-passing)g(paradigm)f (at)g(least)h(partially)h(stems)e(from)g(its)h(wide)75 203 y(p)q(ortabilit)o(y)l(.)k(Programs)13 b(expressed)j(this)f(w)o(a)o (y)f(ma)o(y)g(run)h(on)g(distributed-memory)h(m)o(ultipro)q(cessors,)75 259 y(net)o(w)o(orks)i(of)i(w)o(orkstations,)e(and)i(com)o(binations)g (of)g(all)g(of)f(these.)34 b(In)20 b(addition,)h(shared-memory)75 316 y(implemen)o(tations)g(are)e(p)q(ossible.)34 b(The)20 b(paradigm)g(will)h(not)e(b)q(e)h(made)f(obsolete)h(b)o(y)g(arc)o (hitectures)75 372 y(com)o(bining)15 b(the)e(shared-)h(and)g (distributed-memory)h(views,)f(or)f(b)o(y)g(increases)i(in)f(net)o(w)o (ork)f(sp)q(eeds.)20 b(It)75 429 y(th)o(us)d(should)i(b)q(e)f(b)q(oth)g (p)q(ossible)h(and)f(useful)g(to)f(implemen)o(t)i(this)f(standard)f(on) h(a)f(great)g(v)m(ariet)o(y)g(of)75 485 y(mac)o(hines,)i(including)h (those)e(\\mac)o(hines")g(consisting)g(of)g(collections)h(of)e(other)h (mac)o(hines,)g(parallel)75 542 y(or)d(not,)f(connected)i(b)o(y)f(a)g (comm)o(unication)h(net)o(w)o(ork.)166 598 y(The)21 b(in)o(terface)f (is)h(suitable)h(for)e(use)h(b)o(y)f(fully)i(general)f(MIMD)f (programs,)g(as)g(w)o(ell)h(as)f(those)75 655 y(written)d(in)g(the)g (more)f(restricted)h(st)o(yle)g(of)f(SPMD.)g(Although)h(no)f(explicit)j (supp)q(ort)e(for)f(threads)g(is)75 711 y(pro)o(vided,)e(the)f(in)o (terface)h(has)f(b)q(een)h(designed)g(so)f(as)g(not)f(to)h(prejudice)i (their)e(use.)20 b(With)13 b(this)h(v)o(ersion)75 768 y(of)h Fw(MPI)f Fv(no)h(supp)q(ort)h(is)g(pro)o(vided)g(for)e(dynamic)i (spa)o(wning)g(of)e(tasks.)166 824 y Fw(MPI)e Fv(pro)o(vides)h(man)o(y) g(features)f(in)o(tended)i(to)e(impro)o(v)o(e)h(p)q(erformance)g(on)g (scalable)h(parallel)g(com-)75 880 y(puters)23 b(with)g(sp)q(ecialized) j(in)o(terpro)q(cessor)c(comm)o(unication)i(hardw)o(are.)42 b(Th)o(us,)24 b(w)o(e)e(exp)q(ect)i(that)75 937 y(nativ)o(e,)18 b(high-p)q(erformance)h(implemen)o(tations)g(of)f Fw(MPI)f Fv(will)i(b)q(e)g(pro)o(vided)f(on)g(suc)o(h)g(mac)o(hines.)28 b(A)o(t)75 993 y(the)17 b(same)f(time,)h(implemen)o(tations)h(of)e Fw(MPI)g Fv(on)g(top)h(of)f(standard)g(Unix)i(in)o(terpro)q(cessor)e (comm)o(uni-)75 1050 y(cation)e(proto)q(cols)g(will)i(pro)o(vide)e(p)q (ortabilit)o(y)h(to)f(w)o(orkstation)e(clusters)j(and)f(heterogenous)g (net)o(w)o(orks)75 1106 y(of)h(w)o(orkstations.)j(Sev)o(eral)e (proprietary)l(,)e(nativ)o(e)i(implemen)o(tations)g(of)f Fw(MPI)p Fv(,)f(and)h(a)g(public)i(domain,)75 1163 y(p)q(ortable)f (implemen)o(tation)g(of)f Fw(MPI)g Fv(are)f(in)i(progress)f(at)g(the)g (time)g(of)g(this)h(writing)f([17,)f(13].)75 1306 y Fn(1.4)59 b(What)20 b(Is)f(Included)g(In)g(The)g(Standa)n(rd?)75 1407 y Fv(The)c(standard)g(includes:)143 1514 y Fx(\017)23 b Fv(P)o(oin)o(t-to-p)q(oin)o(t)15 b(comm)o(unication)143 1608 y Fx(\017)23 b Fv(Collectiv)o(e)16 b(op)q(erations)143 1701 y Fx(\017)23 b Fv(Pro)q(cess)15 b(groups)143 1795 y Fx(\017)23 b Fv(Comm)o(unication)15 b(con)o(texts)143 1889 y Fx(\017)23 b Fv(Pro)q(cess)15 b(top)q(ologies)143 1983 y Fx(\017)23 b Fv(Bindings)17 b(for)d(F)l(ortran)g(77)h(and)g(C) 143 2077 y Fx(\017)23 b Fv(En)o(vironmen)o(tal)15 b(Managemen)o(t)f (and)i(inquiry)143 2170 y Fx(\017)23 b Fv(Pro\014ling)16 b(in)o(terface)75 2314 y Fn(1.5)59 b(What)20 b(Is)f(Not)h(Included)e (In)i(The)f(Standa)n(rd?)75 2415 y Fv(The)c(standard)g(do)q(es)h(not)f (sp)q(ecify:)143 2521 y Fx(\017)23 b Fv(Explicit)17 b(shared-memory)e (op)q(erations)143 2615 y Fx(\017)23 b Fv(Op)q(erations)c(that)e (require)i(more)f(op)q(erating)h(system)f(supp)q(ort)g(than)g(is)h (curren)o(tly)g(standard;)189 2672 y(for)14 b(example,)i(in)o (terrupt-driv)o(en)g(receiv)o(es,)g(remote)f(execution,)h(or)e(activ)o (e)i(messages)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 12 4 11 bop 75 -100 a Fv(4)903 b Fm(CHAPTER)16 b(1.)34 b(INTR)o(ODUCTION) 17 b(TO)e(MPI)143 45 y Fx(\017)23 b Fv(Program)13 b(construction)j(to)q (ols)143 139 y Fx(\017)23 b Fv(Debugging)15 b(facilities)143 233 y Fx(\017)23 b Fv(Explicit)17 b(supp)q(ort)e(for)g(threads)143 327 y Fx(\017)23 b Fv(Supp)q(ort)15 b(for)g(task)f(managemen)o(t)143 420 y Fx(\017)23 b Fv(I/O)15 b(functions)166 527 y(There)g(are)f(man)o (y)h(features)f(that)g(ha)o(v)o(e)g(b)q(een)i(considered)g(and)f(not)g (included)i(in)f(this)f(standard.)75 583 y(This)i(happ)q(ened)h(for)e (a)g(n)o(um)o(b)q(er)h(of)f(reasons,)g(one)h(of)f(whic)o(h)h(is)g(the)g (time)g(constrain)o(t)f(that)g(w)o(as)f(self-)75 640 y(imp)q(osed)h(in)h(\014nishing)g(the)e(standard.)20 b(F)l(eatures)15 b(that)f(are)h(not)g(included)j(can)e(alw)o(a)o(ys)e (b)q(e)i(o\013ered)f(as)75 696 y(extensions)j(b)o(y)f(sp)q(eci\014c)h (implemen)o(tations.)27 b(P)o(erhaps)17 b(future)g(v)o(ersions)g(of)f Fw(MPI)h Fv(will)h(address)f(some)75 752 y(of)e(these)g(issues.)75 896 y Fn(1.6)59 b(Organization)20 b(of)g(this)f(Do)r(cument)75 997 y Fv(The)k(follo)o(wing)h(is)f(a)f(list)i(of)e(the)h(remaining)h(c) o(hapters)e(in)i(this)f(do)q(cumen)o(t,)i(along)e(with)g(a)f(brief)75 1054 y(description)17 b(of)d(eac)o(h.)143 1160 y Fx(\017)23 b Fv(Chapter)c(2,)i Fw(MPI)e(T)l(erms)h(and)g(Conventions)p Fv(,)j(explains)e(notational)f(terms)g(and)g(con)o(v)o(en)o(tions)189 1216 y(used)15 b(throughout)g(the)g Fw(MPI)g Fv(do)q(cumen)o(t.)143 1310 y Fx(\017)23 b Fv(Chapter)16 b(3,)g Fw(P)o(oint)h(to)f(P)o(oint)h (Communication)p Fv(,)f(de\014nes)h(the)g(basic,)g(pairwise)g(comm)o (unication)189 1367 y(subset)d(of)g Fw(MPI)p Fv(.)f Fl(send)h Fv(and)g Fl(r)n(e)n(c)n(eive)f Fv(are)h(found)h(here,)f(along)g(with)h (man)o(y)f(asso)q(ciated)g(functions)189 1423 y(designed)i(to)f(mak)o (e)f(basic)i(comm)o(unication)g(p)q(o)o(w)o(erful)g(and)f(e\016cien)o (t.)143 1517 y Fx(\017)23 b Fv(Chapter)12 b(4,)h Fw(Collective)g (Communications)p Fv(,)f(de\014nes)i(pro)q(cess-group)f(collectiv)o(e)h (comm)o(unication)189 1573 y(op)q(erations.)28 b(W)l(ell)19 b(kno)o(wn)f(examples)h(of)e(this)i(are)e(barrier)i(and)f(broadcast)f (o)o(v)o(er)g(a)h(group)f(of)189 1630 y(pro)q(cesses)e(\(not)g (necessarily)h(all)g(the)g(pro)q(cesses\).)143 1724 y Fx(\017)23 b Fv(Chapter)16 b(5,)g Fw(Groups,)h(Contexts,)i(and)e (Communicato)o(rs)p Fv(,)e(sho)o(ws)h(ho)o(w)g(groups)g(of)g(pro)q (cesses)h(are)189 1780 y(formed)c(and)h(manipulated,)i(ho)o(w)d(unique) i(comm)o(unication)g(con)o(texts)e(are)h(obtained,)g(and)g(ho)o(w)189 1836 y(the)h(t)o(w)o(o)f(are)h(b)q(ound)h(together)e(in)o(to)h(a)g Fl(c)n(ommunic)n(ator)p Fv(.)143 1930 y Fx(\017)23 b Fv(Chapter)18 b(6,)h Fw(Pro)q(cess)h(T)l(op)q(ologies)p Fv(,)f(explains)h(a)f(set)f(of)h(utilit)o(y)h(functions)f(mean)o(t)f (to)h(assist)f(in)189 1987 y(the)13 b(mapping)i(of)e(pro)q(cess)g (groups)h(\(a)f(linearly)i(ordered)f(set\))f(to)g(ric)o(her)h(top)q (ological)g(structures)189 2043 y(suc)o(h)h(as)g(m)o(ulti-dimensional)j (grids.)143 2137 y Fx(\017)23 b Fv(Chapter)10 b(7,)h Fw(MPI)g(Environmental)g(Management)p Fv(,)g(explains)h(ho)o(w)f(the)g (programmer)e(can)i(manage)189 2193 y(and)g(mak)o(e)f(inquiries)j(of)d (the)h(curren)o(t)f Fw(MPI)g Fv(en)o(vironmen)o(t.)19 b(These)11 b(functions)g(are)g(needed)h(for)e(the)189 2250 y(writing)k(of)f(correct,)g(robust)g(programs,)f(and)i(are)f(esp)q (ecially)j(imp)q(ortan)o(t)d(for)g(the)h(construction)189 2306 y(of)g(highly-p)q(ortable)k(message-passing)d(programs.)143 2400 y Fx(\017)23 b Fv(Chapter)d(8,)h Fw(Pro\014ling)f(Interface)p Fv(,)i(explains)g(a)e(simple)i(name-shifting)g(con)o(v)o(en)o(tion)e (that)g(an)o(y)189 2457 y Fw(MPI)h Fv(implemen)o(tation)i(m)o(ust)f (supp)q(ort.)40 b(One)23 b(motiv)m(ation)f(for)f(this)i(is)f(the)g (abilit)o(y)h(to)f(put)189 2513 y(p)q(erformance)c(pro\014ling)i(calls) g(in)o(to)f Fw(MPI)f Fv(without)g(the)h(need)g(for)f(access)h(to)f(the) h Fw(MPI)f Fv(source)189 2570 y(co)q(de.)31 b(The)19 b(name)g(shift)g(is)h(merely)f(an)g(in)o(terface,)h(it)f(sa)o(ys)f (nothing)i(ab)q(out)e(ho)o(w)h(the)g(actual)189 2626 y(pro\014ling)d(should)g(b)q(e)f(done)h(and)f(in)h(fact,)e(the)h(name)g (shift)g(can)g(b)q(e)h(useful)g(for)e(other)h(purp)q(oses.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 13 5 12 bop 75 -100 a Fm(1.6.)34 b(OR)o(GANIZA)l(TION)17 b(OF)e(THIS)h(DOCUMENT)810 b Fv(5)143 45 y Fx(\017)23 b Fv(Annex)18 b(A,)g Fw(Language)g(Bindings)p Fv(,)i(giv)o(es)d(sp)q (eci\014c)j(syn)o(tax)d(in)i(F)l(ortran)d(77)i(and)g(C,)f(for)g(all)i Fw(MPI)189 102 y Fv(functions,)c(constan)o(ts,)f(and)h(t)o(yp)q(es.)143 195 y Fx(\017)23 b Fv(The)12 b Fw(MPI)g(F)o(unction)i(Index)f Fv(is)g(a)f(simple)i(index)g(sho)o(wing)e(the)h(lo)q(cation)g(of)f(the) h(precise)g(de\014nition)189 252 y(of)h(eac)o(h)i Fw(MPI)e Fv(function,)i(together)e(with)i(b)q(oth)f(C)g(and)h(F)l(ortran)e (bindings.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 14 6 13 bop 75 356 a Fo(Chapter)34 b(2)75 564 y Fr(MPI)40 b(T)-10 b(erms)39 b(and)g(Con)m(v)m(en)m(tions)75 786 y Fv(This)12 b(c)o(hapter)f(explains)i(notational)e(terms)g(and)g(con)o (v)o(en)o(tions)g(used)h(throughout)f(the)g Fw(MPI)g Fv(do)q(cumen)o(t,)75 843 y(some)k(of)g(the)g(c)o(hoices)h(that)e(ha)o (v)o(e)h(b)q(een)i(made,)d(and)i(the)f(rationale)h(b)q(ehind)h(those)e (c)o(hoices.)75 986 y Fn(2.1)59 b(Do)r(cument)18 b(Notation)189 1088 y Fl(R)n(ationale.)45 b Fv(Throughout)15 b(this)i(do)q(cumen)o(t,) f(the)g(rationale)h(for)e(the)h(design)h(c)o(hoices)g(made)f(in)189 1144 y(the)j(in)o(terface)g(sp)q(eci\014cation)i(is)e(set)g(o\013)f(in) i(this)f(format.)30 b(Some)19 b(readers)g(ma)o(y)f(wish)i(to)e(skip)189 1201 y(these)i(sections,)h(while)h(readers)e(in)o(terested)g(in)h(in)o (terface)f(design)h(ma)o(y)f(w)o(an)o(t)f(to)g(read)h(them)189 1257 y(carefully)l(.)h(\()p Fl(End)16 b(of)g(r)n(ationale.)p Fv(\))189 1363 y Fl(A)n(dvic)n(e)21 b(to)h(users.)78 b Fv(Throughout)21 b(this)h(do)q(cumen)o(t,)h(material)f(that)f(sp)q (eaks)g(to)g(users)h(and)189 1420 y(illustrates)12 b(usage)g(is)g(set)f (o\013)g(in)i(this)f(format.)17 b(Some)12 b(readers)f(ma)o(y)g(wish)i (to)e(skip)h(these)g(sections,)189 1476 y(while)g(readers)e(in)o (terested)h(in)g(programming)f(in)h Fw(MPI)f Fv(ma)o(y)f(w)o(an)o(t)h (to)f(read)i(them)f(carefully)l(.)20 b(\()p Fl(End)189 1533 y(of)c(advic)n(e)g(to)h(users.)p Fv(\))189 1639 y Fl(A)n(dvic)n(e)k(to)i(implementors.)80 b Fv(Throughout)22 b(this)g(do)q(cumen)o(t,)i(material)e(that)f(is)i(primarily)189 1695 y(commen)o(tary)13 b(to)g(implemen)o(tors)i(is)f(set)g(o\013)f(in) i(this)g(format.)j(Some)c(readers)g(ma)o(y)f(wish)i(to)e(skip)189 1752 y(these)20 b(sections,)i(while)g(readers)f(in)o(terested)g(in)g Fw(MPI)f Fv(implemen)o(tations)i(ma)o(y)e(w)o(an)o(t)f(to)h(read)189 1808 y(them)15 b(carefully)l(.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)g (to)h(implementors.)p Fv(\))75 1951 y Fn(2.2)59 b(Pro)r(cedure)19 b(Sp)r(eci\014cation)75 2053 y Fw(MPI)g Fv(pro)q(cedures)i(are)e(sp)q (eci\014ed)j(using)f(a)f(language)g(indep)q(enden)o(t)i(notation.)33 b(The)20 b(argumen)o(ts)f(of)75 2109 y(pro)q(cedure)d(calls)g(are)f (mark)o(ed)g(as)g Fk(IN)p Fv(,)g Fk(OUT)g Fv(or)g Fk(INOUT)p Fv(.)g(The)h(meanings)f(of)g(these)h(are:)143 2203 y Fx(\017)23 b Fv(the)15 b(call)h(uses)g(but)f(do)q(es)h(not)e(up)q(date) i(an)f(argumen)o(t)g(mark)o(ed)f Fk(IN)p Fv(,)143 2297 y Fx(\017)23 b Fv(the)15 b(call)h(ma)o(y)f(up)q(date)h(an)f(argumen)o (t)f(mark)o(ed)h Fk(OUT)p Fv(,)143 2391 y Fx(\017)23 b Fv(the)15 b(call)h(b)q(oth)g(uses)f(and)g(up)q(dates)h(an)f(argumen)o (t)g(mark)o(ed)f Fk(INOUT)p Fv(.)166 2485 y(There)20 b(is)g(one)g(sp)q(ecial)i(case)e(|)g(if)h(an)e(argumen)o(t)h(is)g(a)g (handle)h(to)e(an)h(opaque)g(ob)s(ject)f(\(these)75 2541 y(terms)d(are)h(de\014ned)h(in)g(Section)f(2.4.1\),)e(and)i(the)g(ob)s (ject)f(is)i(up)q(dated)f(b)o(y)g(the)g(pro)q(cedure)h(call,)g(then)75 2598 y(the)g(argumen)o(t)g(is)g(mark)o(ed)g Fk(OUT)p Fv(.)g(It)h(is)f(mark)o(ed)g(this)h(w)o(a)o(y)e(ev)o(en)i(though)f(the) g(handle)h(itself)h(is)e(not)75 2654 y(mo)q(di\014ed)c(|)f(w)o(e)f(use) h(the)g Fk(OUT)g Fv(attribute)f(to)g(denote)h(that)f(what)g(the)g (handle)i Fl(r)n(efer)n(enc)n(es)c Fv(is)j(up)q(dated.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 15 7 14 bop 75 -100 a Fm(2.3.)34 b(SEMANTIC)15 b(TERMS)1244 b Fv(7)166 45 y(The)17 b(de\014nition)i(of)d Fw(MPI)g Fv(tries)h(to)f(a)o(v)o(oid,)h(to)f(the)h(largest)f(p)q(ossible)j (exten)o(t,)d(the)h(use)g(of)f Fk(INOUT)75 102 y Fv(argumen)o(ts,)e(b)q (ecause)i(suc)o(h)g(use)f(is)h(error-prone,)e(esp)q(ecially)k(for)d (scalar)g(argumen)o(ts.)166 158 y(A)k(common)f(o)q(ccurrence)h(for)f Fw(MPI)g Fv(functions)i(is)f(an)f(argumen)o(t)g(that)g(is)h(used)g(as)f Fk(IN)h Fv(b)o(y)f(some)75 214 y(pro)q(cesses)12 b(and)g Fk(OUT)f Fv(b)o(y)h(other)f(pro)q(cesses.)19 b(Suc)o(h)12 b(argumen)o(t)f(is,)h(syn)o(tactically)l(,)h(an)f Fk(INOUT)g Fv(argumen)o(t)75 271 y(and)j(is)h(mark)o(ed)e(as)g(suc)o(h,)h (although,)g(seman)o(tically)l(,)h(it)f(is)h(not)e(used)i(in)f(one)g (call)i(b)q(oth)e(for)f(input)i(and)75 327 y(for)f(output.)166 384 y(Another)h(frequen)o(t)g(situation)h(arises)g(when)f(an)g(argumen) o(t)g(v)m(alue)h(is)g(needed)h(only)f(b)o(y)f(a)g(subset)75 440 y(of)f(the)g(pro)q(cesses.)20 b(When)c(an)f(argumen)o(t)f(is)i(not) f(signi\014can)o(t)h(at)e(a)h(pro)q(cess)h(then)f(an)g(arbitrary)g(v)m (alue)75 497 y(can)g(b)q(e)h(passed)g(as)e(argumen)o(t.)166 553 y(Unless)h(sp)q(eci\014ed)g(otherwise,)f(an)g(argumen)o(t)f(of)g(t) o(yp)q(e)h Fk(OUT)g Fv(or)f(t)o(yp)q(e)h Fk(INOUT)g Fv(cannot)g(b)q(e)g (aliased)75 610 y(with)h(an)o(y)f(other)g(argumen)o(t)g(passed)g(to)g (an)g Fw(MPI)g Fv(pro)q(cedure.)21 b(An)15 b(example)g(of)f(argumen)o (t)f(aliasing)j(in)75 666 y(C)f(app)q(ears)g(b)q(elo)o(w.)21 b(If)15 b(w)o(e)g(de\014ne)h(a)f(C)g(pro)q(cedure)h(lik)o(e)h(this,)75 760 y Ft(void)23 b(copyIntBuffer\()f(int)i(*pin,)f(int)g(*pout,)g(int)h (len)f(\))75 816 y({)71 b(int)24 b(i;)170 873 y(for)g(\(i=0;)f(i)75 1194 y Fv(t)o(yp)q(e.)20 b(Comparisons)15 b(to)f(this)i (constan)o(t)e(are)h(used)h(to)f(test)f(for)h(v)m(alidit)o(y)i(of)d (the)i(handle.)-1783 b Fx(?)166 1250 y Fv(A)15 b(call)h(to)f(deallo)q (cate)h(in)o(v)m(alidates)h(the)e(handle)h(and)g(marks)e(the)h(ob)s (ject)g(for)f(deallo)q(cation.)22 b(The)75 1307 y(ob)s(ject)c(is)h(not) f(accessible)j(to)d(the)g(user)h(after)f(the)g(call.)32 b(Ho)o(w)o(ev)o(er,)18 b Fw(MPI)g Fv(need)h(not)f(deallo)q(cate)i(the) 75 1363 y(ob)s(ject)13 b(immediatly)l(.)21 b(An)o(y)13 b(op)q(eration)h(p)q(ending)h(\(at)d(the)i(time)f(of)g(the)h(deallo)q (cate\))g(that)f(in)o(v)o(olv)o(es)h(this)75 1420 y(ob)s(ject)h(will)h (complete)g(normally;)g(the)f(ob)s(ject)g(will)i(b)q(e)e(deallo)q (cated)i(afterw)o(ards.)-1678 b Fx(>)166 1476 y Fw(MPI)15 b Fv(calls)h(do)g(not)f(c)o(hange)g(the)g(v)m(alue)i(of)e(handles,)h (with)g(the)g(exception)g(of)f(calls)i(that)d(allo)q(cate)75 1533 y(and)h(deallo)q(cate)i(ob)s(jects,)d(and)h(of)g(the)g(call)i Fw(MPI)p 932 1533 14 2 v 16 w(TYPE)p 1065 1533 V 16 w(COMMIT)p Fv(,)e(in)h(Section)g(3.12.4.)166 1589 y(A)c(n)o(ull)h(handle)g (argumen)o(t)d(is)i(an)g(erroneous)f Fk(IN)h Fv(argumen)o(t)f(in)h Fw(MPI)f Fv(calls,)i(unless)g(an)f(exception)g(is)75 1645 y(explicitly)k(stated)d(in)i(the)f(text)f(that)g(de\014nes)i(the)e (function.)20 b(Suc)o(h)15 b(exception)f(is)h(allo)o(w)o(ed)f(for)f (handles)75 1702 y(to)f(request)g(ob)s(jects)g(in)i(W)l(ait)e(and)h(T)l (est)f(calls)i(\(sections)e(3.7.3)f(and)i(3.7.5\).)k(Otherwise,)d(a)e (n)o(ull)i(handle)75 1758 y(can)j(only)g(b)q(e)g(passed)g(to)g(a)f (function)h(that)f(allo)q(cates)i(a)e(new)h(ob)s(ject)f(and)h(returns)g (a)f(reference)i(to)e(it)75 1815 y(in)g(the)f(handle.)-509 b Fx(?)166 1871 y Fv(An)17 b(opaque)h(ob)s(ject)e(and)h(its)h(handle)g (are)f(signi\014can)o(t)h(only)g(at)e(the)i(pro)q(cess)f(where)g(the)h (ob)s(ject)75 1928 y(w)o(as)c(created,)h(and)h(cannot)f(b)q(e)g (transferred)g(to)g(another)g(pro)q(cess.)166 1984 y Fw(MPI)e Fv(pro)o(vides)h(certain)g(prede\014ned)i(opaque)e(ob)s(jects) f(and)h(prede\014ned,)h(static)f(handles)h(to)e(these)75 2041 y(ob)s(jects.)19 b(Suc)o(h)d(ob)s(jects)f(ma)o(y)f(not)h(b)q(e)h (destro)o(y)o(ed.)189 2147 y Fl(R)n(ationale.)37 b Fv(This)12 b(design)g(hides)g(the)g(in)o(ternal)g(represen)o(tation)f(used)h(for)e Fw(MPI)h Fv(data)g(structures,)189 2203 y(th)o(us)16 b(allo)o(wing)h(similar)h(calls)g(in)f(C)g(and)f(F)l(ortran.)23 b(It)17 b(also)f(a)o(v)o(oids)h(con\015icts)g(with)g(the)g(t)o(yping) 189 2260 y(rules)k(in)h(these)f(languages,)h(and)f(easily)h(allo)o(ws)f (future)g(extensions)g(of)g(functionalit)o(y)l(.)38 b(The)189 2316 y(mec)o(hanism)17 b(for)g(opaque)g(ob)s(jects)g(used)h(here)f(lo)q (osely)h(follo)o(ws)g(the)f(POSIX)h(F)l(ortran)e(binding)189 2373 y(standard.)189 2448 y(The)c(explicit)i(separating)f(of)e(handles) j(in)f(user)f(space,)h(ob)s(jects)f(in)h(system)e(space,)i(allo)o(ws)g (space-)189 2504 y(reclaiming,)18 b(deallo)q(cation)g(calls)f(to)f(b)q (e)h(made)g(at)f(appropriate)g(p)q(oin)o(ts)h(in)g(the)g(user)f (program.)189 2561 y(If)j(the)f(opaque)h(ob)s(jects)f(w)o(ere)g(in)i (user)f(space,)g(one)g(w)o(ould)g(ha)o(v)o(e)f(to)g(b)q(e)i(v)o(ery)e (careful)h(not)g(to)189 2617 y(go)d(out)h(of)g(scop)q(e)h(b)q(efore)f (an)o(y)g(p)q(ending)i(op)q(eration)f(requiring)g(that)f(ob)s(ject)g (completed.)27 b(The)189 2674 y(sp)q(eci\014ed)16 b(design)f(allo)o(ws) g(an)f(ob)s(ject)f(to)h(b)q(e)h(mark)o(ed)f(for)f(deallo)q(cation,)j (the)e(user)h(program)e(can)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 9 17 9 16 bop 75 -100 a Fm(2.4.)34 b(D)o(A)l(T)l(A)15 b(TYPES)1375 b Fv(9)189 45 y(then)16 b(go)g(out)g(of)g(scop)q(e,)g(and)h(the)f(ob)s (ject)g(itself)h(still)h(p)q(ersists)e(un)o(til)i(an)o(y)e(p)q(ending)i (op)q(erations)189 102 y(are)d(complete.)189 177 y(The)22 b(requiremen)o(t)g(that)g(handles)h(supp)q(ort)f(assignmen)o (t/comparison)g(is)g(made)g(since)i(suc)o(h)189 233 y(op)q(erations)16 b(are)f(common.)22 b(This)16 b(restricts)g(the)g(domain)g(of)g(p)q (ossible)i(implemen)o(tations.)23 b(The)189 289 y(alternativ)o(e)14 b(w)o(ould)h(ha)o(v)o(e)f(b)q(een)h(to)f(allo)o(w)g(handles)i(to)e(ha)o (v)o(e)f(b)q(een)j(an)e(arbitrary)l(,)g(opaque)g(t)o(yp)q(e.)189 346 y(This)d(w)o(ould)g(force)g(the)f(in)o(tro)q(duction)i(of)e (routines)i(to)e(do)g(assignmen)o(t)h(and)g(comparison,)g(adding)189 402 y(complexit)o(y)l(,)16 b(and)f(w)o(as)f(therefore)h(ruled)h(out.)k (\()p Fl(End)c(of)g(r)n(ationale.)p Fv(\))189 508 y Fl(A)n(dvic)n(e)e (to)i(users.)39 b Fv(A)15 b(user)f(ma)o(y)g(acciden)o(tly)i(create)e(a) g(dangling)h(reference)g(b)o(y)f(assigning)i(to)d(a)189 564 y(handle)j(the)f(v)m(alue)i(of)e(another)f(handle,)j(and)e(then)g (deallo)q(cating)i(the)f(ob)s(ject)e(asso)q(ciated)i(with)189 621 y(these)j(handles.)33 b(Con)o(v)o(ersely)l(,)20 b(if)f(a)g(handle)h (v)m(ariable)h(is)f(deallo)q(cated)g(b)q(efore)g(the)f(asso)q(ciated) 189 677 y(ob)s(ject)c(is)h(freed,)g(then)g(the)g(ob)s(ject)f(b)q (ecomes)i(inaccessible)h(\(this)e(ma)o(y)f(o)q(ccur,)h(for)f(example,)h (if)189 734 y(the)e(handle)i(is)f(a)f(lo)q(cal)h(v)m(ariable)h(within)g (a)e(subroutine,)h(and)g(the)f(subroutine)h(is)g(exited)h(b)q(efore)189 790 y(the)i(asso)q(ciated)h(ob)s(ject)f(is)h(deallo)q(cated\).)30 b(It)19 b(is)g(the)f(user's)g(resp)q(onsibilit)o(y)j(to)d(a)o(v)o(oid)g (adding)189 847 y(or)g(deleting)i(references)f(to)f(opaque)h(ob)s (jects,)f(except)h(as)f(a)h(result)g(of)f(calls)h(that)f(allo)q(cate)h (or)189 903 y(deallo)q(cate)d(suc)o(h)g(ob)s(jects.)j(\()p Fl(End)c(of)i(advic)n(e)f(to)g(users.)p Fv(\))189 1009 y Fl(A)n(dvic)n(e)k(to)i(implementors.)72 b Fv(The)21 b(in)o(tended)h(seman)o(tics)f(of)f(opaque)h(ob)s(jects)f(is)h(that)f (eac)o(h)189 1065 y(opaque)15 b(ob)s(ject)f(is)h(separate)g(from)f(eac) o(h)h(other;)f(eac)o(h)h(call)h(to)e(allo)q(cate)i(suc)o(h)f(an)g(ob)s (ject)f(copies)189 1122 y(all)22 b(the)g(information)g(required)g(for)f (the)h(ob)s(ject.)38 b(Implemen)o(tations)23 b(ma)o(y)e(a)o(v)o(oid)g (excessiv)o(e)189 1178 y(cop)o(ying)g(b)o(y)g(substituting)h (referencing)h(for)d(cop)o(ying.)39 b(F)l(or)20 b(example,)j(a)e(deriv) o(ed)h(datat)o(yp)q(e)189 1234 y(ma)o(y)d(con)o(tain)h(references)h(to) e(its)i(comp)q(onen)o(ts,)f(rather)g(then)g(copies)h(of)f(its)g(comp)q (onen)o(ts;)i(a)189 1291 y(call)17 b(to)e Fw(MPI)p 414 1291 14 2 v 16 w(COMM)p 572 1291 V 16 w(GROUP)j Fv(ma)o(y)d(return)h(a) g(reference)h(to)e(the)h(group)g(asso)q(ciated)g(with)h(the)189 1347 y(comm)o(unicator,)g(rather)f(than)h(a)g(cop)o(y)g(of)g(this)h (group.)26 b(In)17 b(suc)o(h)h(cases,)f(the)h(implemen)o(tation)189 1404 y(m)o(ust)h(main)o(tain)i(reference)h(coun)o(ts,)f(and)f(allo)q (cate)h(and)g(deallo)q(cate)h(ob)s(jects)d(suc)o(h)i(that)f(the)189 1460 y(visible)d(e\013ect)e(is)h(as)f(if)g(the)g(ob)s(jects)g(w)o(ere)g (copied.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)g(to)h(implementors.)p Fv(\))75 1582 y Fj(2.4.2)49 b(Arra)o(y)16 b(a)o(rguments)75 1668 y Fv(An)21 b Fw(MPI)g Fv(call)h(ma)o(y)e(need)i(an)e(argumen)o(t)h (that)f(is)h(an)g(arra)o(y)f(of)g(opaque)h(ob)s(jects,)g(or)g(an)g (arra)o(y)e(of)75 1724 y(handles.)31 b(The)19 b(arra)o(y-of-handles)f (is)h(a)f(regular)g(arra)o(y)g(with)g(en)o(tries)h(that)f(are)g (handles)i(to)e(ob)s(jects)75 1781 y(of)g(the)h(same)g(t)o(yp)q(e)f(in) i(consecutiv)o(e)g(lo)q(cations)f(in)h(the)f(arra)o(y)l(.)29 b(Whenev)o(er)19 b(suc)o(h)g(an)g(arra)o(y)e(is)j(used,)75 1837 y(an)d(additional)h Fw(len)f Fv(argumen)o(t)f(is)h(required)h(to)e (indicate)i(the)f(n)o(um)o(b)q(er)g(of)f(v)m(alid)j(en)o(tries)e (\(unless)h(this)75 1894 y(n)o(um)o(b)q(er)f(can)h(b)q(e)g(deriv)o(ed)g (otherwise\).)25 b(The)18 b(v)m(alid)h(en)o(tries)e(are)g(at)g(the)g(b) q(egining)i(of)e(the)g(arra)o(y;)f Fw(len)75 1950 y Fv(indicates)j(ho)o (w)e(man)o(y)h(of)f(them)h(there)g(are,)f(and)h(need)h(not)e(b)q(e)h (the)g(en)o(tire)g(size)h(of)e(the)h(arra)o(y)l(.)27 b(The)75 2006 y(same)15 b(approac)o(h)g(is)g(follo)o(w)o(ed)h(for)f (other)f(arra)o(y)g(argumen)o(ts.)75 2128 y Fj(2.4.3)49 b(State)75 2214 y Fw(MPI)15 b Fv(pro)q(cedures)i(use)f(at)f(v)m(arious) h(places)h(argumen)o(ts)d(with)i Fl(state)g Fv(t)o(yp)q(es.)21 b(The)16 b(v)m(alues)h(of)f(suc)o(h)g(data)75 2270 y(t)o(yp)q(e)i(are)h (all)g(iden)o(ti\014ed)h(b)o(y)f(names,)f(and)h(no)f(op)q(eration)h(is) g(de\014ned)h(on)e(them.)29 b(F)l(or)18 b(example,)i(the)75 2327 y Fw(MPI)p 160 2327 V 16 w(ERRHANDLER)p 469 2327 V 18 w(SET)10 b Fv(routine)h(has)f(a)g(state)f(t)o(yp)q(e)h(argumen)o (t)f(with)i(v)m(alues)g Fi(MPI)p 1545 2327 13 2 v 15 w(ERRORS)p 1720 2327 V 13 w(ARE)p 1813 2327 V 15 w(F)m(A-)75 2383 y(T)m(AL)p Fv(,)k Fi(MPI)p 255 2383 V 14 w(ERRORS)p 429 2383 V 14 w(RETURN)p Fv(,)f(etc.)75 2505 y Fj(2.4.4)49 b(Named)16 b(constants)75 2591 y Fw(MPI)e Fv(pro)q(cedures)g(sometimes) h(assign)f(a)g(sp)q(ecial)i(meaning)e(to)g(a)g(sp)q(ecial)h(v)m(alue)h (of)e(a)f(basic)i(t)o(yp)q(e)f(argu-)75 2647 y(men)o(t;)j(e.g.)25 b Fw(tag)17 b Fv(is)g(an)g(in)o(teger-v)m(alued)i(argumen)o(t)d(of)h(p) q(oin)o(t-to-p)q(oin)o(t)g(comm)o(unication)h(op)q(erations,)75 2704 y(with)e(a)g(sp)q(ecial)i(wild-card)f(v)m(alue,)g Fi(MPI)p 775 2704 V 15 w(ANY)p 875 2704 V 14 w(T)m(A)o(G)p Fv(.)e(Suc)o(h)i(argumen)o(ts)e(will)i(ha)o(v)o(e)f(a)g(range)f(of)h (regular)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 10 18 10 17 bop 75 -100 a Fv(10)692 b Fm(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y Fv(v)m(alues,)j(whic)o(h)g (is)f(a)f(prop)q(er)h(subrange)g(of)f(the)h(range)g(of)f(v)m(alues)i (of)e(the)h(corresp)q(onding)h(basic)f(t)o(yp)q(e;)75 102 y(sp)q(ecial)c(v)m(alues)g(\(suc)o(h)f(as)f Fi(MPI)p 597 102 13 2 v 15 w(ANY)p 697 102 V 14 w(T)m(A)o(G)p Fv(\))g(will)i(b)q(e)f(outside)g(the)g(regular)g(range.)18 b(The)12 b(range)g(of)f(regular)75 158 y(v)m(alues)16 b(can)g(b)q(e)g(queried)g(using)g(en)o(vironmen)o(tal)g(inquiry)h (functions)e(\(Chapter)g(7\).)-1693 b Fx(>)166 214 y Fw(MPI)23 b Fv(also)g(pro)o(vides)h(prede\014ned)h(named)e(constan)o(t) f(handles,)27 b(suc)o(h)c(as)g Fi(MPI)p 1570 214 V 14 w(COMM)p 1714 214 V 15 w(W)o(ORLD)75 271 y Fv(whic)o(h)17 b(is)g(a)f(handle)i(to)d(an)i(ob)s(ject)f(that)f(represen)o(ts)h(all)i (pro)q(cesses)f(a)o(v)m(ailable)h(at)d(start-up)h(time)h(and)75 327 y(allo)o(w)o(ed)f(to)e(comm)o(unicate)i(with)f(an)o(y)g(of)g(them.) 166 384 y(All)k(named)f(constan)o(ts,)f(with)h(the)g(exception)g(of)g Fi(MPI)p 1133 384 V 14 w(BOTTOM)f Fv(in)i(F)l(ortran,)e(can)g(b)q(e)i (used)f(in)75 440 y(initialization)25 b(expressions)e(or)e(assignmen)o (ts.)40 b(These)22 b(constan)o(ts)f(do)h(not)g(c)o(hange)g(v)m(alues)h (during)75 497 y(execution.)35 b(Opaque)20 b(ob)s(jects)g(accessed)g(b) o(y)g(constan)o(t)f(handles)i(are)e(de\014ned)j(and)e(do)f(not)h(c)o (hange)75 553 y(v)m(alue)g(b)q(et)o(w)o(een)e Fw(MPI)g Fv(initialization)j(\()p Fw(MPI)p 845 553 14 2 v 16 w(INIT\(\))c Fv(call\))i(and)g Fw(MPI)e Fv(completion)j(\()p Fw(MPI)p 1634 553 V 15 w(FINALIZE\(\))75 610 y Fv(call\).)-333 b Fx(?)75 728 y Fj(2.4.5)49 b(Choice)75 813 y Fw(MPI)15 b Fv(functions)i(sometimes)f(use)g(argumen)o(ts)f(with)h(a)f Fl(choic)n(e)h Fv(\(or)f(union\))h(data)g(t)o(yp)q(e.)21 b(Distinct)c(calls)75 870 y(to)c(the)h(same)g(routine)h(ma)o(y)e(pass)h (b)o(y)f(reference)i(actual)f(argumen)o(ts)f(of)h(di\013eren)o(t)g(t)o (yp)q(es.)20 b(The)14 b(mec)o(ha-)75 926 y(nism)j(for)e(pro)o(viding)i (suc)o(h)f(argumen)o(ts)f(will)i(di\013er)g(from)e(language)h(to)f (language.)22 b(F)l(or)15 b(F)l(ortran,)g(the)75 983 y(do)q(cumen)o(t)h(uses)f Fp(<)p Fw(t)o(yp)q(e)p Fp(>)i Fv(to)e(represen)o(t)g(a)g(c)o(hoice)h(v)m(ariable,)g(for)f(C,)f(w)o(e) h(use)h Fw(\(void)e(*\))p Fv(.)75 1101 y Fj(2.4.6)49 b(Addresses)75 1186 y Fv(Some)22 b Fw(MPI)f Fv(pro)q(cedures)h(use)g Fl(addr)n(ess)g Fv(argumen)o(ts)e(that)h(represen)o(t)h(an)g(absolute)g (address)f(in)i(the)75 1243 y(calling)16 b(program.)i(The)d(datat)o(yp) q(e)e(of)h(suc)o(h)g(an)g(argumen)o(t)g(is)g(an)g(in)o(teger)g(of)g (the)g(size)h(needed)h(to)d(hold)75 1299 y(an)o(y)i(v)m(alid)i(address) e(in)h(the)f(execution)i(en)o(vironmen)o(t.)75 1439 y Fn(2.5)59 b(Language)19 b(Binding)75 1540 y Fv(This)e(section)g (de\014nes)h(the)f(rules)g(for)f Fw(MPI)g Fv(language)h(binding)i(in)e (general)g(and)g(for)f(F)l(ortran)g(77)g(and)75 1597 y(ANSI)d(C)e(in)i(particular.)19 b(De\014ned)13 b(here)f(are)g(v)m (arious)g(ob)s(ject)g(represen)o(tations,)g(as)f(w)o(ell)i(as)f(the)g (naming)75 1653 y(con)o(v)o(en)o(tions)21 b(used)g(for)e(expressing)j (this)f(standard.)35 b(The)21 b(actual)g(calling)h(sequences)f(are)g (de\014ned)75 1710 y(elsewhere.)166 1766 y(It)i(is)g(exp)q(ected)h (that)e(an)o(y)g(F)l(ortran)g(90)g(and)h(C++)g(implemen)o(tations)h (use)f(the)g(F)l(ortran)f(77)75 1822 y(and)f(ANSI)h(C)e(bindings,)k (resp)q(ectiv)o(ely)l(.)39 b(Although)22 b(w)o(e)e(consider)i(it)f (premature)g(to)f(de\014ne)i(other)75 1879 y(bindings)e(to)d(F)l (ortran)g(90)g(and)h(C++,)h(the)f(curren)o(t)f(bindings)j(are)e (designed)h(to)e(encourage,)i(rather)75 1935 y(than)c(discourage,)g (exp)q(erimen)o(tation)i(with)e(b)q(etter)g(bindings)i(that)e(migh)o(t) g(b)q(e)h(adopted)f(later.)166 1992 y(Since)i(the)e(w)o(ord)g(P)l (ARAMETER)h(is)g(a)f(k)o(eyw)o(ord)g(in)h(the)g(F)l(ortran)e(language,) h(w)o(e)g(use)h(the)g(w)o(ord)75 2048 y(\\argumen)o(t")21 b(to)i(denote)g(the)g(argumen)o(ts)e(to)i(a)f(subroutine.)44 b(These)23 b(are)f(normally)i(referred)f(to)75 2105 y(as)e(parameters)f (in)h(C,)g(ho)o(w)o(ev)o(er,)g(w)o(e)f(exp)q(ect)i(that)e(C)h (programmers)f(will)i(understand)g(the)f(w)o(ord)75 2161 y(\\argumen)o(t")c(\(whic)o(h)i(has)f(no)g(sp)q(eci\014c)i(meaning)f (in)g(C\),)e(th)o(us)h(allo)o(wing)h(us)g(to)e(a)o(v)o(oid)h (unnecessary)75 2218 y(confusion)e(for)f(F)l(ortran)f(programmers.)166 2274 y(There)k(are)g(sev)o(eral)g(imp)q(ortan)o(t)f(language)h(binding) i(issues)f(not)e(addressed)i(b)o(y)e(this)i(standard.)75 2331 y(This)11 b(standard)g(do)q(es)g(not)g(discuss)g(the)g(in)o(terop) q(erabilit)o(y)i(of)e(message)f(passing)h(b)q(et)o(w)o(een)g (languages.)19 b(It)75 2387 y(is)14 b(fully)g(exp)q(ected)h(that)d(man) o(y)h(implemen)o(tations)i(will)f(ha)o(v)o(e)f(suc)o(h)h(features,)f (and)g(that)g(suc)o(h)g(features)75 2443 y(are)i(a)g(sign)g(of)g(the)h (qualit)o(y)f(of)g(the)g(implemen)o(tation.)75 2561 y Fj(2.5.1)49 b(F)o(o)o(rtran)15 b(77)i(Binding)h(Issues)75 2647 y Fv(All)j Fw(MPI)d Fv(names)h(ha)o(v)o(e)g(an)g Ft(MPI)p 648 2647 15 2 v 36 w Fv(pre\014x,)h(and)g(all)g(c)o(haracters) e(are)h(capitals.)33 b(Programs)18 b(m)o(ust)g(not)75 2704 y(declare)13 b(v)m(ariables)h(or)e(functions)h(with)f(names)g(b)q (eginning)j(with)d(the)h(pre\014x,)g Ft(MPI)p 1489 2704 V 17 w Fv(.)18 b(This)13 b(is)g(mandated)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 11 19 11 18 bop 75 -100 a Fm(2.5.)34 b(LANGUA)o(GE)15 b(BINDING)1165 b Fv(11)290 81 y Ft(double)23 b(precision)f(a)290 138 y(integer)h(b)290 194 y(...)290 251 y(call)g(MPI_send\(a,...\))290 307 y(call)g(MPI_send\(b,...\))75 455 y Fv(Figure)14 b(2.1:)k(An)c(example)h(of)e(calling)i(a)f(routine)g(with)g(mismatc)o (hed)g(formal)f(and)h(actual)g(argumen)o(ts.)75 586 y(to)h(a)o(v)o(oid) g(p)q(ossible)h(name)g(collisions.)166 643 y(All)h Fw(MPI)f Fv(F)l(ortran)e(subroutines)j(ha)o(v)o(e)f(a)g(return)g(co)q(de)g(in)h (the)f(last)g(argumen)o(t.)22 b(A)16 b(few)f Fw(MPI)h Fv(op-)75 699 y(erations)e(are)f(functions,)i(whic)o(h)f(do)g(not)f(ha) o(v)o(e)h(the)g(return)g(co)q(de)g(argumen)o(t.)19 b(The)14 b(return)f(co)q(de)i(v)m(alue)75 756 y(for)d(successful)h(completion)g (is)g Fi(MPI)p 694 756 13 2 v 14 w(SUCCESS)p Fv(.)e(Other)h(error)g(co) q(des)g(are)g(implemen)o(tation)i(dep)q(enden)o(t;)75 812 y(see)h(Chapter)g(7.)166 869 y(Handles)k(are)e(represen)o(ted)h(in) g(F)l(ortran)f(as)g Ft(INTEGER)p Fv(s.)26 b(Binary-v)m(alued)20 b(v)m(ariables)e(are)g(of)f(t)o(yp)q(e)75 925 y Ft(LOGICAL)p Fv(.)166 982 y(Arra)o(y)d(argumen)o(ts)h(are)f(indexed)j(from)e(one.) 166 1038 y(Unless)e(explicitly)i(stated,)d(the)g Fw(MPI)g Fv(F77)f(binding)k(is)d(consisten)o(t)h(with)f(ANSI)h(standard)f(F)l (ortran)75 1095 y(77.)21 b(There)16 b(are)f(sev)o(eral)h(p)q(oin)o(ts)g (where)g(this)h(standard)e(div)o(erges)h(from)f(the)h(ANSI)h(F)l (ortran)d(77)h(stan-)75 1151 y(dard.)20 b(These)14 b(exceptions)i(are)e (consisten)o(t)g(with)h(common)f(practice)h(in)g(the)f(F)l(ortran)g (comm)o(unit)o(y)l(.)19 b(In)75 1207 y(particular:)143 1299 y Fx(\017)k Fw(MPI)14 b Fv(iden)o(ti\014ers)j(are)e(limited)i(to)d (thirt)o(y)l(,)h(not)g(six,)g(signi\014can)o(t)h(c)o(haracters.)143 1392 y Fx(\017)23 b Fw(MPI)14 b Fv(iden)o(ti\014ers)j(ma)o(y)d(con)o (tain)i(underscores)f(after)g(the)g(\014rst)g(c)o(haracter.)143 1485 y Fx(\017)23 b Fv(An)c Fw(MPI)g Fv(subroutine)h(with)g(a)f(c)o (hoice)h(argumen)o(t)f(ma)o(y)g(b)q(e)h(called)g(with)g(di\013eren)o(t) g(argumen)o(t)189 1541 y(t)o(yp)q(es.)32 b(An)20 b(example)g(is)g(sho)o (wn)f(in)h(Figure)g(2.1.)32 b(This)20 b(violates)g(the)f(letter)h(of)f (the)g(F)l(ortran)189 1598 y(standard,)c(but)h(suc)o(h)h(a)f(violation) h(is)f(common)g(practice.)23 b(An)16 b(alternativ)o(e)h(w)o(ould)f(b)q (e)h(to)e(ha)o(v)o(e)189 1654 y(a)g(separate)f(v)o(ersion)i(of)e Fw(MPI)p 697 1654 14 2 v 16 w(SEND)i Fv(for)e(eac)o(h)i(data)e(t)o(yp)q (e.)143 1747 y Fx(\017)23 b Fv(Although)10 b(not)g(required,)i(it)e(is) h(strongly)f(suggested)g(that)f(named)i Fw(MPI)e Fv(constan)o(ts)g(\()p Ft(PARAMETER)p Fv(s\))189 1804 y(b)q(e)14 b(pro)o(vided)g(in)g(an)g (include)i(\014le,)e(called)h Ft(mpif.h)p Fv(.)k(On)14 b(systems)f(that)f(do)i(not)f(supp)q(ort)h(include)189 1860 y(\014les,)h(the)h(implemen)o(tation)g(should)g(sp)q(ecify)h(the)e (v)m(alues)i(of)d(named)i(constan)o(ts.)143 1953 y Fx(\017)23 b Fv(V)l(endors)c(are)g(encouraged)h(to)f(pro)o(vide)h(t)o(yp)q(e)f (declarations)h(in)g(the)g Ft(mpif.h)e Fv(\014le)j(on)e(F)l(ortran)189 2010 y(systems)i(that)f(supp)q(ort)i(user-de\014ned)h(t)o(yp)q(es.)39 b(One)22 b(should)h(de\014ne,)h(if)e(p)q(ossible,)i(the)e(t)o(yp)q(e) 189 2066 y Fw(MPI)p 274 2066 V 15 w(ADDRESS)p Fv(,)e(whic)o(h)f(is)g (an)g Fw(INTEGER)g Fv(of)f(the)h(size)h(needed)g(to)e(hold)h(an)g (address)f(in)i(the)189 2123 y(execution)14 b(en)o(vironmen)o(t.)19 b(On)14 b(systems)e(where)i(t)o(yp)q(e)f(de\014nition)h(is)g(not)f (supp)q(orted,)g(it)h(is)f(up)h(to)189 2179 y(the)h(user)g(to)f(use)h (an)g Fw(INTEGER)h Fv(of)f(the)g(righ)o(t)f(kind)i(to)f(represen)o(t)g (addresses)g(\(i.e.,)f Ft(INTEGER*4)189 2235 y Fv(on)h(a)g(32)f(bit)i (mac)o(hine,)g Ft(INTEGER*8)e Fv(on)h(a)g(64)f(bit)i(mac)o(hine,)g (etc.\).)1937 2282 y Fx(>)166 2327 y Fv(All)f Fw(MPI)e Fv(named)h(constan)o(ts)f(can)g(b)q(e)i(used)f(wherev)o(er)g(an)f(en)o (tit)o(y)h(declared)g(with)g(the)g Ft(PARAMETER)75 2384 y Fv(attribute)19 b(can)f(b)q(e)i(used)f(in)h(F)l(ortran.)29 b(There)19 b(is)g(one)g(exception)h(to)e(this)h(rule:)28 b(the)19 b Fw(MPI)f Fv(constan)o(t)75 2440 y Fi(MPI)p 152 2440 13 2 v 14 w(BOTTOM)d Fv(\(section)h(3.12.2\))d(can)i(only)h(b) q(e)f(used)h(as)f(a)g(bu\013er)g(argumen)o(t.)483 b Fx(?)75 2561 y Fj(2.5.2)49 b(C)16 b(Binding)i(Issues)75 2647 y Fv(W)l(e)f(use)g(the)g(ANSI)h(C)f(declaration)g(format.)24 b(All)18 b Fw(MPI)e Fv(names)h(ha)o(v)o(e)g(an)f Ft(MPI)p 1460 2647 15 2 v 34 w Fv(pre\014x,)h(de\014ned)i(con-)75 2704 y(stan)o(ts)12 b(are)h(in)i(all)f(capital)g(letters,)g(and)f (de\014ned)i(t)o(yp)q(es)f(and)f(functions)h(ha)o(v)o(e)f(one)h (capital)g(letter)g(after)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 12 20 12 19 bop 75 -100 a Fv(12)692 b Fm(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y Fv(the)j(pre\014x.)30 b(Programs)17 b(m)o(ust)h(not)g(declare)h(v)m(ariables)g(or)f (functions)h(with)g(names)f(b)q(eginning)j(with)75 102 y(the)15 b(pre\014x,)h Ft(MPI)p 369 102 15 2 v 16 w Fv(.)k(This)c(is)g (mandated)f(to)f(a)o(v)o(oid)h(p)q(ossible)i(name)e(collisions.)166 158 y(The)i(de\014nition)h(of)e(named)h(constan)o(ts,)e(function)i (protot)o(yp)q(es,)f(and)h(t)o(yp)q(e)f(de\014nitions)i(m)o(ust)e(b)q (e)75 214 y(supplied)i(in)e(an)f(include)i(\014le)g Fw(mpi.h)p Fv(.)166 271 y(Almost)24 b(all)h(C)e(functions)i(return)f(an)g(error)f (co)q(de.)47 b(The)24 b(successful)h(return)f(co)q(de)h(will)g(b)q(e)75 327 y Ft(MPI)p 150 327 V 17 w(SUCCESS)p Fv(,)16 b(but)i(failure)h (return)f(co)q(des)g(are)f(implemen)o(tation)i(dep)q(enden)o(t.)29 b(A)18 b(few)g(C)f(functions)75 384 y(do)e(not)g(return)g(v)m(alues,)h (so)f(that)f(they)i(can)f(b)q(e)h(implemen)o(ted)h(as)e(macros.)166 440 y(T)o(yp)q(e)g(declarations)h(are)e(pro)o(vided)i(for)e(handles)i (to)e(eac)o(h)h(category)f(of)g(opaque)h(ob)s(jects.)20 b(Either)75 497 y(a)15 b(p)q(oin)o(ter)h(or)e(an)h(in)o(teger)h(t)o(yp) q(e)f(is)h(used.)166 553 y(Arra)o(y)e(argumen)o(ts)h(are)f(indexed)j (from)e(zero.)166 610 y(Logical)j(\015ags)e(are)g(in)o(tegers)h(with)g (v)m(alue)h(0)f(meaning)g(\\false")g(and)g(a)f(non-zero)h(v)m(alue)h (meaning)75 666 y(\\true.")166 723 y(Choice)e(argumen)o(ts)e(are)h(p)q (oin)o(ters)h(of)f(t)o(yp)q(e)g Ft(void*)p Fv(.)166 779 y(Address)f(argumen)o(ts)e(are)h(of)g Fw(MPI)g Fv(de\014ned)i(t)o(yp)q (e)e Fi(MPI)p 1108 779 13 2 v 14 w(Aint)p Fv(.)20 b(This)14 b(is)f(de\014ned)i(to)e(b)q(e)h(an)f Fi(int)h Fv(of)f(the)75 835 y(size)j(needed)h(to)d(hold)i(an)o(y)f(v)m(alid)i(address)e(on)g (the)h(target)e(arc)o(hitecture.)-1512 b Fx(>)166 892 y Fv(All)17 b(named)g Fw(MPI)e Fv(constan)o(ts)g(can)h(b)q(e)h(used)g (in)g(initialization)i(expressions)d(or)g(assignmen)o(ts)g(lik)o(e)75 948 y(C)f(constan)o(ts.)-483 b Fx(?)75 1092 y Fn(2.6)59 b(Pro)r(cesses)75 1193 y Fv(An)15 b Fw(MPI)e Fv(program)g(consists)i (of)f(autonomous)f(pro)q(cesses,)i(executing)g(their)g(o)o(wn)f(co)q (de,)g(in)h(an)g(MIMD)75 1250 y(st)o(yle.)29 b(The)18 b(co)q(des)h(executed)g(b)o(y)f(eac)o(h)g(pro)q(cess)h(need)g(not)f(b)q (e)h(iden)o(tical.)30 b(The)19 b(pro)q(cesses)f(comm)o(u-)75 1306 y(nicate)h(via)f(calls)h(to)e Fw(MPI)g Fv(comm)o(unication)i (primitiv)o(es.)29 b(T)o(ypically)l(,)20 b(eac)o(h)e(pro)q(cess)g (executes)h(in)g(its)75 1362 y(o)o(wn)g(address)h(space,)h(although)f (shared-memory)f(implemen)o(tations)i(of)f Fw(MPI)f Fv(are)g(p)q (ossible.)35 b(This)75 1419 y(do)q(cumen)o(t)20 b(sp)q(eci\014es)i(the) e(b)q(eha)o(vior)h(of)e(a)h(parallel)i(program)d(assuming)h(that)f (only)i Fw(MPI)e Fv(calls)i(are)75 1475 y(used)16 b(for)f(comm)o (unication.)22 b(The)16 b(in)o(teraction)g(of)g(an)f Fw(MPI)g Fv(program)g(with)h(other)f(p)q(ossible)j(means)d(of)75 1532 y(comm)o(unication)h(\(e.g.,)e(shared)h(memory\))f(is)i(not)f(sp)q (eci\014ed.)166 1588 y Fw(MPI)c Fv(do)q(es)h(not)f(sp)q(ecify)i(the)e (execution)i(mo)q(del)f(for)f(eac)o(h)h(pro)q(cess.)19 b(A)11 b(pro)q(cess)h(can)g(b)q(e)g(sequen)o(tial,)75 1645 y(or)e(can)h(b)q(e)h(m)o(ulti-threaded,)g(with)f(threads)g(p)q (ossibly)h(executing)g(concurren)o(tly)l(.)20 b(Care)10 b(has)g(b)q(een)i(tak)o(en)75 1701 y(to)h(mak)o(e)g Fw(MPI)f Fv(\\thread-safe,")h(b)o(y)g(a)o(v)o(oiding)h(the)f(use)h(of)f (implicit)j(state.)i(The)c(desired)h(in)o(teraction)e(of)75 1758 y Fw(MPI)j Fv(with)g(threads)g(is)h(that)e(concurren)o(t)i (threads)f(b)q(e)h(all)g(allo)o(w)o(ed)f(to)g(execute)h Fw(MPI)e Fv(calls,)i(and)g(calls)75 1814 y(b)q(e)g(reen)o(tran)o(t;)e (a)h(blo)q(c)o(king)h Fw(MPI)f Fv(call)h(blo)q(c)o(ks)g(only)g(the)f (in)o(v)o(oking)h(thread,)f(allo)o(wing)h(the)f(sc)o(heduling)75 1871 y(of)f(another)g(thread.)166 1927 y Fw(MPI)j Fv(do)q(es)g(not)g (pro)o(vide)h(mec)o(hanisms)g(to)e(sp)q(ecify)j(the)e(initial)j(allo)q (cation)e(of)f(pro)q(cesses)g(to)g(an)75 1983 y Fw(MPI)c Fv(computation)g(and)h(their)g(binding)i(to)d(ph)o(ysical)h(pro)q (cessors.)20 b(It)14 b(is)h(exp)q(ected)h(that)e(v)o(endors)g(will)75 2040 y(pro)o(vide)20 b(mec)o(hanisms)h(to)e(do)g(so)h(either)g(at)f (load)h(time)g(or)g(at)f(run)h(time.)34 b(Suc)o(h)20 b(mec)o(hanisms)g(will)75 2096 y(allo)o(w)c(the)f(sp)q(eci\014cation)j (of)d(the)h(initial)i(n)o(um)o(b)q(er)d(of)h(required)g(pro)q(cesses,)g (the)g(co)q(de)g(to)f(b)q(e)h(executed)75 2153 y(b)o(y)k(eac)o(h)h (initial)i(pro)q(cess,)e(and)g(the)g(allo)q(cation)g(of)f(pro)q(cesses) h(to)f(pro)q(cessors.)36 b(Also,)21 b(the)g(curren)o(t)75 2209 y(prop)q(osal)f(do)q(es)h(not)e(pro)o(vide)i(for)e(dynamic)i (creation)f(or)g(deletion)h(of)f(pro)q(cesses)g(during)h(program)75 2266 y(execution)c(\(the)f(total)f(n)o(um)o(b)q(er)i(of)f(pro)q(cesses) g(is)h(\014xed\),)f(although)g(it)h(is)f(in)o(tended)i(to)d(b)q(e)i (consisten)o(t)75 2322 y(with)e(suc)o(h)h(extensions.)k(Finally)l(,)d (w)o(e)e(alw)o(a)o(ys)f(iden)o(tify)i(pro)q(cesses)g(according)f(to)g (their)h(relativ)o(e)f(rank)75 2379 y(in)h(a)f(group,)f(that)h(is,)g (consecutiv)o(e)h(in)o(tegers)g(in)g(the)f(range)g Ft(0..groupsize-1)p Fv(.)75 2522 y Fn(2.7)59 b(Erro)n(r)21 b(Handling)75 2623 y Fw(MPI)11 b Fv(pro)o(vides)i(the)f(user)g(with)h(reliable)h (message)d(transmission.)19 b(A)13 b(message)e(sen)o(t)h(is)g(alw)o(a)o (ys)g(receiv)o(ed)75 2680 y(correctly)l(,)17 b(and)g(the)g(user)g(do)q (es)g(not)f(need)i(to)e(c)o(hec)o(k)h(for)f(transmission)h(errors,)f (time-outs,)h(or)f(other)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 13 21 13 20 bop 75 -100 a Fm(2.7.)34 b(ERR)o(OR)17 b(HANDLING)1217 b Fv(13)75 45 y(error)12 b(conditions.)20 b(In)13 b(other)g(w)o(ords,)f Fw(MPI)g Fv(do)q(es)h(not)f(pro)o(vide)h(mec)o(hanisms)g(for)f(dealing) i(with)f(failures)75 102 y(in)i(the)f(comm)o(unication)g(system.)19 b(If)14 b(the)g Fw(MPI)f Fv(implemen)o(tation)i(is)g(built)g(on)e(an)h (unreliable)i(underly-)75 158 y(ing)g(mec)o(hanism,)f(then)h(it)f(is)h (the)f(job)g(of)g(the)g(implemen)o(tor)h(of)f(the)g Fw(MPI)g Fv(subsystem)g(to)f(insulate)j(the)75 214 y(user)12 b(from)g(this)g (unreliabili)q(t)o(y)l(,)j(or)c(to)h(re\015ect)g(unreco)o(v)o(erable)h (errors)f(as)f(failures.)20 b(Whenev)o(er)13 b(p)q(ossible,)75 271 y(suc)o(h)k(failures)g(will)h(b)q(e)g(re\015ected)f(as)f(errors)g (in)h(the)g(relev)m(an)o(t)g(comm)o(unication)g(call.)25 b(Similarly)l(,)19 b Fw(MPI)75 327 y Fv(itself)e(pro)o(vides)f(no)g (mec)o(hanisms)g(for)g(handling)h(pro)q(cessor)f(failures.)23 b(The)16 b(error)f(handling)j(facilities)75 384 y(describ)q(ed)c(in)f (section)g(7.2)e(can)i(b)q(e)f(used)h(to)f(restrict)g(the)g(scop)q(e)h (of)f(an)g(unreco)o(v)o(erable)h(error,)e(or)h(design)75 440 y(error)i(reco)o(v)o(ery)h(at)g(the)g(application)i(lev)o(el.)166 497 y(Of)g(course,)g Fw(MPI)f Fv(programs)g(ma)o(y)g(still)i(b)q(e)g (erroneous.)24 b(A)17 b Fq(program)i(error)c Fv(can)i(o)q(ccur)g(when) 75 553 y(an)f Fw(MPI)g Fv(call)i(is)e(called)i(with)f(an)f(incorrect)h (argumen)o(t)f(\(non-existing)h(destination)g(in)h(a)e(send)h(op)q(er-) 75 610 y(ation,)g(bu\013er)g(to)q(o)f(small)h(in)h(a)e(receiv)o(e)i(op) q(eration,)f(etc.\))25 b(This)17 b(t)o(yp)q(e)g(of)f(error)g(w)o(ould)h (o)q(ccur)h(in)f(an)o(y)75 666 y(implemen)o(tation.)31 b(In)19 b(addition,)h(a)f Fq(resource)h(error)d Fv(ma)o(y)h(o)q(ccur)h (when)g(a)f(program)f(exceeds)j(the)75 723 y(amoun)o(t)d(of)f(a)o(v)m (ailable)j(system)e(resources)h(\(n)o(um)o(b)q(er)f(of)g(p)q(ending)i (messages,)e(system)g(bu\013ers,)g(etc.\).)75 779 y(The)h(o)q (ccurrence)h(of)f(this)g(t)o(yp)q(e)g(of)g(error)f(dep)q(ends)j(on)e (the)g(amoun)o(t)f(of)g(a)o(v)m(ailable)j(resources)e(in)h(the)75 835 y(system)13 b(and)h(the)g(resource)f(allo)q(cation)i(mec)o(hanism)f (used;)h(this)f(ma)o(y)f(di\013er)h(from)f(system)g(to)g(system.)75 892 y(A)k(high-qualit)o(y)h(implemen)o(tation)h(will)f(pro)o(vide)g (generous)f(limits)h(on)f(the)g(imp)q(ortan)o(t)f(resources)h(so)75 948 y(as)e(to)f(alleviate)j(the)e(p)q(ortabilit)o(y)h(problem)g(this)g (represen)o(ts.)166 1005 y(Almost)11 b(all)h Fw(MPI)f Fv(calls)i(return)e(a)g(co)q(de)h(that)f(indicates)h(successful)h (completion)g(of)e(the)g(op)q(eration.)75 1061 y(Whenev)o(er)17 b(p)q(ossible,)h Fw(MPI)d Fv(calls)j(return)e(an)g(error)g(co)q(de)h (if)g(an)f(error)f(o)q(ccurred)i(during)h(the)e(call.)24 b(In)63 b Fx(>)75 1118 y Fv(certain)12 b(circumstances,)h(when)f(the)g Fw(MPI)f Fv(function)h(ma)o(y)f(complete)h(sev)o(eral)g(distinct)h(op)q (erations,)f(and)75 1174 y(therefore)17 b(ma)o(y)g(generate)g(sev)o (eral)h(indep)q(enden)o(t)i(errors,)d(the)h Fw(MPI)e Fv(function)j(ma)o(y)e(return)g(m)o(ultiple)75 1231 y(error)f(co)q (des.)26 b(By)18 b(default,)f(an)g(error)g(detected)g(during)h(the)f (execution)i(of)d(the)h Fw(MPI)g Fv(library)h(causes)62 b Fx(?)75 1287 y Fv(the)15 b(parallel)i(computation)e(to)g(ab)q(ort.)k (Ho)o(w)o(ev)o(er,)14 b Fw(MPI)h Fv(pro)o(vides)h(mec)o(hanisms)f(for)g (users)g(to)g(c)o(hange)75 1344 y(this)i(default)g(and)g(to)f(handle)i (reco)o(v)o(erable)f(errors.)23 b(The)17 b(user)g(ma)o(y)f(sp)q(ecify)i (that)e(no)h(error)f(is)h(fatal,)75 1400 y(and)j(handle)h(error)e(co)q (des)h(returned)g(b)o(y)g Fw(MPI)f Fv(calls)h(b)o(y)g(himself)h(or)e (herself.)34 b(Also,)21 b(the)f(user)g(ma)o(y)75 1456 y(pro)o(vide)14 b(his)h(or)f(her)g(o)o(wn)f(error-handling)i(routines,) f(whic)o(h)h(will)h(b)q(e)e(in)o(v)o(ok)o(ed)g(whenev)o(er)h(an)f Fw(MPI)f Fv(call)75 1513 y(returns)i(abnormally)l(.)21 b(The)15 b Fw(MPI)g Fv(error)f(handling)j(facilities)g(are)e(describ)q (ed)i(in)f(section)g(7.2.)166 1569 y(Sev)o(eral)21 b(factors)f(limit)j (the)e(abilit)o(y)h(of)f Fw(MPI)f Fv(calls)i(to)e(return)h(with)h (meaningful)g(error)e(co)q(des)75 1626 y(when)15 b(an)g(error)f(o)q (ccurs.)20 b Fw(MPI)14 b Fv(ma)o(y)h(not)f(b)q(e)h(able)h(to)e(detect)h (some)g(errors;)f(other)g(errors)g(ma)o(y)g(b)q(e)i(to)q(o)75 1682 y(exp)q(ensiv)o(e)j(to)e(detect)g(in)i(normal)e(execution)i(mo)q (de;)f(\014nally)h(some)e(errors)g(ma)o(y)g(b)q(e)h(\\catastrophic")75 1739 y(and)d(ma)o(y)g(prev)o(en)o(t)g Fw(MPI)f Fv(from)h(returning)h (con)o(trol)f(to)f(the)h(caller)i(in)f(a)f(consisten)o(t)g(state.)166 1795 y(Another)c(subtle)g(issue)h(arises)f(b)q(ecause)h(of)f(the)f (nature)h(of)g(async)o(hronous)f(comm)o(unications:)19 b Fw(MPI)75 1852 y Fv(calls)13 b(ma)o(y)e(initiate)i(op)q(erations)f (that)f(con)o(tin)o(ue)i(async)o(hronously)f(after)f(the)h(call)h (returned.)19 b(Th)o(us,)12 b(the)75 1908 y(op)q(eration)i(ma)o(y)g (return)g(with)g(a)g(co)q(de)h(indicating)h(successful)f(completion,)g (y)o(et)f(later)g(cause)h(an)f(error)75 1965 y(exception)h(to)e(b)q(e)h (raised.)20 b(If)14 b(there)f(is)i(a)e(subsequen)o(t)h(call)h(that)e (relates)h(to)f(the)g(same)h(op)q(eration)g(\(e.g.,)75 2021 y(a)g(call)h(that)e(v)o(eri\014es)i(that)e(an)h(async)o(hronous)g (op)q(eration)g(has)g(completed\))h(then)f(the)g(error)g(argumen)o(t)75 2077 y(asso)q(ciated)19 b(with)f(this)h(call)g(will)h(b)q(e)f(used)g (to)f(indicate)i(the)e(nature)g(of)g(the)h(error.)28 b(In)19 b(a)f(few)g(cases,)75 2134 y(the)e(error)g(ma)o(y)g(o)q(ccur)g (after)g(all)h(calls)h(that)d(relate)i(to)e(the)i(op)q(eration)f(ha)o (v)o(e)g(completed,)h(so)f(that)g(no)75 2190 y(error)d(v)m(alue)j(can)e (b)q(e)h(used)g(to)e(indicate)j(the)f(nature)f(of)f(the)i(error)e (\(e.g.,)g(an)h(error)g(in)h(a)f(send)g(with)h(the)75 2247 y(ready)f(mo)q(de\).)20 b(Suc)o(h)15 b(an)f(error)g(m)o(ust)g(b)q (e)h(treated)f(as)g(fatal,)g(since)i(information)e(cannot)g(b)q(e)i (returned)75 2303 y(for)f(the)g(user)g(to)g(reco)o(v)o(er)f(from)h(it.) 166 2360 y(This)f(do)q(cumen)o(t)g(do)q(es)g(not)f(sp)q(ecify)i(the)f (state)e(of)h(a)h(computation)f(after)g(an)g(erroneous)h Fw(MPI)f Fv(call)75 2416 y(has)h(o)q(ccurred.)20 b(The)14 b(desired)h(b)q(eha)o(vior)f(is)h(that)e(a)g(relev)m(an)o(t)i(error)e (co)q(de)i(b)q(e)f(returned,)g(and)g(the)g(e\013ect)75 2473 y(of)g(the)g(error)g(b)q(e)h(lo)q(calized)h(to)e(the)g(greatest)f (p)q(ossible)j(exten)o(t.)k(E.g.,)13 b(it)h(is)h(highly)g(desireable)h (that)e(an)75 2529 y(erroneous)j(receiv)o(e)i(call)f(will)h(not)e (cause)h(an)o(y)f(part)f(of)h(the)h(receiv)o(er's)g(memory)e(to)h(b)q (e)h(o)o(v)o(erwritten,)75 2586 y(b)q(ey)o(ond)e(the)f(area)g(sp)q (eci\014ed)i(for)e(receiving)h(the)g(message.)166 2642 y(Implemen)o(tations)f(ma)o(y)f(go)g(b)q(ey)o(ond)h(this)g(do)q(cumen)o (t)g(in)g(supp)q(orting)h(in)f(a)f(meaningful)i(manner)75 2698 y Fw(MPI)k Fv(calls)h(that)e(are)h(de\014ned)i(here)e(to)g(b)q(e)h (erroneous.)35 b(F)l(or)19 b(example,)j Fw(MPI)e Fv(sp)q(eci\014es)i (strict)e(t)o(yp)q(e)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 14 22 14 21 bop 75 -100 a Fv(14)692 b Fm(CHAPTER)15 b(2.)35 b(MPI)15 b(TERMS)h(AND)f(CONVENTIONS)75 45 y Fv(matc)o(hing)20 b(rules)g(b)q(et)o(w)o(een)g(matc)o(hing)f(send)h(and)g(receiv)o(e)g (op)q(erations:)29 b(it)20 b(is)g(erroneous)f(to)g(send)h(a)75 102 y(\015oating)c(p)q(oin)o(t)g(v)m(ariable)i(and)e(receiv)o(e)h(an)f (in)o(teger.)22 b(Implemen)o(tations)17 b(ma)o(y)f(go)f(b)q(ey)o(ond)i (these)f(t)o(yp)q(e)75 158 y(matc)o(hing)d(rules,)i(and)e(pro)o(vide)h (automatic)f(t)o(yp)q(e)h(con)o(v)o(ersion)f(in)i(suc)o(h)e (situations.)20 b(It)14 b(will)h(b)q(e)f(helpful)75 214 y(to)h(generate)g(w)o(arnings)g(for)f(suc)o(h)i(nonconforming)f(b)q (eha)o(vior.)75 358 y Fn(2.8)59 b(Implementation)17 b(issues)75 459 y Fv(There)c(are)g(a)f(n)o(um)o(b)q(er)i(of)e(areas)h(where)g(an)g Fw(MPI)f Fv(implemen)o(tation)i(ma)o(y)f(in)o(teract)f(with)i(the)f(op) q(erating)75 516 y(en)o(vironmen)o(t)k(and)g(system.)24 b(While)19 b Fw(MPI)d Fv(do)q(es)h(not)f(mandate)h(that)f(an)o(y)g (services)i(\(suc)o(h)f(as)f(I/O)h(or)75 572 y(signal)g(handling\))g(b) q(e)f(pro)o(vided,)g(it)g(do)q(es)g(strongly)g(suggest)f(the)h(b)q(eha) o(vior)g(to)f(b)q(e)h(pro)o(vided)h(if)f(those)75 629 y(services)g(are)g(a)o(v)m(ailable.)23 b(This)16 b(is)g(an)f(imp)q (ortan)o(t)h(p)q(oin)o(t)g(in)g(ac)o(hieving)h(p)q(ortabilit)o(y)g (across)d(platforms)75 685 y(that)g(pro)o(vide)i(the)g(same)e(set)h(of) g(services.)75 807 y Fj(2.8.1)49 b(Indep)q(endence)15 b(of)i(Basic)g(Runtime)f(Routines)75 893 y Fw(MPI)d Fv(programs)f (require)i(that)f(library)h(routines)g(that)f(are)g(part)g(of)f(the)i (basic)g(language)g(en)o(vironmen)o(t)75 949 y(\(suc)o(h)h(as)g Ft(date)f Fv(and)i Ft(write)e Fv(in)i(F)l(ortran)e(and)h Ft(printf)f Fv(and)h Ft(malloc)g Fv(in)h(ANSI)f(C\))g(and)g(are)g (executed)75 1005 y(after)g Ft(MPI)p 258 1005 15 2 v 17 w(INIT)g Fv(and)i(b)q(efore)f Ft(MPI)p 685 1005 V 17 w(FINALIZE)f Fv(op)q(erate)g(indep)q(enden)o(tly)k(and)e(that)e (their)h Fl(c)n(ompletion)75 1062 y Fv(is)g(indep)q(enden)o(t)h(of)e (the)g(action)h(of)e(other)h(pro)q(cesses)h(in)g(an)f Fw(MPI)g Fv(program.)166 1118 y(Note)f(that)g(this)h(in)g(no)f(w)o(a)o (y)g(prev)o(en)o(ts)g(the)g(creation)h(of)f(library)h(routines)g(that)e (pro)o(vide)i(parallel)75 1175 y(services)h(whose)f(op)q(eration)h(is)g (collectiv)o(e.)22 b(Ho)o(w)o(ev)o(er,)14 b(the)h(follo)o(wing)h (program)e(is)i(exp)q(ected)g(to)f(com-)75 1231 y(plete)j(in)g(an)f (ANSI)h(C)f(en)o(vironmen)o(t)h(regardless)f(of)g(the)g(size)h(of)f Ft(MPI)p 1312 1231 V 16 w(COMM)p 1424 1231 V 17 w(WORLD)g Fv(\(assuming)g(that)75 1288 y(I/O)f(is)f(a)o(v)m(ailable)i(at)e(the)g (executing)h(no)q(des\).)75 1382 y Ft(int)23 b(rank;)75 1438 y(MPI_Init\()g(argc,)g(argv)g(\);)75 1494 y(MPI_Comm_rank\()f (MPI_COMM_WORLD,)g(&rank)h(\);)75 1551 y(if)h(\(rank)f(==)g(0\))h (printf\()f("Starting)f(program\\n")h(\);)75 1607 y(MPI_Finalize\(\);) 75 1701 y Fv(The)15 b(corresp)q(onding)i(F)l(ortran)d(77)g(program)g (is)i(also)f(exp)q(ected)i(to)d(complete.)166 1758 y(An)k(example)h(of) f(what)f(is)i Fl(not)j Fv(required)d(is)g(an)o(y)f(particular)h (ordering)f(of)g(the)g(action)g(of)g(these)75 1814 y(routines)g(when)g (called)i(b)o(y)d(sev)o(eral)h(tasks.)27 b(F)l(or)17 b(example,)i Fw(MPI)e Fv(mak)o(es)g(neither)i(requiremen)o(ts)f(nor)75 1871 y(recommendations)g(for)f(the)g(output)g(from)g(the)h(follo)o (wing)g(program)e(\(again)h(assuming)h(that)f(I/O)g(is)75 1927 y(a)o(v)m(ailable)g(at)d(the)i(executing)g(no)q(des\).)75 2021 y Ft(MPI_Comm_rank\()22 b(MPI_COMM_WORLD,)g(&rank)h(\);)75 2077 y(printf\()g("Output)g(from)g(task)g(rank)h(\045d\\n",)f(rank)g (\);)166 2171 y Fv(In)18 b(addition,)h(calls)f(that)f(fail)h(b)q (ecause)g(of)f(resource)h(exhaustion)g(or)f(other)g(error)g(are)g(not)g (con-)75 2227 y(sidered)h(a)f(violation)h(of)e(the)h(requiremen)o(ts)h (here)f(\(ho)o(w)o(ev)o(er,)f(they)h(are)g(required)h(to)e(complete,)i (just)75 2284 y(not)d(to)f(complete)i(successfully\).)75 2406 y Fj(2.8.2)49 b(Interaction)16 b(with)g(signals)i(in)f(POSIX)75 2491 y Fw(MPI)10 b Fv(do)q(es)h(not)g(sp)q(ecify)h(either)f(the)g(in)o (teraction)g(of)f(pro)q(cesses)i(with)f(signals,)h(in)f(a)g(UNIX)g(en)o (vironmen)o(t,)75 2548 y(or)19 b(with)g(other)g(ev)o(en)o(ts)g(that)f (do)h(not)g(relate)g(to)g Fw(MPI)f Fv(comm)o(unication.)33 b(That)18 b(is,)j(signals)f(are)e(not)75 2604 y(signi\014can)o(t)j (from)f(the)g(view)h(p)q(oin)o(t)g(of)f Fw(MPI)p Fv(,)f(and)h(implemen) o(tors)h(should)h(attempt)d(to)h(implemen)o(t)75 2661 y Fw(MPI)15 b Fv(so)f(that)h(signals)h(are)f(transparen)o(t:)k(an)c Fw(MPI)g Fv(call)h(susp)q(ended)h(b)o(y)e(a)g(signal)h(should)g(resume) g(and)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 15 23 15 22 bop 75 -100 a Fm(2.9.)34 b(EXAMPLES)1393 b Fv(15)75 45 y(complete)18 b(after)e(the)h(signal)h(is)f(handled.)27 b(Generally)l(,)19 b(the)e(state)f(of)g(a)h(computation)g(that)f(is)i (visible)75 102 y(or)d(signi\014can)o(t)h(from)e(the)i(view-p)q(oin)o (t)g(of)f Fw(MPI)f Fv(should)j(only)e(b)q(e)h(a\013ected)f(b)o(y)g Fw(MPI)g Fv(calls.)166 158 y(The)20 b(in)o(ten)o(t)f(of)g Fw(MPI)g Fv(to)f(b)q(e)i(thread)g(and)f(signal)h(safe)f(has)h(a)f(n)o (um)o(b)q(er)g(of)g(subtle)i(e\013ects.)32 b(F)l(or)75 214 y(example,)20 b(on)e(Unix)h(systems,)g(a)f(catc)o(hable)h(signal)g (suc)o(h)g(as)f(SIGALRM)h(\(an)f(alarm)h(signal\))g(m)o(ust)75 271 y(not)f(cause)h(an)g Fw(MPI)f Fv(routine)h(to)f(b)q(eha)o(v)o(e)h (di\013eren)o(tly)h(than)e(it)h(w)o(ould)h(ha)o(v)o(e)e(in)h(the)g (absence)h(of)e(the)75 327 y(signal.)39 b(Of)21 b(course,)h(if)g(the)f (signal)h(handler)h(issues)f Fw(MPI)e Fv(calls)j(or)d(c)o(hanges)h(the) h(en)o(vironmen)o(t)f(in)75 384 y(whic)o(h)d(the)f Fw(MPI)g Fv(routine)g(is)h(op)q(erating)g(\(for)e(example,)i(consuming)g(all)g (a)o(v)m(ailable)h(memory)d(space\),)75 440 y(the)e Fw(MPI)g Fv(routine)g(should)i(b)q(eha)o(v)o(e)e(as)g(appropriate)g(for)f(that)h (situation)g(\(in)h(particular,)g(in)g(this)f(case,)75 497 y(the)h(b)q(eha)o(vior)h(should)g(b)q(e)g(the)f(same)g(as)g(for)g (a)g(m)o(ultithreaded)h Fw(MPI)e Fv(implemen)o(tation\).)166 553 y(A)20 b(second)h(e\013ect)g(is)g(that)e(a)h(signal)i(handler)f (that)f(p)q(erforms)g Fw(MPI)g Fv(calls)i(m)o(ust)e(not)g(in)o(terfere) 75 610 y(with)d(the)g(op)q(eration)h(of)e Fw(MPI)p Fv(.)g(F)l(or)h (example,)h(an)f Fw(MPI)f Fv(receiv)o(e)i(of)f(an)o(y)f(t)o(yp)q(e)h (that)g(o)q(ccurs)g(within)h(a)75 666 y(signal)13 b(handler)h(m)o(ust)d (not)h(cause)h(erroneous)f(b)q(eha)o(vior)h(b)o(y)f(the)h Fw(MPI)e Fv(implemen)o(tation.)20 b(Note)12 b(that)g(an)75 723 y(implemen)o(tation)17 b(is)f(p)q(ermitted)g(to)f(prohibit)i(the)f (use)f(of)h Fw(MPI)e Fv(calls)j(from)e(within)i(a)e(signal)h(handler,) 75 779 y(and)f(is)h(not)f(required)h(to)f(detect)g(suc)o(h)h(use.)166 835 y(It)e(is)g(highly)h(desirable)g(that)e Fw(MPI)g Fv(not)g(use)h Ft(SIGALRM)p Fv(,)f Ft(SIGFPE)p Fv(,)f(or)h Ft(SIGIO)p Fv(.)g(An)h(implemen)o(tation)75 892 y(is)h Fl(r)n(e)n(quir)n(e)n(d)j Fv(to)c(clearly)i(do)q(cumen)o(t)f(all)g(of)f (the)h(signals)g(that)f(the)h Fw(MPI)e Fv(implemen)o(tation)j(uses;)f (a)f(go)q(o)q(d)75 948 y(place)i(for)f(this)g(information)h(is)f(a)g (Unix)h(`)p Ft(man)p Fv(')e(page)h(on)h Ft(MPI)p Fv(.)780 b Fx(>)75 1092 y Fn(2.9)59 b(Examples)75 1193 y Fv(The)18 b(examples)g(in)g(this)g(do)q(cumen)o(t)g(are)f(for)g(illustration)i (purp)q(oses)f(only)l(.)28 b(They)17 b(are)g(not)h(in)o(tended)75 1250 y(to)i(sp)q(ecify)h(the)g(standard.)35 b(F)l(urthermore,)20 b(the)h(examples)g(ha)o(v)o(e)f(not)g(b)q(een)h(carefully)h(c)o(hec)o (k)o(ed)e(or)75 1306 y(v)o(eri\014ed.)1706 b Fx(?)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 16 24 16 23 bop 75 356 a Fo(Chapter)34 b(3)75 564 y Fr(P)m(oin)m(t-to-P)m (oin)m(t)42 b(Comm)m(unication)75 805 y Fn(3.1)59 b(Intro)r(duction)75 906 y Fv(Sending)14 b(and)e(receiving)h(of)f(messages)g(b)o(y)g(pro)q (cesses)g(is)h(the)f(basic)g Fw(MPI)g Fv(comm)o(unication)h(mec)o (hanism.)75 963 y(The)20 b(basic)g(p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o (unication)h(op)q(erations)e(are)g Fq(send)h Fv(and)f Fq(receiv)o(e)p Fv(.)32 b(Their)21 b(use)e(is)75 1019 y(illustrated)e(in)f(the)f(example)h(b)q(elo)o(w.)75 1125 y Ft(#include)23 b("mpi.h")75 1182 y(main\()g(argc,)g(argv)h(\))75 1238 y(int)f(argc;)75 1295 y(char)g(**argv;)75 1351 y({)170 1408 y(char)h(message[20];)170 1464 y(int)g(myrank;)170 1521 y(MPI_Status)f(status;)170 1577 y(MPI_Init\()g(&argc,)g(&argv)g (\);)170 1633 y(MPI_Comm_rank\()f(MPI_COMM_WORLD,)g(&myrank)h(\);)170 1690 y(if)h(\(myrank)f(==)h(0\))95 b(/*)23 b(code)h(for)f(process)g (zero)g(*/)170 1746 y({)266 1803 y(strcpy\(message,"Hello,)e(there"\);) 266 1859 y(MPI_Send\(message,)g(strlen\(message\),)h(MPI_CHAR,)h(1,)g (99,)h(MPI_COMM_WORLD\);)170 1916 y(})170 1972 y(else)382 b(/*)23 b(code)h(for)f(process)g(one)g(*/)170 2029 y({)266 2085 y(MPI_Recv\(message,)e(20,)j(MPI_CHAR,)f(0,)g(99,)h (MPI_COMM_WORLD,)d(&status\);)266 2142 y(printf\("received)h (:\045s:\\n",)g(message\);)170 2198 y(})170 2254 y(MPI_Finalize\(\);)75 2311 y(})166 2417 y Fv(In)17 b(this)h(example,)f(pro)q(cess)h(zero)e (\()p Fw(myrank)g(=)h(0)p Fv(\))f(sends)i(a)e(message)h(to)f(pro)q (cess)h(one)g(using)h(the)75 2474 y Fq(send)f Fv(op)q(eration)g Fw(MPI)p 485 2474 14 2 v 16 w(SEND)p Fv(.)g(The)g(op)q(eration)g(sp)q (eci\014es)i(a)e Fq(send)i(bu\013er)d Fv(in)i(the)f(sender)h(memory)75 2530 y(from)f(whic)o(h)i(the)f(message)g(data)f(is)h(tak)o(en.)28 b(In)19 b(the)f(example)h(ab)q(o)o(v)o(e,)f(the)g(send)g(bu\013er)g (consists)g(of)75 2587 y(the)f(storage)f(con)o(taining)i(the)f(v)m (ariable)i Fw(message)e Fv(in)h(the)f(memory)g(of)f(pro)q(cess)i(zero.) 25 b(The)18 b(lo)q(cation,)75 2643 y(size)k(and)f(t)o(yp)q(e)g(of)g (the)g(send)h(bu\013er)f(are)g(sp)q(eci\014ed)i(b)o(y)e(the)g(\014rst)g (three)g(parameters)g(of)f(the)h(send)75 2699 y(op)q(eration.)29 b(The)19 b(message)f(sen)o(t)g(will)h(con)o(tain)g(the)f(13)g(c)o (haracters)f(of)h(this)h(v)m(ariable.)30 b(In)19 b(addition,)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 17 25 17 24 bop 75 -100 a Fm(3.2.)34 b(BLOCKING)17 b(SEND)e(AND)g(RECEIVE)g (OPERA)l(TIONS)598 b Fv(17)75 45 y(the)17 b(send)h(op)q(eration)g(asso) q(ciates)f(an)g Fq(en)o(v)o(elop)q(e)g Fv(with)h(the)f(message.)26 b(This)17 b(en)o(v)o(elop)q(e)i(sp)q(eci\014es)g(the)75 102 y(message)10 b(destination)h(and)f(con)o(tains)h(distinguishing)h (information)f(that)e(can)i(b)q(e)f(used)h(b)o(y)f(the)h Fq(receiv)o(e)75 158 y Fv(op)q(eration)17 b(to)f(select)i(a)e (particular)i(message.)24 b(The)17 b(last)f(three)h(parameters)f(of)h (the)f(send)i(op)q(eration)75 214 y(sp)q(ecify)f(the)e(en)o(v)o(elop)q (e)h(for)f(the)g(message)g(sen)o(t.)166 271 y(Pro)q(cess)d(one)h(\()p Fw(myrank)e(=)i(1)p Fv(\))f(receiv)o(es)i(this)f(message)f(with)h(the)f Fq(receiv)o(e)g Fv(op)q(eration)h Fw(MPI)p 1734 271 14 2 v 16 w(RECV)p Fv(.)75 327 y(The)e(message)f(to)h(b)q(e)g(receiv)o(ed) h(is)f(selected)h(according)g(to)e(the)h(v)m(alue)h(of)e(its)h(en)o(v)o (elop)q(e,)i(and)e(the)f(message)75 384 y(data)17 b(is)h(stored)f(in)o (to)g(the)h Fq(receiv)o(e)h(bu\013er)p Fv(.)26 b(In)18 b(the)g(example)g(ab)q(o)o(v)o(e,)f(the)h(receiv)o(e)g(bu\013er)g (consists)75 440 y(of)d(the)h(storage)e(con)o(taining)j(the)e(string)h Fw(message)f Fv(in)i(the)e(memory)h(of)f(pro)q(cess)h(one.)21 b(The)16 b(\014rst)f(three)75 497 y(parameters)e(of)h(the)g(receiv)o(e) h(op)q(eration)f(sp)q(ecify)i(the)e(lo)q(cation,)g(size)h(and)g(t)o(yp) q(e)f(of)f(the)h(receiv)o(e)h(bu\013er.)75 553 y(The)e(next)g(three)g (parameters)f(are)h(used)g(for)g(selecting)h(the)f(incoming)h(message.) 19 b(The)13 b(last)g(parameter)75 610 y(is)j(used)f(to)g(return)g (information)h(on)f(the)g(message)g(just)g(receiv)o(ed.)166 666 y(The)g(next)g(sections)g(describ)q(e)i(the)e(blo)q(c)o(king)h (send)g(and)f(receiv)o(e)h(op)q(erations.)k(W)l(e)15 b(discuss)h(send,)75 723 y(receiv)o(e,)e(blo)q(c)o(king)f(comm)o (unication)g(seman)o(tics,)g(t)o(yp)q(e)g(matc)o(hing)f(requiremen)o (ts,)h(t)o(yp)q(e)g(con)o(v)o(ersion)f(in)75 779 y(heterogeneous)18 b(en)o(vironmen)o(ts,)g(and)f(more)g(general)h(comm)o(unication)g(mo)q (des.)27 b(Non)o(blo)q(c)o(king)19 b(com-)75 835 y(m)o(unication)i(is)f (addressed)g(next,)h(follo)o(w)o(ed)f(b)o(y)g(c)o(hannel-lik)o(e)i (constructs)d(and)h(send-receiv)o(e)i(op)q(er-)75 892 y(ations.)32 b(W)l(e)19 b(then)h(consider)g(general)g(datat)o(yp)q(es)f (that)f(allo)o(w)i(one)f(to)g(transfer)f(e\016cien)o(tly)j(hetero-)75 948 y(geneous)g(and)f(noncon)o(tiguous)h(data.)35 b(W)l(e)21 b(conclude)h(with)f(the)f(description)i(of)e(calls)i(for)e(explicit)75 1005 y(pac)o(king)c(and)f(unpac)o(king)h(of)f(messages.)75 1145 y Fn(3.2)59 b(Blo)r(cking)20 b(Send)f(and)g(Receive)f(Op)r (erations)75 1248 y Fj(3.2.1)49 b(Blo)q(cking)18 b(send)75 1334 y Fv(The)d(syn)o(tax)g(of)g(the)g(blo)q(c)o(king)h(send)g(op)q (eration)g(is)f(giv)o(en)h(b)q(elo)o(w.)75 1485 y Fw(MPI)p 160 1485 V 16 w(SEND\(buf,)g(count,)g(datat)o(yp)q(e,)g(dest,)g(tag,)f (comm\))117 1562 y Fk(IN)155 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1633 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts)g(in)g(send)h(bu\013er)h(\(nonnegativ)o(e)e(in)o (te-)905 1690 y(ger\))117 1761 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e (\(handle\))117 1833 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1905 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1976 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))75 2101 y Ft(int)23 b(MPI)p 245 2101 15 2 v 17 w(Send\(void*)g(buf,)g(int) g(count,)h(MPI)p 979 2101 V 16 w(Datatype)f(datatype,)g(int)g(dest,)393 2157 y(int)h(tag,)f(MPI)p 683 2157 V 17 w(Comm)g(comm\))75 2244 y(MPI)p 150 2244 V 17 w(SEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)h(COMM,)f(IERROR\))170 2300 y()g(BUF\(*\))170 2356 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 2443 y Fv(The)15 b(blo)q(c)o(king)i(seman)o(tics)e(of)g(this)h(call)g (are)f(describ)q(ed)i(in)f(Sec.)k(3.4.)75 2561 y Fj(3.2.2)49 b(Message)16 b(data)75 2647 y Fv(The)e(send)g(bu\013er)g(sp)q (eci\014ed)i(b)o(y)e(the)f Fw(MPI)p 803 2647 14 2 v 16 w(SEND)h Fv(op)q(eration)g(consists)g(of)f Fw(count)i Fv(successiv)o(e)g(en)o(tries)f(of)75 2704 y(the)f(t)o(yp)q(e)h (indicated)h(b)o(y)e Fw(datat)o(yp)q(e)p Fv(,)i(starting)e(with)h(the)f (en)o(try)g(at)g(address)h Fw(buf)p Fv(.)20 b(Note)13 b(that)g(w)o(e)g(sp)q(ecify)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 18 26 18 25 bop 75 -100 a Fv(18)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(the)i(message)f (length)i(in)g(terms)e(of)g(n)o(um)o(b)q(er)h(of)g Fl(elements)p Fv(,)f(not)g(n)o(um)o(b)q(er)i(of)e Fl(bytes)p Fv(.)30 b(The)19 b(former)f(is)75 102 y(mac)o(hine)e(indep)q(enden)o(t)i(and)d (closer)h(to)e(the)h(application)i(lev)o(el.)166 158 y(The)g(data)f(part)g(of)g(the)g(message)h(consists)f(of)h(a)f (sequence)i(of)e Fw(count)i Fv(v)m(alues,)f(eac)o(h)g(of)f(the)h(t)o (yp)q(e)75 214 y(indicated)j(b)o(y)e Fw(datat)o(yp)q(e)p Fv(.)31 b Fw(count)20 b Fv(ma)o(y)d(b)q(e)i(zero,)g(in)g(whic)o(h)g (case)g(the)f(data)g(part)f(of)h(the)h(message)e(is)75 271 y(empt)o(y)l(.)i(The)13 b(basic)h(datat)o(yp)q(es)e(that)h(can)g(b) q(e)h(sp)q(eci\014ed)h(for)d(message)h(data)f(v)m(alues)i(corresp)q (ond)g(to)e(the)75 327 y(basic)k(datat)o(yp)q(es)f(of)g(the)g(host)g (language.)20 b(P)o(ossible)d(v)m(alues)f(of)f(this)h(argumen)o(t)e (for)h(F)l(ortran)f(and)i(the)75 384 y(corresp)q(onding)g(F)l(ortran)e (t)o(yp)q(es)h(are)g(listed)i(b)q(elo)o(w.)p 472 532 1006 2 v 471 589 2 57 v 497 572 a(MPI)e(datat)o(yp)q(e)p 1046 589 V 294 w(F)l(ortran)f(datat)o(yp)q(e)p 1477 589 V 472 591 1006 2 v 471 647 2 57 v 497 630 a Fw(MPI)p 582 630 14 2 v 16 w(INTEGER)p 1046 647 2 57 v 287 w Ft(INTEGER)p 1477 647 V 471 703 V 497 687 a Fw(MPI)p 582 687 14 2 v 16 w(REAL)p 1046 703 2 57 v 365 w Ft(REAL)p 1477 703 V 471 760 V 497 743 a Fw(MPI)p 582 743 14 2 v 16 w(DOUBLE)p 777 743 V 17 w(PRECISION)p 1046 760 2 57 v 50 w Ft(DOUBLE)23 b(PRECISION)p 1477 760 V 471 816 V 497 799 a Fw(MPI)p 582 799 14 2 v 16 w(COMPLEX)p 1046 816 2 57 v 263 w Ft(COMPLEX)p 1477 816 V 471 873 V 497 856 a Fw(MPI)p 582 856 14 2 v 16 w(LOGICAL)p 1046 873 2 57 v 291 w Ft(LOGICAL)p 1477 873 V 471 929 V 497 912 a Fw(MPI)p 582 912 14 2 v 16 w(CHARA)o(CTER)p 1046 929 2 57 v 211 w Ft(CHARACTER\(1\))p 1477 929 V 471 986 V 497 969 a Fw(MPI)p 582 969 14 2 v 16 w(BYTE)p 1046 986 2 57 v 1477 986 V 471 1042 V 497 1025 a(MPI)p 582 1025 14 2 v 16 w(P)l(A)o(CKED)p 1046 1042 2 57 v 1477 1042 V 472 1044 1006 2 v 75 1214 a Fv(P)o(ossible)16 b(v)m(alues)g(for)f(this)h(argumen)o(t)e(for)h(C)g(and)g(the)g(corresp) q(onding)h(C)f(t)o(yp)q(es)h(are)f(listed)h(b)q(elo)o(w.)p 466 1354 1019 2 v 465 1410 2 57 v 491 1393 a(MPI)f(datat)o(yp)q(e)p 1004 1410 V 259 w(C)g(datat)o(yp)q(e)p 1483 1410 V 466 1412 1019 2 v 465 1468 2 57 v 491 1451 a Fw(MPI)p 576 1451 14 2 v 16 w(CHAR)p 1004 1468 2 57 v 321 w Ft(signed)23 b(char)p 1483 1468 V 465 1525 V 491 1508 a Fw(MPI)p 576 1508 14 2 v 16 w(SHORT)p 1004 1525 2 57 v 291 w Ft(signed)g(short)g (int)p 1483 1525 V 465 1581 V 491 1564 a Fw(MPI)p 576 1564 14 2 v 16 w(INT)p 1004 1581 2 57 v 365 w Ft(signed)g(int)p 1483 1581 V 465 1638 V 491 1621 a Fw(MPI)p 576 1621 14 2 v 16 w(LONG)p 1004 1638 2 57 v 321 w Ft(signed)g(long)g(int)p 1483 1638 V 465 1694 V 491 1677 a Fw(MPI)p 576 1677 14 2 v 16 w(UNSIGNED)p 815 1677 V 17 w(CHAR)p 1004 1694 2 57 v 81 w Ft(unsigned)f(char)p 1483 1694 V 465 1750 V 491 1733 a Fw(MPI)p 576 1733 14 2 v 16 w(UNSIGNED)p 815 1733 V 17 w(SHORT)p 1004 1750 2 57 v 51 w Ft(unsigned)g(short)i (int)p 1483 1750 V 465 1807 V 491 1790 a Fw(MPI)p 576 1790 14 2 v 16 w(UNSIGNED)p 1004 1807 2 57 v 218 w Ft(unsigned)e(int)p 1483 1807 V 465 1863 V 491 1846 a Fw(MPI)p 576 1846 14 2 v 16 w(UNSIGNED)p 815 1846 V 17 w(LONG)p 1004 1863 2 57 v 81 w Ft(unsigned)g(long)i(int)p 1483 1863 V 465 1920 V 491 1903 a Fw(MPI)p 576 1903 14 2 v 16 w(FLO)o(A)l(T)p 1004 1920 2 57 v 301 w Ft(float)p 1483 1920 V 465 1976 V 491 1959 a Fw(MPI)p 576 1959 14 2 v 16 w(DOUBLE)p 1004 1976 2 57 v 262 w Ft(double)p 1483 1976 V 465 2033 V 491 2016 a Fw(MPI)p 576 2016 14 2 v 16 w(LONG)p 712 2016 V 16 w(DOUBLE)p 1004 2033 2 57 v 126 w Ft(long)f(double)p 1483 2033 V 465 2089 V 491 2072 a Fw(MPI)p 576 2072 14 2 v 16 w(BYTE)p 1004 2089 2 57 v 1483 2089 V 465 2146 V 491 2129 a(MPI)p 576 2129 14 2 v 16 w(P)l(A)o(CKED)p 1004 2146 2 57 v 1483 2146 V 466 2147 1019 2 v 166 2308 a Fv(The)f(datat)o(yp)q(es)f Fw(MPI)p 566 2308 14 2 v 16 w(BYTE)h Fv(and)g Fw(MPI)p 899 2308 V 15 w(P)l(A)o(CKED)g Fv(do)f(not)g(corresp)q(ond)h(to)f(a)h(F)l(ortran)e(or)h(C)75 2365 y(datat)o(yp)q(e.)42 b(A)23 b(v)m(alue)h(of)e(t)o(yp)q(e)h Fw(MPI)p 742 2365 V 15 w(BYTE)h Fv(consists)f(of)f(a)g(b)o(yte)h(\(8)f (binary)h(digits\).)43 b(A)23 b(b)o(yte)f(is)75 2421 y(unin)o(terpreted)h(and)f(is)h(di\013eren)o(t)f(from)f(a)h(c)o (haracter.)39 b(Di\013eren)o(t)22 b(mac)o(hines)h(ma)o(y)e(ha)o(v)o(e)h (di\013eren)o(t)75 2478 y(represen)o(tations)15 b(for)g(c)o(haracters,) f(or)h(ma)o(y)g(use)h(more)f(than)g(one)g(b)o(yte)h(to)e(represen)o(t)i (c)o(haracters.)j(On)75 2534 y(the)g(other)g(hand,)h(a)f(b)o(yte)g(has) g(the)g(same)g(binary)g(v)m(alue)i(on)e(all)h(mac)o(hines.)32 b(The)20 b(use)f(of)g(the)g(t)o(yp)q(e)75 2591 y Fw(MPI)p 160 2591 V 16 w(P)l(A)o(CKED)c Fv(is)h(explained)h(in)f(Section)g (3.13.)166 2647 y Fw(MPI)c Fv(requires)h(supp)q(ort)g(of)f(the)h(datat) o(yp)q(es)f(listed)i(ab)q(o)o(v)o(e,)e(whic)o(h)h(matc)o(h)f(the)h (basic)g(datat)o(yp)q(es)f(of)75 2704 y(F)l(ortran)h(77)g(and)h(ANSI)h (C.)f(Additional)h Fw(MPI)f Fv(datat)o(yp)q(es)f(should)i(b)q(e)g(pro)o (vided)g(if)f(the)g(host)g(language)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 19 27 19 26 bop 75 -100 a Fm(3.2.)34 b(BLOCKING)17 b(SEND)e(AND)g(RECEIVE)g (OPERA)l(TIONS)598 b Fv(19)75 45 y(has)15 b(additional)i(data)e(t)o(yp) q(es:)20 b Fw(MPI)p 698 45 14 2 v 16 w(LONG)p 834 45 V 16 w(LONG)p 970 45 V 17 w(INT)p Fv(,)15 b(for)g(\(64)f(bit\))i(C)f (in)o(tegers)g(declared)i(to)d(b)q(e)i(of)75 102 y(t)o(yp)q(e)11 b Fw(longlong)f(int)p Fv(;)i Fw(MPI)p 503 102 V 16 w(DOUBLE)p 698 102 V 17 w(COMPLEX)f Fv(for)f(double)i(precision)g(complex)g(in)f (F)l(ortran)f(declared)63 b Fx(>)75 158 y Fv(to)17 b(b)q(e)i(of)e(t)o (yp)q(e)h Fw(DOUBLE)h(COMPLEX)p Fv(;)f Fw(MPI)p 884 158 V 16 w(REAL2)p Fv(,)g Fw(MPI)p 1147 158 V 16 w(REAL4)g Fv(and)g Fw(MPI)p 1488 158 V 16 w(REAL8)g Fv(for)f(F)l(ortran)61 b Fx(?)75 214 y Fv(reals,)18 b(declared)h(to)e(b)q(e)h(of)f(t)o(yp)q(e) h Fw(REAL*2)p Fv(,)g Fw(REAL*4)f Fv(and)h Fw(REAL*8)p Fv(,)g(resp)q(ectiv)o(ely;)i Fw(MPI)p 1650 214 V 16 w(INTEGER1)75 271 y(MPI)p 160 271 V 16 w(INTEGER2)10 b Fv(and)h Fw(MPI)p 564 271 V 16 w(INTEGER4)f Fv(for)g(F)l(ortran)f(in)o(tegers,)i (declared)h(to)d(b)q(e)i(of)f(t)o(yp)q(e)g Fw(INTEGER*1)p Fv(,)75 327 y Fw(INTEGER*2)15 b Fv(and)h Fw(INTEGER*4)p Fv(,)f(resp)q(ectiv)o(ely;)h(etc.)189 407 y Fl(R)n(ationale.)80 b Fv(One)22 b(goal)g(of)f(the)h(design)h(is)g(to)e(allo)o(w)h(for)f Fw(MPI)g Fv(to)h(b)q(e)g(implemen)o(ted)i(as)d(a)189 464 y(library)l(,)d(with)f(no)g(need)h(for)e(additional)j(prepro)q (cessing)f(or)e(compilation.)27 b(Th)o(us,)17 b(one)g(cannot)189 520 y(assume)c(that)g(a)g(comm)o(unication)h(call)g(has)g(information)f (on)h(the)f(datat)o(yp)q(e)g(of)g(v)m(ariables)i(in)f(the)189 576 y(comm)o(unication)20 b(bu\013er;)i(this)f(information)f(m)o(ust)f (b)q(e)i(supplied)h(b)o(y)e(an)g(explicit)i(argumen)o(t.)189 633 y(The)17 b(need)h(for)f(suc)o(h)h(datat)o(yp)q(e)e(information)i (will)h(b)q(ecome)f(clear)g(in)g(Section)g(3.3.2.)24 b(\()p Fl(End)18 b(of)189 689 y(r)n(ationale.)p Fv(\))75 808 y Fj(3.2.3)49 b(Message)16 b(envelop)q(e)75 893 y Fv(In)j(addition)g(to)e(the)h(data)g(part,)f(messages)h(carry)f (information)i(that)e(can)h(b)q(e)h(used)f(to)g(distinguish)75 950 y(messages)12 b(and)h(selectiv)o(ely)i(receiv)o(e)f(them.)19 b(This)14 b(information)f(consists)g(of)f(a)h(\014xed)g(n)o(um)o(b)q (er)h(of)e(\014elds,)75 1006 y(whic)o(h)k(w)o(e)f(collectiv)o(ely)j (call)e(the)f Fq(message)i(en)o(v)o(elop)q(e)p Fv(.)i(These)d(\014elds) g(are)913 1081 y(source)865 1138 y(destination)943 1194 y(tag)834 1251 y(comm)o(unicator)166 1325 y(The)c(message)g(source)g (is)h(implicitly)i(determined)f(b)o(y)e(the)g(iden)o(tit)o(y)h(of)f (the)g(message)g(sender.)20 b(The)75 1382 y(other)15 b(\014elds)h(are)f(sp)q(eci\014ed)i(b)o(y)f(argumen)o(ts)e(in)i(the)f (send)h(op)q(eration.)166 1438 y(The)f(message)g(destination)h(is)g(sp) q(eci\014ed)h(b)o(y)e(the)h Fw(dest)g Fv(argumen)o(t.)166 1495 y(The)h(in)o(teger-v)m(alued)h(message)e(tag)f(is)i(sp)q (eci\014ed)i(b)o(y)d(the)g Fw(tag)h Fv(argumen)o(t.)22 b(This)17 b(in)o(teger)g(can)f(b)q(e)75 1551 y(used)21 b(b)o(y)f(the)h(program)e(to)h(distinguish)i(di\013eren)o(t)f(t)o(yp)q (es)f(of)g(messages.)35 b(The)21 b(range)f(of)g(v)m(alid)i(tag)75 1608 y(v)m(alues)17 b(is)f Fw(0,...,UB)p Fv(,)e(where)i(the)g(v)m(alue) h(of)f Fw(UB)g Fv(is)g(implemen)o(tation)h(dep)q(enden)o(t.)24 b(It)15 b(can)h(b)q(e)h(found)f(b)o(y)75 1664 y(querying)i(the)g(v)m (alue)g(of)f(the)h(attribute)f Fi(MPI)p 870 1664 13 2 v 14 w(T)m(A)o(G)p 964 1664 V 14 w(UB)p Fv(,)g(as)g(describ)q(ed)i(in)f (Chapter)f(7.)26 b Fw(MPI)17 b Fv(requires)75 1721 y(that)d Fw(UB)i Fv(b)q(e)g(no)f(less)h(than)f(32767.)166 1777 y(The)f Fw(comm)f Fv(argumen)o(t)g(sp)q(eci\014es)j(the)e Fq(comm)o(unicator)h Fv(that)e(is)i(used)g(for)e(the)h(send)h(op)q (eration.)75 1833 y(Comm)o(unicators)f(are)h(explained)j(in)e(Chapter)e (5;)h(b)q(elo)o(w)h(is)f(a)g(brief)h(summary)f(of)g(their)g(usage.)166 1890 y(A)h(comm)o(unicator)f(sp)q(eci\014es)j(the)d(comm)o(unication)i (con)o(text)e(for)g(a)h(comm)o(unication)g(op)q(eration.)75 1946 y(Eac)o(h)f(comm)o(unication)h(con)o(text)f(pro)o(vides)g(a)g (separate)g(\\comm)o(unication)h(univ)o(erse:")21 b(messages)14 b(are)75 2003 y(alw)o(a)o(ys)h(receiv)o(ed)j(within)f(the)f(con)o(text) f(they)h(w)o(ere)g(sen)o(t,)g(and)g(messages)g(sen)o(t)f(in)i (di\013eren)o(t)g(con)o(texts)75 2059 y(do)e(not)g(in)o(terfere.)166 2116 y(The)20 b(comm)o(unicator)f(also)g(sp)q(eci\014es)i(the)f(set)f (of)g(pro)q(cesses)g(that)g(share)g(this)h(comm)o(unication)75 2172 y(con)o(text.)i(This)17 b Fq(pro)q(cess)h(group)e Fv(is)g(ordered)h(and)f(pro)q(cesses)g(are)g(iden)o(ti\014ed)i(b)o(y)e (their)h(rank)f(within)75 2229 y(this)g(group.)21 b(Th)o(us,)15 b(the)g(range)h(of)f(v)m(alid)i(v)m(alues)f(for)f Fw(dest)i Fv(is)f Fw(0,)f(...)20 b(,)c(n-1)p Fv(,)f(where)h Fw(n)g Fv(is)g(the)g(n)o(um)o(b)q(er)f(of)75 2285 y(pro)q(cesses)i(in)h(the)e (group.)25 b(\(If)16 b(the)h(comm)o(unicator)f(is)i(an)e(in)o(ter-comm) o(unicator,)h(then)g(destinations)75 2342 y(are)e(iden)o(ti\014ed)i(b)o (y)e(their)h(rank)f(in)h(the)f(remote)g(group.)k(See)d(Chapter)f(5.\)) 166 2398 y(A)g(prede\014ned)i(comm)o(unicator)d Fi(MPI)p 808 2398 V 15 w(COMM)p 953 2398 V 14 w(W)o(ORLD)h Fv(is)h(pro)o(vided)g (b)o(y)f Fw(MPI)p Fv(.)f(It)h(allo)o(ws)g(comm)o(u-)75 2454 y(nication)22 b(with)f(all)h(pro)q(cesses)f(that)f(are)h (accessible)i(after)d Fw(MPI)g Fv(initialization)k(and)d(pro)q(cesses)g (are)75 2511 y(iden)o(ti\014ed)c(b)o(y)e(their)h(rank)f(in)h(the)f (group)g(of)g Fi(MPI)p 939 2511 V 14 w(COMM)p 1083 2511 V 15 w(W)o(ORLD)p Fv(.)189 2591 y Fl(A)n(dvic)n(e)j(to)i(users.)60 b Fv(Users)18 b(that)g(are)g(comfortable)g(with)h(the)g(notion)g(of)f (a)g(\015at)g(name)h(space)189 2647 y(for)d(pro)q(cesses,)h(and)g(a)g (single)h(comm)o(unication)g(con)o(text,)e(as)h(o\013ered)f(b)o(y)h (most)f(existing)i(com-)189 2704 y(m)o(unication)d(libraries,)h(need)f (only)h(use)e(the)h(prede\014ned)h(v)m(ariable)g Fi(MPI)p 1437 2704 V 15 w(COMM)p 1582 2704 V 14 w(W)o(ORLD)f Fv(as)f(the)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 20 28 20 27 bop 75 -100 a Fv(20)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Fw(comm)f Fv(argumen)o(t.)29 b(This)19 b(will)h(allo)o(w)f(comm)o(unication)g (with)g(all)g(the)g(pro)q(cesses)f(a)o(v)m(ailable)j(at)189 102 y(initialization)d(time.)189 177 y(Users)h(ma)o(y)g(de\014ne)i(new) f(comm)o(unicators,)g(as)g(explained)h(in)g(Chapter)f(5.)33 b(Comm)o(unicators)189 233 y(pro)o(vide)13 b(an)g(imp)q(ortan)o(t)g (encapsulation)h(mec)o(hanism)f(for)g(libraries)h(and)f(mo)q(dules.)21 b(They)13 b(allo)o(w)189 290 y(mo)q(dules)21 b(to)e(ha)o(v)o(e)h(their) g(o)o(wn)g(disjoin)o(t)h(comm)o(unication)f(univ)o(erse)h(and)g(their)f (o)o(wn)g(pro)q(cess)189 346 y(n)o(um)o(b)q(ering)c(sc)o(heme.)k(\()p Fl(End)c(of)g(advic)n(e)g(to)h(users.)p Fv(\))189 452 y Fl(A)n(dvic)n(e)i(to)h(implementors.)65 b Fv(The)20 b(message)f(en)o(v)o(elop)q(e)h(w)o(ould)g(normally)g(b)q(e)g(enco)q (ded)h(b)o(y)e(a)189 509 y(\014xed-length)d(message)e(header.)20 b(Ho)o(w)o(ev)o(er,)13 b(the)i(actual)g(enco)q(ding)h(is)f(implemen)o (tation)g(dep)q(en-)189 565 y(den)o(t.)k(Some)c(of)e(the)i(information) f(\(e.g.,)f(source)h(or)g(destination\))h(ma)o(y)e(b)q(e)i(implicit,)i (and)d(need)189 622 y(not)j(b)q(e)i(explicitly)i(carried)e(b)o(y)f (messages.)28 b(Also,)19 b(pro)q(cesses)f(ma)o(y)f(b)q(e)i(iden)o (ti\014ed)h(b)o(y)e(relativ)o(e)189 678 y(ranks,)c(or)h(absolute)g (ids,)h(etc.)k(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fv(\))75 800 y Fj(3.2.4)49 b(Blo)q(cking)18 b(receive)75 886 y Fv(The)d(syn)o(tax)g(of)g(the)g(blo)q(c)o(king)h(receiv)o(e)h(op) q(eration)e(is)h(giv)o(en)f(b)q(elo)o(w.)75 1037 y Fw(MPI)p 160 1037 14 2 v 16 w(RECV)h(\(buf,)f(count,)h(datat)o(yp)q(e,)h (source,)e(tag,)g(comm,)e(status\))117 1114 y Fk(OUT)108 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g (\(c)o(hoice\))117 1189 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1264 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1339 y(IN)155 b Fw(source)468 b Fk(rank)14 b(of)f(source)j(\(in)o (teger\))117 1414 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1490 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 1565 y(OUT)108 b Fw(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1689 y Ft(int)23 b(MPI)p 245 1689 15 2 v 17 w(Recv\(void*)g(buf,)g(int) g(count,)h(MPI)p 979 1689 V 16 w(Datatype)f(datatype,)g(int)g(source,) 393 1746 y(int)h(tag,)f(MPI)p 683 1746 V 17 w(Comm)g(comm,)g(MPI)p 1034 1746 V 17 w(Status)g(*status\))75 1832 y(MPI)p 150 1832 V 17 w(RECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS,)g(IERROR\))170 1888 y()g(BUF\(*\))170 1945 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS\(MPI)p 1463 1945 V 16 w(STATUS)p 1623 1945 V 17 w(SIZE\),)170 2001 y(IERROR)166 2088 y Fv(The)15 b(blo)q(c)o(king)i (seman)o(tics)e(of)g(this)h(call)g(are)f(describ)q(ed)i(in)f(Sec.)k (3.4.)166 2144 y(The)15 b(receiv)o(e)h(bu\013er)f(consists)h(of)e(the)h (storage)f(con)o(taining)i Fw(count)g Fv(consecutiv)o(e)g(elemen)o(ts)g (of)f(the)75 2201 y(t)o(yp)q(e)e(sp)q(eci\014ed)i(b)o(y)e Fw(datat)o(yp)q(e)p Fv(,)h(starting)e(at)h(address)f Fw(buf)p Fv(.)20 b(The)13 b(length)h(of)e(the)h(receiv)o(ed)h(message)f (m)o(ust)75 2257 y(b)q(e)20 b(less)g(than)f(or)g(equal)h(to)f(the)h (length)g(of)f(the)g(receiv)o(e)h(bu\013er.)33 b(An)20 b(o)o(v)o(er\015o)o(w)e(error)g(o)q(ccurs)i(if)g(all)75 2314 y(incoming)c(data)f(do)q(es)h(not)e(\014t,)h(without)g (truncation,)g(in)o(to)g(the)h(receiv)o(e)g(bu\013er.)166 2370 y(If)i(a)f(message)g(that)f(is)i(shorter)f(than)g(the)h(receiv)o (e)g(bu\013er)f(arriv)o(es,)g(then)h(only)g(those)f(lo)q(cations)75 2427 y(corresp)q(onding)f(to)f(the)g(\(shorter\))f(message)h(are)g(mo)q (di\014ed.)189 2533 y Fl(A)n(dvic)n(e)h(to)i(users.)47 b Fv(The)16 b Fw(MPI)p 733 2533 14 2 v 16 w(PROBE)i Fv(function)f (describ)q(ed)h(in)f(Section)h(3.8)d(can)i(b)q(e)g(used)g(to)189 2589 y(receiv)o(e)f(messages)e(of)h(unkno)o(wn)h(length.)k(\()p Fl(End)c(of)g(advic)n(e)g(to)h(users.)p Fv(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 21 29 21 28 bop 75 -100 a Fm(3.2.)34 b(BLOCKING)17 b(SEND)e(AND)g(RECEIVE)g (OPERA)l(TIONS)598 b Fv(21)189 45 y Fl(A)n(dvic)n(e)16 b(to)i(implementors.)46 b Fv(Ev)o(en)16 b(though)g(no)h(sp)q(eci\014c)h (b)q(eha)o(vior)f(is)f(mandated)h(b)o(y)f Fw(MPI)g Fv(for)189 102 y(erroneous)f(programs,)f(the)h(recommended)h(handling)h(of)e(o)o (v)o(er\015o)o(w)f(situations)h(is)h(to)f(return)g(in)189 158 y Fw(status)21 b Fv(information)e(ab)q(out)g(the)g(source)g(and)g (tag)f(of)h(the)g(incoming)h(message.)31 b(The)19 b(receiv)o(e)189 214 y(op)q(eration)e(will)j(return)d(an)g(error)g(co)q(de.)28 b(A)17 b(qualit)o(y)h(implemen)o(tation)h(will)g(also)f(ensure)g(that) 189 271 y(no)d(memory)f(that)h(is)h(outside)f(the)h(receiv)o(e)g (bu\013er)f(will)i(ev)o(er)e(b)q(e)h(o)o(v)o(erwritten.)189 346 y(In)g(the)f(case)g(of)g(a)g(message)g(shorter)g(than)g(the)g (receiv)o(e)h(bu\013er,)f Fw(MPI)g Fv(is)h(quite)g(strict)f(in)h(that)f (it)189 403 y(allo)o(ws)h(no)g(mo)q(di\014cation)h(of)f(the)g(other)g (lo)q(cations.)23 b(A)16 b(more)g(lenien)o(t)h(statemen)o(t)e(w)o(ould) i(allo)o(w)189 459 y(for)c(some)h(optimizations)g(but)g(this)h(is)f (not)g(allo)o(w)o(ed.)19 b(The)c(implemen)o(tation)g(m)o(ust)e(b)q(e)i (ready)f(to)189 515 y(end)i(a)g(cop)o(y)g(in)o(to)g(the)h(receiv)o(er)f (memory)g(exactly)h(at)e(the)h(end)h(of)f(the)g(receiv)o(e)h(bu\013er,) f(ev)o(en)g(if)189 572 y(it)f(is)h(an)f(o)q(dd)h(address.)k(\()p Fl(End)15 b(of)h(advic)n(e)h(to)f(implementors.)p Fv(\))166 678 y(The)22 b(selection)i(of)d(a)h(message)g(b)o(y)g(a)f(receiv)o(e)i (op)q(eration)g(is)f(go)o(v)o(erned)g(b)o(y)g(the)g(v)m(alue)h(of)f (the)75 735 y(message)13 b(en)o(v)o(elop)q(e.)20 b(A)14 b(message)f(can)g(b)q(e)h(receiv)o(ed)h(b)o(y)e(a)g(receiv)o(e)i(op)q (eration)e(if)h(its)g(en)o(v)o(elop)q(e)g(matc)o(hes)75 791 y(the)e Fw(source)p Fv(,)g Fw(tag)g Fv(and)g Fw(comm)e Fv(v)m(alues)j(sp)q(eci\014ed)h(b)o(y)e(the)f(receiv)o(e)i(op)q (eration.)19 b(The)12 b(receiv)o(er)h(ma)o(y)e(sp)q(ecify)75 848 y(a)18 b(wildcard)h Fi(MPI)p 380 848 13 2 v 15 w(ANY)p 480 848 V 14 w(SOURCE)e Fv(v)m(alue)i(for)f Fw(source)p Fv(,)h(and/or)e(a)h(wildcard)36 b Fi(MPI)p 1491 848 V 14 w(ANY)p 1590 848 V 15 w(T)m(A)o(G)17 b Fv(v)m(alue)j(for)75 904 y Fw(tag)p Fv(,)12 b(indicating)i(that)d(an)o(y)h(source)g(and/or)f (tag)g(are)g(acceptable.)20 b(It)12 b(cannot)g(sp)q(ecify)h(a)f (wildcard)h(v)m(alue)75 960 y(for)h Fw(comm)p Fv(.)k(Th)o(us,)c(a)g (message)g(can)h(b)q(e)g(receiv)o(ed)g(b)o(y)g(a)f(receiv)o(e)h(op)q (eration)g(only)g(if)g(it)g(is)g(addressed)g(to)75 1017 y(the)i(receiving)i(pro)q(cess,)e(has)g(a)g(matc)o(hing)g(comm)o (unicator,)f(has)h(matc)o(hing)g(source)g(unless)i(source=)75 1073 y Fi(MPI)p 152 1073 V 14 w(ANY)p 251 1073 V 15 w(SOURCE)13 b Fv(in)j(the)g(pattern,)e(and)i(has)f(a)f(matc)o(hing)i(tag)e(unless)i (tag=)e Fi(MPI)p 1553 1073 V 14 w(ANY)p 1652 1073 V 15 w(T)m(A)o(G)g Fv(in)i(the)75 1130 y(pattern.)166 1186 y(The)24 b(message)g(tag)f(is)i(sp)q(eci\014ed)h(b)o(y)e(the)g Fw(tag)g Fv(argumen)o(t)g(of)g(the)g(receiv)o(e)h(op)q(eration.)47 b(The)75 1243 y(argumen)o(t)24 b Fw(source)p Fv(,)i(if)f(di\013eren)o (t)g(from)e Fi(MPI)p 879 1243 V 14 w(ANY)p 978 1243 V 15 w(SOURCE)p Fv(,)f(is)j(sp)q(eci\014ed)h(as)e(a)g(rank)g(within)i (the)75 1299 y(pro)q(cess)e(group)f(asso)q(ciated)g(with)h(that)e(same) i(comm)o(unicator)e(\(remote)h(pro)q(cess)h(group,)g(for)f(in-)75 1356 y(tercomm)o(unicators\).)37 b(Th)o(us,)23 b(the)e(range)g(of)g(v)m (alid)j(v)m(alues)e(for)f(the)g Fw(source)h Fv(argumen)o(t)f(is)h Fx(f)p Fw(0,...,n-)75 1412 y(1)p Fx(g[f)p Fw(MPI)p 259 1412 14 2 v 15 w(ANY)p 366 1412 V 17 w(SOURCE)p Fx(g)p Fv(,)16 b(where)g Fw(n)g Fv(is)f(the)h(n)o(um)o(b)q(er)f(of)g(pro)q (cesses)g(in)h(this)g(group.)166 1469 y(Note)i(the)i(asymmetry)d(b)q (et)o(w)o(een)j(send)f(and)g(receiv)o(e)h(op)q(erations:)28 b(A)19 b(receiv)o(e)h(op)q(eration)f(ma)o(y)75 1525 y(accept)13 b(messages)f(from)g(an)h(arbitrary)f(sender,)i(on)f(the)g(other)f (hand,)h(a)g(send)g(op)q(eration)g(m)o(ust)g(sp)q(ecify)75 1581 y(a)i(unique)j(receiv)o(er.)k(This)16 b(matc)o(hes)f(a)h(\\push")f (comm)o(unication)i(mec)o(hanism,)f(where)g(data)f(transfer)75 1638 y(is)i(e\013ected)h(b)o(y)f(the)g(sender)g(\(rather)f(than)h(a)g (\\pull")h(mec)o(hanism,)g(where)f(data)f(transfer)g(is)i(e\013ected)75 1694 y(b)o(y)d(the)g(receiv)o(er\).)166 1751 y(Source)g(=)h (destination)g(is)g(allo)o(w)o(ed,)f(that)f(is,)i(a)e(pro)q(cess)i(can) f(send)h(a)f(message)g(to)f(itself.)21 b(\(Ho)o(w-)75 1807 y(ev)o(er,)c(it)h(is)g(unsafe)f(to)g(do)g(so)g(with)h(the)g(blo)q (c)o(king)g(send)g(and)g(receiv)o(e)g(op)q(erations)f(describ)q(ed)j (ab)q(o)o(v)o(e,)75 1864 y(since)c(this)g(ma)o(y)e(lead)i(to)f(deadlo)q (c)o(k.)21 b(See)16 b(Sec.)k(3.5.\))189 1970 y Fl(A)n(dvic)n(e)15 b(to)i(implementors.)43 b Fv(Message)15 b(con)o(text)g(and)h(other)g (comm)o(unicator)f(information)h(can)189 2026 y(b)q(e)k(implemen)o(ted) h(as)e(an)h(additional)h(tag)d(\014eld.)34 b(It)20 b(di\013ers)g(from)f (the)g(regular)h(message)f(tag)189 2083 y(in)i(that)e(wild)j(card)e (matc)o(hing)h(is)f(not)g(allo)o(w)o(ed)h(on)f(this)h(\014eld,)i(and)d (that)g(v)m(alue)h(setting)g(for)189 2139 y(this)e(\014eld)h(is)g(con)o (trolled)g(b)o(y)f(comm)o(unicator)f(manipulation)j(functions.)32 b(\()p Fl(End)19 b(of)h(advic)n(e)g(to)189 2196 y(implementors.)p Fv(\))75 2318 y Fj(3.2.5)49 b(Return)16 b(status)75 2403 y Fv(The)j(source)g(or)f(tag)f(of)h(a)h(receiv)o(ed)g(message)g(ma)o(y) f(not)g(b)q(e)h(kno)o(wn)f(if)h(wildcard)h(v)m(alues)g(w)o(ere)e(used) 75 2460 y(in)e(the)g(receiv)o(e)h(op)q(eration.)k(Also,)16 b(if)g(m)o(ultiple)i(requests)d(are)h(completed)g(b)o(y)g(a)f(single)i Fw(MPI)e Fv(function)63 b Fx(>)75 2516 y Fv(\(see)19 b(Section)g(3.7.5\),)e(a)i(distinct)h(error)e(co)q(de)h(ma)o(y)f(need)h (to)f(b)q(e)i(returned)f(for)f(eac)o(h)g(request.)31 b(The)75 2573 y(information)18 b(is)h(returned)f(b)o(y)g(the)g Fw(status)i Fv(argumen)o(t)d(of)h Fw(MPI)p 1184 2573 V 16 w(RECV)p Fv(.)g(The)g(t)o(yp)q(e)g(of)g Fw(status)i Fv(is)e Fw(MPI)p Fv(-)75 2629 y(de\014ned.)27 b(Status)16 b(v)m(ariables)i(need)g(to)e(b)q(e)i(explicitly)i(allo)q(cated)e(b)o(y) f(the)g(user,)g(that)f(is,)i(they)f(are)f(not)75 2686 y(system)f(ob)s(jects.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 22 30 22 29 bop 75 -100 a Fv(22)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)166 45 y Fv(In)22 b(C,)e Fw(status)j Fv(is)f(a)f(structure)f(that)h(con)o(tains)g(three)g (\014elds)h(named)g Fi(MPI)p 1488 45 13 2 v 14 w(SOURCE)p Fv(,)d Fi(MPI)p 1771 45 V 14 w(T)m(A)o(G)p Fv(,)75 102 y(and)g Fi(MPI)p 244 102 V 14 w(ERROR)p Fv(;)e(the)i(structure)f(ma)o (y)g(con)o(tain)h(additional)h(\014elds.)31 b(Th)o(us,)19 b Fw(status.MPI)p 1672 102 14 2 v 18 w(SOURCE)p Fv(,)75 158 y Fw(status.MPI)p 284 158 V 17 w(T)l(A)o(G)12 b Fv(and)f Fw(status.MPI)p 689 158 V 17 w(ERROR)i Fv(con)o(tain)e(the)g(source,)g (tag,)f(and)h(error)g(co)q(de,)g(resp)q(ectiv)o(ely)l(,)75 214 y(of)k(the)g(receiv)o(ed)h(message.)166 271 y(In)22 b(F)l(ortran,)g Fw(status)i Fv(is)e(an)g(arra)o(y)e(of)i Ft(INTEGER)p Fv(s)f(of)g(size)i Fi(MPI)p 1291 271 13 2 v 14 w(ST)m(A)m(TUS)p 1458 271 V 14 w(SIZE)p Fv(.)e(The)h(constan)o (ts)75 327 y Fi(MPI)p 152 327 V 14 w(SOURCE)p Fv(,)14 b Fi(MPI)p 430 327 V 14 w(T)m(A)o(G)h Fv(and)h Fi(MPI)p 703 327 V 14 w(ERROR)e Fv(are)i(the)g(indices)h(of)f(the)f(en)o(tries)h (that)f(store)g(the)h(source,)75 384 y(tag)11 b(and)i(error)f (\014elds.)20 b(Th)o(us,)12 b Fw(status\(MPI)p 817 384 14 2 v 18 w(SOURCE\))p Fv(,)h Fw(status\(MPI)p 1264 384 V 17 w(T)l(A)o(G\))g Fv(and)f Fw(status\(MPI)p 1694 384 V 18 w(ERROR\))75 440 y Fv(con)o(tain,)j(resp)q(ectiv)o(ely)l(,)i(the)e (source,)g(tag)f(and)i(error)e(co)q(de)i(of)f(the)g(receiv)o(ed)h (message.)166 497 y(In)21 b(general,)g(message)e(passing)i(calls)g(do)f (not)g(mo)q(dify)g(the)g(v)m(alue)h(of)f(the)g(error)f(co)q(de)i (\014eld)g(of)75 553 y(status)c(v)m(ariables.)30 b(This)19 b(\014eld)g(ma)o(y)e(b)q(e)i(up)q(dated)g(only)g(b)o(y)f(the)g (functions)h(in)g(Section)g(3.7.5)d(whic)o(h)75 610 y(return)g(m)o (ultiple)i(statuses.)k(The)17 b(\014eld)g(is)g(up)q(dated)g(if)f(and)h (only)f(if)h(suc)o(h)f(function)h(returns)g(with)f(an)75 666 y(error)e(co)q(de)i(of)f Fi(MPI)p 420 666 13 2 v 14 w(ERR)p 513 666 V 15 w(IN)p 569 666 V 15 w(ST)m(A)m(TUS)p Fv(.)189 758 y Fl(R)n(ationale.)37 b Fv(The)13 b(error)e(\014eld)i(in)g (status)e(is)h(not)g(needed)h(for)f(calls)h(that)e(return)h(only)g(one) g(status,)189 815 y(suc)o(h)k(as)g Fw(MPI)p 434 815 14 2 v 16 w(W)l(AIT)p Fv(,)g(since)i(that)e(w)o(ould)g(only)h(duplicate)i (the)d(information)h(returned)g(b)o(y)f(the)189 871 y(function)d (itself.)20 b(The)13 b(curren)o(t)f(design)i(a)o(v)o(oids)e(the)h (additional)h(o)o(v)o(erhead)e(of)g(setting)h(it,)g(in)h(suc)o(h)189 928 y(cases.)27 b(The)18 b(\014eld)h(is)f(needed)h(for)e(calls)h(that)f (return)h(m)o(ultiple)h(statuses,)e(since)i(eac)o(h)f(request)189 984 y(ma)o(y)c(ha)o(v)o(e)h(had)g(a)g(di\013eren)o(t)h(failure.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fv(\))-160 1031 y Fx(?)166 1076 y Fv(The)h(status)e(argumen)o(t)h(also)g(returns)g (information)h(on)f(the)h(length)g(of)f(the)g(message)g(receiv)o(ed.)75 1133 y(Ho)o(w)o(ev)o(er,)12 b(this)i(information)f(is)h(not)f(directly) h(a)o(v)m(ailable)h(as)e(a)g(\014eld)h(of)f(the)h(status)e(v)m(ariable) j(and)e(a)g(call)75 1189 y(to)i Fw(MPI)p 216 1189 V 15 w(GET)p 319 1189 V 17 w(COUNT)h Fv(is)g(required)g(to)f(\\deco)q(de")g (this)h(information.)75 1340 y Fw(MPI)p 160 1340 V 16 w(GET)p 264 1340 V 17 w(COUNT\(status,)h(datat)o(yp)q(e,)f(count\))117 1417 y Fk(IN)155 b Fw(status)476 b Fk(return)15 b(status)g(of)e(receiv) o(e)i(op)q(eration)f(\(Status\))117 1490 y(IN)155 b Fw(datat)o(yp)q(e) 424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(en) o(try)g(\(handle\))117 1564 y(OUT)108 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(en)o(tries)g(\(in)o (teger\))-160 1642 y Fx(>)75 1688 y Ft(int)23 b(MPI)p 245 1688 15 2 v 17 w(Get)p 334 1688 V 17 w(count\(MPI)p 567 1688 V 16 w(Status)g(*status,)g(MPI)p 1037 1688 V 17 w(Datatype)f(datatype,)h(int)g(*count\))-160 1728 y Fx(?)75 1774 y Ft(MPI)p 150 1774 V 17 w(GET)p 239 1774 V 17 w(COUNT\(STATUS,)f(DATATYPE,)g(COUNT,)h(IERROR\))170 1831 y(INTEGER)g(STATUS\(MPI)p 604 1831 V 16 w(STATUS)p 764 1831 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h(IERROR)-160 1867 y Fx(>)166 1917 y Fv(Returns)12 b(the)f(n)o(um)o(b)q(er)h(of)f(en) o(tries)g(receiv)o(ed.)20 b(\(Again,)12 b(w)o(e)f(coun)o(t)g Fl(entries)p Fv(,)g(eac)o(h)g(of)g(t)o(yp)q(e)h Fl(datatyp)n(e)p Fv(,)75 1974 y(not)i Fl(bytes)p Fv(.\))k(The)d Fw(datat)o(yp)q(e)g Fv(argumen)o(t)e(should)i(matc)o(h)f(the)g(argumen)o(t)f(pro)o(vided)i (b)o(y)f(the)g(receiv)o(e)h(call)75 2030 y(that)h(set)h(the)g Fw(status)i Fv(v)m(ariable.)26 b(\(W)l(e)17 b(shall)h(later)f(see,)h (in)f(Section)h(3.12.5,)d(that)i Fw(MPI)p 1601 2030 14 2 v 15 w(GET)p 1704 2030 V 17 w(COUNT)75 2087 y Fv(ma)o(y)d(return,)h (in)h(certain)g(situations,)f(the)g(v)m(alue)i Fi(MPI)p 1023 2087 13 2 v 14 w(UNDEFINED)p Fv(.\))-1459 b Fx(?)189 2179 y Fl(R)n(ationale.)56 b Fv(Some)18 b(message)f(passing)h (libraries)i(use)e Fk(INOUT)h Fw(count)p Fv(,)g Fw(tag)f Fv(and)g Fw(source)g Fv(argu-)189 2235 y(men)o(ts,)f(th)o(us)h(using)g (them)g(b)q(oth)g(to)f(sp)q(ecify)i(the)f(selection)h(criteria)f(for)f (incoming)i(messages)189 2292 y(and)c(return)g(the)h(actual)f(en)o(v)o (elop)q(e)i(v)m(alues)f(of)f(the)g(receiv)o(ed)i(message.)i(The)d(use)g (of)f(a)f(separate)189 2348 y(status)g(argumen)o(t)h(prev)o(en)o(ts)g (errors)g(that)g(are)g(often)g(attac)o(hed)g(with)h Fk(INOUT)g Fv(argumen)o(t)f(\(e.g.,)189 2405 y(using)f(the)f Fi(MPI)p 459 2405 V 15 w(ANY)p 559 2405 V 14 w(T)m(A)o(G)g Fv(constan)o(t)f(as)h (the)h(tag)e(in)i(a)f(receiv)o(e\).)20 b(Some)14 b(libraries)g(use)g (calls)h(that)189 2461 y(refer)g(implicitly)j(to)d(the)g(\\last)g (message)f(receiv)o(ed.")21 b(This)16 b(is)g(not)e(thread)i(safe.)189 2534 y(The)e Fw(datat)o(yp)q(e)i Fv(argumen)o(t)e(is)h(passed)f(to)g Fw(MPI)p 991 2534 14 2 v 16 w(GET)p 1095 2534 V 17 w(COUNT)h Fv(so)f(as)g(to)g(impro)o(v)o(e)g(p)q(erformance.)189 2591 y(A)k(message)g(migh)o(t)g(b)q(e)h(receiv)o(ed)h(without)e(coun)o (ting)h(the)g(n)o(um)o(b)q(er)g(of)e(elemen)o(ts)j(it)e(con)o(tains,) 189 2647 y(and)g(the)h(coun)o(t)f(v)m(alue)h(is)g(often)f(not)g (needed.)31 b(Also,)19 b(this)g(allo)o(ws)g(the)f(same)g(function)i(to) d(b)q(e)189 2704 y(used)e(after)g(a)g(call)h(to)f Fw(MPI)p 662 2704 V 15 w(PROBE)p Fv(.)i(\()p Fl(End)e(of)i(r)n(ationale.)p Fv(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 23 31 23 30 bop 75 -100 a Fm(3.3.)34 b(D)o(A)l(T)l(A)15 b(TYPE)g(MA)l(TCHING) g(AND)g(D)o(A)l(T)l(A)g(CONVERSION)520 b Fv(23)166 45 y(All)19 b(send)g(and)g(receiv)o(e)g(op)q(erations)f(use)h(the)f Fw(buf)p Fv(,)h Fw(count)p Fv(,)h Fw(datat)o(yp)q(e)p Fv(,)g Fw(source)p Fv(,)f Fw(dest)p Fv(,)h Fw(tag)p Fv(,)f Fw(comm)75 102 y Fv(and)12 b Fw(status)i Fv(argumen)o(ts)d(in)h(the)g (same)g(w)o(a)o(y)f(as)g(the)h(blo)q(c)o(king)h Fw(MPI)p 1216 102 14 2 v 16 w(SEND)f Fv(and)g Fw(MPI)p 1528 102 V 16 w(RECV)h Fv(op)q(erations)75 158 y(describ)q(ed)k(in)f(this)g (section.)75 300 y Fn(3.3)59 b(Data)19 b(t)n(yp)r(e)g(matching)f(and)h (data)g(conversion)75 403 y Fj(3.3.1)49 b(T)l(yp)q(e)17 b(matching)f(rules)75 489 y Fv(One)g(can)f(think)h(of)f(message)g (transfer)f(as)h(consisting)h(of)f(the)g(follo)o(wing)h(three)f (phases.)131 578 y(1.)22 b(Data)14 b(is)h(pulled)j(out)d(of)f(the)i (send)f(bu\013er)h(and)f(a)g(message)g(is)g(assem)o(bled.)131 670 y(2.)22 b(A)15 b(message)g(is)g(transferred)g(from)f(sender)i(to)f (receiv)o(er.)131 762 y(3.)22 b(Data)14 b(is)h(pulled)j(from)c(the)h (incoming)i(message)e(and)g(disassem)o(bled)i(in)o(to)e(the)g(receiv)o (e)h(bu\013er.)166 850 y(T)o(yp)q(e)k(matc)o(hing)f(has)h(to)e(b)q(e)j (observ)o(ed)e(at)g(eac)o(h)g(of)h(these)f(three)h(phases:)29 b(The)19 b(t)o(yp)q(e)h(of)f(eac)o(h)75 907 y(v)m(ariable)i(in)f(the)g (sender)g(bu\013er)g(has)g(to)f(matc)o(h)g(the)g(t)o(yp)q(e)h(sp)q (eci\014ed)i(for)d(that)g(en)o(try)g(b)o(y)h(the)f(send)75 963 y(op)q(eration;)d(the)f(t)o(yp)q(e)h(sp)q(eci\014ed)i(b)o(y)e(the)f (send)i(op)q(eration)e(has)h(to)f(matc)o(h)g(the)h(t)o(yp)q(e)f(sp)q (eci\014ed)j(b)o(y)e(the)75 1020 y(receiv)o(e)g(op)q(eration;)e(and)h (the)g(t)o(yp)q(e)f(of)h(eac)o(h)f(v)m(ariable)i(in)g(the)e(receiv)o(e) i(bu\013er)f(has)f(to)g(matc)o(h)g(the)h(t)o(yp)q(e)75 1076 y(sp)q(eci\014ed)f(for)e(that)g(en)o(try)g(b)o(y)g(the)g(receiv)o (e)h(op)q(eration.)20 b(A)12 b(program)f(that)h(fails)h(to)e(observ)o (e)h(these)h(three)75 1133 y(rules)j(is)g(erroneous.)166 1189 y(T)l(o)f(de\014ne)i(t)o(yp)q(e)f(matc)o(hing)g(more)f(precisely)l (,)j(w)o(e)d(need)i(to)e(deal)h(with)g(t)o(w)o(o)f(issues:)22 b(matc)o(hing)15 b(of)75 1246 y(t)o(yp)q(es)e(of)g(the)h(host)f (language)g(with)h(t)o(yp)q(es)g(sp)q(eci\014ed)h(in)f(comm)o (unication)h(op)q(erations;)e(and)h(matc)o(hing)75 1302 y(of)h(t)o(yp)q(es)g(at)g(sender)g(and)h(receiv)o(er.)166 1359 y(The)k(t)o(yp)q(es)f(of)h(a)f(send)h(and)g(receiv)o(e)h(matc)o(h) e(\(phase)g(t)o(w)o(o\))f(if)j(b)q(oth)e(op)q(erations)h(use)g(iden)o (tical)75 1415 y(names.)36 b(That)20 b(is,)i Fw(MPI)p 517 1415 V 16 w(INTEGER)f Fv(matc)o(hes)f Fw(MPI)p 1006 1415 V 16 w(INTEGER)p Fv(,)h Fw(MPI)p 1327 1415 V 16 w(REAL)g Fv(matc)o(hes)f Fw(MPI)p 1738 1415 V 16 w(REAL)p Fv(,)75 1471 y(and)12 b(so)g(on.)18 b(There)12 b(is)h(one)f(exception)h (to)e(this)h(rule,)h(discussed)g(in)g(Sec.)20 b(3.13,)10 b(the)i(t)o(yp)q(e)g Fw(MPI)p 1687 1471 V 16 w(P)l(A)o(CKED)75 1528 y Fv(can)j(matc)o(h)g(an)o(y)g(other)g(t)o(yp)q(e.)166 1584 y(The)k(t)o(yp)q(e)g(of)f(a)g(v)m(ariable)i(in)g(a)e(host)h (program)e(matc)o(hes)h(the)h(t)o(yp)q(e)g(sp)q(eci\014ed)i(in)e(the)g (comm)o(u-)75 1641 y(nication)g(op)q(eration)g(if)f(the)h(datat)o(yp)q (e)f(name)g(used)h(b)o(y)f(that)f(op)q(eration)i(corresp)q(onds)f(to)g (the)g(basic)75 1697 y(t)o(yp)q(e)f(of)g(the)g(host)g(program)f(v)m (ariable.)27 b(F)l(or)17 b(example,)h(an)f(en)o(try)g(with)g(t)o(yp)q (e)g(name)g Fw(MPI)p 1672 1697 V 16 w(INTEGER)75 1754 y Fv(matc)o(hes)c(a)g(F)l(ortran)g(v)m(ariable)i(of)e(t)o(yp)q(e)g Ft(INTEGER)p Fv(.)f(A)i(table)g(giving)g(this)g(corresp)q(ondence)h (for)e(F)l(ortran)75 1810 y(and)20 b(C)f(app)q(ears)h(in)h(Sec.)33 b(3.2.2.)f(There)20 b(are)f(t)o(w)o(o)g(exceptions)h(to)f(this)i(last)e (rule:)30 b(an)19 b(en)o(try)h(with)75 1867 y(t)o(yp)q(e)f(name)g Fw(MPI)p 391 1867 V 16 w(BYTE)h Fv(or)f Fw(MPI)p 687 1867 V 16 w(P)l(A)o(CKED)g Fv(can)g(b)q(e)h(used)g(to)e(matc)o(h)h(an)o (y)g(b)o(yte)g(of)g(storage)e(\(on)i(a)75 1923 y(b)o(yte-addressable)g (mac)o(hine\),)g(irresp)q(ectiv)o(e)h(of)e(the)g(datat)o(yp)q(e)g(of)g (the)g(v)m(ariable)i(that)d(con)o(tains)i(this)75 1980 y(b)o(yte.)f(The)12 b(t)o(yp)q(e)g Fw(MPI)p 466 1980 V 16 w(P)l(A)o(CKED)f Fv(is)h(used)g(to)f(send)i(data)e(that)f(has)i(b) q(een)h(explicitly)h(pac)o(k)o(ed,)e(or)f(receiv)o(e)75 2036 y(data)k(that)g(will)i(b)q(e)g(explicitly)h(unpac)o(k)o(ed,)e(see) g(Section)g(3.13.)k(The)c(t)o(yp)q(e)g Fw(MPI)p 1468 2036 V 16 w(BYTE)g Fv(allo)o(ws)g(one)g(to)75 2092 y(transfer)e(the)i (binary)g(v)m(alue)g(of)f(a)g(b)o(yte)g(in)h(memory)e(unc)o(hanged.)166 2149 y(T)l(o)h(summarize,)g(the)g(t)o(yp)q(e)h(matc)o(hing)f(rules)h (fall)g(in)o(to)f(the)g(three)h(categories)f(b)q(elo)o(w.)143 2238 y Fx(\017)23 b Fv(Comm)o(unication)11 b(of)f(t)o(yp)q(ed)i(v)m (alues)g(\(e.g.,)e(with)h(datat)o(yp)q(e)g(di\013eren)o(t)g(from)21 b Fw(MPI)p 1586 2238 V 16 w(BYTE)p Fv(\),)11 b(where)189 2294 y(the)k(datat)o(yp)q(es)f(of)g(the)h(corresp)q(onding)h(en)o (tries)f(in)h(the)f(sender)g(program,)e(in)j(the)f(send)g(call,)h(in) 189 2351 y(the)f(receiv)o(e)h(call)g(and)g(in)g(the)f(receiv)o(er)h (program)e(m)o(ust)h(all)h(matc)o(h.)143 2442 y Fx(\017)23 b Fv(Comm)o(unication)16 b(of)g(un)o(t)o(yp)q(ed)h(v)m(alues)g(\(e.g.,) e(of)g(datat)o(yp)q(e)h Fw(MPI)p 1330 2442 V 16 w(BYTE)p Fv(\),)g(where)g(b)q(oth)g(sender)189 2499 y(and)g(receiv)o(er)h(use)f (the)g(datat)o(yp)q(e)g Fw(MPI)p 881 2499 V 15 w(BYTE)p Fv(.)h(In)f(this)h(case,)f(there)g(are)g(no)f(requiremen)o(ts)i(on)189 2555 y(the)d(t)o(yp)q(es)h(of)f(the)g(corresp)q(onding)h(en)o(tries)g (in)h(the)e(sender)h(and)g(the)f(receiv)o(er)h(programs,)e(nor)h(is)189 2612 y(it)h(required)h(that)f(they)g(b)q(e)h(the)f(same.)143 2704 y Fx(\017)23 b Fv(Comm)o(unication)15 b(in)o(v)o(olving)i(pac)o(k) o(ed)e(data,)f(where)h Fw(MPI)p 1195 2704 V 16 w(P)l(A)o(CKED)g Fv(is)h(used.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 24 32 24 31 bop 75 -100 a Fv(24)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)166 45 y Fv(The)e(follo)o(wing)h (examples)g(illustrate)h(the)e(\014rst)g(t)o(w)o(o)e(cases.)75 151 y Fq(Example)18 b(3.1)k Fv(Sender)16 b(and)g(receiv)o(er)g(sp)q (ecify)g(matc)o(hing)g(t)o(yp)q(es.)75 245 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 302 y(IF\(rank.EQ.0\))f (THEN)170 358 y(CALL)i(MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g(tag,) h(comm,)f(ierr\))75 415 y(ELSE)170 471 y(CALL)h(MPI_RECV\(b\(1\),)e (15,)h(MPI_REAL,)g(0,)g(tag,)h(comm,)f(status,)g(ierr\))75 527 y(END)g(IF)166 621 y Fv(This)16 b(co)q(de)h(is)f(correct)f(if)h(b)q (oth)g Fw(a)g Fv(and)g Fw(b)g Fv(are)f(real)h(arra)o(ys)f(of)g(size)i Fx(\025)d Fv(10.)21 b(\(In)16 b(F)l(ortran,)e(it)i(migh)o(t)75 678 y(b)q(e)e(correct)e(to)h(use)g(this)g(co)q(de)h(ev)o(en)f(if)h Fw(a)f Fv(or)f Fw(b)i Fv(ha)o(v)o(e)e(size)i Fp(<)f Fv(10:)19 b(e.g.,)12 b(when)h Fw(a\(1\))f Fv(can)h(b)q(e)h(equiv)m(alenced)75 734 y(to)h(an)g(arra)o(y)f(with)h(ten)g(reals.\))75 840 y Fq(Example)j(3.2)k Fv(Sender)16 b(and)g(receiv)o(er)g(do)f(not)g(sp)q (ecify)h(matc)o(hing)f(t)o(yp)q(es.)75 947 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1003 y(IF\(rank.EQ.0\))f (THEN)170 1060 y(CALL)i(MPI_SEND\(a\(1\),)e(10,)h(MPI_REAL,)g(1,)g (tag,)h(comm,)f(ierr\))75 1116 y(ELSE)170 1173 y(CALL)h (MPI_RECV\(b\(1\),)e(40,)h(MPI_BYTE,)g(0,)g(tag,)h(comm,)f(status,)g (ierr\))75 1229 y(END)g(IF)166 1335 y Fv(This)d(co)q(de)g(is)f (erroneous,)h(since)g(sender)g(and)g(receiv)o(er)g(do)f(not)f(pro)o (vide)i(matc)o(hing)f(datat)o(yp)q(e)75 1392 y(argumen)o(ts.)75 1498 y Fq(Example)f(3.3)k Fv(Sender)16 b(and)g(receiv)o(er)g(sp)q (ecify)g(comm)o(unication)g(of)f(un)o(t)o(yp)q(ed)h(v)m(alues.)75 1592 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1648 y(IF\(rank.EQ.0\))f(THEN)170 1705 y(CALL)i(MPI_SEND\(a\(1\),)e (40,)h(MPI_BYTE,)g(1,)g(tag,)h(comm,)f(ierr\))75 1761 y(ELSE)170 1818 y(CALL)h(MPI_RECV\(b\(1\),)e(60,)h(MPI_BYTE,)g(0,)g (tag,)h(comm,)f(status,)g(ierr\))75 1874 y(END)g(IF)166 1968 y Fv(This)16 b(co)q(de)g(is)g(correct,)e(irresp)q(ectiv)o(e)j(of)e (the)h(t)o(yp)q(e)f(and)h(size)g(of)f Fw(a)g Fv(and)h Fw(b)f Fv(\(unless)i(this)e(results)h(in)75 2024 y(an)f(out)g(of)g(b)q (ound)h(memory)e(access\).)189 2131 y Fl(A)n(dvic)n(e)e(to)i(users.)38 b Fv(If)12 b(a)g(bu\013er)g(of)f(t)o(yp)q(e)h Fi(MPI)p 965 2131 13 2 v 15 w(BYTE)f Fv(is)i(passed)f(as)g(an)g(argumen)o(t)f (to)g Fw(MPI)p 1732 2131 14 2 v 16 w(SEND)p Fv(,)189 2187 y(then)j Fw(MPI)g Fv(will)i(send)f(the)g(data)f(stored)g(at)f(con) o(tiguous)i(lo)q(cations,)g(starting)f(from)f(the)i(address)189 2244 y(indicated)22 b(b)o(y)e(the)h Fw(buf)g Fv(argumen)o(t.)35 b(This)21 b(ma)o(y)f(ha)o(v)o(e)g(unexp)q(ected)i(results)f(when)g(the) g(data)189 2300 y(la)o(y)o(out)f(is)i(not)f(as)f(a)h(casual)h(user)f(w) o(ould)h(exp)q(ect)f(it)h(to)e(b)q(e.)39 b(F)l(or)20 b(example,)j(some)e(F)l(ortran)189 2356 y(compilers)e(implemen)o(t)g(v) m(ariables)g(of)f(t)o(yp)q(e)g Fi(CHARA)o(CTER)e Fv(as)i(a)g(structure) f(that)h(con)o(tains)g(the)189 2413 y(c)o(haracter)c(length)i(and)g(a)f (p)q(oin)o(ter)h(to)f(the)g(actual)h(string.)k(In)c(suc)o(h)g(an)f(en)o (vironmen)o(t,)h(sending)189 2469 y(and)g(receiving)h(a)e(F)l(ortran)g Fi(CHARA)o(CTER)f Fv(v)m(ariable)j(using)g(the)e Fi(MPI)p 1378 2469 13 2 v 15 w(BYTE)g Fv(t)o(yp)q(e)h(will)h(not)e(ha)o(v)o(e) 189 2526 y(the)f(an)o(ticipated)i(result)f(of)f(transferring)h(the)f(c) o(haracter)g(string.)20 b(F)l(or)14 b(this)h(reason,)f(the)h(user)f(is) 189 2582 y(advised)i(to)e(use)i(t)o(yp)q(ed)f(comm)o(unications)h (whenev)o(er)g(p)q(ossible.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)h(to)f (users.)p Fv(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 25 33 25 32 bop 75 -100 a Fm(3.3.)34 b(D)o(A)l(T)l(A)15 b(TYPE)g(MA)l(TCHING) g(AND)g(D)o(A)l(T)l(A)g(CONVERSION)520 b Fv(25)75 45 y Fw(T)l(yp)q(e)16 b Fi(MPI)p 260 45 13 2 v 15 w(CHARA)o(CTER)75 131 y Fv(The)c(t)o(yp)q(e)g Fw(MPI)p 349 131 14 2 v 16 w(CHARA)o(CTER)h Fv(matc)o(hes)f(one)g(c)o(haracter)f(of)g(a)h(F)l (ortran)f(v)m(ariable)i(of)f(t)o(yp)q(e)g Ft(CHARACTER)p Fv(,)75 187 y(rather)20 b(then)g(the)h(en)o(tire)g(c)o(haracter)e (string)i(stored)f(in)h(the)f(v)m(ariable.)37 b(F)l(ortran)19 b(v)m(ariables)j(of)e(t)o(yp)q(e)75 244 y Fi(CHARA)o(CTER)g Fv(or)i(substrings)g(are)f(transferred)h(as)f(if)i(they)f(w)o(ere)g (arra)o(ys)e(of)i(c)o(haracters.)39 b(This)22 b(is)75 300 y(illustrated)17 b(in)f(the)f(example)h(b)q(elo)o(w.)75 407 y Fq(Example)i(3.4)k Fv(T)l(ransfer)15 b(of)g(F)l(ortran)f Fi(CHARA)o(CTER)p Fv(s.)75 513 y Ft(CHARACTER*10)22 b(a)75 569 y(CHARACTER*10)g(b)75 682 y(CALL)h(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))75 739 y(IF\(rank.EQ.0\))f(THEN)170 795 y(CALL)i(MPI_SEND\(a,)e (5,)i(MPI_CHARACTER,)e(1,)h(tag,)h(comm,)f(ierr\))75 852 y(ELSE)170 908 y(CALL)h(MPI_RECV\(b\(6:10\),)d(5,)j(MPI_CHARACTER,) e(0,)h(tag,)h(comm,)f(status,)g(ierr\))75 964 y(END)g(IF)166 1071 y Fv(The)14 b(last)h(\014v)o(e)f(c)o(haracters)g(of)f(string)i Fi(b)f Fv(at)g(pro)q(cess)g(1)g(are)g(replaced)i(b)o(y)e(the)g(\014rst) g(\014v)o(e)h(c)o(haracters)75 1127 y(of)g(string)g Fi(a)g Fv(at)g(pro)q(cess)g(0.)189 1233 y Fl(R)n(ationale.)45 b Fv(The)16 b(alternativ)o(e)h(c)o(hoice)g(w)o(ould)f(b)q(e)h(for)e Fw(MPI)p 1246 1233 V 16 w(CHARA)o(CTER)i Fv(to)f(matc)o(h)f(a)h(c)o (har-)189 1290 y(acter)e(of)h(arbitrary)g(length.)20 b(This)c(runs)f(in)o(to)h(problems.)189 1365 y(A)g(F)l(ortran)e(c)o (haracter)h(v)m(ariable)j(is)e(a)g(constan)o(t)f(length)h(string,)g (with)g(no)g(sp)q(ecial)i(termination)189 1421 y(sym)o(b)q(ol.)h(There) 12 b(is)h(no)g(\014xed)f(con)o(v)o(en)o(tion)h(on)f(ho)o(w)g(to)g (represen)o(t)g(c)o(haracters,)g(and)g(ho)o(w)g(to)g(store)189 1478 y(their)h(length.)20 b(Some)13 b(compilers)h(pass)e(a)h(c)o (haracter)f(argumen)o(t)g(to)g(a)h(routine)g(as)f(a)h(pair)g(of)f (argu-)189 1534 y(men)o(ts,)g(one)g(holding)h(the)f(address)h(of)e(the) h(string)g(and)h(the)f(other)f(holding)j(the)e(length)h(of)e(string.) 189 1591 y(Consider)h(the)f(case)h(of)f(an)g Fw(MPI)g Fv(comm)o(unication)h(call)h(that)e(is)h(passed)g(a)f(comm)o(unication) h(bu\013er)189 1647 y(with)17 b(t)o(yp)q(e)f(de\014ned)i(b)o(y)f(a)f (deriv)o(ed)h(datat)o(yp)q(e)f(\(Section)i(3.12\).)k(If)17 b(this)g(comm)o(unicator)f(bu\013er)189 1704 y(con)o(tains)e(v)m (ariables)h(of)e(t)o(yp)q(e)h Ft(CHARACTER)f Fv(then)h(the)g (information)g(on)g(their)h(length)f(will)i(not)d(b)q(e)189 1760 y(passed)i(to)g(the)g Fw(MPI)g Fv(routine.)189 1835 y(This)j(problem)g(forces)f(us)h(to)f(pro)o(vide)h(explicit)i (information)d(on)h(c)o(haracter)f(length)h(with)g(the)189 1892 y Fw(MPI)d Fv(call.)21 b(One)c(could)f(add)g(a)f(length)h (parameter)f(to)g(the)g(t)o(yp)q(e)h Fw(MPI)p 1411 1892 V 15 w(CHARA)o(CTER)p Fv(,)h(but)f(this)189 1948 y(do)q(es)11 b(not)g(add)h(m)o(uc)o(h)f(con)o(v)o(enience)i(and)e(the)h(same)f (functionalit)o(y)h(can)g(b)q(e)g(ac)o(hiev)o(ed)g(b)o(y)f(de\014ning) 189 2005 y(a)k(suitable)h(deriv)o(ed)g(datat)o(yp)q(e.)k(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fv(\))189 2111 y Fl(A)n(dvic)n(e)f(to) i(implementors.)48 b Fv(Some)16 b(compilers)i(pass)e(F)l(ortran)g Fi(CHARA)o(CTER)e Fv(argumen)o(ts)i(as)g(a)189 2167 y(structure)g(with) i(a)f(length)g(and)h(a)e(p)q(oin)o(ter)i(to)e(the)h(actual)h(string.)25 b(In)18 b(suc)o(h)f(an)g(en)o(vironmen)o(t,)189 2224 y(the)h Fw(MPI)g Fv(call)h(needs)h(to)d(dereference)j(the)e(p)q(oin)o (ter)h(in)g(order)g(to)e(reac)o(h)h(the)h(string.)29 b(\()p Fl(End)19 b(of)189 2280 y(advic)n(e)d(to)g(implementors.)p Fv(\))75 2402 y Fj(3.3.2)49 b(Data)17 b(conversion)75 2488 y Fv(One)g(of)e(the)h(goals)f(of)h Fw(MPI)f Fv(is)h(to)f(supp)q (ort)h(parallel)i(computations)d(across)g(heterogeneous)h(en)o(viron-) 75 2544 y(men)o(ts.)j(Comm)o(unication)d(in)g(a)f(heterogeneous)g(en)o (vironmen)o(t)g(ma)o(y)g(require)h(data)e(con)o(v)o(ersions.)20 b(W)l(e)75 2601 y(use)c(the)f(follo)o(wing)h(terminology)l(.)75 2695 y Fq(t)o(yp)q(e)h(con)o(v)o(ersion)22 b Fv(c)o(hanges)12 b(the)h(datat)o(yp)q(e)f(of)g(a)g(v)m(alue,)i(e.g.,)e(b)o(y)g(rounding) h(a)f Ft(REAL)g Fv(to)g(an)g Ft(INTEGER)p Fv(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 26 34 26 33 bop 75 -100 a Fv(26)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fq(represen)o(tation)g (con)o(v)o(ersion)22 b Fv(c)o(hanges)16 b(the)g(binary)g(represen)o (tation)g(of)f(a)g(v)m(alue,)i(e.g.,)e(from)g(Hex)189 102 y(\015oating)g(p)q(oin)o(t)g(to)g(IEEE)g(\015oating)h(p)q(oin)o(t.) 166 178 y(The)e(t)o(yp)q(e)f(matc)o(hing)h(rules)g(imply)h(that)e Fw(MPI)g Fv(comm)o(unication)h(nev)o(er)f(en)o(tails)i(t)o(yp)q(e)e (con)o(v)o(ersion.)75 235 y(On)20 b(the)g(other)f(hand,)i Fw(MPI)e Fv(requires)h(that)f(a)g(represen)o(tation)h(con)o(v)o(ersion) f(b)q(e)h(p)q(erformed)g(when)g(a)75 291 y(t)o(yp)q(ed)h(v)m(alue)h(is) f(transferred)f(across)g(en)o(vironmen)o(ts)h(that)f(use)h(di\013eren)o (t)f(represen)o(tations)h(for)f(the)75 348 y(datat)o(yp)q(e)h(of)f (this)i(v)m(alue.)38 b Fw(MPI)21 b Fv(do)q(es)g(not)g(sp)q(ecify)h (rules)g(for)f(represen)o(tation)g(con)o(v)o(ersion.)37 b(Suc)o(h)75 404 y(con)o(v)o(ersion)21 b(is)g(exp)q(ected)i(to)d (preserv)o(e)h(in)o(teger,)h(logical)g(or)e(c)o(haracter)h(v)m(alues,)i (and)e(to)f(con)o(v)o(ert)g(a)75 461 y(\015oating)15 b(p)q(oin)o(t)h(v)m(alue)g(to)f(the)g(nearest)g(v)m(alue)h(that)f(can)g (b)q(e)h(represen)o(ted)g(on)f(the)g(target)f(system.)166 517 y(Ov)o(er\015o)o(w)h(and)g(under\015o)o(w)g(exceptions)h(ma)o(y)f (o)q(ccur)g(during)h(\015oating)f(p)q(oin)o(t)h(con)o(v)o(ersions.)k (Con-)75 574 y(v)o(ersion)h(of)g(in)o(tegers)g(or)g(c)o(haracters)f(ma) o(y)g(also)i(lead)f(to)g(exceptions)h(when)g(a)e(v)m(alue)j(that)d(can) h(b)q(e)75 630 y(represen)o(ted)16 b(in)g(one)f(system)g(cannot)g(b)q (e)h(represen)o(ted)f(in)h(the)f(other)g(system.)20 b(An)15 b(exception)h(o)q(ccur-)75 686 y(ring)i(during)g(represen)o(tation)g (con)o(v)o(ersion)f(results)h(in)h(a)e(failure)i(of)e(the)g(comm)o (unication.)28 b(An)18 b(error)75 743 y(o)q(ccurs)e(either)f(in)h(the)g (send)g(op)q(eration,)f(or)f(the)i(receiv)o(e)g(op)q(eration,)f(or)f(b) q(oth.)166 799 y(If)22 b(a)f(v)m(alue)i(sen)o(t)e(in)h(a)f(message)g (is)h(un)o(t)o(yp)q(ed)h(\(i.e.,)f(of)f(t)o(yp)q(e)h Fw(MPI)p 1370 799 14 2 v 15 w(BYTE)p Fv(\),)g(then)g(the)f(binary)75 856 y(represen)o(tation)e(of)f(the)h(b)o(yte)g(stored)f(at)g(the)h (receiv)o(er)h(is)f(iden)o(tical)i(to)d(the)h(binary)g(represen)o (tation)75 912 y(of)d(the)h(b)o(yte)g(loaded)g(at)g(the)f(sender.)26 b(This)17 b(holds)h(true,)e(whether)h(sender)h(and)f(receiv)o(er)g(run) g(in)h(the)75 969 y(same)i(or)f(in)i(distinct)g(en)o(vironmen)o(ts.)35 b(No)20 b(represen)o(tation)g(con)o(v)o(ersion)g(is)h(required.)35 b(\(Note)20 b(that)75 1025 y(represen)o(tation)15 b(con)o(v)o(ersion)g (ma)o(y)g(o)q(ccur)g(when)h(v)m(alues)g(of)e(t)o(yp)q(e)i Fw(MPI)p 1307 1025 V 15 w(CHARA)o(CTER)h Fv(or)d Fw(MPI)p 1741 1025 V 16 w(CHAR)75 1082 y Fv(are)h(transferred,)f(for)h(example,) g(from)g(an)g(EBCDIC)g(enco)q(ding)i(to)d(an)h(ASCI)q(I)i(enco)q (ding.\))166 1138 y(No)g(con)o(v)o(ersion)h(need)g(o)q(ccur)g(when)h (an)e Fw(MPI)g Fv(program)f(executes)j(in)f(a)f(homogeneous)h(system,) 75 1195 y(where)d(all)i(pro)q(cesses)e(run)h(in)g(the)f(same)g(en)o (vironmen)o(t.)166 1251 y(Consider)d(the)g(three)g(examples,)h (3.1{3.3.)j(The)c(\014rst)g(program)e(is)j(correct,)e(assuming)h(that)f Fw(a)h Fv(and)75 1307 y Fw(b)i Fv(are)g Ft(REAL)f Fv(arra)o(ys)g(of)g (size)i Fx(\025)e Fv(10.)19 b(If)14 b(the)g(sender)g(and)g(receiv)o(er) h(execute)f(in)h(di\013eren)o(t)f(en)o(vironmen)o(ts,)75 1364 y(then)21 b(the)g(ten)h(real)f(v)m(alues)h(that)e(are)h(fetc)o (hed)g(from)g(the)g(send)g(bu\013er)g(will)i(b)q(e)f(con)o(v)o(erted)e (to)h(the)75 1420 y(represen)o(tation)g(for)f(reals)g(on)h(the)g (receiv)o(er)g(site)g(b)q(efore)g(they)f(are)h(stored)f(in)h(the)g (receiv)o(e)g(bu\013er.)75 1477 y(While)e(the)e(n)o(um)o(b)q(er)h(of)f (real)g(elemen)o(ts)h(fetc)o(hed)g(from)f(the)g(send)h(bu\013er)f (equal)i(the)e(n)o(um)o(b)q(er)h(of)f(real)75 1533 y(elemen)o(ts)d (stored)e(in)i(the)f(receiv)o(e)h(bu\013er,)f(the)g(n)o(um)o(b)q(er)g (of)g(b)o(ytes)g(stored)f(need)i(not)f(equal)h(the)f(n)o(um)o(b)q(er)75 1590 y(of)e(b)o(ytes)g(loaded.)20 b(F)l(or)11 b(example,)i(the)e (sender)h(ma)o(y)f(use)h(a)g(four)f(b)o(yte)g(represen)o(tation)h(and)g (the)f(receiv)o(er)75 1646 y(an)k(eigh)o(t)h(b)o(yte)f(represen)o (tation)g(for)f(reals.)166 1703 y(The)h(second)h(program)e(is)i (erroneous,)f(and)g(its)g(b)q(eha)o(vior)h(is)g(unde\014ned.)166 1759 y(The)e(third)g(program)e(is)i(correct.)19 b(The)13 b(exact)g(same)g(sequence)i(of)e(fort)o(y)f(b)o(ytes)h(that)g(w)o(ere)g (loaded)75 1816 y(from)i(the)h(send)g(bu\013er)g(will)h(b)q(e)g(stored) e(in)h(the)g(receiv)o(e)h(bu\013er,)e(ev)o(en)h(if)g(sender)h(and)f (receiv)o(er)g(run)g(in)75 1872 y(a)g(di\013eren)o(t)g(en)o(vironmen)o (t.)22 b(The)17 b(message)e(sen)o(t)h(has)g(exactly)g(the)g(same)g (length)h(\(in)f(b)o(ytes\))g(and)g(the)75 1928 y(same)g(binary)g (represen)o(tation)g(as)g(the)g(message)f(receiv)o(ed.)24 b(If)16 b Fw(a)f Fv(and)i Fw(b)f Fv(are)g(of)f(di\013eren)o(t)h(t)o(yp) q(es,)g(or)f(if)75 1985 y(they)h(are)f(of)h(the)g(same)f(t)o(yp)q(e)h (but)g(di\013eren)o(t)g(data)f(represen)o(tations)h(are)g(used,)g(then) g(the)g(bits)g(stored)75 2041 y(in)h(the)f(receiv)o(e)h(bu\013er)g(ma)o (y)e(enco)q(de)i(v)m(alues)h(that)d(are)h(di\013eren)o(t)g(from)g(the)g (v)m(alues)h(they)g(enco)q(ded)g(in)75 2098 y(the)e(send)h(bu\013er.) 166 2154 y(Data)e(represen)o(tation)i(con)o(v)o(ersion)f(also)h (applies)h(to)e(the)g(en)o(v)o(elop)q(e)i(of)e(a)g(message:)20 b(source,)15 b(des-)75 2211 y(tination)h(and)f(tag)f(are)h(all)h(in)o (tegers)g(that)e(ma)o(y)h(need)h(to)e(b)q(e)i(con)o(v)o(erted.)189 2293 y Fl(A)n(dvic)n(e)g(to)i(implementors.)50 b Fv(The)17 b(curren)o(t)f(de\014nition)j(do)q(es)e(not)g(require)g(messages)f(to)g (carry)189 2350 y(data)h(t)o(yp)q(e)i(information.)29 b(Both)18 b(sender)h(and)f(receiv)o(er)h(pro)o(vide)g(complete)g(data)e (t)o(yp)q(e)i(infor-)189 2406 y(mation.)24 b(In)17 b(a)g(heterogeneous) f(en)o(vironmen)o(t,)h(one)g(can)g(either)h(use)f(a)f(mac)o(hine)i (indep)q(enden)o(t)189 2462 y(enco)q(ding)h(suc)o(h)f(as)g(XDR,)f(or)h (ha)o(v)o(e)f(the)h(receiv)o(er)g(con)o(v)o(ert)g(from)f(the)h(sender)g (represen)o(tation)189 2519 y(to)c(its)i(o)o(wn,)e(or)h(ev)o(en)g(ha)o (v)o(e)g(the)g(sender)h(do)f(the)g(con)o(v)o(ersion.)189 2591 y(Additional)20 b(t)o(yp)q(e)e(information)g(migh)o(t)g(b)q(e)h (added)g(to)f(messages)g(in)h(order)f(to)f(allo)o(w)i(the)f(sys-)189 2647 y(tem)g(to)f(detect)i(mismatc)o(hes)f(b)q(et)o(w)o(een)g(datat)o (yp)q(e)g(at)g(sender)g(and)h(receiv)o(er.)29 b(This)19 b(migh)o(t)f(b)q(e)189 2704 y(particularly)13 b(useful)g(in)f(a)g(slo)o (w)o(er)f(but)h(safer)g(debug)g(mo)q(de.)19 b(\()p Fl(End)13 b(of)g(advic)n(e)g(to)h(implementors.)p Fv(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 27 35 27 34 bop 75 -100 a Fm(3.4.)34 b(COMMUNICA)l(TION)16 b(MODES)1048 b Fv(27)166 45 y Fw(MPI)19 b Fv(do)q(es)g(not)g(require)h (supp)q(ort)g(for)e(in)o(ter-language)i(comm)o(unication.)33 b(The)20 b(b)q(eha)o(vior)f(of)g(a)75 102 y(program)12 b(is)i(unde\014ned)h(if)e(messages)g(are)g(sen)o(t)g(b)o(y)g(a)g(C)g (pro)q(cess)g(and)h(receiv)o(ed)g(b)o(y)f(a)g(F)l(ortran)f(pro)q(cess,) 75 158 y(or)j(vice-v)o(ersa.)189 264 y Fl(R)n(ationale.)39 b Fw(MPI)15 b Fv(do)q(es)g(not)f(handle)i(in)o(ter-language)f(comm)o (unication)h(b)q(ecause)g(there)f(are)f(no)189 321 y(agreed)h (standards)g(for)g(the)g(corresp)q(ondence)i(b)q(et)o(w)o(een)f(C)f(t)o (yp)q(es)g(and)h(F)l(ortran)e(t)o(yp)q(es.)21 b(There-)189 377 y(fore,)14 b Fw(MPI)h Fv(programs)f(that)g(mix)i(languages)f(w)o (ould)h(not)e(p)q(ort.)20 b(\()p Fl(End)c(of)g(r)n(ationale.)p Fv(\))189 483 y Fl(A)n(dvic)n(e)k(to)h(implementors.)71 b Fw(MPI)20 b Fv(implemen)o(tors)h(ma)o(y)f(w)o(an)o(t)f(to)h(supp)q (ort)g(in)o(ter-language)189 540 y(comm)o(unication)13 b(b)o(y)g(allo)o(wing)g(F)l(ortran)f(programs)f(to)h(use)h(\\C)f(MPI)h (t)o(yp)q(es,")g(suc)o(h)g(as)f Fi(MPI)p 1782 540 13 2 v 14 w(INT,)189 596 y(MPI)p 266 596 V 14 w(CHAR)p Fv(,)19 b(etc.,)h(and)f(allo)o(wing)i(C)e(programs)g(to)g(use)h(F)l(ortran)e(t) o(yp)q(es.)33 b(\()p Fl(End)20 b(of)h(advic)n(e)f(to)189 653 y(implementors.)p Fv(\))75 796 y Fn(3.4)59 b(Communication)18 b(Mo)r(des)75 898 y Fv(The)e(send)h(call)h(describ)q(ed)g(in)f(Section) g(3.2.1)d(is)j Fq(blo)q(c)o(king)p Fv(:)23 b(it)16 b(do)q(es)h(not)f (return)g(un)o(til)h(the)f(message)75 954 y(data)j(and)g(en)o(v)o(elop) q(e)h(ha)o(v)o(e)f(b)q(een)h(safely)g(stored)f(a)o(w)o(a)o(y)e(so)i (that)g(the)g(sender)h(is)g(free)f(to)f(access)i(and)75 1010 y(o)o(v)o(erwrite)15 b(the)g(send)h(bu\013er.)k(The)c(message)f (migh)o(t)g(b)q(e)h(copied)g(directly)h(in)o(to)e(the)h(matc)o(hing)f (receiv)o(e)75 1067 y(bu\013er,)g(or)f(it)i(migh)o(t)f(b)q(e)h(copied)g (in)o(to)f(a)g(temp)q(orary)g(system)g(bu\013er.)166 1123 y(Message)c(bu\013ering)i(decouples)g(the)f(send)h(and)f(receiv)o (e)h(op)q(erations.)19 b(A)12 b(blo)q(c)o(king)h(send)g(can)f(com-)75 1180 y(plete)i(as)e(so)q(on)h(as)g(the)f(message)h(w)o(as)f (bu\013ered,)h(ev)o(en)h(if)f(no)g(matc)o(hing)g(receiv)o(e)g(has)g(b)q (een)h(executed)g(b)o(y)75 1236 y(the)f(receiv)o(er.)19 b(On)14 b(the)e(other)g(hand,)i(message)e(bu\013ering)h(can)g(b)q(e)g (exp)q(ensiv)o(e,)h(as)e(it)h(en)o(tails)h(additional)75 1293 y(memory-to-memory)j(cop)o(ying,)h(and)g(it)g(requires)h(the)f (allo)q(cation)h(of)e(memory)h(for)f(bu\013ering.)29 b Fw(MPI)75 1349 y Fv(o\013ers)12 b(the)i(c)o(hoice)g(of)f(sev)o(eral)g (comm)o(unication)i(mo)q(des)e(that)g(allo)o(w)g(one)h(to)e(con)o(trol) h(the)h(c)o(hoice)g(of)f(the)75 1406 y(comm)o(unication)j(proto)q(col.) 166 1462 y(The)g(send)h(call)g(describ)q(ed)h(in)e(Section)h(3.2.1)d (used)j(the)f Fq(standard)g Fv(comm)o(unication)h(mo)q(de.)22 b(In)75 1519 y(this)16 b(mo)q(de,)e(it)i(is)f(up)h(to)e Fw(MPI)h Fv(to)f(decide)j(whether)e(outgoing)g(messages)f(will)j(b)q(e) f(bu\013ered.)k Fw(MPI)15 b Fv(ma)o(y)75 1575 y(bu\013er)j(outgoing)h (messages.)29 b(In)19 b(suc)o(h)g(a)f(case,)h(the)f(send)h(call)h(ma)o (y)e(complete)h(b)q(efore)g(a)f(matc)o(hing)75 1631 y(receiv)o(e)d(is)f (in)o(v)o(ok)o(ed.)20 b(On)14 b(the)g(other)g(hand,)g(bu\013er)g(space) g(ma)o(y)f(b)q(e)h(una)o(v)m(ailable,)i(or)e Fw(MPI)f Fv(ma)o(y)g(c)o(ho)q(ose)75 1688 y(not)k(to)h(bu\013er)f(outgoing)h (messages,)g(for)f(p)q(erformance)h(reasons.)27 b(In)19 b(this)f(case,)g(the)g(send)g(call)h(will)75 1744 y(not)13 b(complete)h(un)o(til)g(a)f(matc)o(hing)g(receiv)o(e)h(has)f(b)q(een)i (p)q(osted,)e(and)g(the)h(data)e(has)h(b)q(een)i(mo)o(v)o(ed)d(to)h (the)75 1801 y(receiv)o(er.)166 1857 y(Th)o(us,)j(a)f(send)i(in)g (standard)f(mo)q(de)g(can)g(b)q(e)h(started)e(whether)h(or)g(not)f(a)h (matc)o(hing)g(receiv)o(e)h(has)75 1914 y(b)q(een)d(p)q(osted.)19 b(It)12 b(ma)o(y)g(complete)h(b)q(efore)g(a)f(matc)o(hing)g(receiv)o(e) i(is)f(p)q(osted.)19 b(The)13 b(standard)f(mo)q(de)g(send)75 1970 y(is)18 b Fq(non-lo)q(cal)p Fv(:)27 b(successful)19 b(completion)g(of)e(the)g(send)h(op)q(eration)g(ma)o(y)f(dep)q(end)i (on)e(the)h(o)q(ccurrence)75 2027 y(of)d(a)g(matc)o(hing)g(receiv)o(e.) 189 2133 y Fl(R)n(ationale.)42 b Fv(The)15 b(reluctance)i(of)e Fw(MPI)g Fv(to)g(mandate)g(whether)h(standard)f(sends)h(are)f (bu\013ering)189 2189 y(or)e(not)h(stems)g(from)f(the)i(desire)g(to)f (ac)o(hiev)o(e)g(p)q(ortable)h(programs.)j(Since)e(an)o(y)e(system)g (will)i(run)189 2246 y(out)d(of)g(bu\013er)g(resources)h(as)f(message)g (sizes)h(are)f(increased,)i(and)f(some)f(implemen)o(tations)i(ma)o(y) 189 2302 y(w)o(an)o(t)i(to)g(pro)o(vide)i(little)h(bu\013ering,)f Fw(MPI)e Fv(tak)o(es)h(the)g(p)q(osition)h(that)f(correct)g(\(and)g (therefore,)189 2359 y(p)q(ortable\))c(programs)e(do)i(not)g(rely)g(on) g(system)g(bu\013ering)g(in)h(standard)e(mo)q(de.)20 b(Bu\013ering)15 b(ma)o(y)189 2415 y(impro)o(v)o(e)g(the)h(p)q (erformance)g(of)g(a)f(correct)h(program,)e(but)i(it)g(do)q(esn't)g (a\013ect)f(the)h(result)g(of)g(the)189 2472 y(program.)33 b(If)21 b(the)f(user)g(wishes)h(to)f(guaran)o(tee)f(a)h(certain)h (amoun)o(t)e(of)h(bu\013ering,)h(the)g(user-)189 2528 y(pro)o(vided)13 b(bu\013er)g(system)f(of)g(Sec.)20 b(3.6)12 b(should)i(b)q(e)f(used,)g(along)g(with)g(the)g(bu\013ered-mo)q(de)h (send.)189 2585 y(\()p Fl(End)h(of)i(r)n(ationale.)p Fv(\))166 2691 y(There)e(are)g(three)h(additional)g(comm)o(unication)g (mo)q(des.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 28 36 28 35 bop 75 -100 a Fv(28)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)166 45 y Fv(A)i Fq(bu\013ered)f Fv(mo)q(de)h(send)g(op)q(eration)g(can)g(b)q(e)h (started)e(whether)h(or)f(not)h(a)f(matc)o(hing)h(receiv)o(e)75 102 y(has)g(b)q(een)h(p)q(osted.)31 b(It)19 b(ma)o(y)f(complete)i(b)q (efore)f(a)f(matc)o(hing)h(receiv)o(e)h(is)g(p)q(osted.)31 b(Ho)o(w)o(ev)o(er,)18 b(unlik)o(e)75 158 y(the)k(standard)f(send,)i (this)f(op)q(eration)g(is)g Fq(lo)q(cal)p Fv(,)j(and)d(its)g (completion)h(do)q(es)f(not)f(dep)q(end)i(on)f(the)75 214 y(o)q(ccurrence)d(of)e(a)h(matc)o(hing)g(receiv)o(e.)28 b(Th)o(us,)18 b(if)g(a)g(send)g(is)g(executed)h(and)f(no)g(matc)o(hing) g(receiv)o(e)g(is)75 271 y(p)q(osted,)13 b(then)f Fw(MPI)g Fv(m)o(ust)f(bu\013er)h(the)g(outgoing)g(message,)g(so)g(as)f(to)h (allo)o(w)g(the)g(send)h(call)g(to)f(complete.)75 327 y(An)h(error)g(will)i(o)q(ccur)e(if)h(there)f(is)g(insu\016cien)o(t)i (bu\013er)e(space.)20 b(The)13 b(amoun)o(t)f(of)h(a)o(v)m(ailable)i (bu\013er)e(space)75 384 y(is)k(con)o(trolled)f(b)o(y)g(the)g(user)h(|) f(see)g(Section)h(3.6.)22 b(Bu\013er)16 b(allo)q(cation)h(b)o(y)f(the)g (user)g(ma)o(y)f(b)q(e)i(required)75 440 y(for)e(the)g(bu\013ered)g(mo) q(de)h(to)f(b)q(e)g(e\013ectiv)o(e.)166 497 y(A)j(send)g(that)f(uses)g (the)h Fq(sync)o(hronous)e Fv(mo)q(de)i(can)g(b)q(e)g(started)f (whether)g(or)g(not)h(a)f(matc)o(hing)75 553 y(receiv)o(e)k(w)o(as)e(p) q(osted.)34 b(Ho)o(w)o(ev)o(er,)19 b(the)h(send)h(will)g(complete)g (successfully)h(only)e(if)g(a)g(matc)o(hing)g(re-)75 610 y(ceiv)o(e)f(is)h(p)q(osted,)f(and)g(the)f(receiv)o(e)i(op)q (eration)f(has)f(started)g(to)g(receiv)o(e)h(the)g(message)f(sen)o(t)g (b)o(y)h(the)75 666 y(sync)o(hronous)c(send.)21 b(Th)o(us,)14 b(the)h(completion)i(of)d(a)h(sync)o(hronous)g(send)h(not)e(only)i (indicates)h(that)d(the)75 723 y(send)g(bu\013er)f(can)g(b)q(e)h (reused,)f(but)g(also)g(indicates)i(that)d(the)h(receiv)o(er)h(has)f (reac)o(hed)g(a)g(certain)h(p)q(oin)o(t)f(in)75 779 y(its)i(execution,) g(namely)g(that)e(it)i(has)f(started)g(executing)h(the)g(matc)o(hing)f (receiv)o(e.)21 b(If)14 b(b)q(oth)h(sends)g(and)75 835 y(receiv)o(es)e(are)e(blo)q(c)o(king)j(op)q(erations)e(then)g(the)g (use)h(of)e(the)h(sync)o(hronous)g(mo)q(de)g(pro)o(vides)h(sync)o (hronous)75 892 y(comm)o(unication)18 b(seman)o(tics:)25 b(a)18 b(comm)o(unication)g(do)q(es)g(not)g(complete)g(at)f(either)h (end)h(b)q(efore)f(b)q(oth)75 948 y(pro)q(cesses)e(rendezv)o(ous)f(at)g (the)g(comm)o(unication.)21 b(A)15 b(send)h(executed)g(in)g(this)g(mo)q (de)f(is)h Fq(non-lo)q(cal)p Fv(.)166 1005 y(A)e(send)h(that)e(uses)i (the)f Fq(ready)g Fv(comm)o(unication)g(mo)q(de)h(ma)o(y)e(b)q(e)i (started)e Fl(only)h Fv(if)h(the)f(matc)o(hing)75 1061 y(receiv)o(e)k(is)g(already)f(p)q(osted.)27 b(Otherwise,)18 b(the)f(op)q(eration)h(is)f(erroneous)g(and)h(its)f(outcome)g(is)h (unde-)75 1118 y(\014ned.)i(On)13 b(some)f(systems,)h(this)g(allo)o(ws) g(the)f(remo)o(v)m(al)h(of)f(a)h(hand-shak)o(e)g(op)q(eration)g(that)f (is)h(otherwise)75 1174 y(required)k(and)f(results)g(in)h(impro)o(v)o (ed)f(p)q(erformance.)22 b(The)16 b(completion)h(of)e(the)h(send)h(op)q (eration)f(do)q(es)75 1231 y(not)g(dep)q(end)i(on)e(the)g(status)f(of)h (a)g(matc)o(hing)g(receiv)o(e,)h(and)f(merely)h(indicates)h(that)d(the) h(send)h(bu\013er)75 1287 y(can)i(b)q(e)g(reused.)30 b(A)19 b(send)g(op)q(eration)g(that)f(uses)g(the)h(ready)f(mo)q(de)h (has)g(the)f(same)g(seman)o(tics)h(as)f(a)75 1344 y(standard)i(send)i (op)q(eration,)g(or)e(a)g(sync)o(hronous)h(send)g(op)q(eration;)j(it)d (is)g(merely)g(that)g(the)f(sender)75 1400 y(pro)o(vides)e(additional)g (information)g(to)e(the)i(system)f(\(namely)g(that)g(a)f(matc)o(hing)i (receiv)o(e)g(is)g(already)75 1456 y(p)q(osted\),)d(that)g(can)h(sa)o (v)o(e)e(some)i(o)o(v)o(erhead.)k(In)c(a)f(correct)g(program,)f (therefore,)h(a)g(ready)h(send)g(could)75 1513 y(b)q(e)i(replaced)g(b)o (y)f(a)f(standard)h(send)h(with)f(no)g(e\013ect)g(on)f(the)i(b)q(eha)o (vior)f(of)g(the)g(program)e(other)i(than)75 1569 y(p)q(erformance.)166 1626 y(Three)g(additional)i(send)f(functions)g(are)f(pro)o(vided)h(for) e(the)h(three)h(additional)g(comm)o(unication)75 1682 y(mo)q(des.)23 b(The)16 b(comm)o(unication)h(mo)q(de)g(is)f(indicated)i (b)o(y)e(a)g(one)g(letter)h(pre\014x:)22 b Fw(B)16 b Fv(for)g(bu\013ered,)g Fw(S)h Fv(for)75 1739 y(sync)o(hronous,)e(and)g Fw(R)h Fv(for)e(ready)l(.)75 1890 y Fw(MPI)p 160 1890 14 2 v 16 w(BSEND)i(\(buf,)f(count,)h(datat)o(yp)q(e,)h(dest,)f(tag,)f (comm\))117 1967 y Fk(IN)155 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 2042 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 2117 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 2192 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 2267 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 2343 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))75 2467 y Ft(int)23 b(MPI)p 245 2467 15 2 v 17 w(Bsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 2467 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2523 y(int)h(tag,)f(MPI)p 683 2523 V 17 w(Comm)g(comm\))75 2610 y(MPI)p 150 2610 V 17 w(BSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 2666 y()f(BUF\(*\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 29 37 29 36 bop 75 -100 a Fm(3.4.)34 b(COMMUNICA)l(TION)16 b(MODES)1048 b Fv(29)170 45 y Ft(INTEGER)23 b(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(IERROR)166 132 y Fv(Send)16 b(in)g(bu\013ered)g (mo)q(de.)75 283 y Fw(MPI)p 160 283 14 2 v 16 w(SSEND)g(\(buf,)f (count,)h(datat)o(yp)q(e,)g(dest,)h(tag,)e(comm\))117 360 y Fk(IN)155 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 435 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 510 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e (\(handle\))117 585 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 660 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 735 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))75 860 y Ft(int)23 b(MPI)p 245 860 15 2 v 17 w(Ssend\(void*)g(buf,)g(int)g (count,)g(MPI)p 1002 860 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 916 y(int)h(tag,)f(MPI)p 683 916 V 17 w(Comm)g(comm\))75 1003 y(MPI)p 150 1003 V 17 w(SSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 1059 y()f(BUF\(*\))170 1116 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 1202 y Fv(Send)16 b(in)g(sync)o(hronous)f(mo)q(de.)75 1353 y Fw(MPI)p 160 1353 14 2 v 16 w(RSEND)h(\(buf,)f(count,)h(datat)o (yp)q(e,)h(dest,)f(tag,)f(comm\))117 1430 y Fk(IN)155 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 1505 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1581 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1656 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1731 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1806 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))75 1930 y Ft(int)23 b(MPI)p 245 1930 15 2 v 17 w(Rsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1930 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 1987 y(int)h(tag,)f(MPI)p 683 1987 V 17 w(Comm)g(comm\))75 2073 y(MPI)p 150 2073 V 17 w(RSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(IERROR\))170 2130 y()f(BUF\(*\))170 2186 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)166 2273 y Fv(Send)16 b(in)g(ready)f(mo)q(de.)166 2329 y(There)20 b(is)g(only)h(one)f(receiv)o(e)h(op)q(eration,)g(whic)o(h)g(can)f(matc) o(h)f(an)o(y)g(of)h(the)g(send)g(mo)q(des.)35 b(The)75 2386 y(receiv)o(e)15 b(op)q(eration)g(describ)q(ed)h(in)f(the)f(last)g (section)h(is)g Fq(blo)q(c)o(king)p Fv(:)20 b(it)15 b(returns)f(only)g (after)g(the)g(receiv)o(e)75 2442 y(bu\013er)k(con)o(tains)h(the)f (newly)i(receiv)o(ed)f(message.)29 b(A)19 b(receiv)o(e)g(can)g (complete)g(b)q(efore)f(the)h(matc)o(hing)75 2498 y(send)d(has)f (completed)h(\(of)e(course,)h(it)h(can)f(complete)h(only)g(after)e(the) i(matc)o(hing)f(send)h(has)f(started\).)166 2555 y(In)f(a)g(m)o (ulti-threaded)h(implemen)o(tation)h(of)d Fw(MPI)p Fv(,)g(the)h(system) g(ma)o(y)f(de-sc)o(hedule)j(a)e(thread)g(that)75 2611 y(is)h(blo)q(c)o(k)o(ed)h(on)f(a)g(send)g(or)f(receiv)o(e)i(op)q (eration,)f(and)g(sc)o(hedule)i(another)d(thread)h(for)f(execution)i (in)g(the)75 2668 y(same)g(address)g(space.)22 b(In)16 b(suc)o(h)h(a)e(case)h(it)g(is)h(the)f(user's)f(resp)q(onsibilit)o(y)k (not)c(to)g(access)h(or)g(mo)q(dify)g(a)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 30 38 30 37 bop 75 -100 a Fv(30)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(comm)o(unication)h (bu\013er)e(un)o(til)i(the)f(comm)o(unication)h(completes.)25 b(Otherwise,)18 b(the)f(outcome)f(of)h(the)75 102 y(computation)e(is)h (unde\014ned.)189 201 y Fl(R)n(ationale.)56 b Fv(W)l(e)18 b(prohibit)i(read)d(accesses)i(to)e(a)h(send)g(bu\013er)g(while)i(it)e (is)g(b)q(eing)i(used,)f(ev)o(en)189 258 y(though)f(the)g(send)g(op)q (eration)h(is)f(not)g(supp)q(osed)h(to)e(alter)h(the)g(con)o(ten)o(t)g (of)f(this)i(bu\013er.)28 b(This)189 314 y(ma)o(y)17 b(seem)h(more)f(stringen)o(t)g(than)h(necessary)l(,)g(but)g(the)g (additional)h(restriction)g(causes)f(little)189 370 y(loss)e(of)g (functionalit)o(y)i(and)f(allo)o(ws)g(b)q(etter)f(p)q(erformance)h(on)f (some)g(systems)g(|)h(consider)h(the)189 427 y(case)13 b(where)g(data)f(transfer)g(is)i(done)f(b)o(y)g(a)g(DMA)f(engine)j (that)d(is)h(not)g(cac)o(he-coheren)o(t)h(with)f(the)189 483 y(main)i(pro)q(cessor.)20 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fv(\))189 583 y Fl(A)n(dvic)n(e)10 b(to)j(implementors.)36 b Fv(Since)12 b(a)e(sync)o(hronous)g(send)h(cannot)f(complete)h(b)q (efore)g(a)f(matc)o(hing)189 639 y(receiv)o(e)16 b(is)f(p)q(osted,)h (one)f(will)i(not)e(normally)g(bu\013er)h(messages)e(sen)o(t)h(b)o(y)g (suc)o(h)h(an)f(op)q(eration.)189 713 y(It)j(is)h(recommended)g(to)e(c) o(ho)q(ose)i(bu\013ering)f(o)o(v)o(er)g(blo)q(c)o(king)h(the)g(sender,) g(whenev)o(er)g(p)q(ossible,)189 770 y(for)13 b(standard)h(sends.)20 b(The)15 b(programmer)e(can)i(signal)g(his)g(or)f(her)g(preference)h (for)f(blo)q(c)o(king)i(the)189 826 y(sender)f(un)o(til)i(a)e(matc)o (hing)g(receiv)o(e)h(o)q(ccurs)g(b)o(y)f(using)h(the)f(sync)o(hronous)g (send)h(mo)q(de.)189 901 y(A)f(p)q(ossible)i(comm)o(unication)g(proto)q (col)e(for)g(the)h(v)m(arious)f(comm)o(unication)i(mo)q(des)e(is)h (outlined)189 957 y(b)q(elo)o(w.)189 1031 y Fw(ready)f(send)p Fv(:)21 b(The)16 b(message)e(is)i(sen)o(t)f(as)g(so)q(on)g(as)g(p)q (ossible.)189 1105 y Fw(synchronous)20 b(send:)28 b Fv(The)19 b(sender)g(sends)g(a)f(request-to-send)h(message.)30 b(The)18 b(receiv)o(er)i(stores)189 1162 y(this)12 b(request.)19 b(When)13 b(a)e(matc)o(hing)i(receiv)o(e)g(is)f(p)q(osted,)h(the)f (receiv)o(er)h(sends)g(bac)o(k)f(a)f(p)q(ermission-)189 1218 y(to-send)k(message,)f(and)i(the)f(sender)h(no)o(w)f(sends)g(the)h (message.)189 1292 y Fw(standa)o(rd)e(send:)21 b Fv(First)13 b(proto)q(col)h(ma)o(y)e(b)q(e)j(used)f(for)f(short)f(messages,)h(and)h (second)g(proto)q(col)g(for)189 1349 y(long)h(messages.)189 1423 y Fw(bu\013ered)j(send:)25 b Fv(The)18 b(sender)g(copies)g(the)f (message)g(in)o(to)g(a)g(bu\013er)g(and)h(then)f(sends)h(it)g(with)f(a) 189 1479 y(non)o(blo)q(c)o(king)f(send)g(\(using)g(the)f(same)g(proto)q (col)g(as)g(for)f(standard)h(send\).)189 1554 y(Additional)i(con)o (trol)f(messages)g(migh)o(t)g(b)q(e)g(needed)i(for)d(\015o)o(w)h(con)o (trol)f(and)i(error)e(reco)o(v)o(ery)l(.)22 b(Of)189 1610 y(course,)15 b(there)g(are)g(man)o(y)f(other)h(p)q(ossible)i (proto)q(cols.)189 1684 y(Ready)i(send)h(can)g(b)q(e)g(implemen)o(ted)h (as)e(a)f(standard)h(send.)33 b(In)20 b(this)g(case)f(there)g(will)i(b) q(e)f(no)189 1741 y(p)q(erformance)15 b(adv)m(an)o(tage)g(\(or)f (disadv)m(an)o(tage\))h(for)g(the)g(use)h(of)e(ready)h(send.)189 1815 y(A)g(standard)f(send)i(can)f(b)q(e)h(implemen)o(ted)h(as)e(a)f (sync)o(hronous)i(send.)k(In)c(suc)o(h)f(a)g(case,)g(no)g(data)189 1871 y(bu\013ering)g(is)h(needed.)21 b(Ho)o(w)o(ev)o(er,)14 b(man)o(y)h(\(most?\))k(users)c(exp)q(ect)h(some)f(bu\013ering.)189 1945 y(In)h(a)g(m)o(ulti-threaded)i(en)o(vironmen)o(t,)e(the)h (execution)g(of)f(a)f(blo)q(c)o(king)j(comm)o(unication)f(should)189 2002 y(blo)q(c)o(k)k(only)g(the)f(executing)i(thread,)f(allo)o(wing)g (the)g(thread)f(sc)o(heduler)i(to)e(de-sc)o(hedule)j(this)189 2058 y(thread)15 b(and)g(sc)o(hedule)i(another)e(thread)g(for)f (execution.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)g(to)h(implementors.)p Fv(\))75 2201 y Fn(3.5)59 b(Semantics)18 b(of)i(p)r(oint-to-p)r(oint)e (communication)75 2302 y Fv(A)g(v)m(alid)i Fw(MPI)e Fv(implemen)o (tation)h(guaran)o(tees)f(certain)h(general)f(prop)q(erties)h(of)f(p)q (oin)o(t-to-p)q(oin)o(t)h(com-)75 2359 y(m)o(unication,)d(whic)o(h)g (are)f(describ)q(ed)i(in)f(this)f(section.)75 2478 y Fw(Order)46 b Fv(Messages)17 b(are)g Fl(non-overtaking)p Fv(:)25 b(If)18 b(a)f(sender)i(sends)f(t)o(w)o(o)f(messages)g(in)i (succession)g(to)e(the)75 2534 y(same)d(destination,)i(and)f(b)q(oth)g (matc)o(h)f(the)h(same)f(receiv)o(e,)h(then)h(this)f(op)q(eration)g (cannot)f(receiv)o(e)i(the)75 2591 y(second)e(message)e(if)i(the)f (\014rst)g(one)h(is)f(still)i(p)q(ending.)21 b(If)13 b(a)g(receiv)o(er)h(p)q(osts)f(t)o(w)o(o)f(receiv)o(es)i(in)g (succession,)75 2647 y(and)k(b)q(oth)g(matc)o(h)f(the)h(same)g (message,)g(then)g(the)g(second)g(receiv)o(e)h(op)q(eration)f(cannot)g (b)q(e)g(satis\014ed)75 2704 y(b)o(y)g(this)g(message,)f(if)i(the)e (\014rst)h(one)g(is)g(still)h(p)q(ending.)29 b(This)19 b(requiremen)o(t)f(facilitates)h(matc)o(hing)e(of)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 31 39 31 38 bop 75 -100 a Fm(3.5.)34 b(SEMANTICS)16 b(OF)f(POINT-TO-POINT)i (COMMUNICA)l(TION)450 b Fv(31)75 45 y(sends)21 b(to)f(receiv)o(es.)36 b(It)21 b(guaran)o(tees)f(that)f(message-passing)i(co)q(de)g(is)g (deterministic,)j(if)d(pro)q(cesses)75 102 y(are)d(single-threaded)j (and)e(the)g(wildcard)h Fi(MPI)p 913 102 13 2 v 14 w(ANY)p 1012 102 V 15 w(SOURCE)c Fv(is)k(not)e(used)h(in)h(receiv)o(es.)31 b(\(Some)18 b(of)75 158 y(the)c(calls)i(describ)q(ed)g(later,)e(suc)o (h)h(as)f Fw(MPI)p 813 158 14 2 v 15 w(CANCEL)h Fv(or)f Fw(MPI)p 1152 158 V 16 w(W)l(AIT)l(ANY)p Fv(,)g(are)g(additional)i (sources)e(of)75 214 y(nondeterminism.\))166 271 y(If)j(a)g(pro)q(cess) h(has)f(a)g(single)h(thread)f(of)g(execution,)i(then)e(an)o(y)g(t)o(w)o (o)f(comm)o(unications)i(executed)75 327 y(b)o(y)e(this)h(pro)q(cess)g (are)f(ordered.)23 b(On)17 b(the)f(other)g(hand,)h(if)g(the)f(pro)q (cess)h(is)g(m)o(ulti-threaded,)g(then)g(the)75 384 y(seman)o(tics)e (of)f(thread)h(execution)h(ma)o(y)e(not)g(de\014ne)i(a)e(relativ)o(e)i (order)e(b)q(et)o(w)o(een)h(t)o(w)o(o)e(send)j(op)q(erations)75 440 y(executed)23 b(b)o(y)f(t)o(w)o(o)f(distinct)i(threads.)40 b(The)23 b(op)q(erations)f(are)g(logically)h(concurren)o(t,)h(ev)o(en)e (if)h(one)75 497 y(ph)o(ysically)f(precedes)e(the)g(other.)33 b(In)20 b(suc)o(h)g(a)f(case,)i(the)e(t)o(w)o(o)g(messages)g(sen)o(t)g (can)h(b)q(e)g(receiv)o(ed)h(in)75 553 y(an)o(y)f(order.)37 b(Similarly)l(,)24 b(if)d(t)o(w)o(o)e(receiv)o(e)j(op)q(erations)f (that)f(are)g(logically)j(concurren)o(t)e(receiv)o(e)g(t)o(w)o(o)75 610 y(successiv)o(ely)i(sen)o(t)d(messages,)h(then)g(the)g(t)o(w)o(o)f (messages)g(can)h(matc)o(h)f(the)h(t)o(w)o(o)e(receiv)o(es)j(in)f (either)75 666 y(order.)75 772 y Fq(Example)d(3.5)k Fv(An)16 b(example)g(of)f(non-o)o(v)o(ertaking)f(messages.)75 879 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 935 y(IF)h(\(rank.EQ.0\))e(THEN)170 992 y(CALL)i(MPI_BSEND\(buf1,)e (count,)h(MPI_REAL,)f(1,)i(tag,)f(comm,)g(ierr\))170 1048 y(CALL)h(MPI_BSEND\(buf2,)e(count,)h(MPI_REAL,)f(1,)i(tag,)f (comm,)g(ierr\))75 1104 y(ELSE)95 b(!)24 b(rank.EQ.1)170 1161 y(CALL)g(MPI_RECV\(buf1,)e(count,)h(MPI_REAL,)f(0,)i(MPI_ANY_TAG,) e(comm,)h(status,)g(ierr\))170 1217 y(CALL)h(MPI_RECV\(buf2,)e(count,)h (MPI_REAL,)f(0,)i(tag,)f(comm,)g(status,)g(ierr\))75 1274 y(END)g(IF)75 1380 y Fv(The)17 b(message)f(sen)o(t)h(b)o(y)g(the)f (\014rst)h(send)g(m)o(ust)f(b)q(e)i(receiv)o(ed)g(b)o(y)f(the)f (\014rst)h(receiv)o(e,)h(and)f(the)f(message)75 1437 y(sen)o(t)f(b)o(y)g(the)g(second)h(send)g(m)o(ust)e(b)q(e)i(receiv)o (ed)h(b)o(y)e(the)g(second)h(receiv)o(e.)75 1557 y Fw(Progress)45 b Fv(If)13 b(a)g(pair)g(of)f(matc)o(hing)h(send)h(and)f(receiv)o(es)g (ha)o(v)o(e)g(b)q(een)h(initiated)g(on)f(t)o(w)o(o)e(pro)q(cesses,)j (then)75 1613 y(at)k(least)h(one)h(of)e(these)h(t)o(w)o(o)f(op)q (erations)h(will)i(complete,)f(indep)q(enden)o(tly)i(of)c(other)h (actions)g(in)h(the)75 1670 y(system:)e(the)13 b(send)h(op)q(eration)f (will)h(complete,)g(unless)g(the)f(receiv)o(e)h(is)f(satis\014ed)h(b)o (y)e(another)h(message,)75 1726 y(and)h(completes;)g(the)g(receiv)o(e)h (op)q(eration)f(will)h(complete,)g(unless)g(the)f(message)f(sen)o(t)g (is)i(consumed)f(b)o(y)75 1782 y(another)h(matc)o(hing)g(receiv)o(e)h (that)f(w)o(as)f(p)q(osted)h(at)g(the)g(same)g(destination)h(pro)q (cess.)75 1889 y Fq(Example)i(3.6)k Fv(An)16 b(example)g(of)f(t)o(w)o (o,)e(in)o(tert)o(wined)j(matc)o(hing)f(pairs.)75 1983 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2039 y(IF)h(\(rank.EQ.0\))e(THEN)170 2095 y(CALL)i(MPI_BSEND\(buf1,)e (count,)h(MPI_REAL,)f(1,)i(tag1,)f(comm,)g(ierr\))170 2152 y(CALL)h(MPI_SSEND\(buf2,)e(count,)h(MPI_REAL,)f(1,)i(tag2,)f (comm,)g(ierr\))75 2208 y(ELSE)95 b(!)24 b(rank.EQ.1)170 2265 y(CALL)g(MPI_RECV\(buf1,)e(count,)h(MPI_REAL,)f(0,)i(tag2,)f (comm,)g(status,)g(ierr\))170 2321 y(CALL)h(MPI_RECV\(buf2,)e(count,)h (MPI_REAL,)f(0,)i(tag1,)f(comm,)g(status,)g(ierr\))75 2378 y(END)g(IF)75 2472 y Fv(Both)18 b(pro)q(cesses)g(in)o(v)o(ok)o(e)h (their)f(\014rst)g(comm)o(unication)h(call.)30 b(Since)19 b(the)f(\014rst)g(send)h(of)f(pro)q(cess)g(zero)75 2528 y(uses)h(the)f(bu\013ered)h(mo)q(de,)g(it)g(m)o(ust)f(complete,)i (irresp)q(ectiv)o(e)g(of)e(the)g(state)g(of)g(pro)q(cess)h(one.)29 b(Since)75 2584 y(no)15 b(matc)o(hing)h(receiv)o(e)g(is)g(p)q(osted,)g (the)g(message)f(will)i(b)q(e)f(copied)h(in)o(to)e(bu\013er)h(space.)21 b(\(If)15 b(insu\016cien)o(t)75 2641 y(bu\013er)h(space)g(is)h(a)o(v)m (ailable,)g(then)f(the)g(program)f(will)j(fail.\))k(The)16 b(second)h(send)f(is)h(then)f(in)o(v)o(ok)o(ed.)22 b(A)o(t)75 2697 y(that)16 b(p)q(oin)o(t,)g(a)h(matc)o(hing)f(pair)h(of)f(send)h (and)f(receiv)o(e)h(op)q(eration)g(is)g(enabled,)h(and)e(b)q(oth)h(op)q (erations)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 32 40 32 39 bop 75 -100 a Fv(32)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(m)o(ust)f(complete.) 23 b(Pro)q(cess)16 b(one)g(next)g(in)o(v)o(ok)o(es)g(its)g(second)h (receiv)o(e)g(call,)g(whic)o(h)g(will)h(b)q(e)e(satis\014ed)h(b)o(y)75 102 y(the)c(bu\013ered)g(message.)19 b(Note)12 b(that)h(pro)q(cess)g (one)g(receiv)o(ed)h(the)f(messages)f(in)i(the)f(rev)o(erse)f(order)h (they)75 158 y(w)o(ere)i(sen)o(t.)75 278 y Fw(F)o(airness)46 b(MPI)15 b Fv(mak)o(es)h(no)f(guaran)o(tee)h(of)f Fl(fairness)g Fv(in)i(the)f(handling)h(of)f(comm)o(unication.)23 b(Supp)q(ose)75 335 y(that)17 b(a)g(send)h(is)g(p)q(osted.)26 b(Then)18 b(it)g(is)g(p)q(ossible)h(that)d(the)i(destination)g(pro)q(cess)g(rep)q (eatedly)g(p)q(osts)f(a)75 391 y(receiv)o(e)i(that)e(matc)o(hes)g(this) h(send,)h(y)o(et)e(the)h(message)f(is)h(nev)o(er)g(receiv)o(ed,)h(b)q (ecause)g(it)f(is)g(eac)o(h)g(time)75 447 y(o)o(v)o(ertak)o(en)c(b)o(y) i(another)f(message,)g(sen)o(t)h(from)f(another)g(source.)21 b(Similarly)l(,)d(supp)q(ose)f(that)e(a)g(receiv)o(e)75 504 y(w)o(as)f(p)q(osted)i(b)o(y)f(a)g(m)o(ulti-threaded)h(pro)q(cess.) k(Then)c(it)f(is)h(p)q(ossible)h(that)d(messages)h(that)f(matc)o(h)h (this)75 560 y(receiv)o(e)20 b(are)g(rep)q(eatedly)g(receiv)o(ed,)i(y)o (et)d(the)g(receiv)o(e)i(is)f(nev)o(er)f(satis\014ed,)i(b)q(ecause)f (it)g(is)g(o)o(v)o(ertak)o(en)75 617 y(b)o(y)15 b(other)h(receiv)o(es)g (p)q(osted)g(at)e(this)i(no)q(de)h(\(b)o(y)e(other)g(executing)h (threads\).)21 b(It)15 b(is)h(the)g(programmer's)75 673 y(resp)q(onsibilit)o(y)i(to)c(prev)o(en)o(t)h(starv)m(ation)g(in)h(suc) o(h)g(situations.)75 793 y Fw(Resource)23 b(limitations)46 b Fv(An)o(y)22 b(p)q(ending)i(comm)o(unication)e(op)q(eration)h (consumes)f(system)g(resources)75 850 y(that)14 b(are)g(limited.)22 b(Errors)14 b(ma)o(y)g(o)q(ccur)h(when)g(lac)o(k)g(of)f(resources)h (prev)o(en)o(t)f(the)h(execution)h(of)e(an)h Fw(MPI)75 906 y Fv(call.)20 b(A)13 b(qualit)o(y)g(implemen)o(tation)h(will)g(use) f(a)f(\(small\))g(\014xed)h(amoun)o(t)f(of)g(resources)h(for)f(eac)o(h) g(p)q(ending)75 963 y(send)i(in)f(the)g(ready)g(or)g(sync)o(hronous)g (mo)q(de)g(and)g(for)g(eac)o(h)g(p)q(ending)i(receiv)o(e.)20 b(Ho)o(w)o(ev)o(er,)12 b(bu\013er)h(space)75 1019 y(ma)o(y)g(b)q(e)h (consumed)g(to)e(store)h(messages)g(sen)o(t)g(in)h(standard)f(mo)q(de,) h(and)f(m)o(ust)g(b)q(e)h(consumed)g(to)f(store)75 1076 y(messages)e(sen)o(t)g(in)i(bu\013ered)f(mo)q(de,)g(when)g(no)f(matc)o (hing)h(receiv)o(e)h(is)f(a)o(v)m(ailable.)20 b(The)12 b(amoun)o(t)f(of)g(space)75 1132 y(a)o(v)m(ailable)17 b(for)e(bu\013ering)i(will)g(b)q(e)f(m)o(uc)o(h)g(smaller)h(than)e (program)g(data)g(memory)g(on)h(man)o(y)f(systems.)75 1189 y(Then,)g(it)h(will)h(b)q(e)e(easy)g(to)g(write)g(programs)f(that) h(o)o(v)o(errun)f(a)o(v)m(ailable)j(bu\013er)f(space.)166 1245 y Fw(MPI)e Fv(allo)o(ws)h(the)g(user)g(to)f(pro)o(vide)i(bu\013er) f(memory)f(for)g(messages)g(sen)o(t)h(in)h(the)f(bu\013ered)g(mo)q(de.) 75 1301 y(F)l(urthermore,)f Fw(MPI)g Fv(sp)q(eci\014es)j(a)e(detailed)h (op)q(erational)g(mo)q(del)g(for)e(the)h(use)h(of)e(this)i(bu\013er.)j (An)d Fw(MPI)75 1358 y Fv(implemen)o(tation)f(is)f(required)h(to)e(do)h (no)f(w)o(orse)g(than)h(implied)i(b)o(y)d(this)h(mo)q(del.)21 b(This)14 b(allo)o(ws)g(users)g(to)75 1414 y(a)o(v)o(oid)h(bu\013er)g (o)o(v)o(er\015o)o(ws)e(when)i(they)g(use)h(bu\013ered)f(sends.)20 b(Bu\013er)15 b(allo)q(cation)h(and)f(use)h(is)f(describ)q(ed)75 1471 y(in)h(Section)g(3.6.)166 1527 y(A)j(bu\013ered)g(send)h(op)q (eration)f(that)f(cannot)h(complete)h(b)q(ecause)f(of)g(a)g(lac)o(k)g (of)f(bu\013er)h(space)g(is)75 1584 y(erroneous.)36 b(When)20 b(suc)o(h)h(a)f(situation)h(is)g(detected,)h(an)f(error)e(is)i (signalled)i(that)c(ma)o(y)h(cause)h(the)75 1640 y(program)f(to)h (terminate)g(abnormally)l(.)38 b(On)22 b(the)f(other)g(hand,)i(a)e (standard)g(send)g(op)q(eration)h(that)75 1697 y(cannot)16 b(complete)g(b)q(ecause)h(of)f(lac)o(k)g(of)f(bu\013er)h(space)g(will)i (merely)e(blo)q(c)o(k,)h(w)o(aiting)f(for)f(bu\013er)h(space)75 1753 y(to)g(b)q(ecome)h(a)o(v)m(ailable)h(or)e(for)f(a)h(matc)o(hing)h (receiv)o(e)g(to)f(b)q(e)h(p)q(osted.)23 b(This)17 b(b)q(eha)o(vior)g (is)g(preferable)g(in)75 1810 y(man)o(y)h(situations.)31 b(Consider)20 b(a)e(situation)h(where)g(a)g(pro)q(ducer)g(rep)q (eatedly)h(pro)q(duces)g(new)f(v)m(alues)75 1866 y(and)j(sends)g(them)f (to)g(a)g(consumer.)39 b(Assume)22 b(that)e(the)i(pro)q(ducer)g(pro)q (duces)g(new)g(v)m(alues)h(faster)75 1922 y(than)18 b(the)g(consumer)g (can)g(consume)g(them.)28 b(If)18 b(bu\013ered)g(sends)g(are)g(used,)h (then)f(a)f(bu\013er)h(o)o(v)o(er\015o)o(w)75 1979 y(will)i(result.)31 b(Additional)20 b(sync)o(hronization)g(has)e(to)g(b)q(e)h(added)h(to)e (the)g(program)g(so)g(as)g(to)g(prev)o(en)o(t)75 2035 y(this)h(from)e(o)q(ccurring.)29 b(If)19 b(standard)e(sends)i(are)f (used,)h(then)f(the)g(pro)q(ducer)h(will)h(b)q(e)f(automatically)75 2092 y(throttled,)c(as)f(its)i(send)g(op)q(erations)f(will)i(blo)q(c)o (k)f(when)f(bu\013er)h(space)f(is)h(una)o(v)m(ailable.)166 2148 y(In)c(some)f(situations,)h(a)f(lac)o(k)g(of)g(bu\013er)h(space)f (leads)h(to)f(deadlo)q(c)o(k)h(situations.)19 b(This)12 b(is)g(illustrated)75 2205 y(b)o(y)j(the)g(examples)h(b)q(elo)o(w.)75 2311 y Fq(Example)i(3.7)k Fv(An)16 b(exc)o(hange)f(of)g(messages.)75 2417 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2474 y(IF)h(\(rank.EQ.0\))e(THEN)170 2530 y(CALL)i(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\))170 2587 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(status,)g(ierr\))75 2643 y(ELSE)95 b(!)24 b(rank.EQ.1)170 2700 y(CALL)g(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(status,)g(ierr\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 33 41 33 40 bop 75 -100 a Fm(3.5.)34 b(SEMANTICS)16 b(OF)f(POINT-TO-POINT)i (COMMUNICA)l(TION)450 b Fv(33)170 45 y Ft(CALL)24 b(MPI_SEND\(sendbuf,) d(count,)i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))75 102 y(END)g(IF)75 204 y Fv(This)16 b(program)f(will)i(succeed)g(ev)o (en)f(if)g(no)g(bu\013er)g(space)g(for)f(data)g(is)h(a)o(v)m(ailable.) 23 b(The)16 b(standard)f(send)75 261 y(op)q(eration)g(can)h(b)q(e)g (replaced,)g(in)g(this)f(example,)h(with)g(a)e(sync)o(hronous)i(send.) 75 364 y Fq(Example)i(3.8)k Fv(An)16 b(attempt)e(to)h(exc)o(hange)g (messages.)75 455 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\)) 75 512 y(IF)h(\(rank.EQ.0\))e(THEN)170 568 y(CALL)i(MPI_RECV\(recvbuf,) d(count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(status,)g(ierr\))170 625 y(CALL)h(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(ierr\))75 681 y(ELSE)95 b(!)24 b(rank.EQ.1)170 738 y(CALL)g(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(status,)g(ierr\))170 794 y(CALL)h(MPI_SEND\(sendbuf,)d(count,) i(MPI_REAL,)g(0,)h(tag,)f(comm,)g(ierr\))75 850 y(END)g(IF)75 942 y Fv(The)16 b(receiv)o(e)g(op)q(eration)f(of)g(the)h(\014rst)f(pro) q(cess)g(m)o(ust)g(complete)h(b)q(efore)g(its)g(send,)f(and)h(can)f (complete)75 998 y(only)g(if)g(the)g(matc)o(hing)f(send)h(of)f(the)h (second)g(pro)q(cessor)f(is)h(executed.)21 b(The)15 b(receiv)o(e)g(op)q (eration)g(of)f(the)75 1055 y(second)j(pro)q(cess)g(m)o(ust)e(complete) i(b)q(efore)g(its)g(send)g(and)f(can)h(complete)g(only)g(if)g(the)f (matc)o(hing)g(send)75 1111 y(of)d(the)h(\014rst)f(pro)q(cess)g(is)h (executed.)21 b(This)14 b(program)e(will)j(alw)o(a)o(ys)e(deadlo)q(c)o (k.)20 b(The)14 b(same)f(holds)h(for)f(an)o(y)75 1168 y(other)i(send)h(mo)q(de.)75 1271 y Fq(Example)i(3.9)k Fv(An)16 b(exc)o(hange)f(that)g(relies)h(on)f(bu\013ering.)75 1362 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1418 y(IF)h(\(rank.EQ.0\))e(THEN)170 1475 y(CALL)i(MPI_SEND\(sendbuf,)d (count,)i(MPI_REAL,)g(1,)h(tag,)f(comm,)g(ierr\))170 1531 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i(MPI_REAL,)g(1,)h(tag,)f (comm,)g(status,)g(ierr\))75 1588 y(ELSE)95 b(!)24 b(rank.EQ.1)170 1644 y(CALL)g(MPI_SEND\(sendbuf,)d(count,)i(MPI_REAL,)g(0,)h(tag,)f (comm,)g(ierr\))170 1701 y(CALL)h(MPI_RECV\(recvbuf,)d(count,)i (MPI_REAL,)g(0,)h(tag,)f(comm,)g(status,)g(ierr\))75 1757 y(END)g(IF)75 1849 y Fv(The)17 b(message)g(sen)o(t)g(b)o(y)g(eac)o (h)g(pro)q(cess)g(has)g(to)g(b)q(e)h(copied)g(out)f(b)q(efore)g(the)g (send)h(op)q(eration)f(returns)75 1905 y(and)e(the)g(receiv)o(e)h(op)q (eration)f(starts.)k(F)l(or)14 b(the)h(program)f(to)g(complete,)h(it)h (is)f(necessary)g(that)g(at)f(least)75 1961 y(one)22 b(of)f(the)g(t)o(w)o(o)f(messages)h(sen)o(t)g(b)q(e)h(bu\013ered.)39 b(Th)o(us,)23 b(this)f(program)e(can)h(succeed)i(only)f(if)g(the)75 2018 y(comm)o(unication)16 b(system)f(can)g(bu\013er)g(at)g(least)g Fw(count)i Fv(w)o(ords)d(of)h(data.)189 2121 y Fl(A)n(dvic)n(e)f(to)h (users.)40 b Fv(When)14 b(standard)g(send)g(op)q(erations)g(are)g (used,)g(then)h(a)f(deadlo)q(c)o(k)h(situation)189 2177 y(ma)o(y)c(o)q(ccur)h(where)h(b)q(oth)f(pro)q(cesses)g(are)g(blo)q(c)o (k)o(ed)h(b)q(ecause)g(bu\013er)f(space)g(is)h(not)e(a)o(v)m(ailable.) 21 b(The)189 2234 y(same)15 b(will)i(certainly)g(happ)q(en,)f(if)g(the) g(sync)o(hronous)f(mo)q(de)h(is)g(used.)21 b(If)16 b(the)g(bu\013ered)g (mo)q(de)f(is)189 2290 y(used,)k(and)f(not)g(enough)g(bu\013er)h(space) f(is)h(a)o(v)m(ailable,)h(then)e(the)h(program)e(will)i(not)f(complete) 189 2347 y(either.)34 b(Ho)o(w)o(ev)o(er,)19 b(rather)g(than)h(a)f (deadlo)q(c)o(k)i(situation,)g(w)o(e)e(shall)i(ha)o(v)o(e)e(a)h (bu\013er)f(o)o(v)o(er\015o)o(w)189 2403 y(error.)189 2478 y(A)c(program)g(is)h(\\safe")e(if)i(no)g(message)f(bu\013ering)h (is)g(required)h(for)e(the)g(program)g(to)g(complete.)189 2534 y(One)22 b(can)g(replace)h(all)f(sends)h(in)f(suc)o(h)g(program)f (with)h(sync)o(hronous)g(sends,)h(and)f(the)g(pro-)189 2591 y(gram)16 b(will)i(still)h(run)e(correctly)l(.)25 b(This)18 b(conserv)m(ativ)o(e)f(programming)f(st)o(yle)h(pro)o(vides)h (the)f(b)q(est)189 2647 y(p)q(ortabilit)o(y)l(,)e(since)g(program)f (completion)h(do)q(es)g(not)e(dep)q(end)j(on)f(the)f(amoun)o(t)g(of)f (bu\013er)i(space)189 2704 y(a)o(v)m(ailable)i(or)d(in)i(the)g(comm)o (unication)f(proto)q(col)h(used.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 34 42 34 41 bop 75 -100 a Fv(34)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Fv(Man)o(y)e (programmers)f(prefer)i(to)f(ha)o(v)o(e)h(more)f(leew)o(a)o(y)h(and)g (b)q(e)g(able)h(to)e(use)h(the)g(\\unsafe")f(pro-)189 102 y(gramming)c(st)o(yle)g(sho)o(wn)h(in)g(example)h(3.9.)k(In)12 b(suc)o(h)g(cases,)g(the)g(use)g(of)f(standard)g(sends)h(is)g(lik)o (ely)189 158 y(to)i(pro)o(vide)i(the)g(b)q(est)f(compromise)h(b)q(et)o (w)o(een)f(p)q(erformance)h(and)f(robustness:)20 b(qualit)o(y)c(imple-) 189 214 y(men)o(tations)e(will)j(pro)o(vide)f(su\016cien)o(t)g (bu\013ering)f(so)g(that)f(\\common)h(practice")g(programs)f(will)189 271 y(not)j(deadlo)q(c)o(k.)28 b(The)18 b(bu\013ered)g(send)g(mo)q(de)g (can)f(b)q(e)i(used)f(for)f(programs)f(that)h(require)h(more)189 327 y(bu\013ering,)k(or)e(in)i(situations)f(where)g(the)g(programmer)f (w)o(an)o(ts)f(more)i(con)o(trol.)36 b(This)22 b(mo)q(de)189 384 y(migh)o(t)14 b(also)g(b)q(e)h(used)g(for)e(debugging)j(purp)q (oses,)e(as)g(bu\013er)g(o)o(v)o(er\015o)o(w)f(conditions)j(are)e (easier)h(to)189 440 y(diagnose)g(than)g(deadlo)q(c)o(k)h(conditions.) 189 515 y(Non)o(blo)q(c)o(king)h(message-passing)f(op)q(erations,)h(as) f(describ)q(ed)i(in)f(Section)g(3.7,)e(can)i(b)q(e)g(used)g(to)189 572 y(a)o(v)o(oid)f(the)g(need)h(for)f(bu\013ering)g(outgoing)g (messages.)23 b(This)16 b(prev)o(en)o(ts)g(deadlo)q(c)o(ks)h(due)g(to)f (lac)o(k)189 628 y(of)h(bu\013er)h(space,)g(and)g(impro)o(v)o(es)f(p)q (erformance,)h(b)o(y)g(allo)o(wing)g(o)o(v)o(erlap)g(of)f(computation)h (and)189 685 y(comm)o(unication,)12 b(and)f(a)o(v)o(oiding)h(the)f(o)o (v)o(erheads)g(of)f(allo)q(cating)j(bu\013ers)e(and)g(cop)o(ying)g (messages)189 741 y(in)o(to)k(bu\013ers.)20 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fv(\))75 884 y Fn(3.6)59 b(Bu\013er)20 b(allo)r(cation)e(and)i(usage)75 986 y Fv(A)13 b(user)g(ma)o(y)f(sp)q(ecify)i(a)e(bu\013er)h(to)f(b)q(e)h (used)h(for)e(bu\013ering)h(messages)f(sen)o(t)h(in)g(bu\013ered)h(mo)q (de.)19 b(Bu\013er-)75 1042 y(ing)d(is)f(done)h(b)o(y)f(the)g(sender.) 75 1193 y Fw(MPI)p 160 1193 14 2 v 16 w(BUFFER)p 345 1193 V 17 w(A)l(TT)l(A)o(CH\()h(bu\013er,)f(size\))117 1271 y Fk(IN)155 b Fw(bu\013er)478 b Fk(initial)12 b(bu\013er)j (address)g(\(c)o(hoice\))117 1346 y(IN)155 b Fw(size)519 b Fk(bu\013er)15 b(size,)f(in)g(b)o(ytes)g(\(in)o(teger\))75 1470 y Ft(int)23 b(MPI)p 245 1470 15 2 v 17 w(Buffer)p 406 1470 V 17 w(attach\()f(void*)i(buffer,)f(int)g(size\))75 1557 y(MPI)p 150 1557 V 17 w(BUFFER)p 311 1557 V 16 w(ATTACH\()g (BUFFER,)g(SIZE,)g(IERROR\))170 1613 y()g(BUFFER\(*\))170 1669 y(INTEGER)g(SIZE,)h(IERROR)166 1756 y Fv(Pro)o(vides)16 b(to)f Fw(MPI)g Fv(a)h(bu\013er)g(in)h(the)e(user's)h(memory)f(to)h(b)q (e)g(used)h(for)e(bu\013ering)h(outgoing)g(mes-)75 1812 y(sages.)22 b(The)16 b(bu\013er)g(is)h(used)g(only)f(b)o(y)g(messages)g (sen)o(t)f(in)i(bu\013ered)g(mo)q(de.)23 b(Only)17 b(one)f(bu\013er)g (can)g(b)q(e)75 1869 y(attac)o(hed)f(to)f(a)h(pro)q(cess)h(at)e(a)h (time.)-867 b Fx(>)75 2020 y Fw(MPI)p 160 2020 14 2 v 16 w(BUFFER)p 345 2020 V 17 w(DET)l(A)o(CH\()15 b(bu\013er)p 681 2020 V 17 w(addr,)g(size\))117 2097 y Fk(OUT)108 b Fw(bu\013er)p 430 2097 V 17 w(addr)377 b Fk(initial)12 b(bu\013er)j(address)g(\(c)o(hoice\))117 2172 y(OUT)108 b Fw(size)519 b Fk(bu\013er)15 b(size,)f(in)g(b)o(ytes)g(\(in)o (teger\))75 2297 y Ft(int)23 b(MPI)p 245 2297 15 2 v 17 w(Buffer)p 406 2297 V 17 w(detach\()f(void*)i(buffer)p 901 2297 V 16 w(addr,)f(int*)h(size\))75 2383 y(MPI)p 150 2383 V 17 w(BUFFER)p 311 2383 V 16 w(DETACH\()f(BUFFER)p 662 2383 V 16 w(ADDR,)h(SIZE,)f(IERROR\))170 2440 y()g(BUFFER)p 484 2440 V 17 w(ADDR\(*\))170 2496 y(INTEGER)g(SIZE,)h(IERROR)166 2582 y Fv(Detac)o(h)14 b(the)h(bu\013er)g(curren)o(tly)h(asso)q(ciated) f(with)g Fw(MPI)p Fv(.)f(The)h(call)h(returns)f(the)g(address)g(and)g (the)75 2639 y(size)20 b(of)e(the)h(detac)o(hed)g(bu\013er.)31 b(This)20 b(op)q(eration)f(will)h(blo)q(c)o(k)g(un)o(til)g(all)g (messages)e(curren)o(tly)h(in)h(the)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 35 43 35 42 bop 75 -100 a Fm(3.6.)34 b(BUFFER)15 b(ALLOCA)l(TION)i(AND)f(USA) o(GE)843 b Fv(35)75 45 y(bu\013er)12 b(ha)o(v)o(e)f(b)q(een)i (transmitted.)18 b(Up)q(on)13 b(return)e(of)h(this)g(function,)h(the)f (user)f(ma)o(y)g(reuse)i(or)e(deallo)q(cate)75 102 y(the)k(space)h(tak) o(en)e(b)o(y)i(the)f(bu\013er.)75 206 y Fq(Example)j(3.10)23 b Fv(Calls)15 b(to)g(attac)o(h)f(and)h(detac)o(h)h(bu\013ers.)75 299 y Ft(#define)23 b(BUFFSIZE)g(10000)75 355 y(int)g(size)75 411 y(char)g(*buff;)75 468 y(MPI_Buffer_attach\()e(malloc\(BUFFSIZE\),) h(BUFFSIZE\);)75 524 y(/*)i(a)f(buffer)g(of)h(10000)f(bytes)g(can)h (now)f(be)h(used)f(by)h(MPI_Bsend)e(*/)75 581 y(MPI_Buffer_detach\()f (&buff,)i(&size\);)75 637 y(/*)h(Buffer)f(size)g(reduced)g(to)g(zero)h (*/)75 694 y(MPI_Buffer_attach\()d(buff,)j(size\);)75 750 y(/*)g(Buffer)f(of)g(10000)g(bytes)h(available)e(again)h(*/)189 855 y Fl(A)n(dvic)n(e)10 b(to)i(users.)37 b Fv(Ev)o(en)10 b(though)g(the)g(C)g(functions)h Fw(MPI)p 1182 855 14 2 v 16 w(Bu\013er)p 1314 855 V 16 w(attach)h Fv(and)e Fw(MPI)p 1626 855 V 16 w(Bu\013er)p 1758 855 V 16 w(detach)189 911 y Fv(b)q(oth)21 b(ha)o(v)o(e)f(a)g(\014rst)h(argumen)o(t)f(of)g(t)o (yp)q(e)h Fi(void*)p Fv(,)h(these)f(argumen)o(ts)f(are)h(used)g (di\013eren)o(tly:)32 b(A)189 968 y(p)q(oin)o(ter)22 b(to)e(the)i(bu\013er)f(is)h(passed)g(to)f Fw(MPI)p 984 968 V 15 w(Bu\013er)p 1115 968 V 17 w(attach)p Fv(;)k(the)d(address)f (of)g(the)h(p)q(oin)o(ter)f(is)189 1024 y(passed)c(to)g Fw(MPI)p 478 1024 V 16 w(Bu\013er)p 610 1024 V 16 w(detach)p Fv(,)i(so)e(that)g(this)h(call)g(can)g(return)f(the)h(p)q(oin)o(ter)g (v)m(alue.)27 b(\()p Fl(End)18 b(of)189 1080 y(advic)n(e)e(to)g(users.) p Fv(\))189 1185 y Fl(R)n(ationale.)59 b Fv(Both)18 b(argumen)o(ts)g (are)g(de\014ned)i(to)e(b)q(e)h(of)f(t)o(yp)q(e)g Fi(void*)h Fv(\(rather)e(than)h Fi(void*)h Fv(and)189 1241 y Fi(void**)p Fv(,)h(resp)q(ectiv)o(ely\),)i(so)d(as)g(to)g(a)o(v)o(oid)h(complex)g (t)o(yp)q(e)g(casts.)33 b(E.g.,)19 b(in)i(the)f(last)f(example,)189 1298 y Fw(&bu\013)p Fv(,)12 b(whic)o(h)g(is)g(of)f(t)o(yp)q(e)g Fi(cha)o(r**)p Fv(,)h(can)f(b)q(e)h(passed)f(as)g(argumen)o(t)f(to)h Fw(MPI)p 1439 1298 V 16 w(Bu\013er)p 1571 1298 V 16 w(detach)i Fv(without)189 1354 y(t)o(yp)q(e)h(casting.)19 b(If)14 b(the)g(formal)g(parameter)f(had)h(t)o(yp)q(e)g Fi(void**)g Fv(then)g(w)o(e)g(w)o(ould)g(need)h(a)e(t)o(yp)q(e)h(cast)189 1411 y(b)q(efore)h(and)h(after)e(the)h(call.)21 b(\()p Fl(End)16 b(of)g(r)n(ationale.)p Fv(\))1937 1470 y Fx(?)166 1515 y Fv(The)i(statemen)o(ts)f(made)g(in)i(this)f(section)h(describ)q (e)g(the)f(b)q(eha)o(vior)g(of)g Fw(MPI)f Fv(for)g(bu\013ered-mo)q(de) 75 1572 y(sends.)38 b(When)22 b(no)f(bu\013er)g(is)g(curren)o(tly)h (asso)q(ciated,)g Fw(MPI)f Fv(b)q(eha)o(v)o(es)g(as)g(if)g(a)g (zero-sized)h(bu\013er)f(is)75 1628 y(asso)q(ciated)15 b(with)h(the)f(pro)q(cess.)166 1685 y Fw(MPI)20 b Fv(m)o(ust)g(pro)o (vide)h(as)f(m)o(uc)o(h)h(bu\013ering)g(for)f(outgoing)g(messages)g Fl(as)h(if)30 b Fv(outgoing)20 b(message)75 1741 y(data)d(w)o(ere)h (bu\013ered)h(b)o(y)f(the)g(sending)h(pro)q(cess,)g(in)g(the)f(sp)q (eci\014ed)i(bu\013er)e(space,)h(using)f(a)g(circular,)75 1797 y(con)o(tiguous-space)e(allo)q(cation)h(p)q(olicy)l(.)24 b(W)l(e)16 b(outline)h(b)q(elo)o(w)f(a)g(mo)q(del)h(implemen)o(tation)g (that)e(de\014nes)75 1854 y(this)i(p)q(olicy)l(.)24 b Fw(MPI)16 b Fv(ma)o(y)g(pro)o(vide)g(more)g(bu\013ering,)h(and)f(ma)o (y)g(use)g(a)g(b)q(etter)g(bu\013er)h(allo)q(cation)g(algo-)75 1910 y(rithm)i(than)g(describ)q(ed)i(b)q(elo)o(w.)31 b(On)19 b(the)g(other)g(hand,)h Fw(MPI)e Fv(ma)o(y)g(signal)i(an)e (error)h(whenev)o(er)g(the)75 1967 y(simple)14 b(bu\013ering)f(allo)q (cator)f(describ)q(ed)i(b)q(elo)o(w)f(w)o(ould)g(run)g(out)f(of)g (space.)19 b(In)13 b(particular,)g(if)g(no)f(bu\013er)75 2023 y(is)k(explicitly)i(asso)q(ciated)d(with)h(the)f(pro)q(cess,)g (then)h(an)o(y)e(bu\013ered)i(send)g(ma)o(y)e(cause)i(an)f(error.)166 2080 y Fw(MPI)10 b Fv(do)q(es)g(not)g(pro)o(vide)h(mec)o(hanisms)f(for) g(querying)h(or)f(con)o(trolling)h(bu\013ering)g(done)f(b)o(y)g (standard)75 2136 y(mo)q(de)16 b(sends.)22 b(It)15 b(is)h(exp)q(ected)h (that)e(v)o(endors)h(will)h(pro)o(vide)f(suc)o(h)g(information)g(for)f (their)h(implemen-)75 2193 y(tations.)189 2297 y Fl(R)n(ationale.)53 b Fv(There)18 b(is)g(a)f(wide)h(sp)q(ectrum)g(of)f(p)q(ossible)i (implemen)o(tations)g(of)e(bu\013ered)h(com-)189 2353 y(m)o(unication:)32 b(bu\013ering)22 b(can)f(b)q(e)h(done)g(at)e (sender,)j(at)d(receiv)o(er,)j(or)e(b)q(oth;)j(bu\013ers)d(can)g(b)q(e) 189 2410 y(dedicated)d(to)e(one)g(sender-receiv)o(er)i(pair,)f(or)g(b)q (e)g(shared)g(b)o(y)f(all)i(comm)o(unications;)f(bu\013ering)189 2466 y(can)g(b)q(e)h(done)g(in)g(real)g(or)e(in)j(virtual)f(memory;)f (it)h(can)f(use)h(dedicated)h(memory)l(,)e(or)g(memory)189 2523 y(shared)d(b)o(y)h(other)f(pro)q(cesses;)h(bu\013er)f(space)h(ma)o (y)f(b)q(e)h(allo)q(cated)h(statically)f(or)f(b)q(e)i(c)o(hanged)f(dy-) 189 2579 y(namically;)f(etc.)19 b(It)12 b(do)q(es)g(not)g(seem)g (feasible)h(to)f(pro)o(vide)g(a)g(p)q(ortable)g(mec)o(hanism)h(for)e (querying)189 2636 y(or)16 b(con)o(trolling)h(bu\013ering)h(that)d(w)o (ould)i(b)q(e)h(compatible)f(with)g(all)h(these)f(c)o(hoices,)g(y)o(et) f(pro)o(vide)189 2692 y(meaningful)g(information.)k(\()p Fl(End)c(of)g(r)n(ationale.)p Fv(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 36 44 36 43 bop 75 -100 a Fv(36)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fj(3.6.1)49 b(Mo)q(del)18 b(implementation)e(of)g(bu\013ered)g(mo)q(de)75 131 y Fv(The)22 b(mo)q(del)g(implemen)o(tation)h(uses)f(the)f(pac)o (king)h(and)g(unpac)o(king)g(functions)g(describ)q(ed)i(in)e(Sec-)75 187 y(tion)15 b(3.13)f(and)i(the)f(non)o(blo)q(c)o(king)i(comm)o (unication)f(functions)f(describ)q(ed)j(in)e(Section)g(3.7.)166 244 y(W)l(e)21 b(assume)f(that)g(a)g(circular)i(queue)f(of)f(p)q (ending)j(message)d(en)o(tries)h(\(PME\))e(is)i(main)o(tained.)75 300 y(Eac)o(h)14 b(en)o(try)h(con)o(tains)f(a)g(comm)o(unication)i (request)e(handle)i(that)e(iden)o(ti\014es)i(a)e(p)q(ending)j(non)o (blo)q(c)o(king)75 357 y(send,)h(a)e(p)q(oin)o(ter)i(to)e(the)h(next)g (en)o(try)f(and)i(the)f(pac)o(k)o(ed)g(message)f(data.)25 b(The)17 b(en)o(tries)g(are)g(stored)f(in)75 413 y(successiv)o(e)j(lo)q (cations)h(in)f(the)f(bu\013er.)29 b(F)l(ree)19 b(space)f(is)h(a)o(v)m (ailable)h(b)q(et)o(w)o(een)f(the)f(queue)h(tail)g(and)g(the)75 470 y(queue)d(head.)166 526 y(A)f(bu\013ered)h(send)g(call)g(results)f (in)i(the)e(execution)h(of)f(the)g(follo)o(wing)h(co)q(de.)143 620 y Fx(\017)23 b Fv(T)l(ra)o(v)o(erse)14 b(sequen)o(tially)k(the)e (PME)f(queue)h(from)f(head)h(to)o(w)o(ards)e(the)i(tail,)g(deleting)h (all)g(en)o(tries)189 676 y(for)f(comm)o(unications)h(that)f(ha)o(v)o (e)g(completed,)i(up)f(to)f(the)g(\014rst)h(en)o(try)f(with)h(an)f (uncompleted)189 733 y(request;)e(up)q(date)i(queue)g(head)g(to)e(p)q (oin)o(t)i(to)f(that)f(en)o(try)l(.)143 827 y Fx(\017)23 b Fv(Compute)10 b(the)h(n)o(um)o(b)q(er,)h Fw(n)p Fv(,)g(of)f(b)o(ytes) f(needed)j(to)d(store)g(an)h(en)o(try)g(for)f(the)h(new)g(message.)18 b(An)11 b(up-)-2034 b Fx(>)189 883 y Fv(p)q(er)41 b(b)q(ound)h(on)e Fw(n)i Fv(can)e(b)q(e)i(computed)f(as)f(follo)o(ws:)71 b(A)41 b(call)h(to)e(the)h(function)189 940 y Fw(MPI)p 274 940 14 2 v 15 w(P)l(A)o(CK)p 404 940 V 17 w(SIZE\(count,)19 b(datat)o(yp)q(e,)h(comm,)c(size\))p Fv(,)i(with)g(the)h Ft(count,)k(datatype)16 b Fv(and)j Ft(comm)189 996 y Fv(argumen)o(ts)i(used)j(in)f(the)g Fw(MPI)p 760 996 V 16 w(BSEND)g Fv(call,)i(returns)e(an)f(upp)q(er)i(b)q(ound)f(on)g (the)g(amoun)o(t)189 1052 y(of)c(space)h(needed)i(to)d(bu\013er)h(the)g (message)g(data)f(\(see)h(Section)h(3.13\).)33 b(The)20 b(MPI)g(constan)o(t)189 1109 y Fi(MPI)p 266 1109 13 2 v 14 w(BSEND)p 415 1109 V 15 w(O)o(VERHEAD)c Fv(pro)o(vides)j(an)f(upp) q(er)h(b)q(ound)g(on)f(the)g(additional)i(space)e(consumed)189 1165 y(b)o(y)d(the)g(en)o(try)g(\(e.g.,)f(for)g(p)q(oin)o(ters)i(or)e (en)o(v)o(elop)q(e)j(information\).)-1469 b Fx(?)143 1259 y(\017)23 b Fv(Find)14 b(the)g(next)g(con)o(tiguous)f(empt)o(y)h (space)g(of)f Fw(n)h Fv(b)o(ytes)g(in)g(bu\013er)g(\(space)g(follo)o (wing)g(queue)h(tail,)189 1316 y(or)h(space)i(at)e(start)g(of)g (bu\013er)h(if)h(queue)g(tail)g(is)f(to)q(o)g(close)g(to)g(end)g(of)g (bu\013er\).)25 b(If)17 b(space)h(is)f(not)189 1372 y(found)e(then)h (raise)f(bu\013er)g(o)o(v)o(er\015o)o(w)f(error.)143 1466 y Fx(\017)23 b Fv(App)q(end)14 b(to)d(end)j(of)e(PME)g(queue)h(in) g(con)o(tiguous)g(space)f(the)h(new)g(en)o(try)f(that)f(con)o(tains)i (request)189 1522 y(handle,)j(next)f(p)q(oin)o(ter)h(and)f(pac)o(k)o (ed)g(message)g(data;)f Fw(MPI)p 1221 1522 14 2 v 16 w(P)l(A)o(CK)h Fv(is)h(used)g(to)e(pac)o(k)h(data.)143 1616 y Fx(\017)23 b Fv(P)o(ost)14 b(non)o(blo)q(c)o(king)i(send)g (\(standard)f(mo)q(de\))g(for)f(pac)o(k)o(ed)i(data.)143 1710 y Fx(\017)23 b Fv(Return)75 1853 y Fn(3.7)59 b(Nonblo)r(cking)20 b(communication)75 1955 y Fv(One)e(can)g(impro)o(v)o(e)f(p)q (erformance)g(on)h(man)o(y)e(systems)h(b)o(y)h(o)o(v)o(erlapping)f (comm)o(unication)i(and)e(com-)75 2011 y(putation.)35 b(This)21 b(is)g(esp)q(ecially)i(true)d(on)g(systems)g(where)g(comm)o (unication)h(can)g(b)q(e)g(executed)g(au-)75 2068 y(tonomously)12 b(b)o(y)h(an)g(in)o(telligen)o(t)h(comm)o(unication)g(con)o(troller.)19 b(Ligh)o(t-w)o(eigh)o(t)13 b(threads)f(are)h(one)g(mec)o(h-)75 2124 y(anism)21 b(for)g(ac)o(hieving)h(suc)o(h)g(o)o(v)o(erlap.)37 b(An)21 b(alternativ)o(e)h(mec)o(hanism)g(that)e(often)h(leads)g(to)g (b)q(etter)75 2180 y(p)q(erformance)15 b(is)g(to)g(use)g Fq(non)o(blo)q(c)o(king)j(comm)o(unication)p Fv(.)j(A)15 b(non)o(blo)q(c)o(king)h Fq(send)g(start)f Fv(call)h(ini-)75 2237 y(tiates)g(the)h(send)g(op)q(eration,)f(but)h(do)q(es)f(not)g (complete)h(it.)24 b(The)17 b(send)g(start)e(call)i(will)h(return)f(b)q (efore)75 2293 y(the)g(message)f(w)o(as)f(copied)j(out)e(of)g(the)g (send)h(bu\013er.)24 b(A)17 b(separate)e Fq(send)k(complete)e Fv(call)h(is)f(needed)75 2350 y(to)12 b(complete)i(the)f(comm)o (unication,)h(i.e.,)g(to)e(v)o(erify)h(that)g(the)g(data)f(has)h(b)q (een)h(copied)h(out)e(of)f(the)h(send)75 2406 y(bu\013er.)19 b(With)13 b(suitable)i(hardw)o(are,)d(the)h(transfer)g(of)f(data)h(out) g(of)f(the)h(sender)h(memory)f(ma)o(y)f(pro)q(ceed)75 2463 y(concurren)o(tly)g(with)h(computations)e(done)h(at)g(the)g (sender)g(after)f(the)h(send)g(w)o(as)f(initiated)j(and)e(b)q(efore)g (it)75 2519 y(completed.)21 b(Similarly)l(,)16 b(a)e(non)o(blo)q(c)o (king)i Fq(receiv)o(e)g(start)h(call)f Fv(initiates)g(the)f(receiv)o(e) g(op)q(eration,)g(but)75 2576 y(do)q(es)f(not)g(complete)h(it.)k(The)14 b(call)i(will)f(return)f(b)q(efore)g(a)g(message)g(is)g(stored)f(in)o (to)h(the)g(receiv)o(e)h(bu\013er.)75 2632 y(A)i(separate)f Fq(receiv)o(e)i(complete)f Fv(call)h(is)f(needed)h(to)e(complete)h(the) g(receiv)o(e)g(op)q(eration)g(and)g(v)o(erify)75 2689 y(that)11 b(the)i(data)e(has)h(b)q(een)h(receiv)o(ed)h(in)o(to)e(the)g (receiv)o(e)h(bu\013er.)19 b(With)12 b(suitable)i(hardw)o(are,)d(the)i (transfer)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 37 45 37 44 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fv(37)75 45 y(of)16 b(data)g(in)o(to)g(the)h(receiv)o(er)g(memory)f (ma)o(y)f(pro)q(ceed)j(concurren)o(tly)f(with)g(computations)f(done)h (after)75 102 y(the)d(receiv)o(e)g(w)o(as)f(initiated)j(and)e(b)q (efore)g(it)g(completed.)20 b(The)14 b(use)g(of)f(non)o(blo)q(c)o(king) i(receiv)o(es)g(ma)o(y)e(also)75 158 y(a)o(v)o(oid)18 b(system)g(bu\013ering)h(and)f(memory-to-memory)f(cop)o(ying,)i(as)f (information)h(is)g(pro)o(vided)g(early)75 214 y(on)c(the)g(lo)q (cation)h(of)f(the)g(receiv)o(e)i(bu\013er.)166 271 y(Non)o(blo)q(c)o (king)f(send)f(start)f(calls)h(can)g(use)g(the)g(same)f(four)h(mo)q (des)g(as)f(blo)q(c)o(king)i(sends:)k Fw(standa)o(rd)p Fv(,)75 327 y Fw(bu\013ered)p Fv(,)c Fw(synchronous)i Fv(and)e Fw(ready)p Fv(.)22 b(These)16 b(carry)f(the)h(same)f(meaning.) 23 b(Sends)16 b(of)g(all)g(mo)q(des,)g Fw(ready)75 384 y Fv(excepted,)f(can)f(b)q(e)g(started)f(whether)h(a)g(matc)o(hing)g (receiv)o(e)g(has)g(b)q(een)h(p)q(osted)f(or)g(not;)f(a)h(non)o(blo)q (c)o(king)75 440 y Fw(ready)e Fv(send)g(can)f(b)q(e)i(started)d(only)j (if)f(a)f(matc)o(hing)h(receiv)o(e)g(is)g(p)q(osted.)19 b(In)12 b(all)h(cases,)f(the)f(send)h(start)f(call)75 497 y(is)g(lo)q(cal:)19 b(it)11 b(returns)f(immediately)l(,)j(irresp)q (ectiv)o(e)f(of)e(the)h(status)f(of)g(other)g(pro)q(cesses.)19 b(If)10 b(the)h(call)h(causes)75 553 y(some)17 b(system)f(resource)h (to)f(b)q(e)i(exhausted,)f(then)g(it)g(will)i(fail)f(and)f(return)g(an) f(error)h(co)q(de.)25 b(Qualit)o(y)75 610 y(implemen)o(tations)14 b(of)f Fw(MPI)f Fv(should)i(ensure)f(that)g(this)g(happ)q(ens)h(only)f (in)h(\\pathological")f(cases.)20 b(That)75 666 y(is,)13 b(an)f Fw(MPI)f Fv(implemen)o(tation)i(should)g(b)q(e)g(able)f(to)g (supp)q(ort)g(a)f(large)h(n)o(um)o(b)q(er)g(of)g(p)q(ending)i(non)o (blo)q(c)o(king)75 723 y(op)q(erations.)166 779 y(The)j(send-complete)i (call)f(returns)f(when)h(data)e(has)h(b)q(een)i(copied)f(out)f(of)f (the)i(send)f(bu\013er.)26 b(It)75 835 y(ma)o(y)14 b(carry)h (additional)i(meaning,)e(dep)q(ending)j(on)d(the)g(send)h(mo)q(de.)166 892 y(If)e(the)g(send)h(mo)q(de)f(is)h Fw(synchronous)p Fv(,)g(then)g(the)f(send)g(can)h(complete)f(only)h(if)f(a)g(matc)o (hing)g(receiv)o(e)75 948 y(has)j(started.)24 b(That)17 b(is,)g(a)g(receiv)o(e)h(has)f(b)q(een)h(p)q(osted,)f(and)g(has)g(b)q (een)h(matc)o(hed)f(with)g(the)h(send.)25 b(In)75 1005 y(this)18 b(case,)f(the)g(send-complete)i(call)f(is)g(non-lo)q(cal.)27 b(Note)16 b(that)h(a)g(sync)o(hronous,)g(non)o(blo)q(c)o(king)h(send)75 1061 y(ma)o(y)13 b(complete,)h(if)g(matc)o(hed)g(b)o(y)f(a)g(non)o(blo) q(c)o(king)i(receiv)o(e,)g(b)q(efore)f(the)f(receiv)o(e)i(complete)f (call)h(o)q(ccurs.)75 1118 y(\(It)h(can)h(complete)g(as)f(so)q(on)g(as) g(the)h(sender)g(\\kno)o(ws")e(the)i(transfer)e(will)j(complete,)g(but) e(b)q(efore)h(the)75 1174 y(receiv)o(er)f(\\kno)o(ws")e(the)h(transfer) g(will)i(complete.\))166 1231 y(If)f(the)g(send)h(mo)q(de)f(is)h Fw(bu\013ered)g Fv(then)f(the)g(message)g(m)o(ust)g(b)q(e)g(bu\013ered) h(if)f(there)h(is)f(no)g(p)q(ending)75 1287 y(receiv)o(e.)27 b(In)18 b(this)g(case,)f(the)g(send-complete)i(call)f(is)g(lo)q(cal,)h (and)e(m)o(ust)g(succeed)h(irresp)q(ectiv)o(e)h(of)e(the)75 1344 y(status)d(of)h(a)g(matc)o(hing)g(receiv)o(e.)166 1400 y(If)f(the)g(send)g(mo)q(de)g(is)g Fw(standa)o(rd)h Fv(then)g(the)e(send-complete)j(call)f(ma)o(y)e(return)g(b)q(efore)h(a) g(matc)o(hing)75 1456 y(receiv)o(e)h(o)q(ccurred,)f(if)g(the)g(message) f(is)i(bu\013ered.)20 b(On)14 b(the)g(other)f(hand,)h(the)g (send-complete)h(ma)o(y)e(not)75 1513 y(complete)19 b(un)o(til)h(a)e (matc)o(hing)g(receiv)o(e)h(o)q(ccurred,)h(and)f(the)f(message)g(w)o (as)g(copied)h(in)o(to)g(the)f(receiv)o(e)75 1569 y(bu\013er.)166 1626 y(Non)o(blo)q(c)o(king)e(sends)g(can)f(b)q(e)h(matc)o(hed)f(with)h (blo)q(c)o(king)g(receiv)o(es,)g(and)f(vice-v)o(ersa.)189 1732 y Fl(A)n(dvic)n(e)h(to)i(users.)50 b Fv(The)17 b(completion)g(of)g (a)f(send)h(op)q(eration)g(ma)o(y)f(b)q(e)i(dela)o(y)o(ed,)f(for)f (standard)189 1789 y(mo)q(de,)d(and)g(m)o(ust)f(b)q(e)h(dela)o(y)o(ed,) h(for)e(sync)o(hronous)g(mo)q(de,)h(un)o(til)h(a)f(matc)o(hing)f (receiv)o(e)i(is)f(p)q(osted.)189 1845 y(The)k(use)g(of)f(non)o(blo)q (c)o(king)j(sends)e(in)h(these)f(t)o(w)o(o)e(cases)i(allo)o(ws)g(the)g (sender)h(to)e(pro)q(ceed)i(ahead)189 1901 y(of)c(the)g(receiv)o(er,)h (so)f(that)g(the)h(computation)f(is)h(more)f(toleran)o(t)g(of)g (\015uctuations)h(in)g(the)g(sp)q(eeds)189 1958 y(of)f(the)i(t)o(w)o(o) e(pro)q(cesses.)189 2033 y(Non)o(blo)q(c)o(king)k(sends)g(in)f(the)h (bu\013ered)f(and)h(ready)f(mo)q(des)g(ha)o(v)o(e)g(a)f(more)h(limited) i(impact.)26 b(A)189 2090 y(non)o(blo)q(c)o(king)16 b(send)g(will)h (return)e(as)g(so)q(on)g(as)g(p)q(ossible,)i(whereas)e(a)g(blo)q(c)o (king)h(send)g(will)h(return)189 2146 y(after)h(the)h(data)g(has)g(b)q (een)h(copied)h(out)e(of)f(the)i(sender)f(memory)l(.)32 b(The)19 b(use)h(of)f(non)o(blo)q(c)o(king)189 2202 y(sends)j(is)g(adv) m(an)o(tageous)f(in)h(these)g(cases)g(only)g(if)g(data)f(cop)o(ying)h (can)g(b)q(e)g(concurren)o(t)g(with)189 2259 y(computation.)189 2334 y(The)e(message-passing)g(mo)q(del)g(implies)i(that)d(comm)o (unication)i(is)f(initiated)h(b)o(y)f(the)g(sender.)189 2390 y(The)d(comm)o(unication)g(will)h(generally)g(ha)o(v)o(e)e(lo)o(w) o(er)g(o)o(v)o(erhead)h(if)g(a)f(receiv)o(e)i(is)f(already)g(p)q(osted) 189 2447 y(when)22 b(the)h(sender)g(initiates)g(the)f(comm)o(unication) h(\(data)f(can)g(b)q(e)h(mo)o(v)o(ed)f(directly)h(to)f(the)189 2503 y(receiv)o(e)e(bu\013er,)f(and)g(there)g(is)h(no)f(need)h(to)e (queue)i(a)e(p)q(ending)j(send)f(request\).)31 b(Ho)o(w)o(ev)o(er,)18 b(a)189 2560 y(receiv)o(e)g(op)q(eration)f(can)h(complete)g(only)g (after)e(the)i(matc)o(hing)f(send)h(has)f(o)q(ccurred.)27 b(The)18 b(use)189 2616 y(of)13 b(non)o(blo)q(c)o(king)i(receiv)o(es)f (allo)o(ws)g(one)g(to)f(ac)o(hiev)o(e)h(lo)o(w)o(er)f(comm)o(unication) i(o)o(v)o(erheads)e(without)189 2673 y(blo)q(c)o(king)j(the)f(receiv)o (er)h(while)h(it)f(w)o(aits)e(for)h(the)g(send.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fv(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 38 46 38 45 bop 75 -100 a Fv(38)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fj(3.7.1)49 b(Communication)17 b(Objects)75 131 y Fv(Non)o(blo)q(c)o(king)i(comm)o (unications)f(use)g(opaque)g Fw(request)h Fv(ob)s(jects)e(to)g(iden)o (tify)i(comm)o(unication)g(op)q(er-)75 187 y(ations)e(and)g(matc)o(h)f (the)i(op)q(eration)f(that)f(initiates)i(the)f(comm)o(unication)h(with) f(the)g(op)q(eration)h(that)75 244 y(terminates)f(it.)27 b(These)17 b(are)g(system)g(ob)s(jects)g(that)f(are)h(accessed)h(via)g (a)f(handle.)27 b(A)17 b(request)h(ob)s(ject)75 300 y(iden)o(ti\014es)c (v)m(arious)f(prop)q(erties)h(of)e(a)g(comm)o(unication)h(op)q (eration,)g(suc)o(h)g(as)f(the)h(send)g(mo)q(de,)g(the)g(com-)75 357 y(m)o(unication)i(bu\013er)f(that)g(is)h(asso)q(ciated)f(with)h (it,)f(its)g(con)o(text,)g(the)g(tag)f(and)i(destination)g(argumen)o (ts)75 413 y(to)e(b)q(e)h(used)f(for)g(a)g(send,)h(or)e(the)i(tag)e (and)i(source)f(argumen)o(ts)f(to)h(b)q(e)h(used)g(for)e(a)h(receiv)o (e.)20 b(In)14 b(addition,)75 470 y(this)i(ob)s(ject)e(stores)h (information)g(ab)q(out)g(the)g(status)g(of)f(the)i(p)q(ending)h(comm)o (unication)f(op)q(eration.)75 591 y Fj(3.7.2)49 b(Communication)17 b(initiation)75 677 y Fv(W)l(e)g(use)h(the)g(same)f(naming)g(con)o(v)o (en)o(tions)h(as)f(for)g(blo)q(c)o(king)h(comm)o(unication:)25 b(a)17 b(pre\014x)h(of)f Fw(B)p Fv(,)g Fw(S)p Fv(,)g(or)75 734 y Fw(R)f Fv(is)h(used)f(for)g Fw(bu\013ered)p Fv(,)g Fw(synchronous)i Fv(or)e Fw(ready)g Fv(mo)q(de.)22 b(In)17 b(addition)g(a)f(pre\014x)g(of)g Fw(I)f Fv(\(for)g Fw(immediate)p Fv(\))75 790 y(indicates)i(that)d(the)h(call)i(is)e(non)o(blo)q(c)o (king.)75 941 y Fw(MPI)p 160 941 14 2 v 16 w(ISEND\(buf,)g(count,)h (datat)o(yp)q(e,)h(dest,)f(tag,)f(comm,)e(request\))117 1018 y Fk(IN)155 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 1093 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 1169 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e (\(handle\))117 1244 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1319 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1394 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 1469 y(OUT)108 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1593 y Ft(int)23 b(MPI)p 245 1593 15 2 v 17 w(Isend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1593 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 1650 y(int)h(tag,)f(MPI)p 683 1650 V 17 w(Comm)g(comm,)g(MPI)p 1034 1650 V 17 w(Request)g(*request\))75 1736 y(MPI)p 150 1736 V 17 w(ISEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g (COMM,)h(REQUEST,)e(IERROR\))170 1793 y()h(BUF\(*\))170 1849 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1936 y Fv(Start)14 b(a)h(standard)g(mo)q(de,)g(non)o(blo)q (c)o(king)h(send.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 39 47 39 46 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fv(39)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(IBSEND\(buf,)15 b(count,)i(datat)o(yp)q(e,)f(dest,)g(tag,)f(comm,)e(request\))117 122 y Fk(IN)155 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 197 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 273 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e (\(handle\))117 348 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 423 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 498 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Ft(int)23 b(MPI)p 245 698 15 2 v 17 w(Ibsend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 698 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(IBSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR\))170 897 y()g(BUF\(*\))170 953 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1040 y Fv(Start)14 b(a)h(bu\013ered)h(mo)q(de,)f(non)o(blo) q(c)o(king)h(send.)75 1191 y Fw(MPI)p 160 1191 14 2 v 16 w(ISSEND\(buf,)f(count,)i(datat)o(yp)q(e,)f(dest,)g(tag,)f(comm,)e (request\))117 1268 y Fk(IN)155 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1343 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1418 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h(bu\013er)g(elemen)o(t)e(\(handle\))117 1493 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1569 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1644 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 1719 y(OUT)108 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1843 y Ft(int)23 b(MPI)p 245 1843 15 2 v 17 w(Issend\(void*)f(buf,)i (int)f(count,)g(MPI)p 1026 1843 V 17 w(Datatype)g(datatype,)f(int)i (dest,)393 1900 y(int)g(tag,)f(MPI)p 683 1900 V 17 w(Comm)g(comm,)g (MPI)p 1034 1900 V 17 w(Request)g(*request\))75 1986 y(MPI)p 150 1986 V 17 w(ISSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2043 y()g(BUF\(*\))170 2099 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 2186 y Fv(Start)14 b(a)h(sync)o(hronous)g(mo)q(de,)g(non)o (blo)q(c)o(king)i(send.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 40 48 40 47 bop 75 -100 a Fv(40)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(IRSEND\(buf,)f(count,)g(datat)o(yp)q(e,)g(dest,)g (tag,)f(comm,)e(request\))117 122 y Fk(IN)155 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 197 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 273 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(send)h (bu\013er)g(elemen)o(t)e(\(handle\))117 348 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 423 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 498 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Ft(int)23 b(MPI)p 245 698 15 2 v 17 w(Irsend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 698 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(IRSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR\))170 897 y()g(BUF\(*\))170 953 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)166 1040 y Fv(Start)14 b(a)h(ready)g(mo)q(de)h(non)o(blo)q(c)o (king)g(send.)75 1191 y Fw(MPI)p 160 1191 14 2 v 16 w(IRECV)f(\(buf,)h (count,)g(datat)o(yp)q(e,)g(source,)g(tag,)e(comm,)f(request\))117 1268 y Fk(OUT)108 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(receiv) o(e)h(bu\013er)g(\(c)o(hoice\))117 1343 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 1418 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(receiv)o(e)h(bu\013er)g(elemen) o(t)e(\(handle\))117 1493 y(IN)155 b Fw(source)468 b Fk(rank)14 b(of)f(source)j(\(in)o(teger\))117 1569 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 1644 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 1719 y(OUT)108 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1843 y Ft(int)23 b(MPI)p 245 1843 15 2 v 17 w(Irecv\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1843 V 17 w(Datatype)g(datatype,)g(int)g(source,)393 1900 y(int)h(tag,)f(MPI)p 683 1900 V 17 w(Comm)g(comm,)g(MPI)p 1034 1900 V 17 w(Request)g(*request\))75 1986 y(MPI)p 150 1986 V 17 w(IRECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g (COMM,)g(REQUEST,)g(IERROR\))170 2043 y()g(BUF\(*\))170 2099 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR)166 2186 y Fv(Start)14 b(a)h(non)o(blo)q(c)o(king)i (receiv)o(e.)166 2242 y(These)f(calls)h(allo)q(cate)g(a)e(comm)o (unication)i(request)f(ob)s(ject)f(and)h(asso)q(ciate)g(it)g(with)g (the)g(request)75 2298 y(handle)j(\(the)f(argumen)o(t)g Fw(request)p Fv(\).)30 b(The)18 b(request)h(can)f(b)q(e)h(used)f(later) h(to)e(query)i(the)f(status)f(of)h(the)75 2355 y(comm)o(unication)e(or) f(w)o(ait)f(for)h(its)g(completion.)166 2411 y(A)i(non)o(blo)q(c)o (king)i(send)f(call)g(indicates)h(that)e(the)g(system)g(ma)o(y)g(start) f(cop)o(ying)h(data)g(out)g(of)g(the)75 2468 y(send)g(bu\013er.)24 b(The)17 b(sender)g(should)g(not)g(access)f(an)o(y)g(part)g(of)g(the)h (send)g(bu\013er)g(after)e(a)i(non)o(blo)q(c)o(king)75 2524 y(send)f(op)q(eration)f(is)h(called,)g(un)o(til)h(the)e(send)h (completes.)166 2581 y(A)d(non)o(blo)q(c)o(king)h(receiv)o(e)g(call)g (indicates)g(that)e(the)h(system)f(ma)o(y)g(start)g(writing)h(data)f (in)o(to)h(the)g(re-)75 2637 y(ceiv)o(e)f(bu\013er.)19 b(The)11 b(receiv)o(er)h(should)h(not)e(access)g(an)o(y)g(part)g(of)g (the)g(receiv)o(e)i(bu\013er)e(after)g(a)g(non)o(blo)q(c)o(king)75 2694 y(receiv)o(e)16 b(op)q(eration)g(is)f(called,)i(un)o(til)f(the)f (receiv)o(e)h(completes.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 41 49 41 48 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fv(41)75 45 y Fj(3.7.3)49 b(Communication)17 b(Completion)75 131 y Fv(The)g(functions)g Fw(MPI)p 452 131 14 2 v 16 w(W)l(AIT)f Fv(and)g Fw(MPI)p 768 131 V 16 w(TEST)h Fv(are)f(used)h(to) f(complete)h(a)f(non)o(blo)q(c)o(king)h(comm)o(unica-)75 187 y(tion.)i(The)13 b(completion)h(of)e(a)g(send)h(op)q(eration)g (indicates)h(that)e(the)g(sender)h(is)g(no)o(w)f(free)h(to)f(up)q(date) h(the)75 244 y(lo)q(cations)k(in)f(the)h(send)f(bu\013er)g(\(the)g (send)g(op)q(eration)g(itself)h(lea)o(v)o(es)f(the)g(con)o(ten)o(t)g (of)f(the)h(send)h(bu\013er)75 300 y(unc)o(hanged\).)25 b(It)17 b(do)q(es)g(not)f(indicate)i(that)e(the)h(message)f(has)h(b)q (een)g(receiv)o(ed,)h(rather,)e(it)h(ma)o(y)f(ha)o(v)o(e)75 357 y(b)q(een)g(bu\013ered)g(b)o(y)f(the)h(comm)o(unication)g (subsystem.)k(Ho)o(w)o(ev)o(er,)14 b(if)i(a)f Fw(synchronous)i Fv(mo)q(de)f(send)g(w)o(as)75 413 y(used,)f(the)g(completion)h(of)f (the)f(send)i(op)q(eration)f(indicates)h(that)e(a)h(matc)o(hing)g (receiv)o(e)h(w)o(as)e(initiated,)75 470 y(and)h(that)g(the)g(message)g (will)i(ev)o(en)o(tually)f(b)q(e)g(receiv)o(ed)g(b)o(y)f(this)h(matc)o (hing)f(receiv)o(e.)166 526 y(The)20 b(completion)h(of)e(a)g(receiv)o (e)i(op)q(eration)f(indicates)h(that)e(the)h(receiv)o(e)g(bu\013er)g (con)o(tains)g(the)75 583 y(receiv)o(ed)d(message,)e(the)h(receiv)o(er) h(is)f(no)o(w)f(free)h(to)f(access)h(it,)g(and)g(that)f(the)h(status)f (ob)s(ject)g(is)h(set.)22 b(It)75 639 y(do)q(es)13 b(not)g(indicate)i (that)d(the)h(matc)o(hing)g(send)h(op)q(eration)f(has)g(completed)h (\(but)f(indicates,)i(of)d(course,)75 695 y(that)i(the)i(send)g(w)o(as) e(initiated\).)166 752 y(W)l(e)28 b(shall)g(use)g(the)g(follo)o(wing)g (terminology:)45 b(A)28 b Fq(n)o(ull)g Fv(handle)h(is)f(a)g(handle)h (with)f(v)m(alue)75 808 y Fi(MPI)p 152 808 13 2 v 14 w(REQUEST)p 354 808 V 14 w(NULL)p Fv(.)17 b(A)g(p)q(ersisten)o(t)g (request)f(and)h(the)g(handle)h(to)e(it)g(are)h Fq(inactiv)o(e)g Fv(if)g(the)g(request)75 865 y(is)i(not)f(asso)q(ciated)g(with)h(an)o (y)f(ongoing)h(comm)o(unication)g(\(see)f(Section)h(3.9\).)28 b(A)19 b(handle)h(is)e Fq(activ)o(e)75 921 y Fv(if)g(it)f(is)h(neither) g(n)o(ull)h(nor)e(inactiv)o(e.)27 b(A)17 b Fq(empt)o(y)f Fv(status)h(is)h(a)e(status)h(whic)o(h)h(is)g(set)f(to)f(return)h Fw(tag)h(=)62 b Fx(>)75 978 y Fw(MPI)p 160 978 14 2 v 16 w(ANY)p 268 978 V 17 w(T)l(A)o(G)p Fv(,)15 b Fw(source)g(=)g(MPI)p 665 978 V 15 w(ANY)p 772 978 V 18 w(SOURCE)p Fv(,)g(and)g(is)g(also)g (in)o(ternally)h(con\014gured)f(so)f(that)g(calls)75 1034 y(to)f Fw(MPI)p 214 1034 V 16 w(GET)p 318 1034 V 17 w(COUNT)i Fv(and)f Fw(MPI)p 675 1034 V 16 w(GET)p 779 1034 V 16 w(ELEMENTS)g Fv(return)g Fw(count)i(=)e(0)p Fv(.)19 b(W)l(e)14 b(set)g(a)f(status)h(v)m(ariable)75 1091 y(to)g(empt)o(y)h(when)g(the)g(v)m(alue)h(returned)f(b)o(y)g(it)g (is)h(not)e(signi\014can)o(t.)21 b(Status)14 b(is)h(set)g(in)h(this)f (w)o(a)o(y)f(so)g(as)h(to)75 1147 y(prev)o(en)o(t)g(errors)f(due)i(to)f (accesses)g(of)g(stale)g(information.)854 b Fx(?)75 1298 y Fw(MPI)p 160 1298 V 16 w(W)l(AIT\(request,)16 b(status\))117 1375 y Fk(INOUT)62 b Fw(request)452 b Fk(request)16 b(\(handle\))117 1450 y(OUT)108 b Fw(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\)) 75 1575 y Ft(int)23 b(MPI)p 245 1575 15 2 v 17 w(Wait\(MPI)p 454 1575 V 16 w(Request)g(*request,)g(MPI)p 972 1575 V 17 w(Status)g(*status\))75 1661 y(MPI)p 150 1661 V 17 w(WAIT\(REQUEST,)f(STATUS,)h(IERROR\))170 1718 y(INTEGER)g(REQUEST,) g(STATUS\(MPI)p 819 1718 V 16 w(STATUS)p 979 1718 V 16 w(SIZE\),)g(IERROR)166 1804 y Fv(A)12 b(call)i(to)d Fw(MPI)p 430 1804 14 2 v 16 w(W)l(AIT)h Fv(returns)g(when)h(the)f(op)q(eration)h (iden)o(ti\014ed)h(b)o(y)e Fw(request)i Fv(is)f(complete.)19 b(If)13 b(the)75 1861 y(comm)o(unication)19 b(ob)s(ject)e(asso)q (ciated)i(with)f(this)g(request)g(w)o(as)g(created)g(b)o(y)g(a)f(non)o (blo)q(c)o(king)j(send)e(or)75 1917 y(receiv)o(e)d(call,)h(then)e(the)h (ob)s(ject)f(is)h(deallo)q(cated)h(b)o(y)e(the)h(call)g(to)f Fw(MPI)p 1281 1917 V 16 w(W)l(AIT)g Fv(and)h(the)f(request)h(handle)75 1974 y(is)h(set)f(to)f Fi(MPI)p 324 1974 13 2 v 15 w(REQUEST)p 527 1974 V 13 w(NULL)p Fv(.)i Fw(MPI)p 754 1974 14 2 v 16 w(W)l(AIT)f Fv(is)h(a)f(non-lo)q(cal)h(op)q(eration.)166 2030 y(The)j(call)h(returns,)f(in)g Fw(status)p Fv(,)j(information)c (on)h(the)g(completed)h(op)q(eration.)30 b(The)19 b(con)o(ten)o(t)f(of) 75 2086 y(the)d(status)f(ob)s(ject)h(for)f(a)h(receiv)o(e)h(op)q (eration)f(can)h(b)q(e)f(accessed)h(as)f(describ)q(ed)i(in)f(section)f (3.2.5.)k(The)75 2143 y(status)13 b(ob)s(ject)g(for)f(a)i(send)g(op)q (eration)f(ma)o(y)g(b)q(e)h(queried)h(b)o(y)f(a)f(call)h(to)f Fw(MPI)p 1385 2143 V 16 w(TEST)p 1515 2143 V 17 w(CANCELLED)g Fv(\(see)75 2199 y(Section)j(3.8\).)1613 b Fx(>)166 2256 y Fv(One)15 b(is)g(allo)o(w)o(ed)f(to)g(call)h Fw(MPI)p 688 2256 V 16 w(W)l(AIT)f Fv(with)h(a)e(n)o(ull)j(or)e(inactiv)o(e)h Fw(request)h Fv(argumen)o(t.)j(In)14 b(this)h(case)75 2312 y(the)g(op)q(eration)h(returns)f(immediately)i(with)e(empt)o(y)g Fw(status)p Fv(.)798 b Fx(?)189 2419 y Fl(A)n(dvic)n(e)17 b(to)j(users.)56 b Fv(Successful)20 b(return)e(of)f Fw(MPI)p 1064 2419 V 16 w(W)l(AIT)h Fv(after)f(a)h Fw(MPI)p 1444 2419 V 15 w(IBSEND)h Fv(implies)h(that)189 2475 y(the)f(user)g(send)g (bu\013er)g(can)g(b)q(e)h(reused)f(|)h(i.e.,)f(data)g(has)f(b)q(een)j (sen)o(t)d(out)h(or)f(copied)i(in)o(to)f(a)189 2531 y(bu\013er)10 b(attac)o(hed)h(with)g Fw(MPI)p 680 2531 V 16 w(BUFFER)p 865 2531 V 17 w(A)l(TT)l(A)o(CH)p Fv(.)g(Note)g(that,)f(at)g(this)i(p)q (oin)o(t,)f(w)o(e)g(can)g(no)g(longer)189 2588 y(cancel)17 b(the)f(send)h(\(see)f(Sec.)23 b(3.8\).)e(If)16 b(a)g(matc)o(hing)g (receiv)o(e)h(is)g(nev)o(er)f(p)q(osted,)g(then)h(the)f(bu\013er)189 2644 y(cannot)k(b)q(e)h(freed.)37 b(This)21 b(runs)g(somewhat)f(coun)o (ter)g(to)g(the)h(stated)f(goal)h(of)f Fw(MPI)p 1690 2644 V 16 w(CANCEL)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 42 50 42 49 bop 75 -100 a Fv(42)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Fv(\(alw)o(a)o(ys)e(b)q (eing)i(able)g(to)e(free)h(program)f(space)h(that)g(w)o(as)f(committed) h(to)f(the)h(comm)o(unication)189 102 y(subsystem\).)j(\()p Fl(End)d(of)g(advic)n(e)g(to)h(users.)p Fv(\))189 208 y Fl(A)n(dvic)n(e)j(to)h(implementors.)72 b Fv(In)21 b(a)f(m)o(ulti-threaded)i(en)o(vironmen)o(t,)f(a)f(call)i(to)40 b Fw(MPI)p 1749 208 14 2 v 16 w(W)l(AIT)189 264 y Fv(should)12 b(blo)q(c)o(k)g(only)g(the)g(calling)h(thread,)f(allo)o(wing)g(the)g (thread)f(sc)o(heduler)i(to)e(sc)o(hedule)i(another)189 321 y(thread)i(for)f(execution.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)g (to)h(implementors.)p Fv(\))75 521 y Fw(MPI)p 160 521 V 16 w(TEST\(request,)f(\015ag,)f(status\))117 599 y Fk(INOUT)62 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))117 674 y(OUT)108 b Fw(\015ag)518 b Fi(true)15 b Fk(if)e(op)q(eration)h(completed)f(\(logical\))117 749 y(OUT)108 b Fw(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\)) 75 873 y Ft(int)23 b(MPI)p 245 873 15 2 v 17 w(Test\(MPI)p 454 873 V 16 w(Request)g(*request,)g(int)g(*flag,)g(MPI)p 1234 873 V 17 w(Status)g(*status\))75 960 y(MPI)p 150 960 V 17 w(TEST\(REQUEST,)f(FLAG,)h(STATUS,)g(IERROR\))170 1016 y(LOGICAL)g(FLAG)170 1073 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 1073 V 16 w(STATUS)p 979 1073 V 16 w(SIZE\),)g(IERROR)166 1159 y Fv(A)17 b(call)i(to)e Fw(MPI)p 446 1159 14 2 v 15 w(TEST)h Fv(returns)f Fw(\015ag)h(=)f(true)h Fv(if)g(the)g(op)q (eration)f(iden)o(ti\014ed)j(b)o(y)d Fw(request)i Fv(is)f(com-)75 1216 y(plete.)37 b(In)21 b(suc)o(h)g(a)f(case,)i(the)e(status)g(ob)s (ject)g(is)h(set)f(to)g(con)o(tain)h(information)g(on)f(the)h (completed)75 1272 y(op)q(eration;)15 b(if)h(the)f(comm)o(unication)h (ob)s(ject)e(w)o(as)h(created)g(b)o(y)g(a)g(non)o(blo)q(c)o(king)h (send)g(or)f(receiv)o(e,)h(then)75 1329 y(it)g(is)f(deallo)q(cated)i (and)f(the)f(request)h(handle)g(is)g(set)f(to)g Fi(MPI)p 1127 1329 13 2 v 14 w(REQUEST)p 1329 1329 V 14 w(NULL)p Fv(.)h(The)f(call)i(returns)e Fw(\015ag)75 1385 y(=)g(false)p Fv(,)g(otherwise.)20 b(In)c(this)g(case,)f(the)g(v)m(alue)h(of)f(the)g (status)f(ob)s(ject)h(is)h(unde\014ned.)21 b Fw(MPI)p 1664 1385 14 2 v 16 w(TEST)16 b Fv(is)f(a)75 1442 y(lo)q(cal)h(op)q (eration.)166 1498 y(The)11 b(return)g(status)f(ob)s(ject)h(for)f(a)h (receiv)o(e)h(op)q(eration)f(carries)g(information)g(that)g(can)g(b)q (e)g(accessed)75 1554 y(as)19 b(describ)q(ed)h(in)g(section)g(3.2.5.)29 b(The)19 b(status)g(ob)s(ject)f(for)g(a)h(send)g(op)q(eration)h (carries)f(information)75 1611 y(that)14 b(can)i(b)q(e)g(accessed)f(b)o (y)h(a)e(call)j(to)d Fw(MPI)p 823 1611 V 16 w(TEST)p 953 1611 V 17 w(CANCELLED)h Fv(\(see)g(Section)h(3.8\).)-1739 b Fx(>)166 1667 y Fv(One)18 b(is)f(allo)o(w)o(ed)g(to)g(call)h Fw(MPI)p 702 1667 V 15 w(TEST)f Fv(with)h(a)e(n)o(ull)j(or)d(inactiv)o (e)i Fw(request)g Fv(argumen)o(t.)25 b(In)17 b(suc)o(h)g(a)75 1724 y(case)e(the)g(op)q(eration)h(returns)f(with)h Fw(\015ag)f(=)g (true)h Fv(and)f(empt)o(y)g Fw(status)p Fv(.)-1450 b Fx(?)166 1780 y Fv(The)22 b(functions)h Fw(MPI)p 554 1780 V 15 w(W)l(AIT)f Fv(and)g Fw(MPI)p 881 1780 V 16 w(TEST)g Fv(can)g(b)q(e)g(used)h(to)e(complete)h(b)q(oth)g(sends)h(and) 75 1837 y(receiv)o(es.)189 1943 y Fl(A)n(dvic)n(e)e(to)h(users.)79 b Fv(The)22 b(use)g(of)f(the)h(non)o(blo)q(c)o(king)h Fw(MPI)p 1267 1943 V 16 w(TEST)f Fv(call)g(allo)o(ws)g(the)g(user)g(to) 189 1999 y(sc)o(hedule)e(alternativ)o(e)f(activities)h(within)g(a)f (single)h(thread)f(of)f(execution.)32 b(An)19 b(ev)o(en)o(t-driv)o(en) 189 2056 y(thread)14 b(sc)o(heduler)h(can)f(b)q(e)h(em)o(ulated)g(with) f(p)q(erio)q(dic)i(calls)f(to)f Fw(MPI)p 1382 2056 V 16 w(TEST)p Fv(.)g(\()p Fl(End)g(of)i(advic)n(e)f(to)189 2112 y(users.)p Fv(\))-160 2174 y Fx(>)189 2268 y Fl(R)n(ationale.)55 b Fv(The)18 b(function)g Fw(MPI)p 807 2268 V 16 w(TEST)g Fv(returns)f(with)h Fw(\015ag)g(=)g(true)g Fv(exactly)g(in)h(those)e (situ-)189 2325 y(ations)f(where)g(the)g(function)h Fw(MPI)p 799 2325 V 16 w(W)l(AIT)f Fv(returns;)g(b)q(oth)g(functions)h(return)f (in)h(suc)o(h)g(case)f(the)189 2381 y(same)f(v)m(alue)i(in)g Fw(status)p Fv(.)24 b(Th)o(us,)15 b(a)h(blo)q(c)o(king)h(W)l(ait)f(can) g(b)q(e)g(easily)h(replaced)g(b)o(y)f(a)g(non)o(blo)q(c)o(king)189 2438 y(T)l(est.)j(\()p Fl(End)d(of)g(r)n(ationale.)p Fv(\))-160 2499 y Fx(?)75 2594 y Fq(Example)i(3.11)23 b Fv(Simple)16 b(usage)f(of)g(non)o(blo)q(c)o(king)i(op)q(erations)e (and)g Fw(MPI)p 1378 2594 V 16 w(W)l(AIT)p Fv(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 43 51 43 50 bop 75 -100 a Fm(3.7.)29 b(NONBLOCKING)17 b(COMMUNICA)l(TION)868 b Fv(43)75 45 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 102 y(IF\(rank.EQ.0\))f(THEN)170 158 y(CALL)i(MPI_ISEND\(a\(1\),)e(10,) h(MPI_REAL,)g(1,)g(tag,)h(comm,)f(request,)g(ierr\))170 214 y(****)h(do)f(some)h(computation)e(to)i(mask)f(latency)g(****)170 271 y(CALL)h(MPI_WAIT\(request,)d(status,)i(ierr\))75 327 y(ELSE)170 384 y(CALL)h(MPI_IRECV\(a\(1\),)e(15,)h(MPI_REAL,)g(0,)g (tag,)h(comm,)f(request,)g(ierr\))170 440 y(****)h(do)f(some)h (computation)e(to)i(mask)f(latency)g(****)170 497 y(CALL)h (MPI_WAIT\(request,)d(status,)i(ierr\))75 553 y(END)g(IF)166 659 y Fv(A)14 b(request)g(ob)s(ject)f(can)h(b)q(e)g(deallo)q(cated)h (without)f(w)o(aiting)g(for)f(the)h(asso)q(ciated)g(comm)o(unication)75 716 y(to)h(complete,)g(b)o(y)g(using)h(the)f(follo)o(wing)h(op)q (eration.)75 867 y Fw(MPI)p 160 867 14 2 v 16 w(REQUEST)p 379 867 V 18 w(FREE\(request\))117 944 y Fk(INOUT)62 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1069 y Ft(int)23 b(MPI)p 245 1069 15 2 v 17 w(Request)p 430 1069 V 16 w(free\(MPI)p 638 1069 V 17 w(Request)f(*request\))75 1155 y(MPI)p 150 1155 V 17 w(REQUEST)p 335 1155 V 16 w(FREE\(REQUEST,)g(IERROR\))170 1211 y(INTEGER)h(REQUEST,)g(IERROR)166 1298 y Fv(Mark)18 b(the)h(request)g(ob)s(ject)g(for)f(deallo)q(cation)j (and)e(set)g Fw(request)i Fv(to)d Fi(MPI)p 1467 1298 13 2 v 14 w(REQUEST)p 1669 1298 V 14 w(NULL)p Fv(.)i(An)75 1354 y(ongoing)g(comm)o(unication)h(that)e(is)i(asso)q(ciated)f(with)h (the)f(request)g(will)i(b)q(e)f(allo)o(w)o(ed)f(to)g(complete.)75 1411 y(The)15 b(request)h(will)h(b)q(e)e(deallo)q(cated)i(only)f(after) e(its)i(completion.)189 1517 y Fl(R)n(ationale.)51 b Fv(The)17 b Fw(MPI)p 622 1517 14 2 v 16 w(REQUEST)p 841 1517 V 18 w(FREE)g Fv(mec)o(hanism)h(is)f(pro)o(vided)h(for)f(reasons)f (of)h(p)q(erfor-)189 1574 y(mance)e(and)g(con)o(v)o(enience)i(on)e(the) h(sending)g(side.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fv(\))189 1680 y Fl(A)n(dvic)n(e)i(to)i(users.)68 b Fv(Once)21 b(a)f(request)f(is)i(freed)f(b)o(y)g(a)f(call)i(to)f Fw(MPI)p 1419 1680 V 15 w(REQUEST)p 1637 1680 V 18 w(FREE)p Fv(,)g(it)g(is)189 1736 y(not)c(p)q(ossible)j(to)e(c)o(hec)o(k)g(for)g (the)g(successful)i(completion)f(of)f(the)g(asso)q(ciated)g(comm)o (unication)189 1793 y(with)h(calls)h(to)e Fw(MPI)p 542 1793 V 16 w(W)l(AIT)h Fv(or)g Fw(MPI)p 830 1793 V 15 w(TEST)p Fv(.)g(Also,)h(if)f(an)g(error)g(o)q(ccurs)g(subsequen)o(tly)h (during)189 1849 y(the)f(comm)o(unication,)i(an)f(error)f(co)q(de)i (cannot)e(b)q(e)i(returned)f(to)f(the)h(user)g(|)g(suc)o(h)g(an)g (error)189 1906 y(m)o(ust)d(b)q(e)i(treated)f(as)f(fatal.)26 b(Questions)18 b(arise)f(as)g(to)f(ho)o(w)h(one)g(kno)o(ws)g(when)h (the)f(op)q(erations)189 1962 y(ha)o(v)o(e)f(completed)i(when)g(using)f Fw(MPI)p 840 1962 V 16 w(REQUEST)p 1059 1962 V 18 w(FREE)p Fv(.)g(Dep)q(ending)i(on)e(the)g(program)e(logic,)189 2019 y(there)j(ma)o(y)g(b)q(e)g(other)g(w)o(a)o(ys)f(in)j(whic)o(h)f (the)f(program)f(kno)o(ws)h(that)f(certain)i(op)q(erations)f(ha)o(v)o (e)189 2075 y(completed)f(and)f(this)h(mak)o(es)e(usage)h(of)g Fw(MPI)p 986 2075 V 16 w(REQUEST)p 1205 2075 V 18 w(FREE)g Fv(practical.)24 b(F)l(or)15 b(example,)i(an)189 2131 y(activ)o(e)j(send)g(request)g(could)h(b)q(e)g(freed)g(when)f(the)g (logic)h(of)f(the)g(program)f(is)h(suc)o(h)h(that)e(the)189 2188 y(receiv)o(er)i(sends)h(a)f(reply)h(to)e(the)h(message)g(sen)o(t)g (|)g(the)g(arriv)m(al)h(of)f(the)g(reply)h(informs)f(the)189 2244 y(sender)f(that)f(the)g(send)h(has)g(completed)g(and)g(the)g(send) g(bu\013er)f(can)h(b)q(e)g(reused.)34 b(An)20 b(activ)o(e)189 2301 y(receiv)o(e)e(request)f(should)h(nev)o(er)f(b)q(e)g(freed)h(as)e (the)h(receiv)o(er)h(will)h(ha)o(v)o(e)d(no)h(w)o(a)o(y)f(to)g(v)o (erify)i(that)189 2357 y(the)h(receiv)o(e)i(has)e(completed)h(and)g (the)g(receiv)o(e)g(bu\013er)g(can)f(b)q(e)h(reused.)34 b(\()p Fl(End)19 b(of)i(advic)n(e)f(to)189 2414 y(users.)p Fv(\))75 2520 y Fq(Example)e(3.12)23 b Fv(An)15 b(example)h(using)g Fw(MPI)p 863 2520 V 16 w(REQUEST)p 1082 2520 V 18 w(FREE)p Fv(.)75 2626 y Ft(CALL)23 b(MPI_COMM_RANK\(MPI_COMM_WORLD)o(,)e(rank\)) 75 2683 y(IF\(rank.EQ.0\))h(THEN)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 44 52 44 51 bop 75 -100 a Fv(44)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Ft(DO)24 b(i=1,)f(n)218 102 y(CALL)g(MPI_ISEND\(outval,)f(1,)i(MPI_REAL,)e(1,)i (0,)f(req,)h(ierr\))218 158 y(CALL)f(MPI_REQUEST_FREE\(req,)e(ierr\)) 218 214 y(CALL)i(MPI_IRECV\(inval,)f(1,)i(MPI_REAL,)e(1,)i(0,)g(req,)f (ierr\))218 271 y(CALL)g(MPI_WAIT\(req,)g(status,)f(ierr\))170 327 y(END)i(DO)75 384 y(ELSE)95 b(!)24 b(rank.EQ.1)170 440 y(CALL)g(MPI_IRECV\(inval,)d(1,)j(MPI_REAL,)f(0,)g(0,)h(req,)f (ierr\))170 497 y(CALL)h(MPI_WAIT\(req,)e(status\))170 553 y(DO)i(I=1,)f(n-1)242 610 y(CALL)g(MPI_ISEND\(outval,)f(1,)h (MPI_REAL,)g(0,)h(0,)f(req,)h(ierr\))242 666 y(CALL)f (MPI_REQUEST_FREE\(req,)e(ierr\))242 723 y(CALL)i(MPI_IRECV\(inval,)f (1,)i(MPI_REAL,)e(0,)i(0,)f(req,)h(ierr\))242 779 y(CALL)f (MPI_WAIT\(req,)f(status,)h(ierr\))170 835 y(END)h(DO)170 892 y(CALL)g(MPI_ISEND\(outval,)d(1,)j(MPI_REAL,)f(0,)g(0,)h(req,)f (ierr\))170 948 y(CALL)h(MPI_WAIT\(req,)e(status\))75 1005 y(END)h(IF)75 1124 y Fj(3.7.4)49 b(Semantics)15 b(of)i(Nonblo)q(cking)h(Communications)75 1210 y Fv(The)11 b(seman)o(tics)f(of)h(non)o(blo)q(c)o(king)g(comm)o(unication)h(is)f (de\014ned)h(b)o(y)e(suitably)i(extending)g(the)e(de\014nitions)75 1266 y(in)16 b(Section)g(3.5.)75 1383 y Fw(Order)46 b Fv(Non)o(blo)q(c)o(king)12 b(comm)o(unication)f(op)q(erations)g(are)g (ordered)g(according)g(to)f(the)h(execution)h(order)75 1440 y(of)h(the)g(calls)h(that)e(initiate)i(the)f(comm)o(unication.)20 b(The)13 b(non-o)o(v)o(ertaking)g(requiremen)o(t)g(of)g(Section)h(3.5) 75 1496 y(is)i(extended)g(to)e(non)o(blo)q(c)o(king)j(comm)o (unication,)f(with)f(this)h(de\014nition)h(of)d(order)h(b)q(eing)i (used.)75 1582 y Fq(Example)h(3.13)23 b Fv(Message)14 b(ordering)i(for)e(non)o(blo)q(c)o(king)j(op)q(erations.)75 1661 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 1717 y(IF)h(\(RANK.EQ.0\))e(THEN)218 1773 y(CALL)h(MPI_ISEND\(a,)g(1,)g (MPI_REAL,)g(1,)h(0,)f(comm,)g(r1,)h(ierr\))218 1830 y(CALL)f(MPI_ISEND\(b,)g(1,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(r2,)h (ierr\))75 1886 y(ELSE)95 b(!)24 b(rank.EQ.1)218 1943 y(CALL)f(MPI_IRECV\(a,)g(1,)g(MPI_REAL,)g(0,)h(MPI_ANY_TAG,)e(comm,)h (r1,)g(ierr\))218 1999 y(CALL)g(MPI_IRECV\(b,)g(1,)g(MPI_REAL,)g(0,)h (0,)f(comm,)g(r2,)h(ierr\))75 2056 y(END)f(IF)75 2112 y(CALL)g(MPI_WAIT\(r1,status\))75 2169 y(CALL)g(MPI_WAIT\(r2,status\)) 75 2248 y Fv(The)11 b(\014rst)f(send)h(of)f(pro)q(cess)h(zero)f(will)j (matc)o(h)d(the)g(\014rst)g(receiv)o(e)i(of)e(pro)q(cess)h(one,)g(ev)o (en)g(if)g(b)q(oth)f(messages)75 2304 y(are)15 b(sen)o(t)g(b)q(efore)g (pro)q(cess)h(one)f(executes)h(either)g(receiv)o(e.)75 2421 y Fw(Progress)45 b Fv(A)12 b(call)h(to)f Fw(MPI)p 545 2421 14 2 v 15 w(W)l(AIT)g Fv(that)g(completes)g(a)g(receiv)o(e)h (will)h(ev)o(en)o(tually)f(terminate)f(and)g(return)75 2478 y(if)21 b(a)f(matc)o(hing)g(send)h(has)f(b)q(een)h(started,)g (unless)g(the)g(send)g(is)f(satis\014ed)h(b)o(y)f(another)g(receiv)o (e.)36 b(In)75 2534 y(particular,)20 b(if)f(the)g(matc)o(hing)g(send)g (is)g(non)o(blo)q(c)o(king,)i(then)e(the)g(receiv)o(e)g(should)h (complete)f(ev)o(en)g(if)75 2591 y(no)c(call)i(is)e(executed)i(b)o(y)e (the)g(sender)h(to)f(complete)h(the)f(send.)20 b(Similarly)l(,)e(a)d (call)h(to)f Fw(MPI)p 1651 2591 V 15 w(W)l(AIT)h Fv(that)75 2647 y(completes)i(a)e(send)i(will)h(ev)o(en)o(tually)f(return)f(if)g (a)g(matc)o(hing)g(receiv)o(e)h(has)f(b)q(een)h(started,)f(unless)h (the)75 2704 y(receiv)o(e)e(is)g(satis\014ed)f(b)o(y)g(another)g(send,) g(and)g(ev)o(en)h(if)f(no)g(call)h(is)g(executed)g(to)e(complete)i(the) f(receiv)o(e.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 45 53 45 52 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fv(45)75 45 y Fq(Example)18 b(3.14)23 b Fv(An)15 b(illustration)i(of) d(progress)h(seman)o(tics.)75 138 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f (rank,)h(ierr\))75 194 y(IF)h(\(RANK.EQ.0\))e(THEN)218 251 y(CALL)h(MPI_SSEND\(a,)g(1,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\)) 218 307 y(CALL)g(MPI_SEND\(b,)g(1,)g(MPI_REAL,)g(1,)h(1,)f(comm,)g (ierr\))75 363 y(ELSE)95 b(!)24 b(rank.EQ.1)218 420 y(CALL)f (MPI_IRECV\(a,)g(1,)g(MPI_REAL,)g(0,)h(0,)f(comm,)g(r,)h(ierr\))218 476 y(CALL)f(MPI_RECV\(b,)g(1,)g(MPI_REAL,)g(0,)h(1,)f(comm,)g(ierr\)) 218 533 y(CALL)g(MPI_WAIT\(r,)g(status,)g(ierr\))75 589 y(END)g(IF)166 682 y Fv(This)14 b(co)q(de)f(should)h(not)f(deadlo)q(c)o (k)h(in)g(a)e(correct)h Fw(MPI)g Fv(implemen)o(tation.)20 b(The)13 b(\014rst)g(sync)o(hronous)75 738 y(send)22 b(of)e(pro)q(cess)h(zero)g(m)o(ust)f(complete)i(after)e(pro)q(cess)i (one)f(p)q(osts)f(the)h(matc)o(hing)g(\(non)o(blo)q(c)o(king\))75 795 y(receiv)o(e)15 b(ev)o(en)f(if)h(pro)q(cess)f(one)g(has)g(not)f(y)o (et)h(reac)o(hed)g(the)g(completing)h(w)o(ait)f(call.)20 b(Th)o(us,)14 b(pro)q(cess)g(zero)75 851 y(will)j(con)o(tin)o(ue)f(and) f(execute)h(the)f(second)h(send,)f(allo)o(wing)h(pro)q(cess)g(one)f(to) g(complete)h(execution.)166 956 y(If)d(an)f Fw(MPI)p 354 956 14 2 v 16 w(TEST)h Fv(that)f(completes)h(a)g(receiv)o(e)g(is)g (rep)q(eatedly)h(called)g(with)f(the)g(same)f(argumen)o(ts,)75 1012 y(and)g(a)g(matc)o(hing)g(send)h(has)f(b)q(een)h(started,)f(then)g (the)g(call)i(will)f(ev)o(en)o(tually)g(return)f Fw(\015ag)g(=)h(true)p Fv(,)g(unless)75 1069 y(the)g(send)g(is)g(satis\014ed)g(b)o(y)f (another)h(receiv)o(e.)19 b(If)13 b(an)g Fw(MPI)p 1046 1069 V 16 w(TEST)f Fv(that)g(completes)h(a)g(send)g(is)g(rep)q(eatedly) 75 1125 y(called)i(with)f(the)g(same)f(argumen)o(ts,)g(and)h(a)f(matc)o (hing)g(receiv)o(e)i(has)e(b)q(een)i(started,)e(then)h(the)g(call)g (will)75 1182 y(ev)o(en)o(tually)i(return)f Fw(\015ag)g(=)h(true)p Fv(,)f(unless)i(the)e(receiv)o(e)h(is)g(satis\014ed)g(b)o(y)f(another)f (send.)75 1303 y Fj(3.7.5)49 b(Multiple)17 b(Completions)75 1389 y Fv(It)h(is)h(con)o(v)o(enien)o(t)g(to)f(b)q(e)h(able)g(to)f(w)o (ait)g(for)f(the)i(completion)g(of)f(an)o(y)l(,)h(some,)f(or)g(all)h (the)g(op)q(erations)75 1445 y(in)h(a)f(list,)i(rather)d(than)i(ha)o (ving)f(to)g(w)o(ait)g(for)f(a)h(sp)q(eci\014c)i(message.)32 b(A)19 b(call)i(to)d Fw(MPI)p 1600 1445 V 16 w(W)l(AIT)l(ANY)i Fv(or)75 1502 y Fw(MPI)p 160 1502 V 16 w(TEST)l(ANY)d Fv(can)f(b)q(e)g(used)g(to)f(w)o(ait)h(for)f(the)h(completion)h(of)e (one)h(out)f(of)g(sev)o(eral)h(op)q(erations.)22 b(A)75 1558 y(call)d(to)d Fw(MPI)p 303 1558 V 16 w(W)l(AIT)l(ALL)h Fv(or)g Fw(MPI)p 665 1558 V 16 w(TEST)l(ALL)h Fv(can)f(b)q(e)h(used)g (to)f(w)o(ait)g(for)g(all)h(p)q(ending)i(op)q(erations)d(in)75 1615 y(a)e(list.)23 b(A)16 b(call)h(to)e Fw(MPI)p 484 1615 V 15 w(W)l(AITSOME)i Fv(or)e Fw(MPI)p 892 1615 V 16 w(TESTSOME)h Fv(can)g(b)q(e)h(used)f(to)f(complete)i(all)f(enabled) 75 1671 y(op)q(erations)f(in)h(a)f(list.)75 1822 y Fw(MPI)p 160 1822 V 16 w(W)l(AIT)l(ANY)h(\(count,)g(a)o(rra)o(y)p 640 1822 V 14 w(of)p 691 1822 V 16 w(requests,)h(index,)f(status\))117 1899 y Fk(IN)155 b Fw(count)482 b Fk(list)14 b(length)f(\(in)o(teger\)) 117 1974 y(INOUT)62 b Fw(a)o(rra)o(y)p 416 1974 V 15 w(of)p 468 1974 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 2049 y(OUT)108 b Fw(index)489 b Fk(index)12 b(of)f(handle)g(for)g(op)q (eration)h(that)f(completed)g(\(in)o(teger\))117 2124 y(OUT)108 b Fw(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 2248 y Ft(int)23 b(MPI)p 245 2248 15 2 v 17 w(Waitany\(int)g(count,)g (MPI)p 788 2248 V 16 w(Request)g(*array)p 1139 2248 V 17 w(of)p 1204 2248 V 17 w(requests,)f(int)i(*index,)393 2305 y(MPI)p 468 2305 V 17 w(Status)f(*status\))75 2391 y(MPI)p 150 2391 V 17 w(WAITANY\(COUNT,)f(ARRAY)p 645 2391 V 16 w(OF)p 709 2391 V 17 w(REQUESTS,)h(INDEX,)g(STATUS,)g (IERROR\))170 2448 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2448 V 17 w(OF)p 716 2448 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 2448 V 16 w(STATUS)p 1610 2448 V 16 w(SIZE\),)170 2504 y(IERROR)166 2591 y Fv(Blo)q(c)o(ks)16 b(un)o(til)g(one)f(of)g (the)h(op)q(erations)f(asso)q(ciated)g(with)h(the)f(activ)o(e)h (requests)f(in)h(the)f(arra)o(y)f(has)75 2647 y(completed.)36 b(If)21 b(more)f(then)h(one)f(op)q(eration)h(is)g(enabled)h(and)e(can)h (terminate,)g(one)f(is)h(arbitrarily)75 2704 y(c)o(hosen.)28 b(Returns)18 b(in)h Fw(index)g Fv(the)f(index)h(of)e(that)h(request)f (in)i(the)f(arra)o(y)f(and)h(returns)f(in)i Fw(status)h Fv(the)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 46 54 46 53 bop 75 -100 a Fv(46)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(status)f(of)h(the)f (completing)j(comm)o(unication.)25 b(\(The)17 b(arra)o(y)f(is)h (indexed)i(from)d(zero)h(in)g(C,)g(and)g(from)75 102 y(one)g(in)i(F)l(ortran.\))24 b(If)18 b(the)f(request)g(w)o(as)g(allo)q (cated)h(b)o(y)f(a)g(non)o(blo)q(c)o(king)i(comm)o(unication)f(op)q (eration,)75 158 y(then)e(it)f(is)h(deallo)q(cated)g(and)g(the)f (request)g(handle)i(is)e(set)g(to)g Fi(MPI)p 1229 158 13 2 v 14 w(REQUEST)p 1431 158 V 14 w(NULL)p Fv(.)-1718 b Fx(>)166 214 y Fv(The)17 b Fw(a)o(rra)o(y)p 359 214 14 2 v 15 w(of)p 411 214 V 16 w(requests)i Fv(list)e(ma)o(y)g(con)o (tain)g(n)o(ull)h(or)f(inactiv)o(e)h(handles.)26 b(If)17 b(the)g(list)h(con)o(tains)f(no)75 271 y(activ)o(e)e(handles)i(\(list)f (has)f(length)h(zero)f(or)g(all)h(en)o(tries)g(are)f(n)o(ull)i(or)e (inactiv)o(e\),)g(then)h(the)g(call)g(returns)75 327 y(immediately)h(with)e Fw(index)i(=)e(MPI)p 687 327 V 16 w(UNDEFINED)p Fv(,)g(and)g(a)g(empt)o(y)g Fw(status)p Fv(.)166 384 y(The)k(execution)h(of)e Fw(MPI)p 610 384 V 16 w(W)l(AIT)l(ANY\(count,)j(a)o(rra)o(y)p 1079 384 V 14 w(of)p 1130 384 V 16 w(requests,)h(index,)e(status\))g Fv(has)f(the)f(same)75 440 y(e\013ect)k(as)g(the)g(execution)i(of)44 b Fw(MPI)p 727 440 V 16 w(W)l(AIT\(&a)o(rra)o(y)p 1003 440 V 14 w(of)p 1054 440 V 16 w(requests[i],)26 b(status\))p Fv(,)g(where)c Fw(i)g Fv(is)h(the)g(v)m(alue)75 497 y(returned)c(b)o(y) g Fw(index)h Fv(\(unless)g(the)f(v)m(alue)h(of)f Fw(index)h Fv(is)f Fi(MPI)p 1107 497 13 2 v 15 w(UNDEFINED)p Fv(\).)e Fw(MPI)p 1486 497 14 2 v 16 w(W)l(AIT)l(ANY)j Fv(with)f(an)75 553 y(arra)o(y)14 b(con)o(taining)i(one)f(activ)o(e)h(en)o(try)e(is)i (equiv)m(alen)o(t)h(to)e Fw(MPI)p 1151 553 V 15 w(W)l(AIT)p Fv(.)-1449 b Fx(?)75 704 y Fw(MPI)p 160 704 V 16 w(TEST)l(ANY\(count,) 17 b(a)o(rra)o(y)p 626 704 V 14 w(of)p 677 704 V 16 w(requests,)g (index,)f(\015ag,)f(status\))117 781 y Fk(IN)155 b Fw(count)482 b Fk(list)14 b(length)f(\(in)o(teger\))117 856 y(INOUT)62 b Fw(a)o(rra)o(y)p 416 856 V 15 w(of)p 468 856 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 932 y(OUT)108 b Fw(index)489 b Fk(index)23 b(of)g(op)q(eration)g(that)g (completed,)i(or)e Fi(MPI)p 1736 932 13 2 v 14 w(UNDE-)905 988 y(FINED)13 b Fk(if)g(none)i(completed)e(\(in)o(teger\))117 1063 y(OUT)108 b Fw(\015ag)518 b Fi(true)15 b Fk(if)e(one)h(of)g(the)g (op)q(erations)g(is)g(complete)f(\(logical\))117 1138 y(OUT)108 b Fw(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1263 y Ft(int)23 b(MPI)p 245 1263 15 2 v 17 w(Testany\(int)g(count,)g (MPI)p 788 1263 V 16 w(Request)g(*array)p 1139 1263 V 17 w(of)p 1204 1263 V 17 w(requests,)f(int)i(*index,)393 1319 y(int)g(*flag,)f(MPI)p 731 1319 V 16 w(Status)g(*status\))75 1406 y(MPI)p 150 1406 V 17 w(TESTANY\(COUNT,)f(ARRAY)p 645 1406 V 16 w(OF)p 709 1406 V 17 w(REQUESTS,)h(INDEX,)g(FLAG,)g (STATUS,)g(IERROR\))170 1462 y(LOGICAL)g(FLAG)170 1519 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1519 V 17 w(OF)p 716 1519 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 1519 V 16 w(STATUS)p 1610 1519 V 16 w(SIZE\),)170 1575 y(IERROR)166 1662 y Fv(T)l(ests)d(for)h(completion)h(of)e(either)i(one) f(or)f(none)h(of)g(the)g(op)q(erations)g(asso)q(ciated)g(with)g(activ)o (e)75 1718 y(handles.)g(In)14 b(the)g(former)f(case,)h(it)g(returns)g Fw(\015ag)g(=)g(true)p Fv(,)h(returns)e(in)i Fw(index)g Fv(the)f(index)h(of)f(this)g(request)75 1774 y(in)h(the)g(arra)o(y)l(,) f(and)g(returns)h(in)g Fw(status)i Fv(the)e(status)f(of)g(that)g(op)q (eration;)h(if)g(the)f(request)h(w)o(as)f(allo)q(cated)75 1831 y(b)o(y)i(a)f(non)o(blo)q(c)o(king)i(comm)o(unication)g(call)f (then)g(the)g(request)g(is)g(deallo)q(cated)h(and)f(the)g(handle)h(is)f (set)75 1887 y(to)i Fi(MPI)p 211 1887 13 2 v 14 w(REQUEST)p 413 1887 V 14 w(NULL)p Fv(.)h(\(The)g(arra)o(y)e(is)j(indexed)g(from)e (zero)h(in)g(C,)f(and)h(from)f(one)h(in)h(F)l(ortran.\))75 1944 y(In)j(the)g(latter)f(case)h(\(no)f(op)q(eration)h(completed\),)i (it)e(returns)f Fw(\015ag)h(=)f(false)p Fv(,)j(returns)d(a)h(v)m(alue)g (of)75 2000 y Fi(MPI)p 152 2000 V 14 w(UNDEFINED)15 b Fv(in)h Fw(index)g Fv(and)f Fw(status)i Fv(is)f(unde\014ned.)-1204 b Fx(>)166 2057 y Fv(The)14 b(arra)o(y)e(ma)o(y)h(con)o(tain)g(n)o(ull) i(or)e(inactiv)o(e)h(handles.)21 b(If)14 b(the)f(arra)o(y)f(con)o (tains)i(no)f(activ)o(e)h(handles)75 2113 y(then)h(the)f(call)i (returns)e(immediately)i(with)f Fw(\015ag)f(=)g(true)p Fv(,)h Fw(index)g Fv(=)g Fi(MPI)p 1330 2113 V 14 w(UNDEFINED)p Fv(,)e(and)i(an)f(empt)o(y)75 2170 y Fw(status)p Fv(.)166 2226 y(If)c(the)h(arra)o(y)d(of)i(requests)g(con)o(tains)g(activ)o(e)h (handles)g(then)g(the)f(execution)h(of)f Fw(MPI)p 1572 2226 14 2 v 16 w(TEST)l(ANY\(count,)-2083 b Fx(?)75 2283 y Fw(a)o(rra)o(y)p 173 2283 V 15 w(of)p 225 2283 V 16 w(requests,)23 b(index,)g(status\))g Fv(has)d(the)h(same)g(e\013ect)f (as)h(the)g(execution)g(of)g Fw(MPI)p 1624 2283 V 16 w(TEST\()f(&a)o(r-)75 2339 y(ra)o(y)p 136 2339 V 15 w(of)p 188 2339 V 16 w(requests[i],)d(\015ag,)d(status\))p Fv(,)j(for)d Fw(i=0,)g(1)g(,...,)f(count-1)p Fv(,)j(in)f(some)g(arbitrary)f(order,)g (un)o(til)i(one)f(call)75 2395 y(returns)h Fw(\015ag)g(=)g(true)p Fv(,)g(or)g(all)g(fail.)23 b(In)17 b(the)f(former)f(case,)h Fw(index)h Fv(is)f(set)g(to)f(the)h(last)g(v)m(alue)h(of)e Fw(i)p Fv(,)h(and)g(in)75 2452 y(the)i(latter)f(case,)h(it)f(is)i(set)e (to)g Fi(MPI)p 698 2452 13 2 v 14 w(UNDEFINED)p Fv(.)f Fw(MPI)p 1057 2452 14 2 v 16 w(TEST)l(ANY)j Fv(with)f(an)f(arra)o(y)g (con)o(taining)h(one)75 2508 y(activ)o(e)d(en)o(try)g(is)h(equiv)m (alen)o(t)h(to)d Fw(MPI)p 727 2508 V 16 w(TEST)p Fv(.)-1027 b Fx(>)189 2615 y Fl(R)n(ationale.)61 b Fv(The)19 b(function)h Fw(MPI)p 816 2615 V 16 w(TEST)l(ANY)g Fv(returns)e(with)h Fw(\015ag)g(=)g(true)g Fv(exactly)h(in)f(those)189 2671 y(situations)f(where)g(the)g(function)g Fw(MPI)p 879 2671 V 16 w(W)l(AIT)l(ANY)h Fv(returns;)g(b)q(oth)f(functions)g(return) g(in)h(that)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 47 55 47 54 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fv(47)189 45 y(case)16 b(the)h(same)g(v)m(alues)g(in)h(the)f (remaining)h(parameters.)23 b(Th)o(us,)17 b(a)f(blo)q(c)o(king)i Fw(MPI)p 1660 45 14 2 v 16 w(W)l(AIT)l(ANY)189 102 y Fv(can)e(b)q(e)g(easily)h(replaced)g(b)o(y)e(a)h(non)o(blo)q(c)o(king)h Fw(MPI)p 1081 102 V 16 w(TEST)l(ANY)p Fv(.)f(The)g(same)g(relation)g (holds)h(for)189 158 y(the)d(other)g(pairs)h(of)f(W)l(ait)h(and)f(T)l (est)g(functions)i(de\014ned)g(in)f(this)g(section.)20 b(\()p Fl(End)15 b(of)h(r)n(ationale.)p Fv(\))1937 219 y Fx(?)75 359 y Fw(MPI)p 160 359 V 16 w(W)l(AIT)l(ALL\()e(count,)i(a)o (rra)o(y)p 626 359 V 15 w(of)p 678 359 V 16 w(requests,)h(a)o(rra)o(y)p 971 359 V 15 w(of)p 1023 359 V 15 w(statuses\))117 436 y Fk(IN)155 b Fw(count)482 b Fk(lists)14 b(length)g(\(in)o(teger\))117 511 y(INOUT)62 b Fw(a)o(rra)o(y)p 416 511 V 15 w(of)p 468 511 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o (y)e(of)f(handles\))117 586 y(OUT)108 b Fw(a)o(rra)o(y)p 416 586 V 15 w(of)p 468 586 V 16 w(statuses)276 b Fk(arra)o(y)14 b(of)f(status)i(ob)r(jects)g(\(arra)o(y)f(of)f(Status\))75 711 y Ft(int)23 b(MPI)p 245 711 15 2 v 17 w(Waitall\(int)g(count,)g (MPI)p 788 711 V 16 w(Request)g(*array)p 1139 711 V 17 w(of)p 1204 711 V 17 w(requests,)393 767 y(MPI)p 468 767 V 17 w(Status)g(*array)p 796 767 V 16 w(of)p 860 767 V 17 w(statuses\))75 854 y(MPI)p 150 854 V 17 w(WAITALL\(COUNT,)f (ARRAY)p 645 854 V 16 w(OF)p 709 854 V 17 w(REQUESTS,)h(ARRAY)p 1085 854 V 16 w(OF)p 1149 854 V 17 w(STATUSES,)g(IERROR\))170 910 y(INTEGER)g(COUNT,)g(ARRAY)p 651 910 V 17 w(OF)p 716 910 V 17 w(REQUESTS\(*\))170 967 y(INTEGER)g(ARRAY)p 484 967 V 17 w(OF)p 549 967 V 17 w(STATUSES\(MPI)p 854 967 V 15 w(STATUS)p 1013 967 V 17 w(SIZE,*\),)f(IERROR)166 1053 y Fv(Blo)q(c)o(ks)e(un)o(til)h(all)f(comm)o(unication)g(op)q (erations)g(asso)q(ciated)g(with)g(activ)o(e)f(handles)i(in)g(the)e (list)75 1109 y(complete,)j(and)e(return)g(the)g(status)f(of)g(all)i (these)f(op)q(erations)g(\(this)h(includes)h(the)e(case)g(where)g(no)75 1166 y(handle)e(in)g(the)f(list)h(is)f(activ)o(e\).)25 b(Both)17 b(arra)o(ys)e(ha)o(v)o(e)i(the)g(same)f(n)o(um)o(b)q(er)h(of) g(v)m(alid)i(en)o(tries.)25 b(The)17 b Fw(i)p Fv(-th)75 1222 y(en)o(try)12 b(in)h Fw(a)o(rra)o(y)p 339 1222 14 2 v 15 w(of)p 391 1222 V 16 w(statuses)i Fv(is)e(set)f(to)g(the)g (return)h(status)e(of)h(the)h Fw(i)p Fv(-th)f(op)q(eration.)19 b(Requests)13 b(that)f(w)o(ere)75 1279 y(created)18 b(b)o(y)h(non)o (blo)q(c)o(king)g(comm)o(unication)h(op)q(erations)e(are)g(deallo)q (cated)i(and)f(the)f(corresp)q(onding)75 1335 y(handles)h(in)f(the)g (arra)o(y)e(are)i(set)f(to)g Fi(MPI)p 786 1335 13 2 v 14 w(REQUEST)p 988 1335 V 14 w(NULL)p Fv(.)h(The)g(list)g(ma)o(y)f(con) o(tain)h(n)o(ull)h(or)e(inactiv)o(e)63 b Fx(>)75 1392 y Fv(handles.)21 b(The)16 b(call)g(sets)f(to)f(empt)o(y)h(the)g(status) g(of)g(eac)o(h)g(suc)o(h)g(en)o(try)l(.)166 1448 y(The)10 b(error-free)g(execution)h(of)f Fw(MPI)p 778 1448 14 2 v 16 w(W)l(AIT)l(ALL\(count,)i(a)o(rra)o(y)p 1226 1448 V 14 w(of)p 1277 1448 V 16 w(requests,)h(a)o(rra)o(y)p 1566 1448 V 15 w(of)p 1618 1448 V 16 w(statuses\))f Fv(has)75 1505 y(the)201 b(same)f(e\013ect)h(as)g(the)f(execution)i(of)75 1561 y Fw(MPI)p 160 1561 V 16 w(W)l(AIT\(&a)o(rra)o(y)p 436 1561 V 15 w(of)p 488 1561 V 15 w(request[i],)16 b(&a)o(rra)o(y)p 833 1561 V 15 w(of)p 885 1561 V 16 w(statuses[i]\))p Fv(,)h(for)d Fw(i=0)g(,...,)e(count-1)p Fv(,)j(in)h(some)e(arbitrary)75 1618 y(order.)20 b Fw(MPI)p 297 1618 V 15 w(W)l(AIT)l(ALL)15 b Fv(with)h(an)f(arra)o(y)f(of)h(length)h(one)f(is)h(equiv)m(alen)o(t)h (to)d Fw(MPI)p 1475 1618 V 16 w(W)l(AIT)p Fv(.)166 1674 y(When)d(one)g(or)f(more)h(of)f(the)h(comm)o(unications)g(completed)h (b)o(y)f(a)f(call)i(to)e Fw(MPI)p 1505 1674 V 16 w(W)l(AIT)l(ALL)g Fv(fail,)i(it)f(is)75 1730 y(desireable)h(to)d(return)i(sp)q(eci\014c)h (information)e(on)g(eac)o(h)g(comm)o(unication.)19 b(The)10 b(function)h Fw(MPI)p 1693 1730 V 16 w(W)l(AIT)l(ALL)75 1787 y Fv(will)j(return)e(in)h(suc)o(h)f(case)g(the)h(error)e(co)q(de)i Fi(MPI)p 899 1787 13 2 v 14 w(ERR)p 992 1787 V 14 w(IN)p 1047 1787 V 15 w(ST)m(A)m(TUS)e Fv(and)i(will)g(set)f(the)g(error)g (\014eld)h(of)f(eac)o(h)75 1843 y(status)f(to)g(a)g(sp)q(eci\014c)i (error)e(co)q(de.)19 b(This)13 b(co)q(de)f(will)h(b)q(e)f Fi(MPI)p 1094 1843 V 14 w(SUCCESS)p Fv(,)e(if)i(the)g(sp)q(eci\014c)h (comm)o(unication)75 1900 y(completed;)h(it)e(will)h(b)q(e)g(another)f (sp)q(eci\014c)i(error)d(co)q(de,)i(if)f(it)g(failed;)i(or)e(it)g(can)g (b)q(e)h Fi(MPI)p 1552 1900 V 14 w(ERR)p 1645 1900 V 14 w(PENDING)e Fv(if)75 1956 y(it)k(has)f(neither)i(failed)f(nor)g (completed.)20 b(The)15 b(function)g Fw(MPI)p 1145 1956 14 2 v 16 w(W)l(AIT)l(ALL)f Fv(will)j(return)d Fw(MPI)p 1670 1956 V 16 w(SUCCESS)75 2013 y Fv(if)19 b(no)f(request)h(had)f(an)g (error,)h(or)e(will)k(return)d(another)g(error)g(co)q(de)g(if)h(it)g (failed)h(for)d(other)h(reasons)75 2069 y(\(suc)o(h)13 b(as)f(in)o(v)m(alid)j(argumen)o(ts\).)j(In)c(suc)o(h)f(cases,)g(it)g (will)h(not)e(up)q(date)i(the)f(error)f(\014elds)i(of)e(the)h (statuses.)189 2175 y Fl(R)n(ationale.)37 b Fv(This)13 b(design)f(streamlines)h(error)e(handling)i(in)f(the)g(application.)20 b(The)12 b(application)189 2232 y(co)q(de)f(need)h(only)g(test)e(the)h (\(single\))h(function)g(result)f(to)f(determine)i(if)g(an)f(error)f (has)h(o)q(ccurred.)19 b(It)189 2288 y(needs)13 b(to)f(c)o(hec)o(k)g (eac)o(h)h(individual)i(status)d(only)h(when)f(an)h(error)e(o)q (ccurred.)20 b(\()p Fl(End)13 b(of)h(r)n(ationale.)p Fv(\))1937 2350 y Fx(?)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 48 56 48 55 bop 75 -100 a Fv(48)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(TEST)l(ALL\(count,)f(a)o(rra)o(y)p 613 45 V 14 w(of)p 664 45 V 16 w(requests,)h(\015ag,)e(a)o(rra)o(y)p 1054 45 V 14 w(of)p 1105 45 V 16 w(statuses\))117 122 y Fk(IN)155 b Fw(count)482 b Fk(lists)14 b(length)g(\(in)o(teger\))117 196 y(INOUT)62 b Fw(a)o(rra)o(y)p 416 196 V 15 w(of)p 468 196 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o (y)e(of)f(handles\))117 270 y(OUT)108 b Fw(\015ag)518 b Fk(\(logical\))117 344 y(OUT)108 b Fw(a)o(rra)o(y)p 416 344 V 15 w(of)p 468 344 V 16 w(statuses)276 b Fk(arra)o(y)14 b(of)f(status)i(ob)r(jects)g(\(arra)o(y)f(of)f(Status\))75 468 y Ft(int)23 b(MPI)p 245 468 15 2 v 17 w(Testall\(int)g(count,)g (MPI)p 788 468 V 16 w(Request)g(*array)p 1139 468 V 17 w(of)p 1204 468 V 17 w(requests,)f(int)i(*flag,)393 525 y(MPI)p 468 525 V 17 w(Status)f(*array)p 796 525 V 16 w(of)p 860 525 V 17 w(statuses\))75 611 y(MPI)p 150 611 V 17 w(TESTALL\(COUNT,)f(ARRAY)p 645 611 V 16 w(OF)p 709 611 V 17 w(REQUESTS,)h(FLAG,)g(ARRAY)p 1228 611 V 16 w(OF)p 1292 611 V 17 w(STATUSES,)g(IERROR\))170 667 y(LOGICAL)g(FLAG)170 724 y(INTEGER)g(COUNT,)g(ARRAY)p 651 724 V 17 w(OF)p 716 724 V 17 w(REQUESTS\(*\),)170 780 y(ARRAY)p 293 780 V 17 w(OF)p 358 780 V 17 w(STATUSES\(MPI)p 663 780 V 15 w(STATUS)p 822 780 V 17 w(SIZE,*\),)g(IERROR)166 867 y Fv(Returns)17 b Fw(\015ag)g(=)g(true)g Fv(if)h(all)f(comm)o (unications)h(asso)q(ciated)f(with)g(activ)o(e)g(handles)h(in)f(the)g (arra)o(y)75 923 y(ha)o(v)o(e)f(completed)i(\(this)e(includes)j(the)d (case)h(where)g(no)f(handle)i(in)f(the)g(list)g(is)g(activ)o(e\).)23 b(In)17 b(this)g(case,)75 980 y(eac)o(h)h(status)f(en)o(try)g(that)g (corresp)q(onds)i(to)e(an)h(activ)o(e)f(handle)j(request)d(is)i(set)e (to)h(the)f(status)g(of)h(the)75 1036 y(corresp)q(onding)12 b(comm)o(unication;)i(if)e(the)f(request)h(w)o(as)e(allo)q(cated)j(b)o (y)e(a)g(non)o(blo)q(c)o(king)i(comm)o(unication)75 1093 y(call)k(then)g(it)f(is)h(deallo)q(cated,)g(and)f(the)g(handle)i(is)e (set)g(to)g Fi(MPI)p 1177 1093 13 2 v 14 w(REQUEST)p 1379 1093 V 14 w(NULL)p Fv(.)g(Eac)o(h)g(status)f(en)o(try)-2035 b Fx(>)75 1149 y Fv(that)14 b(corresp)q(onds)i(to)f(a)f(n)o(ull)j(or)e (inactiv)o(e)h(handle)g(is)g(set)f(to)g(empt)o(y)l(.)-1443 b Fx(?)166 1206 y Fv(Otherwise,)16 b Fw(\015ag)f(=)g(false)h Fv(is)f(returned,)h(no)f(request)g(is)h(mo)q(di\014ed)h(and)e(the)g(v)m (alues)i(of)d(the)i(status)75 1262 y(en)o(tries)g(are)e(unde\014ned.)22 b(This)16 b(is)g(a)f(lo)q(cal)h(op)q(eration.)-1177 b Fx(>)166 1318 y Fv(Errors)17 b(that)g(o)q(ccurred)i(during)g(the)f (execution)h(of)e Fw(MPI)p 1172 1318 14 2 v 16 w(TEST)l(ALL)h Fv(are)g(handled)h(as)f(errors)f(in)75 1375 y Fw(MPI)p 160 1375 V 16 w(W)l(AIT)l(ALL)p Fv(.)-535 b Fx(?)75 1526 y Fw(MPI)p 160 1526 V 16 w(W)l(AITSOME\(incount,)16 b(a)o(rra)o(y)p 695 1526 V 15 w(of)p 747 1526 V 16 w(requests,)h(outcount,)f(a)o(rra)o (y)p 1236 1526 V 15 w(of)p 1288 1526 V 16 w(indices,)g(a)o(rra)o(y)p 1553 1526 V 14 w(of)p 1604 1526 V 16 w(statuses\))117 1603 y Fk(IN)155 b Fw(incount)448 b Fk(length)14 b(of)f(arra)o(y)p 1176 1603 13 2 v 15 w(of)p 1225 1603 V 14 w(requests)j(\(in)o(teger\)) 117 1677 y(INOUT)62 b Fw(a)o(rra)o(y)p 416 1677 14 2 v 15 w(of)p 468 1677 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 1751 y(OUT)108 b Fw(outcount)420 b Fk(n)o(um)o(b)q(er)13 b(of)h(completed)f (requests)j(\(in)o(teger\))117 1824 y(OUT)108 b Fw(a)o(rra)o(y)p 416 1824 V 15 w(of)p 468 1824 V 16 w(indices)299 b Fk(arra)o(y)12 b(of)f(indices)h(of)f(op)q(erations)g(that)h(completed)f(\(arra)o(y)h (of)905 1881 y(in)o(tegers\))117 1955 y(OUT)108 b Fw(a)o(rra)o(y)p 416 1955 V 15 w(of)p 468 1955 V 16 w(statuses)276 b Fk(arra)o(y)15 b(of)g(status)h(ob)r(jects)g(for)f(op)q(erations)g(that)h(completed)905 2011 y(\(arra)o(y)e(of)f(Status\))75 2135 y Ft(int)23 b(MPI)p 245 2135 15 2 v 17 w(Waitsome\(int)f(incount,)h(MPI)p 859 2135 V 17 w(Request)g(*array)p 1211 2135 V 16 w(of)p 1275 2135 V 17 w(requests,)g(int)g(*outcount,)393 2192 y(int)h(*array)p 636 2192 V 16 w(of)p 700 2192 V 17 w(indices,)f(MPI)p 1004 2192 V 16 w(Status)g(*array)p 1331 2192 V 17 w(of)p 1396 2192 V 17 w(statuses\))75 2278 y(MPI)p 150 2278 V 17 w(WAITSOME\(INCOUNT,)e(ARRAY)p 716 2278 V 17 w(OF)p 781 2278 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 2278 V 16 w(OF)p 1459 2278 V 17 w(INDICES,)393 2335 y(ARRAY)p 516 2335 V 17 w(OF)p 581 2335 V 17 w(STATUSES,)f(IERROR\))170 2391 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 2391 V 17 w(OF)p 764 2391 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 2391 V 17 w(OF)p 1514 2391 V 17 w(INDICES\(*\),)170 2448 y(ARRAY)p 293 2448 V 17 w(OF)p 358 2448 V 17 w(STATUSES\(MPI)p 663 2448 V 15 w(STATUS)p 822 2448 V 17 w(SIZE,*\),)h(IERROR)166 2534 y Fv(W)l(aits)d(un)o(til)h(at)e(least)i(one)f(of)g(the)g(op)q (erations)g(asso)q(ciated)g(with)h(activ)o(e)f(handles)h(in)g(the)f (list)75 2591 y(ha)o(v)o(e)15 b(completed.)21 b(Returns)15 b(in)i Fw(outcount)g Fv(the)e(n)o(um)o(b)q(er)h(of)e(requests)h(from)g (the)g(list)h Fw(a)o(rra)o(y)p 1656 2591 14 2 v 15 w(of)p 1708 2591 V 16 w(requests)75 2647 y Fv(that)c(ha)o(v)o(e)h(completed.) 20 b(Returns)13 b(in)h(the)f(\014rst)g Fw(outcount)i Fv(lo)q(cations)f(of)e(the)h(arra)o(y)f Fw(a)o(rra)o(y)p 1608 2647 V 14 w(of)p 1659 2647 V 16 w(indices)j Fv(the)75 2704 y(indices)21 b(of)d(these)h(op)q(erations)f(\(index)i(within)g (the)e(arra)o(y)g Fw(a)o(rra)o(y)p 1218 2704 V 14 w(of)p 1269 2704 V 16 w(requests)p Fv(;)k(the)d(arra)o(y)e(is)i(indexed)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 49 57 49 56 bop 75 -100 a Fm(3.7.)34 b(NONBLOCKING)17 b(COMMUNICA)l(TION)863 b Fv(49)75 45 y(from)12 b(zero)g(in)i(C)e(and)h(from)e(one)i(in)g(F)l (ortran\).)18 b(Returns)13 b(in)g(the)g(\014rst)f Fw(outcount)i Fv(lo)q(cations)g(of)e(the)g(arra)o(y)75 102 y Fw(a)o(rra)o(y)p 173 102 14 2 v 15 w(of)p 225 102 V 16 w(status)18 b Fv(the)g(status)e (for)g(these)h(completed)h(op)q(erations.)26 b(If)17 b(a)g(request)g(that)f(completed)i(w)o(as)75 158 y(allo)q(cated)g(b)o (y)g(a)f(non)o(blo)q(c)o(king)h(comm)o(unication)g(call,)h(then)f(it)f (is)h(deallo)q(cated,)h(and)e(the)h(asso)q(ciated)75 214 y(handle)e(is)g(set)f(to)g Fi(MPI)p 471 214 13 2 v 14 w(REQUEST)p 673 214 V 14 w(NULL)p Fv(.)1137 b Fx(>)166 271 y Fv(If)16 b(the)g(list)g(con)o(tains)g(no)f(activ)o(e)h(handles,)h (then)f(the)f(call)i(returns)f(immediately)h(with)f Fw(outcount)75 327 y(=)g(MPI)p 211 327 14 2 v 15 w(UNDEFINED)p Fv(.)166 384 y(When)e(one)g(or)g(more)f(of)h(the)g(comm)o(unications)h (completed)g(b)o(y)e Fw(MPI)p 1370 384 V 16 w(W)l(AITSOME)i Fv(fails,)f(then)h(it)75 440 y(is)e(desirable)g(to)f(return)g(sp)q (eci\014c)i(information)f(on)f(eac)o(h)g(comm)o(unication.)20 b(The)12 b(argumen)o(ts)f Fw(outcount)p Fv(,)75 497 y Fw(a)o(rra)o(y)p 173 497 V 15 w(of)p 225 497 V 16 w(indices)j Fv(and)g Fw(a)o(rra)o(y)p 562 497 V 14 w(of)p 613 497 V 16 w(statuses)i Fv(will)f(b)q(e)f(adjusted)f(to)f(indicate)j (completion)f(of)f(all)h(comm)o(uni-)75 553 y(cations)d(that)g(ha)o(v)o (e)f(succeeded)j(or)e(failed.)20 b(The)11 b(call)h(will)h(return)e(the) g(error)g(co)q(de)h Fi(MPI)p 1562 553 13 2 v 14 w(ERR)p 1655 553 V 14 w(IN)p 1710 553 V 15 w(ST)m(A)m(TUS)75 610 y Fv(and)k(the)f(error)g(\014eld)h(of)f(eac)o(h)h(status)e (returned)i(will)h(b)q(e)f(set)g(to)e(indicate)j(success)f(or)f(to)g (indicate)i(the)75 666 y(sp)q(eci\014c)22 b(error)d(that)h(o)q (ccurred.)35 b(The)20 b(call)i(will)f(return)f Fi(MPI)p 1174 666 V 15 w(SUCCESS)e Fv(if)j(no)f(request)g(resulted)h(in)75 723 y(an)d(error,)g(and)h(will)h(return)e(another)g(error)g(co)q(de)h (if)g(it)f(failed)i(for)e(other)f(reasons)h(\(suc)o(h)h(as)f(in)o(v)m (alid)75 779 y(argumen)o(ts\).)h(In)d(suc)o(h)f(cases,)g(it)g(will)i (not)e(up)q(date)h(the)f(error)f(\014elds)j(of)e(the)g(statuses.)347 b Fx(?)75 930 y Fw(MPI)p 160 930 14 2 v 16 w(TESTSOME\(incount,)17 b(a)o(rra)o(y)p 697 930 V 14 w(of)p 748 930 V 16 w(requests,)g (outcount,)f(a)o(rra)o(y)p 1237 930 V 15 w(of)p 1289 930 V 16 w(indices,)g(a)o(rra)o(y)p 1554 930 V 14 w(of)p 1605 930 V 16 w(statuses\))117 1007 y Fk(IN)155 b Fw(incount)448 b Fk(length)14 b(of)f(arra)o(y)p 1176 1007 13 2 v 15 w(of)p 1225 1007 V 14 w(requests)j(\(in)o(teger\))117 1080 y(INOUT)62 b Fw(a)o(rra)o(y)p 416 1080 14 2 v 15 w(of)p 468 1080 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra)o(y)e(of)f(handles\))117 1153 y(OUT)108 b Fw(outcount)420 b Fk(n)o(um)o(b)q(er)13 b(of)h(completed)f (requests)j(\(in)o(teger\))117 1226 y(OUT)108 b Fw(a)o(rra)o(y)p 416 1226 V 15 w(of)p 468 1226 V 16 w(indices)299 b Fk(arra)o(y)12 b(of)f(indices)h(of)f(op)q(erations)g(that)h(completed)f(\(arra)o(y)h (of)905 1282 y(in)o(tegers\))117 1355 y(OUT)108 b Fw(a)o(rra)o(y)p 416 1355 V 15 w(of)p 468 1355 V 16 w(statuses)276 b Fk(arra)o(y)15 b(of)g(status)h(ob)r(jects)g(for)f(op)q(erations)g(that)h(completed)905 1412 y(\(arra)o(y)e(of)f(Status\))75 1536 y Ft(int)23 b(MPI)p 245 1536 15 2 v 17 w(Testsome\(int)f(incount,)h(MPI)p 859 1536 V 17 w(Request)g(*array)p 1211 1536 V 16 w(of)p 1275 1536 V 17 w(requests,)g(int)g(*outcount,)393 1593 y(int)h(*array)p 636 1593 V 16 w(of)p 700 1593 V 17 w(indices,)f(MPI)p 1004 1593 V 16 w(Status)g(*array)p 1331 1593 V 17 w(of)p 1396 1593 V 17 w(statuses\))75 1679 y(MPI)p 150 1679 V 17 w(TESTSOME\(INCOUNT,)e(ARRAY)p 716 1679 V 17 w(OF)p 781 1679 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 1679 V 16 w(OF)p 1459 1679 V 17 w(INDICES,)393 1736 y(ARRAY)p 516 1736 V 17 w(OF)p 581 1736 V 17 w(STATUSES,)f(IERROR\))170 1792 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 1792 V 17 w(OF)p 764 1792 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 1792 V 17 w(OF)p 1514 1792 V 17 w(INDICES\(*\),)170 1849 y(ARRAY)p 293 1849 V 17 w(OF)p 358 1849 V 17 w(STATUSES\(MPI)p 663 1849 V 15 w(STATUS)p 822 1849 V 17 w(SIZE,*\),)h(IERROR)166 1935 y Fv(Beha)o(v)o(es)14 b(lik)o(e)i Fw(MPI)p 508 1935 14 2 v 16 w(W)l(AITSOME)p Fv(,)e(except)h(that)f(it)h(returns)f (immediately)l(.)21 b(If)15 b(no)f(op)q(eration)h(has)75 1992 y(completed)f(it)f(returns)g Fw(outcount)i(=)e(0)p Fv(.)19 b(If)13 b(there)g(is)g(no)g(activ)o(e)g(handle)i(in)e(the)g (list)h(it)f(returns)g Fw(outcount)64 b Fx(>)75 2048 y Fw(=)16 b(MPI)p 211 2048 V 15 w(UNDEFINED)p Fv(.)1447 b Fx(?)166 2104 y Fw(MPI)p 251 2104 V 16 w(TESTSOME)15 b Fv(is)h(a)e(lo)q(cal)i(op)q(eration,)f(whic)o(h)g(returns)g (immediately)l(,)h(whereas)f Fw(MPI)p 1734 2104 V 16 w(W)l(AIT-)75 2161 y(SOME)i Fv(will)h(blo)q(c)o(k)f(un)o(til)h(a)e (comm)o(unication)h(completes,)g(if)g(it)g(w)o(as)e(passed)i(a)f(list)h (that)f(con)o(tains)h(at)75 2217 y(least)g(one)h(activ)o(e)f(handle.)27 b(Both)17 b(calls)h(ful\014ll)i(a)d Fw(fairness)h Fv(requiremen)o(t:)24 b(If)17 b(a)g(request)h(for)e(a)h(receiv)o(e)75 2274 y(rep)q(eatedly)d(app)q(ears)e(in)i(a)e(list)h(of)g(requests)f(passed)h (to)f Fw(MPI)p 1122 2274 V 16 w(W)l(AITSOME)h Fv(or)f Fw(MPI)p 1524 2274 V 16 w(TESTSOME)p Fv(,)h(and)75 2330 y(a)i(matc)o(hing)g(send)h(has)f(b)q(een)i(p)q(osted,)e(then)h(the)f (receiv)o(e)h(will)h(ev)o(en)o(tually)f(succeed,)h(unless)f(the)f(send) 75 2387 y(is)h(satis\014ed)f(b)o(y)h(another)e(receiv)o(e;)i(and)g (similarly)g(for)f(send)h(requests.)628 b Fx(>)166 2443 y Fv(Errors)29 b(that)g(o)q(ccur)h(during)g(the)g(execution)h(of)e Fw(MPI)p 1191 2443 V 16 w(TESTSOME)i Fv(are)e(handled)i(as)f(for)75 2500 y Fw(MPI)p 160 2500 V 16 w(W)l(AITSOME)p Fv(.)1513 b Fx(?)189 2591 y Fl(A)n(dvic)n(e)11 b(to)i(users.)37 b Fv(The)12 b(use)f(of)g Fw(MPI)p 831 2591 V 16 w(TESTSOME)h Fv(is)f(lik)o(ely)i(to)e(b)q(e)h(more)e(e\016cien)o(t)i(than)f(the)h (use)189 2647 y(of)h Fw(MPI)p 324 2647 V 16 w(TEST)l(ANY)p Fv(.)i(The)f(former)f(returns)g(information)h(on)g(all)h(completed)g (comm)o(unications,)189 2704 y(with)g(the)g(latter,)g(a)g(new)g(call)i (is)e(required)i(for)d(eac)o(h)h(comm)o(unication)h(that)f(completes.) -32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 50 58 50 57 bop 75 -100 a Fv(50)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Fv(A)24 b(serv)o(er)g(with)g(m)o(ultiple)i(clien)o(ts)g(can)e(use)h Fw(MPI)p 1102 45 14 2 v 15 w(W)l(AITSOME)g Fv(so)f(as)g(not)g(to)f (starv)o(e)g(an)o(y)189 102 y(clien)o(t.)35 b(Clien)o(ts)20 b(send)h(messages)e(to)g(the)h(serv)o(er)g(with)g(service)h(requests.) 33 b(The)20 b(serv)o(er)g(calls)189 158 y Fw(MPI)p 274 158 V 15 w(W)l(AITSOME)h Fv(with)g(one)f(receiv)o(e)h(request)f(for)g (eac)o(h)g(clien)o(t,)i(and)f(then)f(handles)i(all)f(re-)189 214 y(ceiv)o(es)g(that)e(completed.)35 b(If)20 b(a)g(call)h(to)f Fw(MPI)p 1002 214 V 16 w(W)l(AIT)l(ANY)h Fv(is)f(used)h(instead,)g (then)g(one)f(clien)o(t)189 271 y(could)d(starv)o(e)e(while)j(requests) e(from)g(another)g(clien)o(t)h(alw)o(a)o(ys)f(sneak)g(in)h(\014rst.)23 b(\()p Fl(End)17 b(of)g(advic)n(e)189 327 y(to)f(users.)p Fv(\))189 434 y Fl(A)n(dvic)n(e)h(to)h(implementors.)52 b Fw(MPI)p 810 434 V 16 w(TESTSOME)18 b Fv(should)g(complete)g(as)f (man)o(y)g(p)q(ending)i(com-)189 490 y(m)o(unications)d(as)e(p)q (ossible.)22 b(\()p Fl(End)16 b(of)g(advic)n(e)g(to)h(implementors.)p Fv(\))75 596 y Fq(Example)h(3.15)23 b Fv(Clien)o(t-serv)o(er)15 b(co)q(de)h(\(starv)m(ation)f(can)g(o)q(ccur\).)75 747 y Ft(CALL)23 b(MPI_COMM_SIZE\(comm,)f(size,)h(ierr\))75 803 y(CALL)g(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 860 y(IF\(rank)g(>)h(0\))f(THEN)214 b(!)24 b(client)f(code)170 916 y(DO)h(WHILE\(.TRUE.\))242 972 y(CALL)f(MPI_ISEND\(a,)g(n,)g (MPI_REAL,)g(0,)g(tag,)h(comm,)f(request,)g(ierr\))242 1029 y(CALL)g(MPI_WAIT\(request,)f(status,)h(ierr\))170 1085 y(END)h(DO)75 1142 y(ELSE)214 b(!)24 b(rank=0)f(--)h(server)f (code)242 1198 y(DO)h(i=1,)f(size-1)314 1255 y(CALL)g (MPI_IRECV\(a\(1,i\),)e(n,)j(MPI_REAL,)f(i)g(tag,)528 1311 y(comm,)h(request_list\(i\),)d(ierr\))242 1368 y(END)i(DO)242 1424 y(DO)h(WHILE\(.TRUE.\))314 1481 y(CALL)f(MPI_WAITANY\(size-1,)e (request_list,)h(index,)h(status,)g(ierr\))314 1537 y(CALL)g (DO_SERVICE\(a\(1,index\)\))45 b(!)23 b(handle)g(one)h(message)314 1593 y(CALL)f(MPI_IRECV\(a\(1,)f(index\),)h(n,)g(MPI_REAL,)g(index,)g (tag,)552 1650 y(comm,)g(request_list\(index\),)f(ierr\))242 1706 y(END)h(DO)75 1763 y(END)g(IF)75 1919 y Fq(Example)18 b(3.16)23 b Fv(Same)15 b(co)q(de,)g(using)h Fw(MPI)p 849 1919 V 16 w(W)l(AITSOME)p Fv(.)75 2069 y Ft(CALL)23 b(MPI_COMM_SIZE\(comm,)f(size,)h(ierr\))75 2126 y(CALL)g (MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))75 2182 y(IF\(rank)g(>)h(0\))f (THEN)214 b(!)24 b(client)f(code)170 2239 y(DO)h(WHILE\(.TRUE.\))242 2295 y(CALL)f(MPI_ISEND\(a,)g(n,)g(MPI_REAL,)g(0,)g(tag,)h(comm,)f (request,)g(ierr\))242 2351 y(CALL)g(MPI_WAIT\(request,)f(status,)h (ierr\))170 2408 y(END)h(DO)75 2464 y(ELSE)214 b(!)24 b(rank=0)f(--)h(server)f(code)170 2521 y(DO)h(i=1,)f(size-1)242 2577 y(CALL)g(MPI_IRECV\(a\(1,i\),)f(n,)h(MPI_REAL,)g(i,)h(tag,)600 2634 y(comm,)f(requests\(i\),)f(ierr\))170 2690 y(END)i(DO)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 51 59 51 58 bop 75 -100 a Fm(3.8.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1161 b Fv(51)170 45 y Ft(DO)24 b(WHILE\(.TRUE.\))242 102 y(CALL)f (MPI_WAITSOME\(size,)f(request_list,)g(numdone,)648 158 y(indices,)g(statuses,)h(ierr\))242 214 y(DO)h(i=1,)f(numdone)314 271 y(CALL)g(DO_SERVICE\(a\(1,)f(indices\(i\)\)\))314 327 y(CALL)h(MPI_IRECV\(a\(1,)f(indices\(i\)\),)g(n,)i(MPI_REAL,)e(0,)i (tag,)624 384 y(comm,)f(requests\(indices\(i\)\),)e(ierr\))242 440 y(END)i(DO)170 497 y(END)h(DO)75 553 y(END)f(IF)75 696 y Fn(3.8)59 b(Prob)r(e)20 b(and)f(Cancel)75 798 y Fv(The)d Fw(MPI)p 254 798 14 2 v 16 w(PROBE)g Fv(and)g Fw(MPI)p 605 798 V 16 w(IPROBE)g Fv(op)q(erations)g(allo)o(w)g (incoming)h(messages)e(to)g(b)q(e)h(c)o(hec)o(k)o(ed)g(for,)75 854 y(without)g(actually)h(receiving)h(them.)k(The)17 b(user)f(can)g(then)h(decide)g(ho)o(w)f(to)g(receiv)o(e)h(them,)f (based)g(on)75 911 y(the)i(information)f(returned)h(b)o(y)f(the)h(prob) q(e)g(\(basically)l(,)h(the)e(information)h(returned)g(b)o(y)f Fw(status)p Fv(\).)28 b(In)75 967 y(particular,)16 b(the)g(user)g(ma)o (y)f(allo)q(cate)h(memory)f(for)g(the)h(receiv)o(e)h(bu\013er,)e (according)h(to)f(the)h(length)g(of)75 1024 y(the)f(prob)q(ed)h (message.)166 1080 y(The)i Fw(MPI)p 347 1080 V 16 w(CANCEL)g Fv(op)q(eration)g(allo)o(ws)g(p)q(ending)i(comm)o(unications)e(to)g(b)q (e)g(canceled.)30 b(This)18 b(is)75 1137 y(required)i(for)f(clean)o (up.)33 b(P)o(osting)19 b(a)g(send)h(or)f(a)g(receiv)o(e)h(ties)f(up)h (user)f(resources)h(\(send)f(or)g(receiv)o(e)75 1193 y(bu\013ers\),)14 b(and)i(a)f(cancel)h(ma)o(y)f(b)q(e)g(needed)i(to)d (free)i(these)f(resources)g(gracefully)l(.)75 1344 y Fw(MPI)p 160 1344 V 16 w(IPROBE\(source,)h(tag,)f(comm,)d(\015ag,)j (status\))117 1421 y Fk(IN)155 b Fw(source)468 b Fk(source)15 b(rank,)f(or)27 b Fi(MPI)p 1281 1421 13 2 v 15 w(ANY)p 1381 1421 V 14 w(SOURCE)13 b Fk(\(in)o(teger\))117 1496 y(IN)155 b Fw(tag)526 b Fk(tag)14 b(v)n(alue)f(or)28 b Fi(MPI)p 1226 1496 V 14 w(ANY)p 1325 1496 V 15 w(T)m(A)o(G)12 b Fk(\(in)o(teger\))117 1571 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 1647 y(OUT)108 b Fw(\015ag)518 b Fk(\(logical\))117 1722 y(OUT)108 b Fw(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1846 y Ft(int)23 b(MPI)p 245 1846 15 2 v 17 w(Iprobe\(int)g(source,)g (int)g(tag,)g(MPI)p 1002 1846 V 17 w(Comm)g(comm,)h(int)f(*flag,)393 1903 y(MPI)p 468 1903 V 17 w(Status)g(*status\))75 1989 y(MPI)p 150 1989 V 17 w(IPROBE\(SOURCE,)f(TAG,)h(COMM,)g(FLAG,)g (STATUS,)g(IERROR\))170 2046 y(LOGICAL)g(FLAG)170 2102 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 2102 V 16 w(STATUS)p 1218 2102 V 16 w(SIZE\),)g(IERROR)166 2188 y Fw(MPI)p 251 2188 14 2 v 16 w(IPROBE\(source,)18 b(tag,)f(comm,)e(\015ag,)j(status\))g Fv(returns)f Fw(\015ag)g(=)h (true)f Fv(if)h(there)f(is)h(a)e(message)75 2245 y(that)f(can)i(b)q(e)f (receiv)o(ed)h(and)g(that)e(matc)o(hes)h(the)g(pattern)f(sp)q (eci\014ed)k(b)o(y)d(the)g(argumen)o(ts)f Fw(source)p Fv(,)h Fw(tag)p Fv(,)75 2301 y(and)f Fw(comm)p Fv(.)k(The)c(call)i (matc)o(hes)e(the)g(same)g(message)g(that)f(w)o(ould)i(ha)o(v)o(e)f(b)q (een)h(receiv)o(ed)h(b)o(y)e(a)g(call)i(to)75 2358 y Fw(MPI)p 160 2358 V 16 w(RECV\(...,)f(source,)i(tag,)f(comm,)e (status\))j Fv(executed)g(at)e(the)h(same)g(p)q(oin)o(t)g(in)h(the)f (program,)f(and)75 2414 y(returns)c(in)i Fw(status)g Fv(the)f(same)f(v)m(alue)i(that)e(w)o(ould)g(ha)o(v)o(e)h(b)q(een)g (returned)g(b)o(y)g Fw(MPI)p 1467 2414 V 15 w(RECV\(\))p Fv(.)19 b(Otherwise,)75 2471 y(the)c(call)i(returns)e Fw(\015ag)g(=)g(false)p Fv(,)g(and)h(lea)o(v)o(es)f Fw(status)i Fv(unde\014ned.)166 2527 y(If)g Fw(MPI)p 298 2527 V 15 w(IPROBE)h Fv(returns)e Fw(\015ag)h(=)f(true)p Fv(,)h(then)g(the)g(con) o(ten)o(t)f(of)g(the)g(status)g(ob)s(ject)g(can)g(b)q(e)h(sub-)75 2584 y(sequen)o(tly)i(accessed)h(as)e(describ)q(ed)i(in)g(section)f (3.2.5)e(to)h(\014nd)h(the)g(source,)g(tag)f(and)g(length)i(of)e(the)75 2640 y(prob)q(ed)e(message.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 52 60 52 59 bop 75 -100 a Fv(52)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)166 45 y Fv(A)c(subsequen)o(t)i (receiv)o(e)f(executed)g(with)g(the)g(same)f(con)o(text,)g(and)g(the)h (source)f(and)h(tag)f(returned)75 102 y(in)19 b Fw(status)i Fv(b)o(y)e Fw(MPI)p 415 102 14 2 v 15 w(IPROBE)h Fv(will)g(receiv)o(e)g (the)f(message)f(that)g(w)o(as)g(matc)o(hed)g(b)o(y)h(the)f(prob)q(e,)i (if)f(no)75 158 y(other)c(in)o(terv)o(ening)h(receiv)o(e)g(o)q(ccurs)f (after)g(the)g(prob)q(e.)20 b(If)15 b(the)g(receiving)i(pro)q(cess)e (is)h(m)o(ulti-threaded,)75 214 y(it)f(is)h(the)f(user's)g(resp)q (onsibilit)o(y)j(to)d(ensure)g(that)g(the)g(last)g(condition)i(holds.) 166 271 y(The)f Fw(source)h Fv(argumen)o(t)e(of)h Fw(MPI)p 738 271 V 16 w(PROBE)h Fv(can)f(b)q(e)h Fi(MPI)p 1140 271 13 2 v 14 w(ANY)p 1239 271 V 15 w(SOURCE)p Fv(,)d(and)i(the)h Fw(tag)f Fv(argumen)o(t)75 327 y(can)g(b)q(e)g Fi(MPI)p 298 327 V 15 w(ANY)p 398 327 V 14 w(T)m(A)o(G)p Fv(,)f(so)g(that)g(one) h(can)g(prob)q(e)g(for)f(messages)h(from)f(an)h(arbitrary)f(source)h (and/or)75 384 y(with)h(an)g(arbitrary)f(tag.)24 b(Ho)o(w)o(ev)o(er,)16 b(a)h(sp)q(eci\014c)h(comm)o(unication)g(con)o(text)e(m)o(ust)g(b)q(e)i (pro)o(vided)g(with)75 440 y(the)d Fw(comm)f Fv(argumen)o(t.)166 497 y(It)i(is)g(not)f(necessary)h(to)f(receiv)o(e)h(a)f(message)h (immediately)h(after)e(it)h(has)f(b)q(een)i(prob)q(ed)f(for,)f(and)75 553 y(the)g(same)g(message)g(ma)o(y)f(b)q(e)i(prob)q(ed)g(for)f(sev)o (eral)g(times)h(b)q(efore)f(it)h(is)f(receiv)o(ed.)75 704 y Fw(MPI)p 160 704 14 2 v 16 w(PROBE\(source,)h(tag,)f(comm,)e (status\))117 781 y Fk(IN)155 b Fw(source)468 b Fk(source)15 b(rank,)f(or)27 b Fi(MPI)p 1281 781 13 2 v 15 w(ANY)p 1381 781 V 14 w(SOURCE)13 b Fk(\(in)o(teger\))117 856 y(IN)155 b Fw(tag)526 b Fk(tag)14 b(v)n(alue,)e(or)28 b Fi(MPI)p 1237 856 V 14 w(ANY)p 1336 856 V 15 w(T)m(A)o(G)13 b Fk(\(in)o(teger\))117 932 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 1007 y(OUT)108 b Fw(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 1131 y Ft(int)23 b(MPI)p 245 1131 15 2 v 17 w(Probe\(int)g(source,)g (int)g(tag,)h(MPI)p 979 1131 V 16 w(Comm)g(comm,)f(MPI)p 1330 1131 V 17 w(Status)g(*status\))75 1218 y(MPI)p 150 1218 V 17 w(PROBE\(SOURCE,)f(TAG,)h(COMM,)g(STATUS,)g(IERROR\))170 1274 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 1274 V 16 w(STATUS)p 1218 1274 V 16 w(SIZE\),)g(IERROR)166 1361 y Fw(MPI)p 251 1361 14 2 v 16 w(PROBE)d Fv(b)q(eha)o(v)o(es)f(lik) o(e)h Fw(MPI)p 777 1361 V 16 w(IPROBE)g Fv(except)f(that)f(it)i(is)f(a) g(blo)q(c)o(king)h(call)g(that)e(returns)75 1417 y(only)e(after)e(a)h (matc)o(hing)g(message)g(has)g(b)q(een)h(found.)166 1473 y(The)11 b Fw(MPI)f Fv(implemen)o(tation)i(of)e Fw(MPI)p 799 1473 V 16 w(PROBE)i Fv(and)e Fw(MPI)p 1140 1473 V 16 w(IPROBE)i Fv(needs)f(to)f(guaran)o(tee)g(progress:)75 1530 y(if)17 b(a)g(call)h(to)e Fw(MPI)p 385 1530 V 16 w(PROBE)i Fv(has)f(b)q(een)h(issued)g(b)o(y)f(a)f(pro)q(cess,)i(and)f (a)f(send)i(that)e(matc)o(hes)h(the)f(prob)q(e)75 1586 y(has)j(b)q(een)i(initiated)g(b)o(y)f(some)f(pro)q(cess,)i(then)f(the)f (call)i(to)e Fw(MPI)p 1244 1586 V 16 w(PROBE)i Fv(will)g(return,)f (unless)h(the)75 1643 y(message)c(is)i(receiv)o(ed)g(b)o(y)f(another)f (concurren)o(t)h(receiv)o(e)h(op)q(eration)f(\(that)e(is)j(executed)g (b)o(y)e(another)75 1699 y(thread)e(at)g(the)h(probing)g(pro)q(cess\).) 21 b(Similarly)l(,)d(if)e(a)f(pro)q(cess)h(busy)g(w)o(aits)f(with)h Fw(MPI)p 1573 1699 V 16 w(IPROBE)g Fv(and)g(a)75 1756 y(matc)o(hing)e(message)g(has)g(b)q(een)h(issued,)g(then)f(the)g(call)h (to)f Fw(MPI)p 1179 1756 V 16 w(IPROBE)g Fv(will)i(ev)o(en)o(tually)f (return)f Fw(\015ag)75 1812 y(=)i(true)f Fv(unless)i(the)e(message)g (is)g(receiv)o(ed)i(b)o(y)e(another)g(concurren)o(t)g(receiv)o(e)h(op)q (eration.)75 1918 y Fq(Example)i(3.17)23 b Fv(Use)15 b(blo)q(c)o(king)h(prob)q(e)g(to)f(w)o(ait)f(for)h(an)g(incoming)i (message.)242 2025 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))242 2081 y(IF)h(\(rank.EQ.0\))e(THEN)361 2138 y(CALL)i(MPI_SEND\(i,)e(1,)i(MPI_INTEGER,)e(2,)h(0,)h(comm,)f(ierr\)) 242 2194 y(ELSE)g(IF\(rank.EQ.1\))f(THEN)361 2251 y(CALL)i (MPI_SEND\(x,)e(1,)i(MPI_REAL,)e(2,)i(0,)f(comm,)h(ierr\))242 2307 y(ELSE)71 b(!)24 b(rank.EQ.2)337 2363 y(DO)g(i=1,)f(2)409 2420 y(CALL)g(MPI_PROBE\(MPI_ANY_SOURCE,)e(0,)791 2476 y(comm,)i(status,)g(ierr\))409 2533 y(IF)h(\(status\(MPI_SOURCE\))d(=)j (0\))f(THEN)75 2589 y(100)381 b(CALL)24 b(MPI_RECV\(i,)e(1,)i (MPI_INTEGER,)e(0,)h(0,)h(status,)f(ierr\))409 2646 y(ELSE)75 2702 y(200)381 b(CALL)24 b(MPI_RECV\(x,)e(1,)i(MPI_REAL,)e(1,)i(0,)f (status,)g(ierr\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 53 61 53 60 bop 75 -100 a Fm(3.8.)34 b(PR)o(OBE)16 b(AND)f(CANCEL)1161 b Fv(53)409 45 y Ft(END)24 b(IF)337 102 y(END)g(DO)242 158 y(END)f(IF)75 264 y Fv(Eac)o(h)15 b(message)g(is)g(receiv)o(ed)i (with)e(the)h(righ)o(t)f(t)o(yp)q(e.)75 371 y Fq(Example)j(3.18)23 b Fv(A)15 b(similar)h(program)e(to)h(the)g(previous)h(example,)g(but)f (no)o(w)g(it)g(has)g(a)g(problem.)242 464 y Ft(CALL)23 b(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\))242 521 y(IF)h(\(rank.EQ.0\))e (THEN)361 577 y(CALL)i(MPI_SEND\(i,)e(1,)i(MPI_INTEGER,)e(2,)h(0,)h (comm,)f(ierr\))242 634 y(ELSE)g(IF\(rank.EQ.1\))f(THEN)361 690 y(CALL)i(MPI_SEND\(x,)e(1,)i(MPI_REAL,)e(2,)i(0,)f(comm,)h(ierr\)) 242 747 y(ELSE)337 803 y(DO)g(i=1,)f(2)409 860 y(CALL)g (MPI_PROBE\(MPI_ANY_SOURCE,)e(0,)791 916 y(comm,)i(status,)g(ierr\))409 972 y(IF)h(\(status\(MPI_SOURCE\))d(=)j(0\))f(THEN)75 1029 y(100)381 b(CALL)24 b(MPI_RECV\(i,)e(1,)i(MPI_INTEGER,)e (MPI_ANY_SOURCE,)862 1085 y(0,)i(status,)f(ierr\))409 1142 y(ELSE)75 1198 y(200)381 b(CALL)24 b(MPI_RECV\(x,)e(1,)i (MPI_REAL,)e(MPI_ANY_SOURCE,)862 1255 y(0,)i(status,)f(ierr\))409 1311 y(END)h(IF)337 1368 y(END)g(DO)242 1424 y(END)f(IF)166 1518 y Fv(W)l(e)16 b(sligh)o(tly)h(mo)q(di\014ed)g(example)f(3.17,)e (using)j Fi(MPI)p 1075 1518 13 2 v 14 w(ANY)p 1174 1518 V 15 w(SOURCE)d Fv(as)h(the)h Fw(source)g Fv(argumen)o(t)f(in)75 1574 y(the)f(t)o(w)o(o)g(receiv)o(e)h(calls)g(in)h(statemen)o(ts)d(lab) q(eled)j(100)e(and)g(200.)19 b(The)c(program)e(is)i(no)o(w)f (incorrect:)20 b(the)75 1631 y(receiv)o(e)f(op)q(eration)f(ma)o(y)f (receiv)o(e)i(a)f(message)f(that)h(is)g(distinct)h(from)f(the)g (message)f(prob)q(ed)i(b)o(y)f(the)75 1687 y(preceding)f(call)f(to)f Fw(MPI)p 504 1687 14 2 v 15 w(PROBE)p Fv(.)189 1794 y Fl(A)n(dvic)n(e)e(to)i(implementors.)38 b Fv(A)13 b(call)i(to)d Fw(MPI)p 971 1794 V 16 w(PROBE\(source,)j(tag,)e(comm,)e(status\))k Fv(will)g(matc)o(h)189 1850 y(the)k(message)g(that)g(w)o(ould)h(ha)o(v) o(e)f(b)q(een)i(receiv)o(ed)f(b)o(y)g(a)f(call)h(to)f Fw(MPI)p 1431 1850 V 16 w(RECV\(...,)h(source,)g(tag,)189 1906 y(comm,)10 b(status\))j Fv(executed)g(at)e(the)h(same)g(p)q(oin)o (t.)19 b(Supp)q(ose)13 b(that)e(this)h(message)g(has)f(source)h Fw(s)p Fv(,)h(tag)189 1963 y Fw(t)i Fv(and)h(comm)o(unicator)f Fw(c)p Fv(.)20 b(If)c(the)f(tag)g(argumen)o(t)f(in)j(the)e(prob)q(e)h (call)g(has)f(v)m(alue)i Fi(MPI)p 1684 1963 13 2 v 14 w(ANY)p 1783 1963 V 15 w(T)m(A)o(G)189 2019 y Fv(then)c(the)g(message)f (prob)q(ed)i(will)g(b)q(e)g(the)f(earliest)g(p)q(ending)i(message)e (from)f(source)h Fw(s)g Fv(with)g(com-)189 2076 y(m)o(unicator)g Fw(c)h Fv(and)f(an)o(y)g(tag;)g(in)i(an)o(y)e(case,)g(the)h(message)f (prob)q(ed)h(will)h(b)q(e)f(the)g(earliest)g(p)q(ending)189 2132 y(message)f(from)g(source)h Fw(s)g Fv(with)g(tag)e Fw(t)j Fv(and)e(comm)o(unicator)h Fw(c)g Fv(\(this)f(is)i(the)e (message)h(that)f(w)o(ould)189 2189 y(ha)o(v)o(e)j(b)q(een)h(receiv)o (ed,)g(so)f(as)g(to)g(preserv)o(e)g(message)g(order\).)22 b(This)17 b(message)f(con)o(tin)o(ues)h(as)f(the)189 2245 y(earliest)g(p)q(ending)h(message)e(from)g(source)g Fw(s)h Fv(with)g(tag)f Fw(t)g Fv(and)h(comm)o(unicator)f Fw(c)p Fv(,)h(un)o(til)g(it)g(is)g(re-)189 2302 y(ceiv)o(ed.)k(A)12 b(receiv)o(e)h(op)q(eration)g(subsequen)o(t)g(to)e(the)i(prob)q(e)g (that)e(uses)i(the)f(same)g(comm)o(unicator)189 2358 y(as)j(the)h(prob)q(e)g(and)g(uses)g(the)f(tag)g(and)h(source)g(v)m (alues)h(returned)f(b)o(y)f(the)h(prob)q(e,)g(m)o(ust)f(receiv)o(e)189 2415 y(this)h(message,)f(unless)i(it)f(has)g(already)g(b)q(een)h (receiv)o(ed)g(b)o(y)f(another)g(receiv)o(e)h(op)q(eration.)22 b(\()p Fl(End)189 2471 y(of)16 b(advic)n(e)g(to)h(implementors.)p Fv(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 54 62 54 61 bop 75 -100 a Fv(54)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(CANCEL\(request\))117 122 y Fk(IN)155 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 247 y Ft(int)23 b(MPI)p 245 247 15 2 v 17 w(Cancel\(MPI)p 502 247 V 16 w(Request)g(*request\))75 333 y(MPI)p 150 333 V 17 w(CANCEL\(REQUEST,)f(IERROR\))170 390 y(INTEGER)h(REQUEST,)g (IERROR)166 476 y Fv(A)15 b(call)i(to)e Fw(MPI)p 440 476 14 2 v 16 w(CANCEL)g Fv(marks)g(for)g(cancellation)i(a)e(p)q (ending,)i(non)o(blo)q(c)o(king)g(comm)o(unication)75 533 y(op)q(eration)d(\(send)g(or)f(receiv)o(e\).)20 b(The)13 b(cancel)i(call)g(is)f(lo)q(cal.)20 b(It)14 b(returns)f(immediately)l (,)j(p)q(ossibly)f(b)q(efore)75 589 y(the)h(comm)o(unication)h(is)g (actually)g(canceled.)25 b(It)16 b(is)h(still)g(necessary)g(to)e (complete)i(a)f(comm)o(unication)75 645 y(that)f(has)g(b)q(een)h(mark)o (ed)f(for)g(cancellation,)h(using)g(a)f(call)i(to)d Fw(MPI)p 1237 645 V 16 w(REQUEST)p 1456 645 V 18 w(FREE)p Fv(,)h Fw(MPI)p 1693 645 V 16 w(W)l(AIT)g Fv(or)75 702 y Fw(MPI)p 160 702 V 16 w(TEST)g Fv(\(or)g(an)o(y)f(of)h(the)g(deriv)o(ed)i(op)q (erations\).)166 758 y(If)i(a)g(comm)o(unication)h(is)g(mark)o(ed)f (for)g(cancellation,)j(then)d(a)g Fw(MPI)p 1370 758 V 16 w(W)l(AIT)g Fv(call)i(for)d(that)h(com-)75 815 y(m)o(unication)i(is) f(guaran)o(teed)f(to)h(return,)g(irresp)q(ectiv)o(e)h(of)f(the)g (activities)h(of)e(other)g(pro)q(cesses)i(\(i.e.,)75 871 y Fw(MPI)p 160 871 V 16 w(W)l(AIT)f Fv(b)q(eha)o(v)o(es)g(as)g(a)g (lo)q(cal)i(function\);)g(similarly)g(if)f Fw(MPI)p 1235 871 V 16 w(TEST)f Fv(is)h(rep)q(eatedly)g(called)h(in)f(a)75 928 y(busy)13 b(w)o(ait)g(lo)q(op)h(for)e(a)h(canceled)h(comm)o (unication,)g(then)g Fw(MPI)p 1173 928 V 15 w(TEST)g Fv(will)g(ev)o(en)o(tually)g(b)q(e)g(successful.)166 984 y Fw(MPI)p 251 984 V 16 w(CANCEL)e Fv(can)f(b)q(e)i(used)f(to)f (cancel)i(a)e(comm)o(unication)h(that)f(uses)h(a)f(p)q(ersisten)o(t)h (request)g(\(see)75 1041 y(Sec.)26 b(3.9\),)15 b(in)j(the)f(same)f(w)o (a)o(y)g(it)h(is)h(used)f(for)g(nonp)q(ersisten)o(t)g(requests.)25 b(A)17 b(successful)h(cancellation)75 1097 y(cancels)c(the)f(activ)o(e) g(comm)o(unication,)h(but)f(not)f(the)h(request)g(itself.)20 b(After)13 b(the)g(call)h(to)e Fw(MPI)p 1690 1097 V 16 w(CANCEL)75 1154 y Fv(and)j(the)g(subsequen)o(t)h(call)g(to)e Fw(MPI)p 697 1154 V 16 w(W)l(AIT)h Fv(or)f Fw(MPI)p 978 1154 V 16 w(TEST)p Fv(,)h(the)g(request)g(b)q(ecomes)h(inactiv)o(e)g (and)f(can)75 1210 y(b)q(e)h(activ)m(ated)g(for)e(a)h(new)g(comm)o (unication.)166 1267 y(The)j(successful)i(cancellation)g(of)e(a)g (bu\013ered)h(send)g(frees)f(the)g(bu\013er)h(space)f(o)q(ccupied)j(b)o (y)d(the)75 1323 y(p)q(ending)f(message.)166 1379 y(Either)i(the)f (cancellation)i(succeeds,)f(or)f(the)g(comm)o(unication)h(succeeds,)h (but)e(not)g(b)q(oth.)29 b(If)18 b(a)75 1436 y(send)h(is)h(mark)o(ed)e (for)g(cancellation,)j(then)e(it)g(m)o(ust)f(b)q(e)i(the)f(case)g(that) f(either)h(the)g(send)g(completes)75 1492 y(normally)l(,)f(in)h(whic)o (h)f(case)f(the)h(message)f(sen)o(t)g(w)o(as)g(receiv)o(ed)h(at)f(the)h (destination)g(pro)q(cess,)g(or)f(that)75 1549 y(the)f(send)g(is)g (successfully)i(canceled,)f(in)g(whic)o(h)f(case)g(no)g(part)f(of)g (the)h(message)f(w)o(as)g(receiv)o(ed)i(at)e(the)75 1605 y(destination.)21 b(Then,)15 b(an)o(y)f(matc)o(hing)h(receiv)o(e)h(has) e(to)h(b)q(e)g(satis\014ed)h(b)o(y)e(another)h(send.)20 b(If)15 b(a)g(receiv)o(e)g(is)75 1662 y(mark)o(ed)d(for)f (cancellation,)k(then)d(it)h(m)o(ust)e(b)q(e)i(the)g(case)f(that)f (either)i(the)g(receiv)o(e)g(completes)g(normally)l(,)75 1718 y(or)18 b(that)g(the)g(receiv)o(e)i(is)f(successfully)h(canceled,) h(in)e(whic)o(h)h(case)e(no)h(part)f(of)g(the)g(receiv)o(e)i(bu\013er)e (is)75 1775 y(altered.)i(Then,)c(an)o(y)e(matc)o(hing)i(send)g(has)f (to)f(b)q(e)i(satis\014ed)g(b)o(y)f(another)g(receiv)o(e.)166 1831 y(If)h(the)f(op)q(eration)h(has)g(b)q(een)g(canceled,)h(then)f (information)g(to)f(that)f(e\013ect)i(will)h(b)q(e)f(returned)g(in)75 1888 y(the)f(status)g(argumen)o(t)f(of)h(the)g(op)q(eration)g(that)g (completes)h(the)f(comm)o(unication.)75 2038 y Fw(MPI)p 160 2038 V 16 w(TEST)p 290 2038 V 16 w(CANCELLED\(status,)i(\015ag\)) 117 2116 y Fk(IN)155 b Fw(status)476 b Fk(status)15 b(ob)r(ject)g (\(Status\))117 2191 y(OUT)108 b Fw(\015ag)518 b Fk(\(logical\))-160 2269 y Fx(>)75 2315 y Ft(int)23 b(MPI)p 245 2315 15 2 v 17 w(Test)p 358 2315 V 17 w(cancelled\(MPI)p 687 2315 V 15 w(Status)g(*status,)g(int)h(*flag\))-160 2355 y Fx(?)75 2402 y Ft(MPI)p 150 2402 V 17 w(TEST)p 263 2402 V 16 w(CANCELLED\(STATUS,)e(FLAG,)h(IERROR\))170 2458 y(LOGICAL)g(FLAG)170 2515 y(INTEGER)g(STATUS\(MPI)p 604 2515 V 16 w(STATUS)p 764 2515 V 17 w(SIZE\),)g(IERROR)166 2601 y Fv(Returns)12 b Fw(\015ag)f(=)h(true)g Fv(if)f(the)h(comm)o (unication)g(asso)q(ciated)f(with)h(the)g(status)e(ob)s(ject)h(w)o(as)f (canceled)75 2658 y(successfully)l(.)27 b(In)18 b(suc)o(h)f(a)g(case,)g (all)g(other)g(\014elds)h(of)e Fw(status)j Fv(\(suc)o(h)e(as)g Fw(count)h Fv(or)e Fw(tag)p Fv(\))h(are)f(unde\014ned.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 55 63 55 62 bop 75 -100 a Fm(3.9.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (REQUESTS)661 b Fv(55)75 45 y(Returns)12 b Fw(\015ag)g(=)g(false)p Fv(,)g(otherwise.)19 b(If)12 b(a)g(receiv)o(e)g(op)q(eration)g(migh)o (t)g(b)q(e)h(canceled)g(then)f(one)g(should)h(call)75 102 y Fw(MPI)p 160 102 14 2 v 16 w(TEST)p 290 102 V 16 w(CANCELLED)g Fv(\014rst,)f(to)g(c)o(hec)o(k)g(whether)h(the)f(op)q (eration)h(w)o(as)e(canceled,)j(b)q(efore)f(c)o(hec)o(king)75 158 y(on)i(the)g(other)g(\014elds)i(of)d(the)i(return)f(status.)189 264 y Fl(A)n(dvic)n(e)k(to)h(users.)64 b Fv(Cancel)20 b(can)g(b)q(e)f(an)h(exp)q(ensiv)o(e)g(op)q(eration)g(that)e(should)j (b)q(e)f(used)f(only)189 321 y(exceptionally)l(.)j(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fv(\))189 427 y Fl(A)n(dvic)n(e)e(to)i(implementors.)39 b Fv(If)15 b(a)f(send)h(op)q(eration)g(uses)f(an)g(\\eager")g(proto)q(col)g (\(data)g(is)g(trans-)189 483 y(ferred)e(to)f(the)h(receiv)o(er)h(b)q (efore)f(a)g(matc)o(hing)f(receiv)o(e)i(is)g(p)q(osted\),)f(then)g(the) g(cancellation)i(of)e(this)189 540 y(send)i(ma)o(y)g(require)h(comm)o (unication)g(with)f(the)g(in)o(tended)i(receiv)o(er)f(in)f(order)g(to)g (free)g(allo)q(cated)189 596 y(bu\013ers.)k(On)13 b(some)f(systems)g (this)h(ma)o(y)f(require)h(an)f(in)o(terrupt)h(to)e(the)i(in)o(tended)h (receiv)o(er.)19 b(Note)189 653 y(that,)14 b(while)j(comm)o(unication)g (ma)o(y)e(b)q(e)h(needed)h(to)e(implemen)o(t)i Fw(MPI)p 1415 653 V 16 w(CANCEL)p Fv(,)e(this)h(is)g(still)h(a)189 709 y(lo)q(cal)g(op)q(eration,)f(since)i(its)e(completion)h(do)q(es)g (not)f(dep)q(end)i(on)e(the)g(co)q(de)h(executed)g(b)o(y)f(other)189 766 y(pro)q(cesses.)21 b(If)15 b(pro)q(cessing)i(is)f(required)g(on)f (another)g(pro)q(cess,)h(this)g(should)g(b)q(e)g(transparen)o(t)f(to) 189 822 y(the)h(application)j(\(hence)e(the)g(need)h(for)e(an)h(in)o (terrupt)g(and)g(an)f(in)o(terrupt)h(handler\).)26 b(\()p Fl(End)17 b(of)189 879 y(advic)n(e)f(to)g(implementors.)p Fv(\))75 1022 y Fn(3.9)59 b(P)n(ersistent)19 b(communication)f (requests)75 1123 y Fv(Often)f(a)f(comm)o(unication)g(with)h(the)f (same)g(argumen)o(t)g(list)h(is)g(rep)q(eatedly)g(executed)g(within)h (the)e(in-)75 1180 y(ner)k(lo)q(op)h(of)f(a)g(parallel)i(computation.) 35 b(In)20 b(suc)o(h)h(a)f(situation,)h(it)g(ma)o(y)e(b)q(e)i(p)q (ossible)h(to)e(optimize)75 1236 y(the)c(comm)o(unication)g(b)o(y)g (binding)h(the)f(list)g(of)f(comm)o(unication)i(argumen)o(ts)d(to)h(a)h Fq(p)q(ersisten)o(t)f Fv(com-)75 1293 y(m)o(unication)k(request)e(once) h(and,)g(then,)g(rep)q(eatedly)h(using)f(the)g(request)g(to)e(initiate) j(and)f(complete)75 1349 y(messages.)g(The)12 b(p)q(ersisten)o(t)g (request)g(th)o(us)f(created)h(can)g(b)q(e)g(though)o(t)f(of)g(as)g(a)h (comm)o(unication)g(p)q(ort)f(or)75 1406 y(a)k(\\half-c)o(hannel.")21 b(It)15 b(do)q(es)h(not)e(pro)o(vide)i(the)f(full)i(functionalit)o(y)f (of)f(a)f(con)o(v)o(en)o(tional)i(c)o(hannel,)g(since)75 1462 y(there)i(is)h(no)f(binding)i(of)e(the)g(send)h(p)q(ort)f(to)f (the)i(receiv)o(e)g(p)q(ort.)28 b(This)19 b(construct)f(allo)o(ws)g (reduction)75 1519 y(of)c(the)g(o)o(v)o(erhead)g(for)g(comm)o (unication)h(b)q(et)o(w)o(een)g(the)f(pro)q(cess)h(and)f(comm)o (unication)h(con)o(troller,)g(but)75 1575 y(not)e(of)g(the)g(o)o(v)o (erhead)g(for)g(comm)o(unication)h(b)q(et)o(w)o(een)g(one)f(comm)o (unication)h(con)o(troller)g(and)g(another.)75 1631 y(It)19 b(is)h(not)e(necessary)i(that)e(messages)h(sen)o(t)f(with)i(a)f(p)q (ersisten)o(t)g(request)g(b)q(e)h(receiv)o(ed)g(b)o(y)f(a)g(receiv)o(e) 75 1688 y(op)q(eration)c(using)h(a)f(p)q(ersisten)o(t)h(request,)f(or)f (vice)j(v)o(ersa.)166 1744 y(A)j(p)q(ersisten)o(t)g(comm)o(unication)g (request)g(is)g(created)g(using)h(one)e(of)h(the)f(four)h(follo)o(wing) g(calls.)75 1801 y(These)c(calls)g(in)o(v)o(olv)o(e)g(no)f(comm)o (unication.)75 1952 y Fw(MPI)p 160 1952 V 16 w(SEND)p 293 1952 V 17 w(INIT\(buf,)f(count,)i(datat)o(yp)q(e,)h(dest,)f(tag,)f (comm,)e(request\))117 2029 y Fk(IN)155 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 2104 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 2179 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)g(\(handle\))117 2254 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 2329 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 2405 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 2480 y(OUT)108 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 2604 y Ft(int)23 b(MPI)p 245 2604 15 2 v 17 w(Send)p 358 2604 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1091 2604 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 2661 y(int)g(tag,)f(MPI)p 683 2661 V 17 w(Comm)g(comm,)g(MPI)p 1034 2661 V 17 w(Request)g(*request\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 56 64 56 63 bop 75 -100 a Fv(56)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(SEND)p 263 45 V 16 w(INIT\(BUF,)23 b(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 102 y()g(BUF\(*\))170 158 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 244 y Fv(Creates)15 b(a)h(p)q(ersisten)o(t)h(comm)o(unication)g (request)f(for)g(a)f(standard)h(mo)q(de)h(send)f(op)q(eration,)h(and)75 301 y(binds)f(to)f(it)g(all)i(the)e(argumen)o(ts)f(of)h(a)g(send)h(op)q (eration.)75 452 y Fw(MPI)p 160 452 14 2 v 16 w(BSEND)p 323 452 V 17 w(INIT\(buf,)f(count,)h(datat)o(yp)q(e,)g(dest,)g(tag,)f (comm,)e(request\))117 529 y Fk(IN)155 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 604 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts) f(sen)o(t)i(\(in)o(teger\))117 679 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 754 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 830 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 905 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 980 y(OUT)108 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1104 y Ft(int)23 b(MPI)p 245 1104 15 2 v 17 w(Bsend)p 382 1104 V 17 w(init\(void*)f(buf,)i(int)f(count,)g(MPI)p 1115 1104 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 1161 y(int)g(tag,)f(MPI)p 683 1161 V 17 w(Comm)g(comm,)g(MPI)p 1034 1161 V 17 w(Request)g(*request\))75 1247 y(MPI)p 150 1247 V 17 w(BSEND)p 287 1247 V 16 w(INIT\(BUF,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1304 y()g(BUF\(*\))170 1360 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 1447 y Fv(Creates)14 b(a)h(p)q(ersisten)o(t)h(comm)o(unication)g (request)f(for)g(a)g(bu\013ered)g(mo)q(de)h(send.)75 1598 y Fw(MPI)p 160 1598 14 2 v 16 w(SSEND)p 318 1598 V 17 w(INIT\(buf,)f(count,)h(datat)o(yp)q(e,)g(dest,)g(tag,)f(comm,)e (request\))117 1675 y Fk(IN)155 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1750 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 1825 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)g(\(handle\))117 1900 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 1975 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 2050 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 2126 y(OUT)108 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 2250 y Ft(int)23 b(MPI)p 245 2250 15 2 v 17 w(Ssend)p 382 2250 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 2250 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 2306 y(int)g(tag,)f(MPI)p 683 2306 V 17 w(Comm)g(comm,)g(MPI)p 1034 2306 V 17 w(Request)g(*request\))75 2393 y(MPI)p 150 2393 V 17 w(SSEND)p 287 2393 V 16 w(INIT\(BUF,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2449 y()g(BUF\(*\))170 2506 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)166 2592 y Fv(Creates)14 b(a)h(p)q(ersisten)o(t)h(comm)o(unication)g(ob)s(ject)f (for)f(a)h(sync)o(hronous)g(mo)q(de)h(send)f(op)q(eration.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 57 65 57 64 bop 75 -100 a Fm(3.9.)34 b(PERSISTENT)16 b(COMMUNICA)l(TION)g (REQUESTS)661 b Fv(57)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(RSEND)p 322 45 V 17 w(INIT\(buf,)15 b(count,)h(datat)o(yp)q(e,)g (dest,)g(tag,)f(comm,)e(request\))117 122 y Fk(IN)155 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o (hoice\))117 197 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(\(in)o(teger\))117 273 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h (elemen)o(t)g(\(handle\))117 348 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 423 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(\(in)o(teger\))117 498 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 573 y(OUT)108 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 698 y Ft(int)23 b(MPI)p 245 698 15 2 v 17 w(Rsend)p 382 698 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 698 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 754 y(int)g(tag,)f(MPI)p 683 754 V 17 w(Comm)g(comm,)g(MPI)p 1034 754 V 17 w(Request)g(*request\))75 840 y(MPI)p 150 840 V 17 w(RSEND)p 287 840 V 16 w(INIT\(BUF,)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 897 y()g (BUF\(*\))170 953 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h (COMM,)f(REQUEST,)f(IERROR)166 1040 y Fv(Creates)14 b(a)h(p)q(ersisten) o(t)h(comm)o(unication)g(ob)s(ject)f(for)f(a)h(ready)g(mo)q(de)h(send)f (op)q(eration.)75 1191 y Fw(MPI)p 160 1191 14 2 v 16 w(RECV)p 291 1191 V 17 w(INIT\(buf,)g(count,)h(datat)o(yp)q(e,)g (source,)g(tag,)e(comm,)f(request\))117 1268 y Fk(OUT)108 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g (\(c)o(hoice\))117 1343 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(receiv)o(ed)j(\(in)o(teger\))117 1418 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(eac)o(h)h(elemen)o(t)g(\(handle\))117 1493 y(IN)155 b Fw(source)468 b Fk(rank)14 b(of)f(source)j(or)d(MPI)p 1309 1493 13 2 v 16 w(ANY)p 1418 1493 V 15 w(SOUR)o(CE)h(\(in)o (teger\))117 1569 y(IN)155 b Fw(tag)526 b Fk(message)14 b(tag)f(or)h(MPI)p 1270 1569 V 15 w(ANY)p 1378 1569 V 16 w(T)m(A)o(G)f(\(in)o(teger\))117 1644 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 1719 y(OUT)108 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 1843 y Ft(int)23 b(MPI)p 245 1843 15 2 v 17 w(Recv)p 358 1843 V 17 w(init\(void*)f(buf,)i(int)f(count,)g(MPI)p 1091 1843 V 17 w(Datatype)g(datatype,)f(int)i(source,)393 1900 y(int)g(tag,)f(MPI)p 683 1900 V 17 w(Comm)g(comm,)g(MPI)p 1034 1900 V 17 w(Request)g(*request\))75 1986 y(MPI)p 150 1986 V 17 w(RECV)p 263 1986 V 16 w(INIT\(BUF,)g(COUNT,)g(DATATYPE,) g(SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2043 y()g(BUF\(*\))170 2099 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g (TAG,)g(COMM,)g(REQUEST,)g(IERROR)166 2186 y Fv(Creates)13 b(a)h(p)q(ersisten)o(t)h(comm)o(unication)f(request)h(for)e(a)h(receiv) o(e)h(op)q(eration.)20 b(The)14 b(argumen)o(t)f Fw(buf)75 2242 y Fv(is)f(mark)o(ed)g(as)f Fw(OUT)h Fv(b)q(ecause)h(the)f(user)g (giv)o(es)g(p)q(ermission)h(to)e(write)h(on)g(the)f(receiv)o(e)i (bu\013er)f(b)o(y)g(passing)75 2298 y(the)j(argumen)o(t)g(to)f Fw(MPI)p 497 2298 14 2 v 16 w(RECV)p 628 2298 V 17 w(INIT)p Fv(.)166 2355 y(A)i(p)q(ersisten)o(t)h(comm)o(unication)f(request)h(is) f(inactiv)o(e)h(after)f(it)g(w)o(as)f(created)h(|)h(no)f(activ)o(e)g (com-)75 2411 y(m)o(unication)g(is)g(attac)o(hed)e(to)h(the)g(request.) 166 2468 y(A)k(comm)o(unication)g(\(send)g(or)g(receiv)o(e\))g(that)f (uses)h(a)f(p)q(ersisten)o(t)i(request)e(is)i(initiated)g(b)o(y)f(the) 75 2524 y(function)d Fw(MPI)p 338 2524 V 16 w(ST)l(ART)p Fv(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 58 66 58 65 bop 75 -100 a Fv(58)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(ST)l(ART\(request\))117 122 y Fk(INOUT)62 b Fw(request)452 b Fk(comm)o(unication)11 b(request)k(\(handle\))75 247 y Ft(int)23 b(MPI)p 245 247 15 2 v 17 w(Start\(MPI)p 478 247 V 16 w(Request)g(*request\))75 333 y(MPI)p 150 333 V 17 w(START\(REQUEST,)f(IERROR\))170 390 y(INTEGER)h(REQUEST,)g (IERROR)166 476 y Fv(The)d(argumen)o(t,)h Fw(request)p Fv(,)h(is)f(a)f(handle)h(returned)g(b)o(y)f(one)g(of)g(the)h(previous)f (\014v)o(e)h(calls.)36 b(The)75 533 y(asso)q(ciated)15 b(request)h(should)g(b)q(e)g(inactiv)o(e.)21 b(The)15 b(request)g(b)q(ecomes)h(activ)o(e)f(once)h(the)f(call)h(is)g(made.)166 589 y(If)e(the)g(request)g(is)h(for)e(a)h(send)h(with)f(ready)g(mo)q (de,)g(then)g(a)g(matc)o(hing)g(receiv)o(e)h(should)g(b)q(e)g(p)q (osted)75 645 y(b)q(efore)j(the)g(call)h(is)g(made.)28 b(The)18 b(comm)o(unication)h(bu\013er)e(should)i(not)f(b)q(e)g (accessed)h(after)e(the)h(call,)75 702 y(and)d(un)o(til)i(the)e(op)q (eration)g(completes.)166 758 y(The)22 b(call)i(is)e(lo)q(cal,)j(with)e (similar)g(seman)o(tics)f(to)g(the)g(non)o(blo)q(c)o(king)i(comm)o (unication)f(op)q(era-)75 815 y(tions)d(describ)q(ed)i(in)e(section)h (3.7.)33 b(That)19 b(is,)i(a)f(call)h(to)e Fw(MPI)p 1174 815 14 2 v 16 w(ST)l(ART)i Fv(with)f(a)f(request)h(created)g(b)o(y)75 871 y Fw(MPI)p 160 871 V 16 w(SEND)p 293 871 V 17 w(INIT)13 b Fv(starts)f(a)h(comm)o(unication)i(in)g(the)e(same)h(manner)g(as)f(a) g(call)i(to)e Fw(MPI)p 1601 871 V 16 w(ISEND)p Fv(;)g(a)h(call)75 928 y(to)h Fw(MPI)p 216 928 V 16 w(ST)l(ART)h Fv(with)g(a)f(request)g (created)h(b)o(y)f Fw(MPI)p 995 928 V 16 w(BSEND)p 1158 928 V 17 w(INIT)g Fv(starts)f(a)h(comm)o(unication)i(in)f(the)75 984 y(same)f(manner)g(as)g(a)g(call)h(to)f Fw(MPI)p 671 984 V 16 w(IBSEND)p Fv(;)g(and)g(so)g(on.)75 1135 y Fw(MPI)p 160 1135 V 16 w(ST)l(ART)l(ALL\(count,)h(a)o(rra)o(y)p 641 1135 V 15 w(of)p 693 1135 V 16 w(requests\))117 1212 y Fk(IN)155 b Fw(count)482 b Fk(list)14 b(length)f(\(in)o(teger\))117 1288 y(INOUT)62 b Fw(a)o(rra)o(y)p 416 1288 V 15 w(of)p 468 1288 V 16 w(requests)272 b Fk(arra)o(y)14 b(of)f(requests)j(\(arra) o(y)e(of)f(handle\))75 1412 y Ft(int)23 b(MPI)p 245 1412 15 2 v 17 w(Startall\(int)f(count,)h(MPI)p 811 1412 V 17 w(Request)g(*array)p 1163 1412 V 17 w(of)p 1228 1412 V 16 w(requests\))75 1498 y(MPI)p 150 1498 V 17 w(STARTALL\(COUNT,)f (ARRAY)p 669 1498 V 16 w(OF)p 733 1498 V 17 w(REQUESTS,)h(IERROR\))170 1555 y(INTEGER)g(COUNT,)g(ARRAY)p 651 1555 V 17 w(OF)p 716 1555 V 17 w(REQUESTS\(*\),)f(IERROR)166 1641 y Fv(Start)i(all)j (comm)o(unications)f(asso)q(ciated)g(with)f(requests)h(in)g Fw(a)o(rra)o(y)p 1399 1641 14 2 v 15 w(of)p 1451 1641 V 16 w(requests)p Fv(.)52 b(A)26 b(call)h(to)75 1698 y Fw(MPI)p 160 1698 V 16 w(ST)l(ART)l(ALL\(count,)18 b(a)o(rra)o(y)p 643 1698 V 15 w(of)p 695 1698 V 16 w(requests\))h Fv(has)d(the)h(same)g(e\013ect)f(as)h(calls)h(to)e Fw(MPI)p 1599 1698 V 16 w(ST)l(ART)i(\(&a)o(r-)75 1754 y(ra)o(y)p 136 1754 V 15 w(of)p 188 1754 V 16 w(requests[i]\))p Fv(,)f(executed)f(for)f Fw(i=0)g(,...,)e(count-1)p Fv(,)j(in)g(some)f (arbitrary)g(order.)166 1811 y(A)i(comm)o(unication)g(started)f(with)h (a)g(call)h(to)e Fw(MPI)p 1063 1811 V 16 w(ST)l(ART)h Fv(or)f Fw(MPI)p 1377 1811 V 16 w(ST)l(ART)l(ALL)i Fv(is)f(completed)75 1867 y(b)o(y)11 b(a)h(call)g(to)f Fw(MPI)p 385 1867 V 16 w(W)l(AIT)p Fv(,)g Fw(MPI)p 620 1867 V 16 w(TEST)p Fv(,)h(or)f(one)g(of)g(the)h(deriv)o(ed)h(functions)f(describ)q(ed)h (in)g(section)f(3.7.5.)75 1924 y(The)18 b(request)f(b)q(ecomes)h (inactiv)o(e)h(after)d(successful)j(completion)g(of)e(suc)o(h)h(call.) 27 b(The)18 b(request)f(is)h(not)75 1980 y(deallo)q(cated)f(and)e(it)g (can)h(b)q(e)g(activ)m(ated)f(anew)h(b)o(y)f(an)g Fw(MPI)p 1116 1980 V 16 w(ST)l(ART)h Fv(or)e Fw(MPI)p 1427 1980 V 16 w(ST)l(ART)l(ALL)i Fv(call.)166 2037 y(A)f(p)q(ersisten)o(t)h (request)f(is)h(deallo)q(cated)g(b)o(y)g(a)e(call)j(to)d Fw(MPI)p 1190 2037 V 16 w(REQUEST)p 1409 2037 V 18 w(FREE)i Fv(\(Section)f(3.7.3\).)166 2093 y(The)g(call)g(to)f Fw(MPI)p 481 2093 V 16 w(REQUEST)p 700 2093 V 17 w(FREE)h Fv(can)g(o)q(ccur)g(at)e(an)o(y)h(p)q(oin)o(t)h(in)h(the)e(program)f (after)h(the)g(p)q(er-)75 2149 y(sisten)o(t)h(request)g(w)o(as)f (created.)20 b(Ho)o(w)o(ev)o(er,)14 b(the)h(request)g(will)h(b)q(e)g (deallo)q(cated)g(only)g(after)e(it)h(b)q(ecomes)75 2206 y(inactiv)o(e.)33 b(Activ)o(e)20 b(receiv)o(e)g(requests)f(should)i (not)e(b)q(e)h(freed.)32 b(Otherwise,)21 b(it)e(will)i(not)e(b)q(e)h(p) q(ossible)75 2262 y(to)c(c)o(hec)o(k)g(that)f(the)i(receiv)o(e)g(has)f (completed.)23 b(It)17 b(is)f(preferable,)h(in)g(general,)g(to)e(free)h (requests)g(when)75 2319 y(they)h(are)g(inactiv)o(e.)27 b(If)17 b(this)g(rule)h(is)g(follo)o(w)o(ed,)g(then)f(the)g(functions)h (describ)q(ed)h(in)f(this)g(section)f(will)75 2375 y(b)q(e)f(in)o(v)o (ok)o(ed)f(in)h(a)f(sequence)i(of)d(the)i(form,)166 2491 y Fq(Create)h Fv(\()p Fq(Start)g(Complete)p Fv(\))724 2474 y Fh(\003)760 2491 y Fq(F)l(ree)f Fp(;)e Fv(where)h Fx(\003)f Fv(indicates)i(zero)f(or)f(more)g(rep)q(etitions.)21 b(If)15 b(the)75 2606 y(same)g(comm)o(unication)i(ob)s(ject)e(is)h (used)g(in)h(sev)o(eral)f(concurren)o(t)f(threads,)h(it)g(is)g(the)f (user's)h(resp)q(onsi-)75 2663 y(bilit)o(y)h(to)d(co)q(ordinate)i (calls)g(so)f(that)f(the)i(correct)e(sequence)j(is)f(ob)q(ey)o(ed.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 59 67 59 66 bop 75 -100 a Fm(3.10.)34 b(SEND-RECEIVE)1277 b Fv(59)166 45 y(A)12 b(send)g(op)q(eration)f(initiated)j(with)d Fw(MPI)p 874 45 14 2 v 16 w(ST)l(ART)i Fv(can)e(b)q(e)i(matc)o(hed)e (with)h(an)o(y)f(receiv)o(e)i(op)q(eration)75 102 y(and,)g(lik)o (ewise,)h(a)e(receiv)o(e)i(op)q(eration)e(initiated)i(with)f Fw(MPI)p 1098 102 V 16 w(ST)l(ART)g Fv(can)g(receiv)o(e)g(messages)f (generated)75 158 y(b)o(y)j(an)o(y)g(send)h(op)q(eration.)75 301 y Fn(3.10)59 b(Send-receive)75 403 y Fv(The)18 b Fq(send-receiv)o(e)g Fv(op)q(erations)g(com)o(bine)h(in)g(one)f(call)h (the)f(sending)i(of)d(a)h(message)g(to)f(one)h(desti-)75 459 y(nation)g(and)f(the)h(receiving)h(of)e(another)g(message,)h(from)e (another)h(pro)q(cess.)28 b(The)17 b(t)o(w)o(o)g(\(source)g(and)75 516 y(destination\))i(are)f(p)q(ossibly)j(the)d(same.)30 b(A)19 b(send-receiv)o(e)h(op)q(eration)f(is)g(v)o(ery)f(useful)i(for)e (executing)75 572 y(a)g(shift)i(op)q(eration)f(across)f(a)g(c)o(hain)i (of)e(pro)q(cesses.)31 b(If)19 b(blo)q(c)o(king)h(sends)f(and)g(receiv) o(es)h(are)e(used)i(for)75 629 y(suc)o(h)f(a)f(shift,)h(then)g(one)f (needs)h(to)f(order)g(the)g(sends)h(and)g(receiv)o(es)g(correctly)g (\(for)e(example,)j(ev)o(en)75 685 y(pro)q(cesses)d(send,)f(then)h (receiv)o(e,)g(o)q(dd)f(pro)q(cesses)h(receiv)o(e)g(\014rst,)f(then)g (send\))h(so)f(as)g(to)f(prev)o(en)o(t)h(cyclic)75 741 y(dep)q(endencies)j(that)c(ma)o(y)g(lead)h(to)f(deadlo)q(c)o(k.)23 b(When)16 b(a)f(send-receiv)o(e)j(op)q(eration)e(is)g(used,)g(the)g (com-)75 798 y(m)o(unication)j(subsystem)e(tak)o(es)g(care)h(of)f (these)h(issues.)29 b(The)18 b(send-receiv)o(e)h(op)q(eration)f(can)g (b)q(e)g(used)75 854 y(in)e(conjunction)f(with)h(the)e(functions)i (describ)q(ed)h(in)e(Chapter)g(6)f(in)i(order)e(to)h(p)q(erform)f (shifts)h(on)g(v)m(ar-)75 911 y(ious)20 b(logical)h(top)q(ologies.)33 b(Also,)21 b(a)e(send-receiv)o(e)j(op)q(eration)d(is)h(useful)h(for)e (implemen)o(ting)j(remote)75 967 y(pro)q(cedure)16 b(calls.)166 1024 y(A)g(message)f(sen)o(t)g(b)o(y)h(a)f(send-receiv)o(e)j(op)q (eration)d(can)h(b)q(e)h(receiv)o(ed)f(b)o(y)g(a)f(regular)h(receiv)o (e)h(op)q(er-)75 1080 y(ation)f(or)f(prob)q(ed)i(b)o(y)f(a)g(prob)q(e)g (op)q(eration;)g(a)g(send-receiv)o(e)i(op)q(eration)e(can)g(receiv)o(e) h(a)f(message)f(sen)o(t)75 1137 y(b)o(y)g(a)g(regular)g(send)h(op)q (eration.)75 1288 y Fw(MPI)p 160 1288 V 16 w(SENDRECV\(sendbuf,)24 b(sendcount,)f(sendt)o(yp)q(e,)g(dest,)f(sendtag,)g(recvbuf,)g (recvcount,)f(recvt)o(yp)q(e,)75 1344 y(source,)15 b(recvtag,)g(comm,)e (status\))117 1421 y Fk(IN)155 b Fw(sendbuf)444 b Fk(initial)12 b(address)j(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 1496 y(IN)155 b Fw(sendcount)399 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1571 y(IN)155 b Fw(sendt)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(bu\013er)g(\(handle\))117 1647 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o (teger\))117 1722 y(IN)155 b Fw(sendtag)443 b Fk(send)15 b(tag)e(\(in)o(teger\))117 1797 y(OUT)108 b Fw(recvbuf)450 b Fk(initial)12 b(address)j(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\)) 117 1872 y(IN)155 b Fw(recvcount)405 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h(bu\013er)g(\(in)o(teger\))117 1947 y(IN)155 b Fw(recvt)o(yp)q(e)430 b Fk(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(receiv)o(e)i(bu\013er)g(\(handle\))117 2022 y(IN)155 b Fw(source)468 b Fk(rank)14 b(of)f(source)j(\(in)o (teger\))117 2097 y(IN)155 b Fw(recvtag)449 b Fk(receiv)o(e)15 b(tag)f(\(in)o(teger\))117 2173 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 2248 y(OUT)108 b Fw(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\))75 2372 y Ft(int)23 b(MPI)p 245 2372 15 2 v 17 w(Sendrecv\(void)f (*sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 2372 V 17 w(Datatype)f(sendtype,)393 2429 y(int)i(dest,)f(int)g(sendtag,)g(void)g (*recvbuf,)g(int)g(recvcount,)393 2485 y(MPI)p 468 2485 V 17 w(Datatype)g(recvtype,)f(int)i(source,)f(MPI)p 1297 2485 V 16 w(Datatype)g(recvtag,)393 2542 y(MPI)p 468 2542 V 17 w(Comm)g(comm,)g(MPI)p 819 2542 V 17 w(Status)g(*status\))75 2628 y(MPI)p 150 2628 V 17 w(SENDRECV\(SENDBUF,)e(SENDCOUNT,)i (SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVBUF,)393 2684 y(RECVCOUNT,)h (RECVTYPE,)f(SOURCE,)h(RECVTAG,)g(COMM,)g(STATUS,)g(IERROR\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 60 68 60 67 bop 75 -100 a Fv(60)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Ft()23 b(SENDBUF\(*\),)g(RECVBUF\(*\))170 102 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVCOUNT,)h(RECVTYPE,)170 158 y(SOURCE,)g(RECV)h(TAG,)f(COMM,)g(STATUS\(MPI)p 986 158 15 2 v 16 w(STATUS)p 1146 158 V 16 w(SIZE\),)g(IERROR)166 244 y Fv(Execute)c(a)e(blo)q(c)o(king)j(send)f(and)f(receiv)o(e)h(op)q (eration.)29 b(Both)18 b(send)h(and)f(receiv)o(e)h(use)g(the)f(same)75 301 y(comm)o(unicator,)i(but)g(p)q(ossibly)i(di\013eren)o(t)e(tags.)33 b(The)20 b(send)g(bu\013er)g(and)g(receiv)o(e)h(bu\013ers)f(m)o(ust)f (b)q(e)75 357 y(disjoin)o(t,)c(and)h(ma)o(y)e(ha)o(v)o(e)h(di\013eren)o (t)h(lengths)f(and)h(datat)o(yp)q(es.)75 508 y Fw(MPI)p 160 508 14 2 v 16 w(SENDRECV)p 408 508 V 17 w(REPLA)o(CE\(buf,)j (count,)f(datat)o(yp)q(e,)h(dest,)f(sendtag,)h(source,)f(recvtag,)f (comm,)e(sta-)75 565 y(tus\))117 642 y Fk(INOUT)62 b Fw(buf)527 b Fk(initial)12 b(address)j(of)f(send)g(and)g(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 715 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)9 b(of)g(elemen)o(ts)h(in)f(send)i(and)e(receiv)o (e)i(bu\013er)g(\(in)o(teger\))117 788 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(t)o(yp)q(e)15 b(of)e(elemen)o(ts)h(in)f(send)i(and)f(receiv)o(e)h (bu\013er)g(\(handle\))117 861 y(IN)155 b Fw(dest)511 b Fk(rank)14 b(of)f(destination)h(\(in)o(teger\))117 934 y(IN)155 b Fw(sendtag)443 b Fk(send)15 b(message)f(tag)f(\(in)o (teger\))117 1007 y(IN)155 b Fw(source)468 b Fk(rank)14 b(of)f(source)j(\(in)o(teger\))117 1080 y(IN)155 b Fw(recvtag)449 b Fk(receiv)o(e)15 b(message)f(tag)f(\(in)o(teger\))117 1153 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 1226 y(OUT)108 b Fw(status)476 b Fk(status)15 b(ob)r(ject)g(\(Status\)) 75 1350 y Ft(int)23 b(MPI)p 245 1350 15 2 v 17 w(Sendrecv)p 454 1350 V 16 w(replace\(void*)f(buf,)i(int)f(count,)g(MPI)p 1258 1350 V 17 w(Datatype)g(datatype,)393 1407 y(int)h(dest,)f(int)g (sendtag,)g(int)g(source,)g(int)h(recvtag,)f(MPI)p 1614 1407 V 16 w(Comm)h(comm,)393 1463 y(MPI)p 468 1463 V 17 w(Status)f(*status\))75 1550 y(MPI)p 150 1550 V 17 w(SENDRECV)p 359 1550 V 16 w(REPLACE\(BUF,)f(COUNT,)h(DATATYPE,)g (DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)393 1606 y(COMM,)g(STATUS,)g (IERROR\))170 1662 y()g(BUF\(*\))170 1719 y(INTEGER)g(COUNT,)g (DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)g(COMM,)170 1775 y(STATUS\(MPI)p 413 1775 V 16 w(STATUS)p 573 1775 V 17 w(SIZE\),)g(IERROR)166 1862 y Fv(Execute)18 b(a)f(blo)q(c)o(king)i (send)f(and)g(receiv)o(e.)27 b(The)18 b(same)f(bu\013er)h(is)g(used)g (b)q(oth)f(for)g(the)h(send)g(and)75 1918 y(for)d(the)g(receiv)o(e,)h (so)e(that)h(the)g(message)g(sen)o(t)g(is)h(replaced)g(b)o(y)f(the)g (message)g(receiv)o(ed.)166 1975 y(The)d(seman)o(tics)h(of)f(a)g (send-receiv)o(e)i(op)q(eration)e(is)h(what)e(w)o(ould)i(b)q(e)g (obtained)g(if)g(the)f(caller)h(fork)o(ed)75 2031 y(t)o(w)o(o)j (concurren)o(t)h(threads,)g(one)g(to)f(execute)i(the)f(send,)g(and)h (one)f(to)f(execute)i(the)f(receiv)o(e,)h(follo)o(w)o(ed)75 2088 y(b)o(y)d(a)g(join)h(of)e(these)i(t)o(w)o(o)e(threads.)189 2179 y Fl(A)n(dvic)n(e)d(to)h(implementors.)37 b Fv(Additional)12 b(in)o(termediate)f(bu\013ering)g(is)g(needed)h(for)e(the)h(\\replace") 189 2235 y(v)m(arian)o(t.)19 b(\()p Fl(End)d(of)g(advic)n(e)h(to)f (implementors.)p Fv(\))75 2376 y Fn(3.11)59 b(Null)21 b(p)n(ro)r(cesses)75 2478 y Fv(In)15 b(man)o(y)e(instances,)i(it)f(is)h (con)o(v)o(enien)o(t)g(to)e(sp)q(ecify)j(a)d(\\dumm)o(y")h(source)g(or) g(destination)h(for)e(comm)o(u-)75 2534 y(nication.)23 b(This)16 b(simpli\014es)i(the)e(co)q(de)g(that)g(is)g(needed)h(for)e (dealing)i(with)f(b)q(oundaries,)h(for)e(example,)75 2591 y(in)h(the)f(case)h(of)e(a)h(non-circular)i(shift)e(done)h(with)g (calls)g(to)e(send-receiv)o(e.)166 2647 y(The)f(sp)q(ecial)h(v)m(alue) 25 b Fi(MPI)p 605 2647 13 2 v 14 w(PROC)p 731 2647 V 13 w(NULL)14 b Fv(can)e(b)q(e)h(used)g(instead)g(of)f(a)g(rank)g (wherev)o(er)h(a)f(source)g(or)g(a)75 2704 y(destination)j(argumen)o(t) e(is)h(required)i(in)e(a)g(call.)21 b(A)14 b(comm)o(unication)g(with)h (pro)q(cess)26 b Fi(MPI)p 1634 2704 V 15 w(PROC)p 1761 2704 V 13 w(NULL)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 61 69 61 68 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fv(61)75 45 y(has)11 b(no)h(e\013ect.)18 b(A)12 b(send)g(to)21 b Fi(MPI)p 633 45 13 2 v 15 w(PROC)p 760 45 V 13 w(NULL)13 b Fv(succeeds)f(and)g(returns)f(as)g(so)q(on)h(as)f (p)q(ossible.)20 b(A)12 b(receiv)o(e)75 102 y(from)24 b Fi(MPI)p 269 102 V 14 w(PROC)p 395 102 V 13 w(NULL)14 b Fv(succeeds)g(and)f(returns)f(as)h(so)q(on)f(as)h(p)q(ossible)h(with) f(no)g(mo)q(di\014cations)h(to)e(the)75 158 y(receiv)o(e)17 b(bu\013er.)22 b(When)17 b(a)e(receiv)o(e)i(with)g Fw(source)f Fv(=)31 b Fi(MPI)p 1080 158 V 15 w(PROC)p 1207 158 V 13 w(NULL)17 b Fv(is)f(executed)h(then)g(the)f(status)75 214 y(ob)s(ject)f(returns)30 b Fw(source)16 b Fv(=)29 b Fi(MPI)p 661 214 V 14 w(PROC)p 787 214 V 14 w(NULL)p Fv(,)15 b Fw(tag)g Fv(=)30 b Fi(MPI)p 1147 214 V 14 w(ANY)p 1246 214 V 15 w(T)m(A)o(G)14 b Fv(and)h Fw(count)i(=)e(0)p Fv(.)75 356 y Fn(3.12)59 b(Derived)19 b(datat)n(yp)r(es)75 458 y Fv(Up)13 b(to)f(here,)h(all)h(p)q(oin)o(t)f(to)f(p)q(oin)o(t)i (comm)o(unication)f(ha)o(v)o(e)f(in)o(v)o(olv)o(ed)i(only)f(con)o (tiguous)g(bu\013ers)g(con)o(tain-)75 514 y(ing)h(a)f(sequence)i(of)e (elemen)o(ts)i(of)e(the)g(same)h(t)o(yp)q(e.)19 b(This)14 b(is)g(to)q(o)f(constraining)h(on)g(t)o(w)o(o)e(accoun)o(ts.)19 b(One)75 571 y(often)d(w)o(an)o(ts)f(to)g(pass)h(messages)f(that)h(con) o(tain)g(v)m(alues)h(with)f(di\013eren)o(t)h(datat)o(yp)q(es)e(\(e.g.,) g(an)h(in)o(teger)75 627 y(coun)o(t,)e(follo)o(w)o(ed)g(b)o(y)g(a)g (sequence)h(of)e(real)i(n)o(um)o(b)q(ers\);)f(and)g(one)g(often)g(w)o (an)o(ts)f(to)g(send)i(noncon)o(tiguous)75 684 y(data)k(\(e.g.,)g(a)g (sub-blo)q(c)o(k)i(of)e(a)h(matrix\).)32 b(One)20 b(solution)h(is)f(to) f(pac)o(k)g(noncon)o(tiguous)h(data)f(in)o(to)g(a)75 740 y(con)o(tiguous)e(bu\013er)h(at)e(the)i(sender)g(site)f(and)h (unpac)o(k)f(it)h(bac)o(k)f(at)g(the)g(receiv)o(er)h(site.)27 b(This)18 b(has)f(the)75 796 y(disadv)m(an)o(tage)12 b(of)g(requiring)h(additional)h(memory-to-memory)d(cop)o(y)h(op)q (erations)g(at)g(b)q(oth)g(sites,)h(ev)o(en)75 853 y(when)j(the)g(comm) o(unication)g(subsystem)g(has)f(scatter-gather)g(capabilities.)24 b(Instead,)15 b Fw(MPI)g Fv(pro)o(vides)75 909 y(mec)o(hanisms)h(to)f (sp)q(ecify)i(more)f(general,)g(mixed,)g(and)g(noncon)o(tiguous)g(comm) o(unication)h(bu\013ers.)k(It)75 966 y(is)16 b(up)g(to)f(the)h (implemen)o(tation)h(to)e(decide)i(whether)f(data)f(should)h(b)q(e)g (\014rst)g(pac)o(k)o(ed)f(in)i(a)e(con)o(tiguous)75 1022 y(bu\013er)c(b)q(efore)h(b)q(eing)h(transmitted,)e(or)g(whether)h(it)g (can)f(b)q(e)h(collected)i(directly)e(from)f(where)h(it)f(resides.)166 1079 y(The)j(general)h(mec)o(hanisms)g(pro)o(vided)g(here)f(allo)o(w)h (one)f(to)g(transfer)f(directly)l(,)j(without)e(cop)o(ying,)75 1135 y(ob)s(jects)j(of)h(v)m(arious)h(shap)q(e)f(and)h(size.)29 b(It)18 b(is)h(not)f(assumed)g(that)f(the)h Fw(MPI)g Fv(library)h(is)f(cognizan)o(t)g(of)75 1192 y(the)d(ob)s(jects)f (declared)i(in)g(the)f(host)f(language.)20 b(Th)o(us,)15 b(if)g(one)g(w)o(an)o(ts)f(to)g(transfer)g(a)g(structure,)h(or)f(an)75 1248 y(arra)o(y)g(section,)h(it)g(will)h(b)q(e)f(necessary)g(to)g(pro)o (vide)g(in)g Fw(MPI)g Fv(a)f(de\014nition)j(of)d(a)g(comm)o(unication)i (bu\013er)75 1305 y(that)d(mimics)j(the)e(de\014nition)i(of)e(the)g (structure)g(or)f(arra)o(y)g(section)i(in)g(question.)20 b(These)15 b(facilities)h(can)75 1361 y(b)q(e)j(used)g(b)o(y)f(library) h(designers)h(to)d(de\014ne)j(comm)o(unication)f(functions)g(that)f (can)g(transfer)g(ob)s(jects)75 1417 y(de\014ned)e(in)g(the)f(host)f (language)h(|)g(b)o(y)g(deco)q(ding)h(their)f(de\014nitions)i(as)d(a)o (v)m(ailable)i(in)g(a)e(sym)o(b)q(ol)i(table)75 1474 y(or)f(a)g(dop)q(e)g(v)o(ector.)20 b(Suc)o(h)15 b(higher-lev)o(el)j (comm)o(unication)e(functions)g(are)f(not)g(part)f(of)h Fw(MPI)p Fv(.)166 1530 y(More)d(general)g(comm)o(unication)i(bu\013ers) e(are)g(sp)q(eci\014ed)i(b)o(y)f(replacing)g(the)g(basic)g(datat)o(yp)q (es)f(that)75 1587 y(ha)o(v)o(e)e(b)q(een)i(used)f(so)f(far)g(with)h (deriv)o(ed)g(datat)o(yp)q(es)f(that)g(are)g(constructed)g(from)g (basic)h(datat)o(yp)q(es)f(using)75 1643 y(the)j(constructors)e (describ)q(ed)j(in)g(this)f(section.)19 b(These)13 b(metho)q(ds)f(of)g (constructing)h(deriv)o(ed)h(datat)o(yp)q(es)75 1700 y(can)h(b)q(e)h(applied)h(recursiv)o(ely)l(.)166 1756 y(A)e Fq(general)j(datat)o(yp)q(e)e Fv(is)g(an)f(opaque)h(ob)s(ject)e (that)h(sp)q(eci\014es)i(t)o(w)o(o)d(things:)143 1843 y Fx(\017)23 b Fv(A)15 b(sequence)h(of)f(basic)h(datat)o(yp)q(es)143 1933 y Fx(\017)23 b Fv(A)15 b(sequence)h(of)f(in)o(teger)g(\(b)o(yte\)) g(displacemen)o(ts)166 2020 y(The)22 b(displacemen)o(ts)g(are)g(not)f (required)h(to)f(b)q(e)h(p)q(ositiv)o(e,)i(distinct,)g(or)d(in)h (increasing)h(order.)75 2076 y(Therefore,)15 b(the)g(order)h(of)e (items)i(need)g(not)f(coincide)j(with)d(their)h(order)g(in)g(store,)e (and)i(an)f(item)h(ma)o(y)75 2133 y(app)q(ear)j(more)g(than)g(once.)33 b(W)l(e)19 b(call)i(suc)o(h)e(a)g(pair)h(of)f(sequences)h(\(or)f (sequence)h(of)f(pairs\))g(a)g Fq(t)o(yp)q(e)75 2189 y(map)p Fv(.)24 b(The)17 b(sequence)h(of)f(basic)g(datat)o(yp)q(es)f (\(displacemen)o(ts)i(ignored\))f(is)g(the)g Fq(t)o(yp)q(e)i(signature) e Fv(of)75 2245 y(the)e(datat)o(yp)q(e.)166 2302 y(Let)189 2388 y Fp(T)6 b(y)r(pemap)12 b Fv(=)h Fx(f)p Fv(\()p Fp(ty)r(pe)562 2395 y Fg(0)581 2388 y Fp(;)8 b(disp)686 2395 y Fg(0)705 2388 y Fv(\))p Fp(;)g(:::;)g Fv(\()o Fp(ty)r(pe)905 2395 y Ff(n)p Fh(\000)p Fg(1)971 2388 y Fp(;)g(disp)1076 2395 y Ff(n)p Fh(\000)p Fg(1)1144 2388 y Fv(\))p Fx(g)p Fp(;)75 2475 y Fv(b)q(e)16 b(suc)o(h)f(a)g(t)o (yp)q(e)h(map,)e(where)i Fp(ty)r(pe)709 2482 y Ff(i)738 2475 y Fv(are)f(basic)h(t)o(yp)q(es,)f(and)g Fp(disp)1233 2482 y Ff(i)1262 2475 y Fv(are)g(displacemen)o(ts.)21 b(Let)189 2561 y Fp(T)6 b(y)r(pesig)13 b Fv(=)g Fx(f)p Fp(ty)r(pe)517 2568 y Fg(0)537 2561 y Fp(;)8 b(:::;)g(ty)r(pe)702 2568 y Ff(n)p Fh(\000)p Fg(1)768 2561 y Fx(g)75 2647 y Fv(b)q(e)14 b(the)f(asso)q(ciated)h(t)o(yp)q(e)f(signature.)19 b(This)14 b(t)o(yp)q(e)f(map,)g(together)g(with)g(a)g(base)g(address)h Fl(buf)p Fv(,)f(sp)q(eci\014es)75 2704 y(a)19 b(comm)o(unication)i (bu\013er:)28 b(the)20 b(comm)o(unication)h(bu\013er)e(that)g(consists) h(of)f Fp(n)h Fv(en)o(tries,)h(where)f(the)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 62 70 62 69 bop 75 -100 a Fv(62)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fp(i)p Fv(-th)f(en)o(try)f(is)i(at)e(address)h Fp(buf)f Fv(+)c Fp(disp)763 52 y Ff(i)793 45 y Fv(and)16 b(has)g(t)o(yp)q(e)g Fp(ty)r(pe)1151 52 y Ff(i)1165 45 y Fv(.)23 b(A)16 b(message)f(assem)o (bled)i(from)e(suc)o(h)h(a)75 102 y(comm)o(unication)g(bu\013er)f(will) i(consist)e(of)g Fp(n)h Fv(v)m(alues,)g(of)e(the)i(t)o(yp)q(es)f (de\014ned)h(b)o(y)g Fp(T)6 b(y)r(pesig)r Fv(.)166 158 y(W)l(e)11 b(can)f(use)h(a)f(handle)i(to)e(a)g(general)h(datat)o(yp)q (e)f(as)h(an)f(argumen)o(t)g(in)h(a)f(send)i(or)e(receiv)o(e)h(op)q (eration,)75 214 y(instead)h(of)e(a)h(basic)g(datat)o(yp)q(e)g(argumen) o(t.)18 b(The)11 b(op)q(eration)g Fw(MPI)p 1198 214 14 2 v 16 w(SEND\(buf,)h(1,)f(datat)o(yp)q(e,...\))19 b Fv(will)13 b(use)75 271 y(the)18 b(send)g(bu\013er)f(de\014ned)i(b)o(y) e(the)h(base)f(address)h Fw(buf)g Fv(and)g(the)f(general)h(datat)o(yp)q (e)f(asso)q(ciated)h(with)75 327 y Fw(datat)o(yp)q(e)p Fv(;)23 b(it)c(will)i(generate)f(a)f(message)g(with)g(the)h(t)o(yp)q(e) f(signature)h(determined)h(b)o(y)e(the)g Fw(datat)o(yp)q(e)75 384 y Fv(argumen)o(t.)28 b Fw(MPI)p 390 384 V 16 w(RECV\(buf,)20 b(1,)e(datat)o(yp)q(e,...\))29 b Fv(will)20 b(use)e(the)h(receiv)o(e)g (bu\013er)f(de\014ned)h(b)o(y)g(the)f(base)75 440 y(address)d Fw(buf)h Fv(and)g(the)f(general)h(datat)o(yp)q(e)e(asso)q(ciated)i (with)f Fw(datat)o(yp)q(e)p Fv(.)166 497 y(General)h(datat)o(yp)q(es)f (can)h(b)q(e)h(used)f(in)h(all)g(send)f(and)h(receiv)o(e)f(op)q (erations.)22 b(W)l(e)16 b(discuss,)h(in)g(Sec.)75 553 y(3.12.5,)c(the)i(case)g(where)h(the)f(second)h(argumen)o(t)e Fw(count)j Fv(has)e(v)m(alue)h Fp(>)d Fv(1.)166 610 y(The)e(basic)g (datat)o(yp)q(es)f(presen)o(ted)i(in)f(section)g(3.2.2)f(are)g (particular)h(cases)g(of)f(a)h(general)g(datat)o(yp)q(e,)75 666 y(and)19 b(are)f(prede\014ned.)32 b(Th)o(us,)19 b Fw(MPI)p 711 666 V 16 w(INT)f Fv(is)h(a)g(prede\014ned)h(handle)g(to)e (a)g(datat)o(yp)q(e)g(with)h(t)o(yp)q(e)g(map)75 723 y Fx(f)p Fv(\()p Fw(int)p Fp(;)8 b Fv(0\))p Fx(g)p Fv(,)13 b(with)j(one)g(en)o(try)f(of)g(t)o(yp)q(e)g Fw(int)h Fv(and)g(displacemen)o(t)h(zero.)j(The)15 b(other)g(basic)i(datat)o(yp) q(es)d(are)75 779 y(similar.)166 835 y(The)h Fq(exten)o(t)f Fv(of)g(a)g(datat)o(yp)q(e)g(is)h(de\014ned)g(to)f(b)q(e)h(the)g(span)f (from)g(the)g(\014rst)g(b)o(yte)h(to)e(the)i(last)f(b)o(yte)75 892 y(o)q(ccupied)19 b(b)o(y)d(en)o(tries)h(in)h(this)f(datat)o(yp)q (e,)f(rounded)h(up)g(to)f(satisfy)h(alignmen)o(t)g(requiremen)o(ts.)25 b(That)75 948 y(is,)15 b(if)189 1042 y Fp(T)6 b(y)r(pemap)12 b Fv(=)h Fx(f)p Fv(\()p Fp(ty)r(pe)562 1049 y Fg(0)581 1042 y Fp(;)8 b(disp)686 1049 y Fg(0)705 1042 y Fv(\))p Fp(;)g(:::;)g Fv(\()o Fp(ty)r(pe)905 1049 y Ff(n)p Fh(\000)p Fg(1)971 1042 y Fp(;)g(disp)1076 1049 y Ff(n)p Fh(\000)p Fg(1)1144 1042 y Fv(\))p Fx(g)p Fp(;)75 1136 y Fv(then)-323 b Fx(>)283 1230 y Fp(l)q(b)p Fv(\()p Fp(T)6 b(y)r(pemap)p Fv(\))40 b(=)h(min)688 1259 y Ff(j)742 1230 y Fp(disp)826 1237 y Ff(j)844 1230 y Fp(;)272 1320 y(ub)p Fv(\()p Fp(T)6 b(y)r(pemap)p Fv(\))40 b(=)h(max)693 1349 y Ff(j)743 1320 y Fv(\()p Fp(disp)845 1327 y Ff(j)873 1320 y Fv(+)10 b Fp(siz)r(eof)5 b Fv(\()p Fp(ty)r(pe)1150 1327 y Ff(j)1169 1320 y Fv(\)\))k(+)i Fp(\017;)22 b Fv(and)189 1410 y Fp(extent)p Fv(\()p Fp(T)6 b(y)r(pemap)p Fv(\))42 b(=)f Fp(ub)p Fv(\()p Fp(T)6 b(y)r(pemap)p Fv(\))j Fx(\000)h Fp(l)q(b)p Fv(\()p Fp(T)c(y)r(pemap)p Fv(\))p Fp(:)528 b Fv(\(3.1\))75 1504 y(If)17 b Fp(ty)r(pe)206 1511 y Ff(i)237 1504 y Fv(requires)g(alignmen)o(t)g(to)f(a)h(b)o(yte)f (address)h(that)f(is)h(is)g(a)f(m)o(ultiple)j(of)d Fp(k)1474 1511 y Ff(i)1488 1504 y Fv(,)g(then)h Fp(\017)g Fv(is)g(the)g(least)75 1560 y(nonnegativ)o(e)f(incremen)o(t)h(needed)g(to)f(round)g Fp(extent)p Fv(\()p Fp(T)6 b(y)r(pemap)p Fv(\))16 b(to)g(the)g(next)g (m)o(ultiple)i(of)d(max)1803 1567 y Ff(i)1825 1560 y Fp(k)1849 1567 y Ff(i)1862 1560 y Fv(.)75 1617 y(The)g(complete)h (de\014nition)h(of)e Fq(exten)o(t)g Fv(is)h(giv)o(en)f(on)h(page)f(71.) -1326 b Fx(?)75 1723 y Fq(Example)18 b(3.19)23 b Fv(Assume)12 b(that)g Fp(T)6 b(y)r(pe)13 b Fv(=)g Fx(f)p Fv(\()p Fw(double)p Fp(;)8 b Fv(0\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(8\))p Fx(g)j Fv(\(a)h Fw(double)i Fv(at)e(displacemen)o(t)i(zero,)75 1780 y(follo)o(w)o(ed)k(b)o(y)g(a)f Fw(cha)o(r)g Fv(at)g(displacemen)o (t)j(eigh)o(t\).)27 b(Assume,)18 b(furthermore,)f(that)g(doubles)i(ha)o (v)o(e)e(to)g(b)q(e)75 1836 y(strictly)d(aligned)g(at)e(addresses)i (that)e(are)h(m)o(ultiples)h(of)f(eigh)o(t.)19 b(Then,)14 b(the)f(exten)o(t)g(of)f(this)i(datat)o(yp)q(e)e(is)75 1893 y(16)e(\(9)g(rounded)h(to)f(the)h(next)g(m)o(ultiple)h(of)f(8\).) 17 b(A)11 b(datat)o(yp)q(e)f(that)g(consists)h(of)f(a)g(c)o(haracter)g (immediately)75 1949 y(follo)o(w)o(ed)16 b(b)o(y)f(a)g(double)h(will)h (also)e(ha)o(v)o(e)g(an)g(exten)o(t)g(of)g(16.)189 2055 y Fl(R)n(ationale.)39 b Fv(The)15 b(de\014nition)h(of)e(exten)o(t)g(is) h(motiv)m(ated)f(b)o(y)h(the)f(assumption)h(that)e(the)i(amoun)o(t)189 2112 y(of)i(padding)i(added)g(at)e(the)h(end)g(of)g(eac)o(h)g (structure)f(in)i(an)f(arra)o(y)e(of)i(structures)f(is)i(the)f(least) 189 2168 y(needed)d(to)e(ful\014ll)j(alignmen)o(t)e(constrain)o(ts.)19 b(More)14 b(explicit)i(con)o(trol)d(of)g(the)h(exten)o(t)g(is)g(pro)o (vided)189 2225 y(in)i(section)g(3.12.3.)k(Suc)o(h)c(explicit)i(con)o (trol)e(is)g(needed)h(in)f(cases)g(where)g(the)g(assumption)g(do)q(es) 189 2281 y(not)e(hold,)i(for)f(example,)g(where)h(union)g(t)o(yp)q(es)f (are)g(used.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fv(\))75 2403 y Fj(3.12.1)49 b(Datat)o(yp)q(e)17 b(constructo)o(rs)75 2489 y Fw(Contiguous)47 b Fv(The)16 b(simplest)i(datat)o(yp)q(e)e (constructor)f(is)i Fw(MPI)p 1171 2489 V 16 w(TYPE)p 1304 2489 V 17 w(CONTIGUOUS)h Fv(whic)o(h)f(allo)o(ws)75 2545 y(replication)g(of)e(a)g(datat)o(yp)q(e)f(in)o(to)h(con)o(tiguous) h(lo)q(cations.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 63 71 63 70 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fv(63)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(TYPE)p 293 45 V 17 w(CONTIGUOUS\(count,)17 b(oldt)o(yp)q(e,)f(newt)o(yp)q(e\)) 117 122 y Fk(IN)155 b Fw(count)482 b Fk(replication)14 b(coun)o(t)g(\(nonnegativ)o(e)g(in)o(teger\))117 194 y(IN)155 b Fw(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i (\(handle\))117 266 y(OUT)108 b Fw(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 390 y Ft(int)23 b(MPI)p 245 390 15 2 v 17 w(Type)p 358 390 V 17 w(contiguous\(int)f (count,)h(MPI)p 972 390 V 17 w(Datatype)f(oldtype,)393 447 y(MPI)p 468 447 V 17 w(Datatype)h(*newtype\))75 533 y(MPI)p 150 533 V 17 w(TYPE)p 263 533 V 16 w(CONTIGUOUS\(COUNT,)f (OLDTYPE,)h(NEWTYPE,)f(IERROR\))170 589 y(INTEGER)h(COUNT,)g(OLDTYPE,)g (NEWTYPE,)g(IERROR)166 676 y Fw(newt)o(yp)q(e)c Fv(is)f(the)f(datat)o (yp)q(e)g(obtained)h(b)o(y)f(concatenating)h Fw(count)h Fv(copies)f(of)f Fw(oldt)o(yp)q(e)p Fv(.)27 b(Concate-)75 732 y(nation)15 b(is)h(de\014ned)h(using)f Fl(extent)e Fv(as)h(the)h(size)g(of)e(the)i(concatenated)f(copies.)75 814 y Fq(Example)j(3.20)23 b Fv(Let)17 b Fw(oldt)o(yp)q(e)g Fv(ha)o(v)o(e)g(t)o(yp)q(e)g(map)f Fx(f)p Fv(\()p Fw(double)q Fp(;)8 b Fv(0\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(8\))p Fx(g)p Fp(;)15 b Fv(with)i(exten)o(t)f(16,)h(and)f(let)75 871 y Fw(count)e Fv(=)f(3.)19 b(The)d(t)o(yp)q(e)f(map)g(of)g(the)g (datat)o(yp)q(e)g(returned)g(b)o(y)h Fw(newt)o(yp)q(e)h Fv(is)189 947 y Fx(f)p Fv(\()p Fw(double)p Fp(;)8 b Fv(0\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(8\))p Fp(;)g Fv(\()p Fw(double)o Fp(;)g Fv(16\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(24\))p Fp(;)g Fv(\()p Fw(double)n Fp(;)g Fv(32\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(40\))p Fx(g)p Fv(;)75 1024 y(i.e.,)15 b(alternating)g Fw(double)i Fv(and)e Fw(cha)o(r)g Fv(elemen)o(ts,)h(with)f(displacemen)o(ts)i(0)p Fp(;)8 b Fv(8)p Fp(;)g Fv(16)p Fp(;)f Fv(24)p Fp(;)g Fv(32)o Fp(;)h Fv(4)o(0.)166 1189 y(In)16 b(general,)f(assume)g(that)g (the)g(t)o(yp)q(e)g(map)g(of)g Fw(oldt)o(yp)q(e)h Fv(is)189 1265 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 1272 y Fg(0)333 1265 y Fp(;)8 b(disp)438 1272 y Fg(0)457 1265 y Fv(\))p Fp(;)g(:::;)f Fv(\()p Fp(ty)r(pe)657 1272 y Ff(n)p Fh(\000)p Fg(1)723 1265 y Fp(;)h(disp)828 1272 y Ff(n)p Fh(\000)p Fg(1)895 1265 y Fv(\))p Fx(g)p Fp(;)75 1342 y Fv(with)16 b(exten)o(t)e Fp(ex)p Fv(.)20 b(Then)c Fw(newt)o(yp)q(e)h Fv(has)e(a)g(t)o(yp)q(e)g(map)g(with)h Fw(count)11 b Fx(\001)f Fw(n)16 b Fv(en)o(tries)f(de\014ned)i(b)o(y:)189 1418 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 1425 y Fg(0)333 1418 y Fp(;)8 b(disp)438 1425 y Fg(0)457 1418 y Fv(\))p Fp(;)g(:::;)f Fv(\()p Fp(ty)r(pe)657 1425 y Ff(n)p Fh(\000)p Fg(1)723 1418 y Fp(;)h(disp)828 1425 y Ff(n)p Fh(\000)p Fg(1)895 1418 y Fv(\))p Fp(;)g Fv(\()p Fp(ty)r(pe)1036 1425 y Fg(0)1055 1418 y Fp(;)g(disp)1160 1425 y Fg(0)1189 1418 y Fv(+)i Fp(ex)p Fv(\))p Fp(;)e(:::;)g Fv(\()p Fp(ty)r(pe)1482 1425 y Ff(n)p Fh(\000)p Fg(1)1548 1418 y Fp(;)g(disp)1653 1425 y Ff(n)p Fh(\000)p Fg(1)1730 1418 y Fv(+)j Fp(ex)p Fv(\))p Fp(;)189 1500 y(:::;)d Fv(\()p Fp(ty)r(pe)351 1507 y Fg(0)368 1500 y Fp(;)g(disp)473 1507 y Fg(0)502 1500 y Fv(+)i Fp(ex)h Fx(\001)e Fv(\()p Fw(count)j Fx(\000)e Fv(1\)\))p Fp(;)e(:::;)f Fv(\()p Fp(ty)r(pe)1048 1507 y Ff(n)p Fh(\000)p Fg(1)1114 1500 y Fp(;)h(disp)1219 1507 y Ff(n)p Fh(\000)p Fg(1)1296 1500 y Fv(+)j Fp(ex)f Fx(\001)g Fv(\()p Fw(count)h Fx(\000)f Fv(1\)\))p Fx(g)p Fp(:)75 1726 y Fw(V)o(ecto)o(r)45 b Fv(The)16 b(function)g Fw(MPI)p 600 1726 14 2 v 16 w(TYPE)p 733 1726 V 17 w(VECTOR)g Fv(is)g(a)f(more)g(general)h(constructor)e(that)h(allo)o(ws)g(repli-)75 1782 y(cation)20 b(of)g(a)g(datat)o(yp)q(e)f(in)o(to)h(lo)q(cations)h (that)e(consist)i(of)e(equally)j(spaced)f(blo)q(c)o(ks.)35 b(Eac)o(h)20 b(blo)q(c)o(k)g(is)75 1838 y(obtained)g(b)o(y)f (concatenating)h(the)f(same)g(n)o(um)o(b)q(er)h(of)f(copies)h(of)f(the) g(old)h(datat)o(yp)q(e.)32 b(The)19 b(spacing)75 1895 y(b)q(et)o(w)o(een)d(blo)q(c)o(ks)f(is)h(a)f(m)o(ultiple)i(of)e(the)g (exten)o(t)g(of)g(the)g(old)h(datat)o(yp)q(e.)75 2046 y Fw(MPI)p 160 2046 V 16 w(TYPE)p 293 2046 V 17 w(VECTOR\()g(count,)g (blo)q(cklength,)g(stride,)g(oldt)o(yp)q(e,)g(newt)o(yp)q(e\))117 2123 y Fk(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(nonnegativ)o(e)f(in)o(teger\))117 2195 y(IN)155 b Fw(blo)q(cklength)371 b Fk(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g(in)f(eac)o(h)i(blo)q(c)o(k)e(\(nonnegativ)o(e)h (in)o(te-)905 2251 y(ger\))117 2323 y(IN)155 b Fw(stride)484 b Fk(n)o(um)o(b)q(er)13 b(of)f(elemen)o(ts)h(b)q(et)o(w)o(een)h(start)g (of)f(eac)o(h)g(blo)q(c)o(k)g(\(in)o(te-)905 2379 y(ger\))117 2451 y(IN)155 b Fw(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i (\(handle\))117 2523 y(OUT)108 b Fw(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 2647 y Ft(int)23 b(MPI)p 245 2647 15 2 v 17 w(Type)p 358 2647 V 17 w(vector\(int)f (count,)h(int)h(blocklength,)e(int)i(stride,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)f(oldtype,)f(MPI)p 986 2704 V 17 w(Datatype)h(*newtype\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 64 72 64 71 bop 75 -100 a Fv(64)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(TYPE)p 263 45 V 16 w(VECTOR\(COUNT,)22 b(BLOCKLENGTH,)h(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR\))170 102 y(INTEGER)g(COUNT,)g(BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h (IERROR)75 238 y Fq(Example)18 b(3.21)23 b Fv(Assume,)15 b(again,)f(that)h Fw(oldt)o(yp)q(e)h Fv(has)f(t)o(yp)q(e)g(map)g Fx(f)p Fv(\()p Fw(double)q Fp(;)8 b Fv(0\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(8\))p Fx(g)p Fp(;)13 b Fv(with)j(ex-)75 294 y(ten)o(t)g(16.)21 b(A)c(call)g(to)e Fw(MPI)p 527 294 14 2 v 16 w(TYPE)p 660 294 V 17 w(VECTOR\()i(2,)e(3,)h(4,)f(oldt)o (yp)q(e,)i(newt)o(yp)q(e\))h Fv(will)g(create)d(the)h(datat)o(yp)q(e)75 351 y(with)g(t)o(yp)q(e)f(map,)189 445 y Fx(f)p Fv(\()p Fw(double)p Fp(;)8 b Fv(0\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(8\))p Fp(;)g Fv(\()p Fw(double)o Fp(;)g Fv(16\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(24\))p Fp(;)g Fv(\()p Fw(double)n Fp(;)g Fv(32\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(40\))p Fp(;)189 551 y Fv(\()p Fw(double)q Fp(;)g Fv(64\))p Fp(;)g Fv(\()p Fw(cha)o(r)m Fp(;)g Fv(72\))p Fp(;)g Fv(\()p Fw(double)o Fp(;)g Fv(80\))p Fp(;)g Fv(\()p Fw(cha)o(r)m Fp(;)g Fv(88\))p Fp(;)g Fv(\()p Fw(double)o Fp(;)g Fv(96\))p Fp(;)g Fv(\()p Fw(cha)o(r)m Fp(;)g Fv(104\))p Fx(g)p Fp(:)75 657 y Fv(That)13 b(is,)h(t)o(w)o(o)e(blo)q(c)o(ks)j (with)e(three)h(copies)h(eac)o(h)e(of)g(the)h(old)g(t)o(yp)q(e,)g(with) g(a)f(stride)h(of)f(4)h(elemen)o(ts)g(\(4)7 b Fx(\001)g Fv(16)75 714 y(b)o(ytes\))15 b(b)q(et)o(w)o(een)g(the)g(blo)q(c)o(ks.) 75 820 y Fq(Example)j(3.22)23 b Fv(A)c(call)i(to)d Fw(MPI)p 692 820 V 16 w(TYPE)p 825 820 V 17 w(VECTOR\(3,)j(1,)e(-2,)h(oldt)o(yp) q(e,)i(newt)o(yp)q(e\))f Fv(will)g(create)e(the)75 876 y(datat)o(yp)q(e,)189 970 y Fx(f)p Fv(\()p Fw(double)p Fp(;)8 b Fv(0\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(8\))p Fp(;)g Fv(\()p Fw(double)o Fp(;)g Fx(\000)p Fv(32\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fx(\000)p Fv(24\))p Fp(;)g Fv(\()p Fw(double)o Fp(;)g Fx(\000)p Fv(64\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fx(\000)p Fv(56\))p Fx(g)p Fp(:)166 1159 y Fv(In)16 b(general,)f(assume)g(that)g Fw(oldt)o(yp)q(e)h Fv(has)f(t)o(yp)q(e)g(map,)189 1253 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 1260 y Fg(0)333 1253 y Fp(;)8 b(disp)438 1260 y Fg(0)457 1253 y Fv(\))p Fp(;)g(:::;)f Fv(\()p Fp(ty)r(pe)657 1260 y Ff(n)p Fh(\000)p Fg(1)723 1253 y Fp(;)h(disp)828 1260 y Ff(n)p Fh(\000)p Fg(1)895 1253 y Fv(\))p Fx(g)p Fp(;)75 1347 y Fv(with)14 b(exten)o(t)f Fp(ex)p Fv(.)19 b(Let)13 b Fw(bl)h Fv(b)q(e)g(the)f Fw(blo)q(cklength)p Fv(.)21 b(The)14 b(newly)g(created)f(datat)o(yp)q(e)g(has)g(a)g(t)o(yp) q(e)h(map)f(with)75 1403 y Fw(count)e Fx(\001)f Fw(bl)h Fx(\001)e Fp(n)16 b Fv(en)o(tries:)189 1497 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 1504 y Fg(0)333 1497 y Fp(;)8 b(disp)438 1504 y Fg(0)457 1497 y Fv(\))p Fp(;)g(:::;)f Fv(\()p Fp(ty)r(pe)657 1504 y Ff(n)p Fh(\000)p Fg(1)723 1497 y Fp(;)h(disp)828 1504 y Ff(n)p Fh(\000)p Fg(1)895 1497 y Fv(\))p Fp(;)189 1603 y Fv(\()p Fp(ty)r(pe)291 1610 y Fg(0)310 1603 y Fp(;)g(disp)415 1610 y Fg(0)444 1603 y Fv(+)i Fp(ex)p Fv(\))p Fp(;)e(:::;)g Fv(\()p Fp(ty)r(pe)737 1610 y Ff(n)p Fh(\000)p Fg(1)803 1603 y Fp(;)g(disp)908 1610 y Ff(n)p Fh(\000)p Fg(1)985 1603 y Fv(+)j Fp(ex)p Fv(\))p Fp(;)d(:::;)189 1709 y Fv(\()p Fp(ty)r(pe)291 1716 y Fg(0)310 1709 y Fp(;)g(disp)415 1716 y Fg(0)444 1709 y Fv(+)i(\()p Fw(bl)g Fx(\000)h Fv(1\))e Fx(\001)h Fp(ex)p Fv(\))p Fp(;)e(:::;)g Fv(\()p Fp(ty)r(pe)918 1716 y Ff(n)p Fh(\000)p Fg(1)984 1709 y Fp(;)g(disp)1089 1716 y Ff(n)p Fh(\000)p Fg(1)1166 1709 y Fv(+)j(\()p Fw(bl)f Fx(\000)g Fv(1\))g Fx(\001)g Fp(ex)p Fv(\))p Fp(;)189 1816 y Fv(\()p Fp(ty)r(pe)291 1823 y Fg(0)310 1816 y Fp(;)e(disp)415 1823 y Fg(0)444 1816 y Fv(+)i Fw(stride)h Fx(\001)f Fp(ex)p Fv(\))p Fp(;)e(:::;)g Fv(\()p Fp(ty)r(pe)874 1823 y Ff(n)p Fh(\000)p Fg(1)940 1816 y Fp(;)g(disp)1045 1823 y Ff(n)p Fh(\000)p Fg(1)1122 1816 y Fv(+)j Fw(stride)g Fx(\001)e Fp(ex)p Fv(\))p Fp(;)f(:::;)189 1922 y Fv(\()p Fp(ty)r(pe)291 1929 y Fg(0)310 1922 y Fp(;)g(disp)415 1929 y Fg(0)444 1922 y Fv(+)i(\()p Fw(stride)h Fv(+)g Fw(bl)f Fx(\000)g Fv(1\))g Fx(\001)g Fp(ex)p Fv(\))p Fp(;)e(:::;)g Fv(\()o Fp(ty)r(pe)1077 1929 y Ff(n)p Fh(\000)p Fg(1)1143 1922 y Fp(;)g(disp)1248 1929 y Ff(n)p Fh(\000)p Fg(1)1326 1922 y Fv(+)i(\()p Fw(stride)h Fv(+)f Fw(bl)h Fx(\000)f Fv(1\))g Fx(\001)f Fp(ex)p Fv(\))p Fp(;)f(::::;)189 2028 y Fv(\()p Fp(ty)r(pe)291 2035 y Fg(0)310 2028 y Fp(;)g(disp)415 2035 y Fg(0)444 2028 y Fv(+)i Fw(stride)h Fx(\001)f Fv(\()p Fw(count)h Fx(\000)g Fv(1\))e Fx(\001)h Fp(ex)p Fv(\))p Fp(;)e(:::;)189 2135 y Fv(\()p Fp(ty)r(pe)291 2142 y Ff(n)p Fh(\000)p Fg(1)359 2135 y Fp(;)g(disp)464 2142 y Ff(n)p Fh(\000)p Fg(1)542 2135 y Fv(+)i Fw(stride)h Fx(\001)f Fv(\()p Fw(count)h Fx(\000)f Fv(1\))g Fx(\001)g Fp(ex)p Fv(\))p Fp(;)e(:::;)189 2241 y Fv(\()p Fp(ty)r(pe)291 2248 y Fg(0)310 2241 y Fp(;)g(disp)415 2248 y Fg(0)444 2241 y Fv(+)i(\()p Fw(stride)h Fx(\001)f Fv(\()p Fw(count)h Fx(\000)f Fv(1\))g(+)g Fw(bl)h Fx(\000)f Fv(1\))g Fx(\001)f Fp(ex)p Fv(\))p Fp(;)f(:::;)189 2347 y Fv(\()p Fp(ty)r(pe)291 2354 y Ff(n)p Fh(\000)p Fg(1)359 2347 y Fp(;)g(disp)464 2354 y Ff(n)p Fh(\000)p Fg(1)542 2347 y Fv(+)i(\()p Fw(stride)h Fx(\001)e Fv(\()p Fw(count)j Fx(\000)e Fv(1\))g(+)g Fw(bl)g Fx(\000)h Fv(1\))e Fx(\001)h Fp(ex)p Fv(\))p Fx(g)p Fp(:)166 2536 y Fv(A)16 b(call)i(to)e Fw(MPI)p 443 2536 V 16 w(TYPE)p 576 2536 V 16 w(CONTIGUOUS\(count,)k(oldt)o(yp)q(e,)d(newt)o(yp)q(e\))h Fv(is)f(equiv)m(alen)o(t)h(to)e(a)g(call)i(to)75 2592 y Fw(MPI)p 160 2592 V 16 w(TYPE)p 293 2592 V 17 w(VECTOR\(count,)g(1,)f (1,)f(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))p Fv(,)h(or)e(to)g(a)h(call)g (to)f Fw(MPI)p 1495 2592 V 16 w(TYPE)p 1628 2592 V 17 w(VECTOR\(1,)75 2649 y(count,)g(n,)f(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))p Fv(,)h Fw(n)e Fv(arbitrary)l(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 65 73 65 72 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fv(65)75 45 y Fw(Hvecto)o(r)45 b Fv(The)19 b(function)h Fw(MPI)p 631 45 14 2 v 16 w(TYPE)p 764 45 V 17 w(HVECTOR)g Fv(is)f(iden)o(tical)i(to)d Fw(MPI)p 1389 45 V 16 w(TYPE)p 1522 45 V 17 w(VECTOR)p Fv(,)h(except)75 102 y(that)h Fw(stride)h Fv(is)f(giv)o(en)h(in)g(b)o(ytes,)g(rather)f(than)g(in)h (elemen)o(ts.)36 b(The)20 b(use)h(for)e(b)q(oth)i(t)o(yp)q(es)f(of)g(v) o(ector)75 158 y(constructors)14 b(is)i(illustrated)h(in)f(Sec.)k (3.12.7.)e(\()p Fw(H)d Fv(stands)g(for)g(\\heterogeneous"\).)75 309 y Fw(MPI)p 160 309 V 16 w(TYPE)p 293 309 V 17 w(HVECTOR\()h(count,) g(blo)q(cklength,)h(stride,)e(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))117 386 y Fk(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(blo)q(c)o(ks)g(\(nonnegativ)o(e)f(in)o(teger\))117 461 y(IN)155 b Fw(blo)q(cklength)371 b Fk(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g(in)f(eac)o(h)i(blo)q(c)o(k)e(\(nonnegativ)o(e)h (in)o(te-)905 518 y(ger\))117 593 y(IN)155 b Fw(stride)484 b Fk(n)o(um)o(b)q(er)13 b(of)g(b)o(ytes)h(b)q(et)o(w)o(een)h(start)f (of)f(eac)o(h)h(blo)q(c)o(k)f(\(in)o(teger\))117 668 y(IN)155 b Fw(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i (\(handle\))117 743 y(OUT)108 b Fw(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 868 y Ft(int)23 b(MPI)p 245 868 15 2 v 17 w(Type)p 358 868 V 17 w(hvector\(int)f (count,)h(int)h(blocklength,)e(MPI)p 1306 868 V 17 w(Aint)h(stride,)393 924 y(MPI)p 468 924 V 17 w(Datatype)g(oldtype,)f(MPI)p 986 924 V 17 w(Datatype)h(*newtype\))75 1011 y(MPI)p 150 1011 V 17 w(TYPE)p 263 1011 V 16 w(HVECTOR\(COUNT,)f(BLOCKLENGTH,)h (STRIDE,)f(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1067 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)166 1236 y Fv(Assume)15 b(that)g Fw(oldt)o(yp)q(e)h Fv(has)f(t)o(yp)q(e)g (map,)189 1330 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 1337 y Fg(0)333 1330 y Fp(;)8 b(disp)438 1337 y Fg(0)457 1330 y Fv(\))p Fp(;)g(:::;)f Fv(\()p Fp(ty)r(pe)657 1337 y Ff(n)p Fh(\000)p Fg(1)723 1330 y Fp(;)h(disp)828 1337 y Ff(n)p Fh(\000)p Fg(1)895 1330 y Fv(\))p Fx(g)p Fp(;)75 1424 y Fv(with)14 b(exten)o(t)f Fp(ex)p Fv(.)19 b(Let)13 b Fw(bl)h Fv(b)q(e)g(the)f Fw(blo)q(cklength)p Fv(.)21 b(The)14 b(newly)g(created)f(datat)o(yp)q(e)g(has)g(a)g(t)o(yp)q(e)h (map)f(with)75 1480 y Fw(count)e Fx(\001)f Fw(bl)h Fx(\001)e Fp(n)16 b Fv(en)o(tries:)189 1574 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 1581 y Fg(0)333 1574 y Fp(;)8 b(disp)438 1581 y Fg(0)457 1574 y Fv(\))p Fp(;)g(:::;)f Fv(\()p Fp(ty)r(pe)657 1581 y Ff(n)p Fh(\000)p Fg(1)723 1574 y Fp(;)h(disp)828 1581 y Ff(n)p Fh(\000)p Fg(1)895 1574 y Fv(\))p Fp(;)189 1680 y Fv(\()p Fp(ty)r(pe)291 1687 y Fg(0)310 1680 y Fp(;)g(disp)415 1687 y Fg(0)444 1680 y Fv(+)i Fp(ex)p Fv(\))p Fp(;)e(:::;)g Fv(\()p Fp(ty)r(pe)737 1687 y Ff(n)p Fh(\000)p Fg(1)803 1680 y Fp(;)g(disp)908 1687 y Ff(n)p Fh(\000)p Fg(1)985 1680 y Fv(+)j Fp(ex)p Fv(\))p Fp(;)d(:::;)189 1787 y Fv(\()p Fp(ty)r(pe)291 1794 y Fg(0)310 1787 y Fp(;)g(disp)415 1794 y Fg(0)444 1787 y Fv(+)i(\()p Fw(bl)g Fx(\000)h Fv(1\))e Fx(\001)h Fp(ex)p Fv(\))p Fp(;)e(:::;)g Fv(\()p Fp(ty)r(pe)918 1794 y Ff(n)p Fh(\000)p Fg(1)984 1787 y Fp(;)g(disp)1089 1794 y Ff(n)p Fh(\000)p Fg(1)1166 1787 y Fv(+)j(\()p Fw(bl)f Fx(\000)g Fv(1\))g Fx(\001)g Fp(ex)p Fv(\))p Fp(;)189 1893 y Fv(\()p Fp(ty)r(pe)291 1900 y Fg(0)310 1893 y Fp(;)e(disp)415 1900 y Fg(0)444 1893 y Fv(+)i Fw(stride)q Fv(\))p Fp(;)e(:::;)g Fv(\()p Fp(ty)r(pe)794 1900 y Ff(n)p Fh(\000)p Fg(1)860 1893 y Fp(;)g(disp)965 1900 y Ff(n)p Fh(\000)p Fg(1)1042 1893 y Fv(+)j Fw(stride)q Fv(\))p Fp(;)d(:::;)189 1999 y Fv(\()p Fp(ty)r(pe)291 2006 y Fg(0)310 1999 y Fp(;)g(disp)415 2006 y Fg(0)444 1999 y Fv(+)i Fw(stride)h Fv(+)g(\()p Fw(bl)f Fx(\000)g Fv(1\))g Fx(\001)g Fp(ex)p Fv(\))p Fp(;)e(:::;)189 2105 y Fv(\()p Fp(ty)r(pe)291 2112 y Ff(n)p Fh(\000)p Fg(1)359 2105 y Fp(;)g(disp)464 2112 y Ff(n)p Fh(\000)p Fg(1)542 2105 y Fv(+)i Fw(stride)h Fv(+)g(\()p Fw(bl)f Fx(\000)g Fv(1\))g Fx(\001)f Fp(ex)p Fv(\))p Fp(;)f(::::;)189 2212 y Fv(\()p Fp(ty)r(pe)291 2219 y Fg(0)310 2212 y Fp(;)g(disp)415 2219 y Fg(0)444 2212 y Fv(+)i Fw(stride)h Fx(\001)f Fv(\()p Fw(count)h Fx(\000)g Fv(1\)\))p Fp(;)d(:::;)f Fv(\()p Fp(ty)r(pe)1047 2219 y Ff(n)p Fh(\000)p Fg(1)1113 2212 y Fp(;)h(disp)1218 2219 y Ff(n)p Fh(\000)p Fg(1)1295 2212 y Fv(+)i Fw(stride)h Fx(\001)f Fv(\()p Fw(count)h Fx(\000)g Fv(1\)\))p Fp(;)d(:::;)189 2318 y Fv(\()p Fp(ty)r(pe)291 2325 y Fg(0)310 2318 y Fp(;)g(disp)415 2325 y Fg(0)444 2318 y Fv(+)i Fw(stride)h Fx(\001)f Fv(\()p Fw(count)h Fx(\000)g Fv(1\))e(+)i(\()p Fw(bl)f Fx(\000)g Fv(1\))g Fx(\001)f Fp(ex)p Fv(\))p Fp(;)f(:::;)189 2424 y Fv(\()p Fp(ty)r(pe)291 2431 y Ff(n)p Fh(\000)p Fg(1)359 2424 y Fp(;)g(disp)464 2431 y Ff(n)p Fh(\000)p Fg(1)542 2424 y Fv(+)i Fw(stride)h Fx(\001)f Fv(\()p Fw(count)h Fx(\000)f Fv(1\))g(+)g(\()p Fw(bl)g Fx(\000)h Fv(1\))e Fx(\001)h Fp(ex)p Fv(\))p Fx(g)p Fp(:)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 66 74 66 73 bop 75 -100 a Fv(66)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fw(Indexed)47 b Fv(The)19 b(function)h Fw(MPI)p 629 45 14 2 v 16 w(TYPE)p 762 45 V 17 w(INDEXED)f Fv(allo)o(ws)h(replication)h(of)d(an)i(old)f (datat)o(yp)q(e)g(in)o(to)g(a)75 102 y(sequence)g(of)f(blo)q(c)o(ks)h (\(eac)o(h)f(blo)q(c)o(k)h(is)g(a)f(concatenation)g(of)g(the)g(old)h (datat)o(yp)q(e\),)e(where)i(eac)o(h)f(blo)q(c)o(k)75 158 y(can)23 b(con)o(tain)f(a)g(di\013eren)o(t)h(n)o(um)o(b)q(er)g(of)f (copies)h(and)g(ha)o(v)o(e)f(a)g(di\013eren)o(t)g(displacemen)o(t.)43 b(All)24 b(blo)q(c)o(k)75 214 y(displacemen)o(ts)17 b(are)e(m)o (ultiples)h(of)f(the)h(old)f(t)o(yp)q(e)h(exten)o(t.)75 365 y Fw(MPI)p 160 365 V 16 w(TYPE)p 293 365 V 17 w(INDEXED\()c(count,) i(a)o(rra)o(y)p 762 365 V 14 w(of)p 813 365 V 16 w(blo)q(cklengths,)h (a)o(rra)o(y)p 1185 365 V 15 w(of)p 1237 365 V 16 w(displacements,)g (oldt)o(yp)q(e,)f(newt)o(yp)q(e\))117 499 y Fk(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)34 b(of)f(blo)q(c)o(ks)h({)g(also)f (n)o(um)o(b)q(er)h(of)f(en)o(tries)i(in)905 556 y Fi(a)o(rra)o(y)p 992 556 13 2 v 15 w(of)p 1041 556 V 15 w(displacements)19 b Fk(and)f Fi(a)o(rra)o(y)p 1483 556 V 15 w(of)p 1532 556 V 15 w(blo)q(cklengths)i Fk(\(non-)905 612 y(negativ)o(e)14 b(in)o(teger\))117 686 y(IN)155 b Fw(a)o(rra)o(y)p 416 686 14 2 v 15 w(of)p 468 686 V 16 w(blo)q(cklengths)191 b Fk(n)o(um)o(b)q(er)18 b(of)f(elemen)o(ts)h(p)q(er)i(blo)q(c)o(k)d (\(arra)o(y)i(of)e(nonnegativ)o(e)905 743 y(in)o(tegers\))117 817 y(IN)155 b Fw(a)o(rra)o(y)p 416 817 V 15 w(of)p 468 817 V 16 w(displacements)164 b Fk(displacemen)o(t)19 b(for)h(eac)o(h)g(blo)q(c)o(k,)g(in)g(m)o(ultiples)e(of)h Fi(oldt)o(yp)q(e)905 873 y Fk(exten)o(t)c(\(arra)o(y)f(of)f(in)o (teger\))117 947 y(IN)155 b Fw(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i(\(handle\))117 1021 y(OUT)108 b Fw(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 1146 y Ft(int)23 b(MPI)p 245 1146 15 2 v 17 w(Type)p 358 1146 V 17 w(indexed\(int)f (count,)h(int)h(*array)p 1068 1146 V 16 w(of)p 1132 1146 V 17 w(blocklengths,)393 1202 y(int)g(*array)p 636 1202 V 16 w(of)p 700 1202 V 17 w(displacements,)e(MPI)p 1147 1202 V 17 w(Datatype)g(oldtype,)393 1259 y(MPI)p 468 1259 V 17 w(Datatype)h(*newtype\))75 1345 y(MPI)p 150 1345 V 17 w(TYPE)p 263 1345 V 16 w(INDEXED\(COUNT,)f(ARRAY)p 757 1345 V 17 w(OF)p 822 1345 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1293 1345 V 16 w(OF)p 1357 1345 V 17 w(DISPLACEMENTS,)393 1402 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1458 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1458 V 17 w(OF)p 716 1458 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1458 V 17 w(OF)p 1323 1458 V 17 w(DISPLACEMENTS\(*\),)170 1515 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 1644 y Fq(Example)18 b(3.23)23 b Fv(Let)14 b Fw(oldt)o(yp)q(e)h Fv(ha)o(v)o(e)f(t)o(yp)q(e)g (map)g Fx(f)p Fv(\()p Fw(double)p Fp(;)8 b Fv(0\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(8\))p Fx(g)p Fp(;)k Fv(with)i(exten)o(t) g(16.)19 b(Let)14 b Fw(B)h(=)75 1700 y(\(3,)e(1\))h Fv(and)g(let)h Fw(D)f(=)g(\(4,)f(0\))p Fv(.)19 b(A)14 b(call)i(to)d Fw(MPI)p 855 1700 14 2 v 16 w(TYPE)p 988 1700 V 17 w(INDEXED\(2,)g(B,)h (D,)f(oldt)o(yp)q(e,)j(newt)o(yp)q(e\))f Fv(returns)75 1757 y(a)g(datat)o(yp)q(e)g(with)g(t)o(yp)q(e)g(map,)189 1845 y Fx(f)p Fv(\()p Fw(double)p Fp(;)8 b Fv(64\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(72\))p Fp(;)g Fv(\()p Fw(double)n Fp(;)g Fv(80\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(88\))p Fp(;)g Fv(\()p Fw(double)n Fp(;)g Fv(96\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(104\))p Fp(;)189 1944 y Fv(\()p Fw(double)q Fp(;)g Fv(0\))p Fp(;)g Fv(\()p Fw(cha)o(r)m Fp(;)g Fv(8\))p Fx(g)p Fp(:)75 2044 y Fv(That)15 b(is,)i(three)f(copies)g(of)g(the)g(old)g(t)o(yp)q(e)g(starting)g(at)f (displacemen)o(t)i(64,)e(and)h(one)h(cop)o(y)e(starting)h(at)75 2100 y(displacemen)o(t)h(0.)166 2282 y(In)f(general,)f(assume)g(that)g Fw(oldt)o(yp)q(e)h Fv(has)f(t)o(yp)q(e)g(map,)189 2371 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 2378 y Fg(0)333 2371 y Fp(;)8 b(disp)438 2378 y Fg(0)457 2371 y Fv(\))p Fp(;)g(:::;)f Fv(\()p Fp(ty)r(pe)657 2378 y Ff(n)p Fh(\000)p Fg(1)723 2371 y Fp(;)h(disp)828 2378 y Ff(n)p Fh(\000)p Fg(1)895 2371 y Fv(\))p Fx(g)p Fp(;)75 2459 y Fv(with)41 b(exten)o(t)f Fl(ex)p Fv(.)96 b(Let)41 b Fw(B)g Fv(b)q(e)g(the)g Fw(a)o(rra)o(y)p 986 2459 V 14 w(of)p 1037 2459 V 16 w(blo)q(cklength)i Fv(argumen)o(t)d(and)g Fw(D)h Fv(b)q(e)g(the)75 2516 y Fw(a)o(rra)o(y)p 173 2516 V 15 w(of)p 225 2516 V 16 w(displacements)17 b Fv(argumen)o(t.)i(The)c(newly)h(created)f(datat)o (yp)q(e)g(has)g Fp(n)c Fx(\001)1450 2484 y Fe(P)1494 2497 y Fi(count)q Fh(\000)p Fg(1)1494 2529 y Ff(i)p Fg(=0)1646 2516 y Fw(B)p Fv([)p Fw(i)p Fv(])j(en)o(tries:)189 2604 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 2611 y Fg(0)333 2604 y Fp(;)8 b(disp)438 2611 y Fg(0)467 2604 y Fv(+)i Fw(D)p Fv([)p Fw(0)p Fv(])f Fx(\001)h Fp(ex)p Fv(\))p Fp(;)e(:::;)g Fv(\()p Fp(ty)r(pe)874 2611 y Ff(n)p Fh(\000)p Fg(1)939 2604 y Fp(;)g(disp)1044 2611 y Ff(n)p Fh(\000)p Fg(1)1122 2604 y Fv(+)i Fw(D)p Fv([)p Fw(0)p Fv(])f Fx(\001)h Fp(ex)p Fv(\))p Fp(;)e(:::;)189 2704 y Fv(\()p Fp(ty)r(pe)291 2711 y Fg(0)310 2704 y Fp(;)g(disp)415 2711 y Fg(0)444 2704 y Fv(+)i(\()p Fw(D)p Fv([)p Fw(0)p Fv(])f(+)h Fw(B)p Fv([)p Fw(0)p Fv(])g Fx(\000)g Fv(1\))g Fx(\001)f Fp(ex)p Fv(\))p Fp(;)f(:::;)g Fv(\()p Fp(ty)r(pe)1098 2711 y Ff(n)p Fh(\000)p Fg(1)1164 2704 y Fp(;)g(disp)1269 2711 y Ff(n)p Fh(\000)p Fg(1)1347 2704 y Fv(+)i(\()p Fw(D)p Fv([)p Fw(0)p Fv(])f(+)h Fw(B)p Fv([)p Fw(0)p Fv(])f Fx(\000)i Fv(1\))e Fx(\001)h Fp(ex)p Fv(\))p Fp(;)e(:::;)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 67 75 67 74 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fv(67)189 45 y(\()p Fp(ty)r(pe)291 52 y Fg(0)310 45 y Fp(;)8 b(disp)415 52 y Fg(0)444 45 y Fv(+)i Fw(D)p Fv([)p Fw(count)h Fx(\000)g Fw(1)p Fv(])e Fx(\001)h Fp(ex)p Fv(\))p Fp(;)e(:::;)g Fv(\()p Fp(ty)r(pe)1013 52 y Ff(n)p Fh(\000)p Fg(1)1079 45 y Fp(;)g(disp)1184 52 y Ff(n)p Fh(\000)p Fg(1)1261 45 y Fv(+)i Fw(D)p Fv([)p Fw(count)h Fx(\000)g Fw(1)p Fv(])e Fx(\001)h Fp(ex)p Fv(\))p Fp(;)e(:::;)189 136 y Fv(\()p Fp(ty)r(pe)291 143 y Fg(0)310 136 y Fp(;)g(disp)415 143 y Fg(0)444 136 y Fv(+)i(\()p Fw(D)p Fv([)p Fw(count)h Fx(\000)f Fw(1)p Fv(])g(+)g Fw(B)p Fv([)p Fw(count)i Fx(\000)e Fw(1)p Fv(])g Fx(\000)g Fv(1\))f Fx(\001)h Fp(ex)p Fv(\))p Fp(;)e(:::;)189 226 y Fv(\()p Fp(ty)r(pe)291 233 y Ff(n)p Fh(\000)p Fg(1)359 226 y Fp(;)g(disp)464 233 y Ff(n)p Fh(\000)p Fg(1)542 226 y Fv(+)i(\()p Fw(D)p Fv([)p Fw(count)h Fx(\000)f Fw(1)p Fv(])g(+)g Fw(B)p Fv([)p Fw(count)h Fx(\000)g Fw(1)p Fv(])e Fx(\000)i Fv(1\))e Fx(\001)h Fp(ex)p Fv(\))p Fx(g)p Fp(:)166 400 y Fv(A)j(call)g(to)f Fw(MPI)p 431 400 14 2 v 16 w(TYPE)p 564 400 V 17 w(VECTOR\(count,)j (blo)q(cklength,)g(stride,)e(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))g Fv(is)f(equiv)m(alen)o(t)75 456 y(to)i(a)f(call)j(to)d Fw(MPI)p 392 456 V 16 w(TYPE)p 525 456 V 17 w(INDEXED\(count,)i(B,)f (D,)f(oldt)o(yp)q(e,)i(newt)o(yp)q(e\))h Fv(where)189 539 y Fw(D)p Fv([)p Fw(j)p Fv(])11 b(=)i Fp(j)g Fx(\001)d Fw(stride)q Fp(;)22 b(j)15 b Fv(=)e(0)p Fp(;)8 b(:::;)g Fw(count)g Fx(\000)j Fv(1)p Fp(;)75 621 y Fv(and)189 704 y Fw(B)p Fv([)p Fw(j)p Fv(])h(=)h Fw(blo)q(cklength)r Fp(;)22 b(j)15 b Fv(=)e(0)p Fp(;)8 b(:::;)g Fw(count)h Fx(\000)h Fv(1)p Fp(:)75 822 y Fw(Hindexed)47 b Fv(The)11 b(function)h Fw(MPI)p 643 822 V 15 w(TYPE)p 775 822 V 17 w(HINDEXED)f Fv(is)g(iden)o(tical)i(to)d Fw(MPI)p 1383 822 V 15 w(TYPE)p 1515 822 V 17 w(INDEXED)p Fv(,)g(except)75 878 y(that)21 b(blo)q(c)o(k)i(displacemen)o(ts)g(in)g Fw(a)o(rra)o(y)p 761 878 V 14 w(of)p 812 878 V 16 w(displacements)h Fv(are)e(sp)q(eci\014ed)i(in)e(b)o(ytes,)h(rather)e(than)h(in)75 935 y(m)o(ultiples)17 b(of)e(the)g Fw(oldt)o(yp)q(e)h Fv(exten)o(t.)75 1086 y Fw(MPI)p 160 1086 V 16 w(TYPE)p 293 1086 V 17 w(HINDEXED\()21 b(count,)k(a)o(rra)o(y)p 814 1086 V 14 w(of)p 865 1086 V 16 w(blo)q(cklengths,)h(a)o(rra)o(y)p 1248 1086 V 15 w(of)p 1300 1086 V 16 w(displacements,)f(oldt)o(yp)q(e,) f(new-)75 1142 y(t)o(yp)q(e\))117 1219 y Fk(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)34 b(of)f(blo)q(c)o(ks)h({)g(also)f (n)o(um)o(b)q(er)h(of)f(en)o(tries)i(in)905 1276 y Fi(a)o(rra)o(y)p 992 1276 13 2 v 15 w(of)p 1041 1276 V 15 w(displacements)19 b Fk(and)f Fi(a)o(rra)o(y)p 1483 1276 V 15 w(of)p 1532 1276 V 14 w(blo)q(cklengths)i Fk(\(in)o(te-)905 1332 y(ger\))117 1405 y(IN)155 b Fw(a)o(rra)o(y)p 416 1405 14 2 v 15 w(of)p 468 1405 V 16 w(blo)q(cklengths)191 b Fk(n)o(um)o(b)q(er)15 b(of)g(elemen)o(ts)h(in)f(eac)o(h)h(blo)q(c)o (k)g(\(arra)o(y)f(of)g(nonnega-)905 1462 y(tiv)o(e)f(in)o(tegers\))117 1535 y(IN)155 b Fw(a)o(rra)o(y)p 416 1535 V 15 w(of)p 468 1535 V 16 w(displacements)164 b Fk(b)o(yte)14 b(displacemen)o(t)g (of)f(eac)o(h)h(blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1608 y(IN)155 b Fw(oldt)o(yp)q(e)450 b Fk(old)13 b(datat)o(yp)q(e)i (\(handle\))117 1680 y(OUT)108 b Fw(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 1805 y Ft(int)23 b(MPI)p 245 1805 15 2 v 17 w(Type)p 358 1805 V 17 w(hindexed\(int)f (count,)h(int)h(*array)p 1092 1805 V 16 w(of)p 1156 1805 V 17 w(blocklengths,)393 1861 y(MPI)p 468 1861 V 17 w(Aint)f(*array)p 748 1861 V 17 w(of)p 813 1861 V 17 w(displacements,)e(MPI)p 1259 1861 V 17 w(Datatype)i(oldtype,)393 1918 y(MPI)p 468 1918 V 17 w(Datatype)g(*newtype\))75 2004 y(MPI)p 150 2004 V 17 w(TYPE)p 263 2004 V 16 w(HINDEXED\(COUNT,)f(ARRAY)p 781 2004 V 17 w(OF)p 846 2004 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1317 2004 V 16 w(OF)p 1381 2004 V 17 w(DISPLACEMENTS,)393 2061 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 2117 y(INTEGER)g(COUNT,)g (ARRAY)p 651 2117 V 17 w(OF)p 716 2117 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 2117 V 17 w(OF)p 1323 2117 V 17 w(DISPLACEMENTS\(*\),)170 2174 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)166 2343 y Fv(Assume)15 b(that)g Fw(oldt)o(yp)q(e)h Fv(has)f(t)o(yp)q(e)g(map,)189 2425 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 2432 y Fg(0)333 2425 y Fp(;)8 b(disp)438 2432 y Fg(0)457 2425 y Fv(\))p Fp(;)g(:::;)f Fv(\()p Fp(ty)r(pe)657 2432 y Ff(n)p Fh(\000)p Fg(1)723 2425 y Fp(;)h(disp)828 2432 y Ff(n)p Fh(\000)p Fg(1)895 2425 y Fv(\))p Fx(g)p Fp(;)75 2508 y Fv(with)41 b(exten)o(t)e Fp(ex)p Fv(.)95 b(Let)41 b Fw(B)f Fv(b)q(e)h(the)g Fw(a)o(rra)o(y)p 988 2508 14 2 v 14 w(of)p 1039 2508 V 16 w(blo)q(cklength)h Fv(argumen)o(t)e(and)g Fw(D)g Fv(b)q(e)h(the)75 2564 y Fw(a)o(rra)o(y)p 173 2564 V 15 w(of)p 225 2564 V 16 w(displacements)23 b Fv(argumen)o(t.)39 b(The)22 b(newly)h(created)f(datat)o(yp)q(e)f(has)h(a)f(t)o(yp)q(e)h (map)f(with)i Fp(n)14 b Fx(\001)75 2589 y Fe(P)119 2602 y Fi(count)q Fh(\000)p Fg(1)119 2634 y Ff(i)p Fg(=0)271 2621 y Fw(B)p Fv([)p Fw(i)p Fv(])g(en)o(tries:)189 2704 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 2711 y Fg(0)333 2704 y Fp(;)8 b(disp)438 2711 y Fg(0)467 2704 y Fv(+)i Fw(D)p Fv([)p Fw(0)p Fv(])o(\))p Fp(;)e(:::;)g Fv(\()p Fp(ty)r(pe)794 2711 y Ff(n)p Fh(\000)p Fg(1)859 2704 y Fp(;)g(disp)964 2711 y Ff(n)p Fh(\000)p Fg(1)1042 2704 y Fv(+)i Fw(D)p Fv([)p Fw(0)p Fv(])o(\))p Fp(;)e(:::;)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 68 76 68 75 bop 75 -100 a Fv(68)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)189 45 y Fv(\()p Fp(ty)r(pe)291 52 y Fg(0)310 45 y Fp(;)8 b(disp)415 52 y Fg(0)444 45 y Fv(+)i Fw(D)p Fv([)p Fw(0)p Fv(])f(+)i(\()p Fw(B)p Fv([)p Fw(0)p Fv(])e Fx(\000)h Fv(1\))g Fx(\001)f Fp(ex)p Fv(\))p Fp(;)f(:::;)189 151 y Fv(\()p Fp(ty)r(pe)291 158 y Ff(n)p Fh(\000)p Fg(1)359 151 y Fp(;)g(disp)464 158 y Ff(n)p Fh(\000)p Fg(1)542 151 y Fv(+)i Fw(D)p Fv([)p Fw(0)p Fv(])f(+)h(\()p Fw(B)p Fv([)p Fw(0)p Fv(])f Fx(\000)i Fv(1\))e Fx(\001)h Fp(ex)p Fv(\))p Fp(;)e(:::;)189 257 y Fv(\()p Fp(ty)r(pe)291 264 y Fg(0)310 257 y Fp(;)g(disp)415 264 y Fg(0)444 257 y Fv(+)i Fw(D)p Fv([)p Fw(count)h Fx(\000)g Fw(1)p Fv(])o(\))p Fp(;)d(:::;)g Fv(\()p Fp(ty)r(pe)933 264 y Ff(n)p Fh(\000)p Fg(1)999 257 y Fp(;)g(disp)1104 264 y Ff(n)p Fh(\000)p Fg(1)1181 257 y Fv(+)i Fw(D)p Fv([)p Fw(count)h Fx(\000)g Fw(1)p Fv(])o(\))p Fp(;)d(:::;)189 363 y Fv(\()p Fp(ty)r(pe)291 370 y Fg(0)310 363 y Fp(;)g(disp)415 370 y Fg(0)444 363 y Fv(+)i Fw(D)p Fv([)p Fw(count)h Fx(\000)g Fw(1)p Fv(])e(+)i(\()p Fw(B)p Fv([)p Fw(count)g Fx(\000)f Fw(1)p Fv(])g Fx(\000)g Fv(1\))f Fx(\001)h Fp(ex)p Fv(\))p Fp(;)e(:::;)189 468 y Fv(\()p Fp(ty)r(pe)291 475 y Ff(n)p Fh(\000)p Fg(1)359 468 y Fp(;)g(disp)464 475 y Ff(n)p Fh(\000)p Fg(1)542 468 y Fv(+)i Fw(D)p Fv([)p Fw(count)h Fx(\000)f Fw(1)p Fv(])g(+)g(\()p Fw(B)p Fv([)p Fw(count)h Fx(\000)g Fw(1)p Fv(])e Fx(\000)i Fv(1\))e Fx(\001)h Fp(ex)p Fv(\))p Fx(g)p Fp(:)75 720 y Fw(Struct)47 b(MPI)p 323 720 14 2 v 16 w(TYPE)p 456 720 V 16 w(STRUCT)20 b Fv(is)g(the)g(most)f(general)g(t)o(yp)q(e)h(constructor.)32 b(It)19 b(further)g(generalizes)75 777 y(the)13 b(previous)h(one)f(in)h (that)e(it)h(allo)o(ws)g(eac)o(h)g(blo)q(c)o(k)h(to)e(consist)h(of)g (replications)h(of)f(di\013eren)o(t)g(datat)o(yp)q(es.)75 928 y Fw(MPI)p 160 928 V 16 w(TYPE)p 293 928 V 17 w(STRUCT\(count,)f(a) o(rra)o(y)p 729 928 V 15 w(of)p 781 928 V 16 w(blo)q(cklengths,)h(a)o (rra)o(y)p 1151 928 V 14 w(of)p 1202 928 V 16 w(displacements,)g(a)o (rra)o(y)p 1599 928 V 15 w(of)p 1651 928 V 15 w(t)o(yp)q(es,)g(new-)75 984 y(t)o(yp)q(e\))117 1062 y Fk(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)18 b(of)h(blo)q(c)o(ks)g(\(in)o(teger\))g({)g (also)f(n)o(um)o(b)q(er)h(of)f(en)o(tries)905 1118 y(in)c(arra)o(ys)h Fi(a)o(rra)o(y)p 1167 1118 13 2 v 15 w(of)p 1216 1118 V 14 w(t)o(yp)q(es)p Fk(,)h Fi(a)o(rra)o(y)p 1432 1118 V 15 w(of)p 1481 1118 V 14 w(displacements)g Fk(and)e Fi(a)o(r-)905 1174 y(ra)o(y)p 959 1174 V 15 w(of)p 1008 1174 V 15 w(blo)q(cklengths)117 1250 y Fk(IN)155 b Fw(a)o(rra)o(y)p 416 1250 14 2 v 15 w(of)p 468 1250 V 16 w(blo)q(cklength)208 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(eac)o(h)g(blo)q(c)o (k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1325 y(IN)155 b Fw(a)o(rra)o(y)p 416 1325 V 15 w(of)p 468 1325 V 16 w(displacements)164 b Fk(b)o(yte)14 b(displacemen)o(t)g(of)f(eac)o(h)h (blo)q(c)o(k)g(\(arra)o(y)g(of)f(in)o(teger\))117 1400 y(IN)155 b Fw(a)o(rra)o(y)p 416 1400 V 15 w(of)p 468 1400 V 16 w(t)o(yp)q(es)327 b Fk(t)o(yp)q(e)20 b(of)f(elemen)o(ts)g(in) g(eac)o(h)h(blo)q(c)o(k)f(\(arra)o(y)g(of)g(handles)g(to)905 1456 y(datat)o(yp)q(e)14 b(ob)r(jects\))117 1531 y(OUT)108 b Fw(newt)o(yp)q(e)433 b Fk(new)15 b(datat)o(yp)q(e)f(\(handle\))75 1656 y Ft(int)23 b(MPI)p 245 1656 15 2 v 17 w(Type)p 358 1656 V 17 w(struct\(int)f(count,)h(int)h(*array)p 1044 1656 V 16 w(of)p 1108 1656 V 17 w(blocklengths,)393 1712 y(MPI)p 468 1712 V 17 w(Aint)f(*array)p 748 1712 V 17 w(of)p 813 1712 V 17 w(displacements,)e(MPI)p 1259 1712 V 17 w(Datatype)i(*array)p 1635 1712 V 16 w(of)p 1699 1712 V 17 w(types,)393 1769 y(MPI)p 468 1769 V 17 w(Datatype)g(*newtype\))75 1855 y(MPI)p 150 1855 V 17 w(TYPE)p 263 1855 V 16 w(STRUCT\(COUNT,)f(ARRAY)p 733 1855 V 17 w(OF)p 798 1855 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1269 1855 V 16 w(OF)p 1333 1855 V 17 w(DISPLACEMENTS,)393 1911 y(ARRAY)p 516 1911 V 17 w(OF)p 581 1911 V 17 w(TYPES,)h(NEWTYPE,)f (IERROR\))170 1968 y(INTEGER)h(COUNT,)g(ARRAY)p 651 1968 V 17 w(OF)p 716 1968 V 17 w(BLOCKLENGTHS\(*\),)e(ARRAY)p 1258 1968 V 17 w(OF)p 1323 1968 V 17 w(DISPLACEMENTS\(*\),)170 2024 y(ARRAY)p 293 2024 V 17 w(OF)p 358 2024 V 17 w(TYPES\(*\),)h (NEWTYPE,)h(IERROR)75 2160 y Fq(Example)18 b(3.24)23 b Fv(Let)15 b Fw(t)o(yp)q(e1)h Fv(ha)o(v)o(e)f(t)o(yp)q(e)g(map,)189 2254 y Fx(f)p Fv(\()p Fw(double)p Fp(;)8 b Fv(0\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(8\))p Fx(g)p Fp(;)75 2347 y Fv(with)k(exten)o(t)f(16.)18 b(Let)11 b Fw(B)h(=)g(\(2,)f(1,)g(3\))p Fv(,)g Fw(D)h(=)f(\(0,)g(16,)h(26\))p Fv(,)e(and)i Fw(T)f(=)h(\(MPI)p 1321 2347 14 2 v 15 w(FLO)o(A)l(T,)f(t)o(yp)q(e1,)i(MPI)p 1711 2347 V 16 w(CHAR\))p Fv(.)75 2404 y(Then)h(a)f(call)h(to)e Fw(MPI)p 447 2404 V 16 w(TYPE)p 580 2404 V 17 w(STRUCT\(3,)h(B,)g(D,)g (T,)g(newt)o(yp)q(e\))h Fv(returns)f(a)g(datat)o(yp)q(e)g(with)g(t)o (yp)q(e)h(map,)189 2497 y Fx(f)p Fv(\()p Fw(\015oat)o Fp(;)8 b Fv(0\))p Fp(;)g Fv(\()p Fw(\015oat)n Fp(;)g Fv(4\))p Fp(;)g Fv(\()p Fw(double)o Fp(;)g Fv(16\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(24\))p Fp(;)g Fv(\()p Fw(cha)o(r)m Fp(;)g Fv(26\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(27\))p Fp(;)g Fv(\()p Fw(cha)o(r)n Fp(;)g Fv(28\))p Fx(g)p Fp(:)75 2591 y Fv(That)16 b(is,)h(t)o(w)o(o)e(copies)i(of)f Fw(MPI)p 610 2591 V 15 w(FLO)o(A)l(T)h Fv(starting)f(at)f(0,)h(follo)o (w)o(ed)h(b)o(y)f(one)h(cop)o(y)f(of)g Fw(t)o(yp)q(e1)h Fv(starting)f(at)75 2647 y(16,)d(follo)o(w)o(ed)h(b)o(y)f(three)h (copies)h(of)e Fw(MPI)p 765 2647 V 16 w(CHAR)p Fv(,)h(starting)f(at)g (26.)18 b(\(W)l(e)c(assume)f(that)g(a)g(\015oat)g(o)q(ccupies)75 2704 y(four)i(b)o(ytes.\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 69 77 69 76 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fv(69)166 45 y(In)16 b(general,)f(let)h Fw(T)f Fv(b)q(e)h(the)f Fw(a)o(rra)o(y)p 742 45 14 2 v 14 w(of)p 793 45 V 16 w(t)o(yp)q(es)i Fv(argumen)o(t,)d(where)i Fw(T[i])f Fv(is)g(a)g(handle)i(to,)189 139 y Fp(ty)r(pemap)360 146 y Ff(i)387 139 y Fv(=)c Fx(f)p Fv(\()p Fp(ty)r(pe)560 120 y Ff(i)560 150 y Fg(0)579 139 y Fp(;)8 b(disp)684 120 y Ff(i)684 150 y Fg(0)702 139 y Fv(\))p Fp(;)g(:::;)g Fv(\()p Fp(ty)r(pe)903 120 y Ff(i)903 150 y(n)924 155 y Fd(i)935 150 y Fh(\000)p Fg(1)982 139 y Fp(;)g(disp)1087 120 y Ff(i)1087 150 y(n)1108 155 y Fd(i)1120 150 y Fh(\000)p Fg(1)1168 139 y Fv(\))p Fx(g)p Fp(;)75 233 y Fv(with)15 b(exten)o(t)g Fp(ex)364 240 y Ff(i)378 233 y Fv(.)20 b(Let)15 b Fw(B)g Fv(b)q(e)g(the)g Fw(a)o(rra)o(y)p 774 233 V 14 w(of)p 825 233 V 16 w(blo)q(cklength)i Fv(argumen)o(t)d(and)h Fw(D)g Fv(b)q(e)g(the)g Fw(a)o(rra)o(y)p 1647 233 V 15 w(of)p 1699 233 V 15 w(displace-)75 289 y(ments)e Fv(argumen)o(t.)19 b(Let)13 b Fw(c)g Fv(b)q(e)g(the)g Fw(count)h Fv(argumen)o(t.)19 b(Then)13 b(the)g(newly)h(created)e(datat)o(yp)q(e)h(has)f(a)h(t)o(yp)q (e)75 346 y(map)i(with)280 314 y Fe(P)323 327 y Fi(c)p Fh(\000)p Fg(1)323 359 y Ff(i)p Fg(=0)397 346 y Fw(B)p Fv([)p Fw(i)p Fv(])9 b Fx(\001)h Fp(n)523 353 y Ff(i)552 346 y Fv(en)o(tries:)189 439 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 421 y Fg(0)314 451 y(0)333 439 y Fp(;)e(disp)438 421 y Fg(0)438 451 y(0)467 439 y Fv(+)i Fw(D)p Fv([)p Fw(0)p Fv(])o(\))p Fp(;)e(:::;)g Fv(\()p Fp(ty)r(pe)794 421 y Fg(0)794 451 y Ff(n)815 456 y Fs(0)832 439 y Fp(;)g(disp)937 421 y Fg(0)937 451 y Ff(n)958 456 y Fs(0)986 439 y Fv(+)j Fw(D)p Fv([)p Fw(0)p Fv(])o(\))p Fp(;)d(:::;)189 546 y Fv(\()p Fp(ty)r(pe)291 527 y Fg(0)291 557 y(0)310 546 y Fp(;)g(disp)415 527 y Fg(0)415 557 y(0)444 546 y Fv(+)i Fw(D)p Fv([)p Fw(0)p Fv(])f(+)i(\()p Fw(B)p Fv([)p Fw(0)p Fv(])e Fx(\000)h Fv(1\))g Fx(\001)f Fp(ex)897 553 y Fg(0)917 546 y Fv(\))p Fp(;)f(:::;)g Fv(\()p Fp(ty)r(pe)1118 527 y Fg(0)1118 557 y Ff(n)1139 562 y Fs(0)1156 546 y Fp(;)g(disp)1261 527 y Fg(0)1261 557 y Ff(n)1282 562 y Fs(0)1311 546 y Fv(+)i Fw(D)p Fv([)p Fw(0)p Fv(])f(+)i(\()p Fw(B)p Fv([)p Fw(0)p Fv(])e Fx(\000)h Fw(1)p Fv(\))g Fx(\001)f Fp(ex)1764 553 y Fg(0)1784 546 y Fv(\))p Fp(;)f(:::;)189 652 y Fv(\()p Fp(ty)r(pe)291 633 y Fi(c)p Fh(\000)p Fg(1)291 664 y(0)356 652 y Fp(;)g(disp)461 633 y Fi(c)p Fh(\000)p Fg(1)461 664 y(0)536 652 y Fv(+)i Fw(D)p Fv([)p Fw(c)g Fx(\000)g Fw(1)p Fv(]\))p Fp(;)e(:::;)g Fv(\()o Fp(ty)r(pe)938 633 y Fi(c)p Fh(\000)p Fg(1)938 664 y Ff(n)959 671 y Fi(c)p Fc(\000)p Fs(1)1017 664 y Fh(\000)p Fg(1)1064 652 y Fp(;)g(disp)1169 633 y Fi(c)p Fh(\000)p Fg(1)1169 664 y Ff(n)1190 671 y Fi(c)p Fc(\000)p Fs(1)1249 664 y Fh(\000)p Fg(1)1306 652 y Fv(+)j Fw(D)p Fv([)p Fw(c)e Fx(\000)i Fw(1)p Fv(])o(\))p Fp(;)d(:::;)189 769 y Fv(\()p Fp(ty)r(pe)291 750 y Fi(c)p Fh(\000)p Fg(1)291 782 y(0)356 769 y Fp(;)g(disp)461 750 y Fi(c)p Fh(\000)p Fg(1)461 782 y(0)536 769 y Fv(+)i Fw(D)p Fv([)p Fw(c)g Fx(\000)g Fw(1)p Fv(])g(+)g(\()p Fw(B)p Fv([)p Fw(c)g Fx(\000)h Fw(1)p Fv(])e Fx(\000)i Fv(1\))e Fx(\001)h Fp(ex)1141 776 y Fi(c)p Fh(\000)p Fg(1)1207 769 y Fv(\))p Fp(;)e(:::;)189 876 y Fv(\()p Fp(ty)r(pe)291 856 y Fi(c)p Fh(\000)p Fg(1)291 888 y Ff(n)312 895 y Fi(c)p Fc(\000)p Fs(1)372 888 y Fh(\000)p Fg(1)419 876 y Fp(;)g(disp)524 856 y Fi(c)p Fh(\000)p Fg(1)524 888 y Ff(n)545 895 y Fi(c)p Fc(\000)p Fs(1)604 888 y Fh(\000)p Fg(1)661 876 y Fv(+)j Fw(D)p Fv([)p Fw(c)e Fx(\000)i Fw(1)p Fv(])e(+)i(\()p Fw(B)p Fv([)p Fw(c)f Fx(\000)g Fw(1)p Fv(])f Fx(\000)i Fw(1)p Fv(\))e Fx(\001)h Fp(ex)1266 883 y Fi(c)p Fh(\000)p Fg(1)1332 876 y Fv(\))p Fx(g)p Fp(:)166 1069 y Fv(A)15 b(call)h(to)f Fw(MPI)p 439 1069 V 16 w(TYPE)p 572 1069 V 16 w(HINDEXED\()g(count,)h (B,)f(D,)g(oldt)o(yp)q(e,)h(newt)o(yp)q(e\))g Fv(is)g(equiv)m(alen)o(t) h(to)d(a)h(call)75 1126 y(to)d Fw(MPI)p 213 1126 V 16 w(TYPE)p 346 1126 V 17 w(STRUCT\()h(count,)i(B,)e(D,)f(T,)h(newt)o(yp)q (e\))p Fv(,)h(where)f(eac)o(h)g(en)o(try)g(of)g Fw(T)g Fv(is)g(equal)h(to)e Fw(oldt)o(yp)q(e)p Fv(.)75 1247 y Fj(3.12.2)49 b(Address)16 b(and)g(extent)f(functions)75 1333 y Fv(The)g(displacemen)o(ts)i(in)e(a)g(general)h(datat)o(yp)q(e)e (are)h(relativ)o(e)g(to)g(some)f(initial)j(bu\013er)e(address.)20 b Fq(Abso-)75 1390 y(lute)15 b(addresses)c Fv(can)h(b)q(e)h (substituted)f(for)g(these)g(displacemen)o(ts:)20 b(w)o(e)12 b(treat)f(them)h(as)g(displacemen)o(ts)75 1446 y(relativ)o(e)17 b(to)e(\\address)h(zero,")g(the)g(start)f(of)h(the)g(address)h(space.) 23 b(This)17 b(initial)h(address)e(zero)g(is)h(indi-)75 1503 y(cated)f(b)o(y)g(the)g(constan)o(t)f Fi(MPI)p 601 1503 13 2 v 14 w(BOTTOM)p Fv(.)g(Th)o(us,)g(a)h(datat)o(yp)q(e)f(can)h (sp)q(ecify)h(the)f(absolute)g(address)g(of)75 1559 y(the)e(en)o(tries) g(in)g(the)g(comm)o(unication)g(bu\013er,)g(in)h(whic)o(h)f(case)g(the) g Fw(buf)g Fv(argumen)o(t)f(is)h(passed)g(the)g(v)m(alue)75 1615 y Fi(MPI)p 152 1615 V 14 w(BOTTOM)p Fv(.)166 1672 y(The)31 b(address)g(of)f(a)h(lo)q(cation)g(in)h(memory)e(can)h(b)q(e)h (found)f(b)o(y)g(in)o(v)o(oking)g(the)g(function)75 1728 y Fw(MPI)p 160 1728 14 2 v 16 w(ADDRESS)p Fv(.)75 1879 y Fw(MPI)p 160 1879 V 16 w(ADDRESS\(lo)q(cation,)16 b(address\))117 1957 y Fk(IN)155 b Fw(lo)q(cation)437 b Fk(lo)q(cation)13 b(in)h(caller)f(memory)f(\(c)o(hoice\))117 2032 y(OUT)108 b Fw(address)449 b Fk(address)15 b(of)f(lo)q(cation)f(\(in)o(teger\))75 2156 y Ft(int)23 b(MPI)p 245 2156 15 2 v 17 w(Address\(void*)f (location,)h(MPI)p 907 2156 V 17 w(Aint)g(*address\))75 2243 y(MPI)p 150 2243 V 17 w(ADDRESS\(LOCATION,)e(ADDRESS,)i(IERROR\)) 170 2299 y()g(LOCATION\(*\))170 2355 y(INTEGER)g(ADDRESS,)g (IERROR)166 2442 y Fv(Returns)16 b(the)f(\(b)o(yte\))f(address)i(of)e Fw(lo)q(cation)p Fv(.)75 2548 y Fq(Example)k(3.25)23 b Fv(Using)15 b Fw(MPI)p 619 2548 14 2 v 16 w(ADDRESS)h Fv(for)f(an)g(arra)o(y)l(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 70 78 70 77 bop 75 -100 a Fv(70)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)147 45 y Ft(REAL)23 b(A\(100,100\))147 102 y(INTEGER)f(I1,)i(I2,)f(DIFF)147 158 y(CALL)g(MPI_ADDRESS\(A\(1,1\),)e(I1,)j(IERROR\))147 214 y(CALL)f(MPI_ADDRESS\(A\(10,10\),)e(I2,)i(IERROR\))147 271 y(DIFF)g(=)h(I2)f(-)h(I1)75 327 y(!)g(The)f(value)g(of)h(DIFF)f(is) h(909*sizeofreal;)e(the)h(values)g(of)h(I1)f(and)h(I2)f(are)75 384 y(!)h(implementation)e(dependent.)189 475 y Fl(A)n(dvic)n(e)e(to)i (users.)73 b Fv(C)20 b(users)h(ma)o(y)f(b)q(e)h(tempted)g(to)f(a)o(v)o (oid)h(the)f(usage)h(of)f Fw(MPI)p 1659 475 14 2 v 16 w(ADDRESS)189 532 y Fv(and)e(rely)g(on)g(the)g(a)o(v)m(ailabilit)o(y)i (of)d(the)h(address)g(op)q(erator)f(&.)29 b(Note,)17 b(ho)o(w)o(ev)o(er,)h(that)f Fi(&)h Fl(c)n(ast-)189 588 y(expr)n(ession)c Fv(is)j(a)e(p)q(oin)o(ter,)i(not)e(an)h(address.)22 b(ANSI)17 b(C)f(do)q(es)g(not)g(require)g(that)g(the)g(v)m(alue)h(of)e (a)189 645 y(p)q(oin)o(ter)f(\(or)f(the)h(p)q(oin)o(ter)g(cast)g(to)f Fi(int)p Fv(\))h(b)q(e)g(the)g(absolute)g(address)g(of)f(the)h(ob)s (ject)g(p)q(oin)o(ted)g(at)g(|)189 701 y(although)g(this)h(is)g (commonly)f(the)h(case.)k(F)l(urthermore,)14 b(referencing)h(ma)o(y)f (not)g(ha)o(v)o(e)f(a)h(unique)189 758 y(de\014nition)19 b(on)f(mac)o(hines)h(with)g(a)e(segmen)o(ted)h(address)g(space.)29 b(The)18 b(use)h(of)e Fw(MPI)p 1659 758 V 16 w(ADDRESS)189 814 y Fv(to)g(\\reference")h(C)f(v)m(ariables)i(guaran)o(tees)e(p)q (ortabilit)o(y)i(to)e(suc)o(h)h(mac)o(hines)g(as)g(w)o(ell.)28 b(\()p Fl(End)18 b(of)189 871 y(advic)n(e)e(to)g(users.)p Fv(\))166 962 y(The)f(follo)o(wing)h(auxiliary)h(functions)f(pro)o (vide)f(useful)i(information)e(on)g(deriv)o(ed)h(datat)o(yp)q(es.)75 1113 y Fw(MPI)p 160 1113 V 16 w(TYPE)p 293 1113 V 17 w(EXTENT\(datat)o(yp)q(e,)g(extent\))117 1190 y Fk(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 1263 y(OUT)108 b Fw(extent)471 b Fk(datat)o(yp)q(e)14 b(exten)o(t)h(\(in)o(teger\))-160 1342 y Fx(>)75 1388 y Ft(int)23 b(MPI)p 245 1388 15 2 v 17 w(Type)p 358 1388 V 17 w(extent\(MPI)p 615 1388 V 16 w(Datatype)f(datatype,)h(MPI)p 1156 1388 V 17 w(Aint)g(*extent\))-160 1428 y Fx(?)75 1474 y Ft(MPI)p 150 1474 V 17 w(TYPE)p 263 1474 V 16 w(EXTENT\(DATATYPE,)f(EXTENT,)h(IERROR\))170 1531 y(INTEGER)g (DATATYPE,)g(EXTENT,)g(IERROR)166 1617 y Fv(Returns)16 b(the)f(exten)o(t)g(of)g(a)f(datat)o(yp)q(e,)h(where)g(exten)o(t)g(is)h (as)f(de\014ned)h(on)f(page)g(71.)75 1768 y Fw(MPI)p 160 1768 14 2 v 16 w(TYPE)p 293 1768 V 17 w(SIZE\(datat)o(yp)q(e,)g (size\))117 1845 y Fk(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 1918 y(OUT)108 b Fw(size)519 b Fk(datat)o(yp)q(e)14 b(size)h(\(in)o(teger\))75 2043 y Ft(int)23 b(MPI)p 245 2043 15 2 v 17 w(Type)p 358 2043 V 17 w(size\(MPI)p 567 2043 V 16 w(Datatype)g(datatype,)f(int) i(*size\))75 2129 y(MPI)p 150 2129 V 17 w(TYPE)p 263 2129 V 16 w(SIZE\(DATATYPE,)e(SIZE,)i(IERROR\))170 2186 y(INTEGER)f(DATATYPE,)g(SIZE,)g(IERROR)166 2272 y Fw(MPI)p 251 2272 14 2 v 16 w(TYPE)p 384 2272 V 17 w(SIZE)18 b Fv(returns)h(the)g(total)f(size,)j(in)f(b)o(ytes,)f(of)g(the)g(en)o (tries)g(in)h(the)f(t)o(yp)q(e)g(signature)75 2329 y(asso)q(ciated)c (with)g Fw(datat)o(yp)q(e)p Fv(;)h(i.e.,)f(the)f(total)g(size)i(of)e (the)h(data)f(in)i(a)e(message)g(that)g(w)o(ould)i(b)q(e)f(created)75 2385 y(with)20 b(this)g(datat)o(yp)q(e.)32 b(En)o(tries)20 b(that)f(o)q(ccur)h(m)o(ultiple)h(times)f(in)h(the)e(datat)o(yp)q(e)g (are)h(coun)o(ted)g(with)75 2442 y(their)c(m)o(ultiplicit)o(y)l(.)-579 b Fx(>)-160 2494 y(?)75 2561 y Fj(3.12.3)49 b(Lo)o(w)o(er-b)q(ound)17 b(and)g(upp)q(er-b)q(ound)g(ma)o(rk)o(ers)75 2647 y Fv(It)e(is)g(often) g(con)o(v)o(enien)o(t)g(to)g(de\014ne)h(explicitly)h(the)e(lo)o(w)o(er) g(b)q(ound)h(and)f(upp)q(er)h(b)q(ound)f(of)g(a)g(t)o(yp)q(e)g(map,)75 2704 y(and)g(o)o(v)o(erride)f(the)h(de\014nition)h(giv)o(en)f(on)g (page)f(71.)19 b(This)c(allo)o(ws)g(one)g(to)e(de\014ne)j(a)e(datat)o (yp)q(e)g(that)g(has)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 71 79 71 78 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fv(71)75 45 y(\\holes")15 b(at)f(its)h(b)q(eginning)i(or)d(its)h (end,)h(or)e(a)g(datat)o(yp)q(e)h(with)g(en)o(tries)g(that)f(extend)i (ab)q(o)o(v)o(e)e(the)h(upp)q(er)75 102 y(b)q(ound)f(or)f(b)q(elo)o(w)g (the)g(lo)o(w)o(er)g(b)q(ound.)20 b(Examples)14 b(of)e(suc)o(h)i(usage) e(are)h(pro)o(vided)h(in)g(Sec.)20 b(3.12.7.)d(Also,)62 b Fx(>)75 158 y Fv(the)16 b(user)h(ma)o(y)e(w)o(an)o(t)g(to)h(o)o(v)o (eride)g(the)h(alignmen)o(t)g(rules)g(that)e(are)h(used)h(to)e(compute) i(upp)q(er)g(b)q(ounds)75 214 y(and)g(exten)o(ts.)25 b(E.g.,)16 b(a)h(C)g(compiler)h(ma)o(y)e(allo)o(w)i(the)f(user)g(to)f (o)o(v)o(eride)h(default)h(alignmen)o(t)g(rules)g(for)75 271 y(some)e(of)g(the)g(structures)g(within)h(a)f(program.)22 b(The)16 b(user)h(has)f(to)f(sp)q(ecify)j(explicitly)h(the)d(b)q(ounds) h(of)75 327 y(the)e(datat)o(yp)q(es)g(that)f(matc)o(h)h(these)g (structures.)1012 b Fx(?)166 384 y Fv(T)l(o)13 b(ac)o(hiev)o(e)h(this,) g(w)o(e)f(add)h(t)o(w)o(o)e(additional)j(\\pseudo-datat)o(yp)q(es,")e Fw(MPI)p 1438 384 14 2 v 16 w(LB)h Fv(and)f Fw(MPI)p 1691 384 V 16 w(UB)p Fv(,)h(that)75 440 y(can)e(b)q(e)h(used,)g(resp)q (ectiv)o(ely)l(,)h(to)d(mark)h(the)g(lo)o(w)o(er)f(b)q(ound)i(or)f(the) g(upp)q(er)h(b)q(ound)g(of)f(a)f(datat)o(yp)q(e.)19 b(These)75 497 y(pseudo-datat)o(yp)q(es)d(o)q(ccup)o(y)g(no)f(space)h(\()p Fp(extent)p Fv(\()p Fw(MPI)p 1022 497 V 16 w(LB)p Fv(\))d(=)h Fp(extent)p Fv(\()p Fw(MPI)p 1400 497 V 16 w(UB)q Fv(\))f(=)g(0\).)20 b(They)c(do)f(not)75 553 y(a\013ect)d(the)h(size)h(or)e(coun)o(t)h(of)f (a)h(datat)o(yp)q(e,)f(and)h(do)g(not)f(a\013ect)g(the)h(the)g(con)o (ten)o(t)g(of)f(a)h(message)f(created)75 610 y(with)k(this)f(datat)o (yp)q(e.)20 b(Ho)o(w)o(ev)o(er,)13 b(they)j(do)f(a\013ect)f(the)i (de\014nition)h(of)e(the)g(exten)o(t)g(of)g(a)f(datat)o(yp)q(e)h(and,) 75 666 y(therefore,)g(a\013ect)f(the)h(outcome)g(of)g(a)g(replication)i (of)d(this)i(datat)o(yp)q(e)f(b)o(y)g(a)g(datat)o(yp)q(e)f (constructor.)75 756 y Fq(Example)k(3.26)23 b Fv(Let)17 b Fw(D)f(=)h(\(-3,)g(0,)g(6\))p Fv(;)f Fw(T)h(=)g(\(MPI)p 1003 756 V 16 w(LB,)f(MPI)p 1185 756 V 16 w(INT,)g(MPI)p 1388 756 V 16 w(UB\))p Fv(,)h(and)g Fw(B)g(=)g(\(1,)f(1,)h(1\))p Fv(.)75 812 y(Then)g(a)e(call)i(to)f Fw(MPI)p 459 812 V 16 w(TYPE)p 592 812 V 16 w(STRUCT\(3,)g(B,)g(D,)g(T,)f(t)o(yp)q(e1\)) i Fv(creates)e(a)h(new)g(datat)o(yp)q(e)g(that)f(has)h(an)75 869 y(exten)o(t)h(of)g(9)g(\(from)f(-3)h(to)g(5,)g(5)g(included\),)i (and)f(con)o(tains)f(an)g(in)o(teger)h(at)e(displacemen)o(t)j(0.)26 b(This)17 b(is)75 925 y(the)f(datat)o(yp)q(e)f(de\014ned)j(b)o(y)d(the) h(sequence)i Fx(f)p Fw(\(lb,)d(-3\),)g(\(int,)h(0\),)f(\(ub,)i(6\))p Fx(g)e Fv(.)22 b(If)16 b(this)g(t)o(yp)q(e)g(is)g(replicated)75 982 y(t)o(wice)i(b)o(y)f(a)g(call)h(to)f Fw(MPI)p 528 982 V 16 w(TYPE)p 661 982 V 17 w(CONTIGUOUS\(2,)h(t)o(yp)q(e1,)g(t)o (yp)q(e2\))g Fv(then)g(the)f(newly)i(created)e(t)o(yp)q(e)75 1038 y(can)d(b)q(e)h(describ)q(ed)i(b)o(y)d(the)g(sequence)i Fx(f)p Fw(\(lb,)e(-3\),)f(\(int,)i(0\),)e(\(int,9\),)h(\(ub,)h(15\))p Fx(g)e Fv(.)19 b(\(An)c(en)o(try)f(of)g(t)o(yp)q(e)g Fw(ub)63 b Fx(>)75 1095 y Fv(can)16 b(b)q(e)g(deleted)h(if)g(there)e (is)i(another)e(en)o(try)g(of)h(t)o(yp)q(e)f Fw(ub)i Fv(with)f(a)g(higher)g(displacemen)o(t;)h(an)f(en)o(try)f(of)75 1151 y(t)o(yp)q(e)g Fw(lb)h Fv(can)f(b)q(e)h(deleted)h(if)e(there)g(is) h(another)f(en)o(try)g(of)g(t)o(yp)q(e)g Fw(lb)g Fv(with)h(a)f(lo)o(w)o (er)g(displacemen)o(t.\))169 b Fx(?)166 1241 y Fv(In)16 b(general,)f(if)189 1323 y Fp(T)6 b(y)r(pemap)12 b Fv(=)h Fx(f)p Fv(\()p Fp(ty)r(pe)562 1330 y Fg(0)581 1323 y Fp(;)8 b(disp)686 1330 y Fg(0)705 1323 y Fv(\))p Fp(;)g(:::;)g Fv(\()o Fp(ty)r(pe)905 1330 y Ff(n)p Fh(\000)p Fg(1)971 1323 y Fp(;)g(disp)1076 1330 y Ff(n)p Fh(\000)p Fg(1)1144 1323 y Fv(\))p Fx(g)p Fp(;)75 1405 y Fv(then)16 b(the)f Fq(lo)o(w)o(er)h(b)q(ound)g Fv(of)f Fp(T)6 b(y)r(pemap)15 b Fv(is)g(de\014ned)i(to)e(b)q(e)189 1517 y Fp(l)q(b)p Fv(\()p Fp(T)6 b(y)r(pemap)p Fv(\))k(=)506 1445 y Fe(\()560 1488 y Fv(min)636 1495 y Ff(j)662 1488 y Fp(disp)746 1495 y Ff(j)1257 1488 y Fv(if)16 b(no)f(en)o(try)f(has)i(basic)f(t)o (yp)q(e)h Fw(lb)560 1545 y Fv(min)636 1552 y Ff(j)654 1545 y Fx(f)p Fp(disp)761 1552 y Ff(j)794 1545 y Fv(suc)o(h)g(that)e Fp(ty)r(pe)1079 1552 y Ff(j)1110 1545 y Fv(=)f Fw(lb)q Fx(g)41 b Fv(otherwise)166 1629 y(Similarly)l(,)17 b(the)e Fq(upp)q(er)j(b)q(ound)e Fv(of)e Fp(T)6 b(y)r(pemap)15 b Fv(is)h(de\014ned)h(to)d(b)q(e)622 b Fx(>)189 1741 y Fp(ub)p Fv(\()p Fp(T)6 b(y)r(pemap)p Fv(\))11 b(=)518 1669 y Fe(\()572 1713 y Fv(max)657 1720 y Ff(j)682 1713 y Fp(disp)766 1720 y Ff(j)794 1713 y Fv(+)g Fp(siz)r(eof)5 b Fv(\()p Fp(ty)r(pe)1072 1720 y Ff(j)1090 1713 y Fv(\))10 b(+)h Fp(\017)108 b Fv(if)16 b(no)f(en)o(try)f(has)i(basic)f(t)o(yp)q (e)h Fw(ub)572 1769 y Fv(max)657 1776 y Ff(j)675 1769 y Fx(f)p Fp(disp)782 1776 y Ff(j)814 1769 y Fv(suc)o(h)g(that)e Fp(ty)r(pe)1099 1776 y Ff(j)1131 1769 y Fv(=)f Fw(ub)q Fx(g)41 b Fv(otherwise)166 1853 y(Then)189 1935 y Fp(extent)p Fv(\()p Fp(T)6 b(y)r(pemap)p Fv(\))13 b(=)g Fp(ub)p Fv(\()p Fp(T)6 b(y)r(pemap)p Fv(\))i Fx(\000)j Fp(l)q(b)p Fv(\()p Fp(T)6 b(y)r(pemap)p Fv(\))1937 1972 y Fx(?)75 2017 y Fv(If)19 b Fp(ty)r(pe)208 2024 y Ff(i)242 2017 y Fv(requires)h (alignmen)o(t)g(to)e(a)h(b)o(yte)g(address)g(that)g(is)h(a)e(m)o (ultiple)k(of)c Fp(k)1460 2024 y Ff(i)1474 2017 y Fv(,)i(then)f Fp(\017)h Fv(is)g(the)f(least)75 2074 y(nonnegativ)o(e)d(incremen)o(t)g (needed)g(to)f(round)g Fp(extent)p Fv(\()p Fp(T)6 b(y)r(pemap)p Fv(\))16 b(to)e(the)h(next)h(m)o(ultiple)h(of)d(max)1794 2081 y Ff(i)1816 2074 y Fp(k)1840 2081 y Ff(i)1854 2074 y Fv(.)166 2130 y(The)g(formal)f(de\014nitions)j(giv)o(en)e(for)f(the)h (v)m(arious)g(datat)o(yp)q(e)f(constructors)g(apply)i(no)o(w,)e(with)h (the)75 2186 y(amended)i(de\014nition)h(of)e Fq(exten)o(t)p Fv(.)166 2243 y(The)f(t)o(w)o(o)e(functions)i(b)q(elo)o(w)h(can)e(b)q (e)i(used)f(for)f(\014nding)i(the)e(lo)o(w)o(er)g(b)q(ound)i(and)f(the) f(upp)q(er)i(b)q(ound)75 2299 y(of)g(a)g(datat)o(yp)q(e.)75 2450 y Fw(MPI)p 160 2450 V 16 w(TYPE)p 293 2450 V 17 w(LB\()f(datat)o(yp)q(e,)j(displacement\))117 2528 y Fk(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 2600 y(OUT)108 b Fw(displacement)344 b Fk(displacemen)o(t)12 b(of)g(lo)o(w)o(er)g(b)q(ound)g(from)f(origin,) g(in)h(b)o(ytes)h(\(in-)905 2657 y(teger\))1937 2735 y Fx(>)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 72 80 72 79 bop 75 -100 a Fv(72)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 91 y Ft(int)23 b(MPI)p 245 91 15 2 v 17 w(Type)p 358 91 V 17 w(lb\(MPI)p 519 91 V 16 w(Datatype)g(datatype,)g(MPI)p 1061 91 V 16 w(Aint*)h(displacement\))-160 131 y Fx(?)75 178 y Ft(MPI)p 150 178 V 17 w(TYPE)p 263 178 V 16 w(LB\()g(DATATYPE,)e (DISPLACEMENT,)h(IERROR\))170 234 y(INTEGER)g(DATATYPE,)g (DISPLACEMENT,)f(IERROR)75 415 y Fw(MPI)p 160 415 14 2 v 16 w(TYPE)p 293 415 V 17 w(UB\()15 b(datat)o(yp)q(e,)h (displacement\))117 492 y Fk(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(\(handle\))117 567 y(OUT)108 b Fw(displacement)344 b Fk(displacemen)o(t)11 b(of)f(upp)q(er)i(b)q (ound)f(from)f(origin,)g(in)h(b)o(ytes)g(\(in-)905 624 y(teger\))-160 702 y Fx(>)75 748 y Ft(int)23 b(MPI)p 245 748 15 2 v 17 w(Type)p 358 748 V 17 w(ub\(MPI)p 519 748 V 16 w(Datatype)g(datatype,)g(MPI)p 1061 748 V 16 w(Aint*)h(displacement\))-160 788 y Fx(?)75 835 y Ft(MPI)p 150 835 V 17 w(TYPE)p 263 835 V 16 w(UB\()g(DATATYPE,)e(DISPLACEMENT,)h (IERROR\))170 891 y(INTEGER)g(DATATYPE,)g(DISPLACEMENT,)f(IERROR)-160 928 y Fx(>)-160 980 y(?)75 1043 y Fj(3.12.4)49 b(Commit)16 b(and)h(free)75 1129 y Fv(A)j(datat)o(yp)q(e)f(ob)s(ject)g(has)g(to)g (b)q(e)h Fq(committed)g Fv(b)q(efore)g(it)g(can)f(b)q(e)i(used)f(in)g (a)f(comm)o(unication.)34 b(A)75 1185 y(committed)15 b(datat)o(yp)q(e)g(can)h(still)h(b)q(e)f(used)g(as)e(a)i(argumen)o(t)e (in)i(datat)o(yp)q(e)f(constructors.)20 b(There)15 b(is)h(no)75 1242 y(need)g(to)f(commit)g(basic)h(datat)o(yp)q(es.)j(They)c(are)g (\\pre-committed.")75 1393 y Fw(MPI)p 160 1393 14 2 v 16 w(TYPE)p 293 1393 V 17 w(COMMIT\(datat)o(yp)q(e\))117 1470 y Fk(INOUT)62 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(that)g(is)g(committed)e(\(handle\))75 1594 y Ft(int)23 b(MPI)p 245 1594 15 2 v 17 w(Type)p 358 1594 V 17 w(commit\(MPI)p 615 1594 V 16 w(Datatype)f(*datatype\))75 1681 y(MPI)p 150 1681 V 17 w(TYPE)p 263 1681 V 16 w(COMMIT\(DATATYPE,)g(IERROR\))170 1737 y(INTEGER)h(DATATYPE,)g(IERROR)166 1824 y Fv(The)14 b(commit)h(op)q(eration)f(commits)g(the)h(datat)o(yp)q(e,)e(that)h(is,) g(the)h(formal)e(description)j(of)e(a)g(com-)75 1880 y(m)o(unication)h(bu\013er,)f(not)g(the)g(con)o(ten)o(t)f(of)h(that)g (bu\013er.)19 b(Th)o(us,)14 b(after)f(a)h(datat)o(yp)q(e)g(has)g(b)q (een)h(commit-)75 1937 y(ted,)e(it)g(can)f(b)q(e)i(rep)q(eatedly)f (reused)h(to)e(comm)o(unicate)h(the)f(c)o(hanging)h(con)o(ten)o(t)f(of) g(a)h(bu\013er)f(or,)h(indeed,)75 1993 y(the)i(con)o(ten)o(t)g(of)g (di\013eren)o(t)g(bu\013ers,)g(with)g(di\013eren)o(t)h(starting)e (addresses.)189 2099 y Fl(A)n(dvic)n(e)19 b(to)i(implementors.)67 b Fv(The)20 b(system)f(ma)o(y)g(\\compile")i(at)e(commit)g(time)h(an)g (in)o(ternal)189 2156 y(represen)o(tation)f(for)f(the)h(datat)o(yp)q(e) g(that)f(facilitates)i(comm)o(unication,)h(e.g.)31 b(c)o(hange)19 b(from)f(a)189 2212 y(compacted)12 b(represen)o(tation)h(to)f(a)g (\015at)h(represen)o(tation)f(of)h(the)f(datat)o(yp)q(e,)h(and)g (select)g(the)g(most)189 2269 y(con)o(v)o(enien)o(t)i(transfer)g(mec)o (hanism.)20 b(\()p Fl(End)c(of)g(advic)n(e)g(to)h(implementors.)p Fv(\))75 2469 y Fw(MPI)p 160 2469 14 2 v 16 w(TYPE)p 293 2469 V 17 w(FREE\(datat)o(yp)q(e\))117 2547 y Fk(INOUT)62 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(that)g(is)g(freed)h (\(handle\))75 2671 y Ft(int)23 b(MPI)p 245 2671 15 2 v 17 w(Type)p 358 2671 V 17 w(free\(MPI)p 567 2671 V 16 w(Datatype)g(*datatype\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 73 81 73 80 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fv(73)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(TYPE)p 263 45 V 16 w(FREE\(DATATYPE,)22 b(IERROR\))170 102 y(INTEGER)h (DATATYPE,)g(IERROR)166 188 y Fv(Marks)14 b(the)h(datat)o(yp)q(e)g(ob)s (ject)f(asso)q(ciated)h(with)h Fw(datat)o(yp)q(e)g Fv(for)f(deallo)q (cation)h(and)g(sets)e Fw(datat)o(yp)q(e)75 244 y Fv(to)e Fi(MPI)p 205 244 13 2 v 14 w(D)o(A)m(T)m(A)m(TYPE)p 431 244 V 12 w(NULL)p Fv(.)h(An)o(y)f(comm)o(unication)h(that)e(is)i (curren)o(tly)g(using)g(this)g(datat)o(yp)q(e)e(will)j(com-)75 301 y(plete)19 b(normally)l(.)31 b(Deriv)o(ed)19 b(datat)o(yp)q(es)f (that)g(w)o(ere)g(de\014ned)i(from)e(the)g(freed)h(datat)o(yp)q(e)f (are)g(not)g(af-)75 357 y(fected.)75 460 y Fq(Example)g(3.27)23 b Fv(The)12 b(follo)o(wing)h(co)q(de)h(fragmen)o(t)d(giv)o(es)i (examples)g(of)f(using)h Fw(MPI)p 1530 460 14 2 v 16 w(TYPE)p 1663 460 V 17 w(COMMIT)p Fv(.)75 551 y Ft(INTEGER)23 b(type1,)g(type2)75 607 y(CALL)g(MPI_TYPE_CONTIGUOUS\(5,)e(MPI_REAL,)i (type1,)g(ierr\))409 664 y(!)h(new)f(type)h(object)f(created)75 720 y(CALL)g(MPI_TYPE_COMMIT\(type1,)e(ierr\))409 777 y(!)j(now)f(type1)g(can)h(be)f(used)h(for)f(communication)75 833 y(type2)g(=)h(type1)409 890 y(!)g(type2)f(can)g(be)h(used)f(for)h (communication)409 946 y(!)g(\(it)f(is)h(a)g(handle)f(to)g(same)h (object)f(as)g(type1\))75 1002 y(CALL)g(MPI_TYPE_VECTOR\(3,)f(5,)h(4,)h (MPI_REAL,)f(type1,)g(ierr\))409 1059 y(!)h(new)f(uncommitted)g(type)g (object)g(created)75 1115 y(CALL)g(MPI_TYPE_COMMIT\(type1,)e(ierr\))409 1172 y(!)j(now)f(type1)g(can)h(be)f(used)h(anew)f(for)h(communication) 166 1274 y Fv(F)l(reeing)19 b(a)f(datat)o(yp)q(e)f(do)q(es)i(not)f (a\013ect)f(an)o(y)h(other)g(datat)o(yp)q(e)f(that)h(w)o(as)f(built)j (from)d(the)i(freed)75 1331 y(datat)o(yp)q(e.)32 b(The)20 b(system)e(b)q(eha)o(v)o(es)i(as)f(if)h(input)g(datat)o(yp)q(e)f (argumen)o(ts)g(to)f(deriv)o(ed)j(datat)o(yp)q(e)e(con-)75 1387 y(structors)14 b(are)h(passed)g(b)o(y)h(v)m(alue.)189 1490 y Fl(A)n(dvic)n(e)g(to)i(implementors.)49 b Fv(The)17 b(implemen)o(tation)h(ma)o(y)e(k)o(eep)h(a)g(reference)g(coun)o(t)g(of) f(activ)o(e)189 1546 y(comm)o(unications)h(that)e(use)i(the)f(datat)o (yp)q(e,)g(in)h(order)f(to)g(decide)i(when)f(to)e(free)i(it.)23 b(Also,)17 b(one)189 1602 y(ma)o(y)d(implemen)o(t)j(constructors)e(of)g (deriv)o(ed)h(datat)o(yp)q(es)f(so)g(that)g(they)g(k)o(eep)h(p)q(oin)o (ters)g(to)e(their)189 1659 y(datat)o(yp)q(e)g(argumen)o(ts,)g(rather)h (then)g(cop)o(ying)h(them.)k(In)15 b(this)h(case,)f(one)g(needs)h(to)e (k)o(eep)i(trac)o(k)189 1715 y(of)f(activ)o(e)g(datat)o(yp)q(e)g (de\014nition)i(references)f(in)g(order)g(to)e(kno)o(w)h(when)h(a)f (datat)o(yp)q(e)g(ob)s(ject)g(can)189 1772 y(b)q(e)h(freed.)k(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fv(\))75 1893 y Fj(3.12.5)49 b(Use)16 b(of)h(general)f(datat)o(yp)q(es)g(in)h (communication)75 1979 y Fv(Handles)g(to)f(deriv)o(ed)h(datat)o(yp)q (es)f(can)h(b)q(e)f(passed)h(to)f(a)g(comm)o(unication)h(call)g(wherev) o(er)f(a)g(datat)o(yp)q(e)75 2035 y(argumen)o(t)11 b(is)h(required.)19 b(A)12 b(call)g(of)f(the)h(form)e Fw(MPI)p 947 2035 V 16 w(SEND\(buf,)j(count,)g(datat)o(yp)q(e)g(,)f(...\))p Fv(,)e(where)i Fw(count)i Fp(>)75 2092 y Fw(1)p Fv(,)e(is)h(in)o (terpreted)g(as)g(if)g(the)f(call)i(w)o(as)d(passed)i(a)f(new)h(datat)o (yp)q(e)f(whic)o(h)h(is)g(the)g(concatenation)g(of)f Fw(count)75 2148 y Fv(copies)j(of)e Fw(datat)o(yp)q(e)p Fv(.)21 b(Th)o(us,)13 b Fw(MPI)p 666 2148 V 16 w(SEND\(buf,)h(count,)h (datat)o(yp)q(e,)h(dest,)f(tag,)e(comm\))e Fv(is)k(equiv)m(alen)o(t)g (to,)75 2239 y Ft(MPI_TYPE_CONTIGUOUS\(count,)20 b(datatype,)j (newtype\))75 2296 y(MPI_TYPE_COMMIT\(newtype\))75 2352 y(MPI_SEND\(buf,)f(1,)i(newtype,)e(dest,)i(tag,)f(comm\).)75 2443 y Fv(Similar)i(statemen)o(ts)e(apply)h(to)g(all)g(other)g(comm)o (unication)g(functions)h(that)e(ha)o(v)o(e)g(a)h Fw(count)h Fv(and)75 2500 y Fw(datat)o(yp)q(e)17 b Fv(argumen)o(t.)166 2556 y(Supp)q(ose)22 b(that)e(a)g(send)h(op)q(eration)g Fw(MPI)p 900 2556 V 16 w(SEND\(buf,)h(count,)h(datat)o(yp)q(e,)g(dest,) g(tag,)f(comm\))c Fv(is)75 2613 y(executed,)e(where)f Fw(datat)o(yp)q(e)i Fv(has)e(t)o(yp)q(e)g(map,)189 2704 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 2711 y Fg(0)333 2704 y Fp(;)8 b(disp)438 2711 y Fg(0)457 2704 y Fv(\))p Fp(;)g(:::;)f Fv(\()p Fp(ty)r(pe)657 2711 y Ff(n)p Fh(\000)p Fg(1)723 2704 y Fp(;)h(disp)828 2711 y Ff(n)p Fh(\000)p Fg(1)895 2704 y Fv(\))p Fx(g)p Fp(;)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 74 82 74 81 bop 75 -100 a Fv(74)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(and)i(exten)o(t)f Fp(extent)p Fv(.)31 b(\(Empt)o(y)18 b(en)o(tries)h(of)f(\\pseudo-t)o (yp)q(e")36 b Fi(MPI)p 1252 45 13 2 v 14 w(UB)18 b Fv(and)35 b Fi(MPI)p 1524 45 V 15 w(LB)18 b Fv(are)h(not)f(listed)75 102 y(in)g(the)e(t)o(yp)q(e)h(map,)g(but)g(they)g(a\013ect)f(the)h(v)m (alue)h(of)e Fp(extent)p Fv(.\))26 b(The)17 b(send)g(op)q(eration)g (sends)g Fp(n)12 b Fx(\001)f Fw(count)75 158 y Fv(en)o(tries,)17 b(where)g(en)o(try)g Fp(i)10 b Fx(\001)h Fp(n)g Fv(+)h Fp(j)19 b Fv(is)e(at)f(lo)q(cation)i Fp(addr)1034 165 y Ff(i;j)1088 158 y Fv(=)e Fw(buf)f Fv(+)c Fp(extent)i Fx(\001)e Fp(i)f Fv(+)i Fp(disp)1581 165 y Ff(j)1615 158 y Fv(and)17 b(has)g(t)o(yp)q(e)75 214 y Fp(ty)r(pe)159 221 y Ff(j)178 214 y Fv(,)d(for)f Fp(i)g Fv(=)g(0)p Fp(;)8 b(:::;)g Fw(co)o(unt)f Fx(\000)i Fv(1)k(and)i Fp(j)g Fv(=)e(0)p Fp(;)8 b(:::;)g(n)d Fx(\000)k Fv(1.)19 b(These)c(en)o(tries) f(need)h(not)f(b)q(e)h(con)o(tiguous,)f(nor)75 271 y(distinct;)i(their) g(order)f(can)g(b)q(e)h(arbitrary)l(.)166 327 y(The)g(v)m(ariable)h (stored)e(at)g(address)g Fp(addr)879 334 y Ff(i;j)933 327 y Fv(in)i(the)e(calling)j(program)c(should)j(b)q(e)f(of)f(a)g(t)o (yp)q(e)h(that)75 384 y(matc)o(hes)11 b Fp(ty)r(pe)331 391 y Ff(j)349 384 y Fv(,)h(where)f(t)o(yp)q(e)g(matc)o(hing)h(is)f (de\014ned)i(as)d(in)i(section)g(3.3.1.)17 b(The)11 b(message)g(sen)o (t)g(con)o(tains)75 440 y Fp(n)f Fx(\001)g Fw(count)17 b Fv(en)o(tries,)e(where)g(en)o(try)g Fp(i)10 b Fx(\001)g Fp(n)g Fv(+)g Fp(j)18 b Fv(has)d(t)o(yp)q(e)g Fp(ty)r(pe)1100 447 y Ff(j)1119 440 y Fv(.)166 497 y(Similarly)l(,)g(supp)q(ose)f(that) e(a)h(receiv)o(e)h(op)q(eration)f Fw(MPI)p 1100 497 14 2 v 16 w(RECV\(buf,)h(count,)g(datat)o(yp)q(e,)h(source,)e(tag,)75 553 y(comm,)g(status\))k Fv(is)e(executed,)h(where)g Fw(datat)o(yp)q(e)g Fv(has)g(t)o(yp)q(e)f(map,)189 647 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 654 y Fg(0)333 647 y Fp(;)8 b(disp)438 654 y Fg(0)457 647 y Fv(\))p Fp(;)g(:::;)f Fv(\()p Fp(ty)r(pe)657 654 y Ff(n)p Fh(\000)p Fg(1)723 647 y Fp(;)h(disp)828 654 y Ff(n)p Fh(\000)p Fg(1)895 647 y Fv(\))p Fx(g)p Fp(;)75 741 y Fv(with)17 b(exten)o(t)g Fp(extent)p Fv(.)25 b(\(Again,)17 b(empt)o(y)g(en)o(tries)g(of)f (\\pseudo-t)o(yp)q(e")32 b Fi(MPI)p 1385 741 13 2 v 15 w(UB)16 b Fv(and)32 b Fi(MPI)p 1653 741 V 14 w(LB)17 b Fv(are)f(not)75 797 y(listed)g(in)f(the)f(t)o(yp)q(e)h(map,)f(but)g (they)h(a\013ect)e(the)i(v)m(alue)h(of)e Fp(extent)p Fv(.\))20 b(This)15 b(receiv)o(e)h(op)q(eration)e(receiv)o(es)75 854 y Fp(n)e Fx(\001)f Fw(count)19 b Fv(en)o(tries,)f(where)f(en)o(try) g Fp(i)12 b Fx(\001)f Fp(n)h Fv(+)g Fp(j)19 b Fv(is)f(at)f(lo)q(cation) h Fw(buf)e Fv(+)c Fp(extent)h Fx(\001)e Fp(i)g Fv(+)h Fp(disp)1579 861 y Ff(j)1614 854 y Fv(and)18 b(has)f(t)o(yp)q(e)75 910 y Fp(ty)r(pe)159 917 y Ff(j)178 910 y Fv(.)i(If)c(the)f(incoming)h (message)f(consists)g(of)g Fp(k)h Fv(elemen)o(ts,)g(then)f(w)o(e)g(m)o (ust)g(ha)o(v)o(e)g Fp(k)f Fx(\024)g Fp(n)c Fx(\001)e Fw(count)p Fv(;)16 b(the)75 967 y Fp(i)10 b Fx(\001)f Fp(n)i Fv(+)f Fp(j)s Fv(-th)15 b(elemen)o(t)h(of)f(the)g(message)g (should)h(ha)o(v)o(e)f(a)f(t)o(yp)q(e)i(that)e(matc)o(hes)h Fp(ty)r(pe)1521 974 y Ff(j)1540 967 y Fv(.)166 1023 y(T)o(yp)q(e)10 b(matc)o(hing)g(is)h(de\014ned)h(according)e(to)g(the)g(t)o(yp)q(e)g (signature)h(of)e(the)h(corresp)q(onding)i(datat)o(yp)q(es,)75 1080 y(that)k(is,)h(the)f(sequence)i(of)e(basic)i(t)o(yp)q(e)e(comp)q (onen)o(ts.)24 b(T)o(yp)q(e)17 b(matc)o(hing)g(do)q(es)f(not)h(dep)q (end)h(on)e(some)75 1136 y(asp)q(ects)k(of)f(the)h(datat)o(yp)q(e)f (de\014nition,)j(suc)o(h)e(as)g(the)f(displacemen)o(ts)j(\(la)o(y)o (out)c(in)j(memory\))e(or)g(the)75 1192 y(in)o(termediate)d(t)o(yp)q (es)f(used.)75 1299 y Fq(Example)j(3.28)23 b Fv(This)c(example)h(sho)o (ws)e(that)h(t)o(yp)q(e)g(matc)o(hing)g(is)h(de\014ned)g(in)g(terms)e (of)h(the)g(basic)75 1355 y(t)o(yp)q(es)c(that)g(a)g(deriv)o(ed)h(t)o (yp)q(e)f(consists)g(of.)75 1449 y Ft(...)75 1505 y(CALL)23 b(MPI_TYPE_CONTIGUOUS\()e(2,)j(MPI_REAL,)f(type2,)g(...\))75 1562 y(CALL)g(MPI_TYPE_CONTIGUOUS\()e(4,)j(MPI_REAL,)f(type4,)g(...\)) 75 1618 y(CALL)g(MPI_TYPE_CONTIGUOUS\()e(2,)j(type2,)f(type22,)g(...\)) 75 1675 y(...)75 1731 y(CALL)g(MPI_SEND\()g(a,)h(4,)f(MPI_REAL,)g (...\))75 1788 y(CALL)g(MPI_SEND\()g(a,)h(2,)f(type2,)g(...\))75 1844 y(CALL)g(MPI_SEND\()g(a,)h(1,)f(type22,)g(...\))75 1901 y(CALL)g(MPI_SEND\()g(a,)h(1,)f(type4,)g(...\))75 1957 y(...)75 2014 y(CALL)g(MPI_RECV\()g(a,)h(4,)f(MPI_REAL,)g(...\))75 2070 y(CALL)g(MPI_RECV\()g(a,)h(2,)f(type2,)g(...\))75 2126 y(CALL)g(MPI_RECV\()g(a,)h(1,)f(type22,)g(...\))75 2183 y(CALL)g(MPI_RECV\()g(a,)h(1,)f(type4,)g(...\))75 2277 y Fv(Eac)o(h)15 b(of)g(the)g(sends)h(matc)o(hes)f(an)o(y)f(of)h (the)g(receiv)o(es.)-160 2335 y Fx(>)166 2383 y Fv(A)i(datat)o(yp)q(e)g (ma)o(y)g(sp)q(ecify)h(o)o(v)o(erlapping)g(en)o(tries.)27 b(The)17 b(use)h(of)f(suc)o(h)g(a)g(datat)o(yp)q(e)g(in)h(a)f(receiv)o (e)75 2439 y(op)q(eration)11 b(is)h(erroneous.)18 b(\(This)11 b(is)h(erroneous)e(ev)o(en)i(if)f(the)g(actual)g(message)g(receiv)o(ed) h(is)f(short)g(enough)75 2496 y(not)k(to)f(write)i(an)o(y)f(en)o(try)f (more)h(than)g(once.\))-1031 b Fx(?)166 2552 y Fv(A)19 b(datat)o(yp)q(e)g(ma)o(y)f(sp)q(ecify)i(o)o(v)o(erlapping)g(en)o (tries.)32 b(If)19 b(suc)o(h)h(a)f(datat)o(yp)q(e)f(is)i(used)f(in)h(a) f(receiv)o(e)75 2609 y(op)q(eration,)13 b(that)f(is,)i(if)f(some)f (part)h(of)f(the)h(receiv)o(e)g(bu\013er)g(is)h(written)e(more)h(than)f (once)h(b)o(y)g(the)g(receiv)o(e)75 2665 y(op)q(eration,)i(then)h(the)f (call)h(is)g(erroneous.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 75 83 75 82 bop 75 -100 a Fm(3.12.)34 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1119 b Fv(75)166 45 y(Supp)q(ose)22 b(that)f Fw(MPI)p 540 45 14 2 v 16 w(RECV\(buf,)j(count,)g(datat)o(yp)q(e,)g(dest,)g (tag,)f(comm,)d(status\))j Fv(is)f(executed,)75 102 y(where)15 b Fw(datat)o(yp)q(e)i Fv(has)e(t)o(yp)q(e)h(map,)189 195 y Fx(f)p Fv(\()p Fp(ty)r(pe)314 202 y Fg(0)333 195 y Fp(;)8 b(disp)438 202 y Fg(0)457 195 y Fv(\))p Fp(;)g(:::;)f Fv(\()p Fp(ty)r(pe)657 202 y Ff(n)p Fh(\000)p Fg(1)723 195 y Fp(;)h(disp)828 202 y Ff(n)p Fh(\000)p Fg(1)895 195 y Fv(\))p Fx(g)p Fp(:)75 289 y Fv(The)14 b(receiv)o(ed)h(message)e (need)i(not)f(\014ll)h(all)g(the)e(receiv)o(e)i(bu\013er,)f(nor)f(do)q (es)i(it)f(need)g(to)f(\014ll)j(a)d(n)o(um)o(b)q(er)h(of)75 346 y(lo)q(cations)g(whic)o(h)h(is)f(a)f(m)o(ultiple)j(of)d Fp(n)p Fv(.)20 b(An)o(y)13 b(n)o(um)o(b)q(er,)h Fp(k)q Fv(,)g(of)f(basic)i(elemen)o(ts)f(can)g(b)q(e)g(receiv)o(ed,)h(where)75 402 y(0)d Fx(\024)h Fp(k)h Fx(\024)f Fw(count)8 b Fx(\001)f Fp(n)p Fv(.)19 b(The)13 b(n)o(um)o(b)q(er)h(of)f(basic)h(elemen)o(ts)g (receiv)o(ed)h(can)e(b)q(e)h(retriev)o(ed)g(from)f Fw(status)i Fv(using)75 459 y(the)g(query)h(function)g Fw(MPI)p 543 459 V 16 w(GET)p 647 459 V 16 w(ELEMENTS)p Fv(.)75 609 y Fw(MPI)p 160 609 V 16 w(GET)p 264 609 V 17 w(ELEMENTS\()f(status,)h (datat)o(yp)q(e,)h(count\))117 687 y Fk(IN)155 b Fw(status)476 b Fk(return)15 b(status)g(of)e(receiv)o(e)i(op)q(eration)f(\(Status\)) 117 762 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(used)h(b)o(y)f(receiv)o(e)h(op)q(eration)f(\(handle\))117 837 y(OUT)108 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(receiv)o(ed)h(basic)f(elemen)o(ts)g(\(in)o(teger\))1937 915 y Fx(>)75 961 y Ft(int)23 b(MPI)p 245 961 15 2 v 17 w(Get)p 334 961 V 17 w(elements\(MPI)p 639 961 V 16 w(Status)g(*status,)f(MPI)p 1108 961 V 17 w(Datatype)h(datatype,)g(int) g(*count\))1937 1002 y Fx(?)75 1048 y Ft(MPI)p 150 1048 V 17 w(GET)p 239 1048 V 17 w(ELEMENTS\(STATUS,)e(DATATYPE,)i(COUNT,)g (IERROR\))170 1104 y(INTEGER)g(STATUS\(MPI)p 604 1104 V 16 w(STATUS)p 764 1104 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h(IERROR) 166 1191 y Fv(The)14 b(previously)i(de\014ned)f(function,)g Fw(MPI)p 905 1191 14 2 v 16 w(GET)p 1009 1191 V 17 w(COUNT)f Fv(\(Sec.)20 b(3.2.5\),)12 b(has)i(a)g(di\013eren)o(t)g(b)q(eha)o(v-)75 1247 y(ior.)19 b(It)12 b(returns)g(the)g(n)o(um)o(b)q(er)g(of)g (\\top-lev)o(el)h(en)o(tries")f(receiv)o(ed,)i(i.e.)19 b(the)12 b(n)o(um)o(b)q(er)g(of)g(\\copies")g(of)g(t)o(yp)q(e)62 b Fx(>)75 1304 y Fw(datat)o(yp)q(e)p Fv(.)20 b(In)13 b(the)e(previous)i(example,)f Fw(MPI)p 849 1304 V 16 w(GET)p 953 1304 V 17 w(COUNT)g Fv(ma)o(y)f(return)h(an)o(y)f(in)o (teger)h(v)m(alue)h Fp(k)q Fv(,)f(where)62 b Fx(?)75 1360 y Fv(0)13 b Fx(\024)g Fp(k)h Fx(\024)f Fw(count)r Fv(.)20 b(If)c Fw(MPI)p 516 1360 V 15 w(GET)p 619 1360 V 17 w(COUNT)g Fv(returns)g Fp(k)q Fv(,)f(then)h(the)f(n)o(um)o(b)q(er) h(of)f(basic)h(elemen)o(ts)g(receiv)o(ed)75 1417 y(\(and)331 b(the)h(v)m(alue)h(returned)f(b)o(y)75 1473 y Fw(MPI)p 160 1473 V 16 w(GET)p 264 1473 V 17 w(ELEMENTS)p Fv(\))19 b(is)h Fp(n)13 b Fx(\001)f Fp(k)q Fv(.)32 b(If)20 b(the)f(n)o(um)o(b)q (er)h(of)f(basic)h(elemen)o(ts)g(receiv)o(ed)g(is)g(not)f(a)g(m)o (ulti-)75 1530 y(ple)g(of)e Fp(n)p Fv(,)i(that)e(is,)h(if)h(the)f (receiv)o(e)h(op)q(eration)f(has)f(not)h(receiv)o(ed)h(an)f(in)o (tegral)g(n)o(um)o(b)q(er)g(of)f Fw(datat)o(yp)q(e)75 1586 y Fv(\\copies,")e(then)h Fw(MPI)p 456 1586 V 16 w(GET)p 560 1586 V 16 w(COUNT)g Fv(returns)f(the)h(v)m(alue)g Fi(MPI)p 1175 1586 13 2 v 14 w(UNDEFINED)p Fv(.)75 1692 y Fq(Example)i(3.29)23 b Fv(Usage)14 b(of)h Fw(MPI)p 676 1692 14 2 v 16 w(GET)p 780 1692 V 17 w(COUNT)h Fv(and)f Fw(MPI)p 1139 1692 V 16 w(GET)p 1243 1692 V 17 w(ELEMENT)p Fv(.)75 1786 y Ft(...)75 1843 y(CALL)23 b(MPI_TYPE_CONTIGUOUS\(2,)e (MPI_REAL,)i(Type2,)g(ierr\))75 1899 y(CALL)g(MPI_TYPE_COMMIT\(Type2,)e (ierr\))75 1955 y(...)75 2012 y(CALL)i(MPI_COMM_RANK\(comm,)f(rank,)h (ierr\))75 2068 y(IF\(rank.EQ.0\))f(THEN)218 2125 y(CALL)h (MPI_SEND\(a,)g(2,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\))218 2181 y(CALL)g(MPI_SEND\(a,)g(3,)g(MPI_REAL,)g(1,)h(0,)f(comm,)g(ierr\)) 75 2238 y(ELSE)218 2294 y(CALL)g(MPI_RECV\(a,)g(2,)g(Type2,)g(0,)h(0,)g (comm,)f(stat,)g(ierr\))218 2351 y(CALL)g(MPI_GET_COUNT\(stat,)f (Type2,)h(i,)g(ierr\))119 b(!)24 b(returns)f(i=1)218 2407 y(CALL)g(MPI_GET_ELEMENTS\(stat,)e(Type2,)i(i,)h(ierr\))47 b(!)24 b(returns)f(i=2)218 2464 y(CALL)g(MPI_RECV\(a,)g(2,)g(Type2,)g (0,)h(0,)g(comm,)f(stat,)g(ierr\))218 2520 y(CALL)g (MPI_GET_COUNT\(stat,)f(Type2,)h(i,)g(ierr\))119 b(!)24 b(returns)f(i=MPI_UNDEFINED)218 2576 y(CALL)g(MPI_GET_ELEMENTS\(stat,)e (Type2,)i(i,)h(ierr\))47 b(!)24 b(returns)f(i=3)75 2633 y(END)g(IF)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 76 84 76 83 bop 75 -100 a Fv(76)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)166 45 y Fv(The)e(function)h Fw(MPI)p 522 45 14 2 v 16 w(GET)p 626 45 V 17 w(ELEMENTS)f Fv(can)g(also)g(b)q(e)g(used)h(after)e(a)h(prob)q(e)g(to)g(\014nd)h (the)f(n)o(um)o(b)q(er)75 102 y(of)20 b(elemen)o(ts)h(in)g(the)f(prob)q (ed)h(message.)34 b(Note)20 b(that)f(the)h(t)o(w)o(o)f(functions)i Fw(MPI)p 1507 102 V 16 w(GET)p 1611 102 V 17 w(COUNT)g Fv(and)75 158 y Fw(MPI)p 160 158 V 16 w(GET)p 264 158 V 17 w(ELEMENTS)15 b Fv(return)g(the)g(same)g(v)m(alues)i(when)e(they)h (are)e(used)i(with)g(basic)g(datat)o(yp)q(es.)189 264 y Fl(R)n(ationale.)38 b Fv(The)14 b(extension)g(giv)o(en)g(to)f(the)h (de\014nition)h(of)e Fw(MPI)p 1303 264 V 16 w(GET)p 1407 264 V 16 w(COUNT)i Fv(seems)e(natural:)189 321 y(one)22 b(w)o(ould)h(exp)q(ect)h(this)f(function)g(to)f(return)h(the)g(v)m (alue)g(of)g(the)f Fw(count)i Fv(argumen)o(t,)f(when)189 377 y(the)c(receiv)o(e)h(bu\013er)g(is)f(\014lled.)34 b(Sometimes)20 b Fw(datat)o(yp)q(e)h Fv(represen)o(ts)e(a)g(basic)h (unit)g(of)f(data)g(one)189 434 y(w)o(an)o(ts)h(to)g(transfer,)i(for)e (example,)j(a)e(record)g(in)h(an)g(arra)o(y)e(of)g(records)h (\(structures\).)37 b(One)189 490 y(should)15 b(b)q(e)h(able)f(to)f (\014nd)i(out)e(ho)o(w)g(man)o(y)g(comp)q(onen)o(ts)h(w)o(ere)g(receiv) o(ed)g(without)g(b)q(othering)h(to)189 547 y(divide)j(b)o(y)e(the)g(n)o (um)o(b)q(er)g(of)g(elemen)o(ts)h(in)g(eac)o(h)f(comp)q(onen)o(t.)26 b(Ho)o(w)o(ev)o(er,)16 b(on)h(other)g(o)q(ccasions,)189 603 y Fw(datat)o(yp)q(e)g Fv(is)f(used)g(to)e(de\014ne)j(a)e(complex)h (la)o(y)o(out)f(of)g(data)g(in)h(the)f(receiv)o(er)h(memory)l(,)f(and)h (do)q(es)189 659 y(not)h(represen)o(t)g(a)g(basic)i(unit)f(of)f(data)g (for)g(transfers.)26 b(In)18 b(suc)o(h)g(cases,)f(one)h(needs)g(to)f (use)h(the)189 716 y(function)e Fw(MPI)p 452 716 V 15 w(GET)p 555 716 V 17 w(ELEMENTS)p Fv(.)f(\()p Fl(End)h(of)g(r)n (ationale.)p Fv(\))189 822 y Fl(A)n(dvic)n(e)j(to)h(implementors.)66 b Fv(The)20 b(de\014nition)h(implies)h(that)c(a)i(receiv)o(e)g(cannot)f (c)o(hange)h(the)189 879 y(v)m(alue)c(of)f(storage)f(outside)h(the)h (en)o(tries)f(de\014ned)i(to)d(comp)q(ose)h(the)h(comm)o(unication)f (bu\013er.)20 b(In)189 935 y(particular,)13 b(the)f(de\014nition)i (implies)g(that)d(padding)i(space)g(in)g(a)e(structure)h(should)h(not)f (b)q(e)h(mo)q(d-)189 992 y(i\014ed)h(when)g(suc)o(h)f(a)g(structure)g (is)h(copied)g(from)f(one)g(pro)q(cess)g(to)g(another.)19 b(This)14 b(w)o(ould)f(prev)o(en)o(t)189 1048 y(the)i(ob)o(vious)h (optimization)g(of)f(cop)o(ying)h(the)f(structure,)g(together)g(with)g (the)h(padding,)g(as)f(one)189 1104 y(con)o(tiguous)d(blo)q(c)o(k.)20 b(The)13 b(implemen)o(tation)g(is)g(free)g(to)f(do)g(this)h (optimization)h(when)f(it)g(do)q(es)f(not)189 1161 y(impact)17 b(the)h(outcome)f(of)g(the)h(computation.)26 b(The)18 b(user)f(can)h(\\force")f(this)h(optimization)g(b)o(y)189 1217 y(explicitly)e(including)h(padding)e(as)e(part)g(of)h(the)f (message.)19 b(\()p Fl(End)c(of)g(advic)n(e)g(to)g(implementors.)p Fv(\))75 1339 y Fj(3.12.6)49 b(Co)o(rrect)15 b(use)h(of)g(addresses)75 1425 y Fv(Successiv)o(ely)23 b(declared)f(v)m(ariables)h(in)e(C)g(or)g (F)l(ortran)e(are)i(not)g(necessarily)h(stored)f(at)f(con)o(tiguous)75 1481 y(lo)q(cations.)i(Th)o(us,)16 b(care)g(m)o(ust)f(b)q(e)h (exercised)i(that)d(displacemen)o(ts)i(do)f(not)f(cross)h(from)f(one)h (v)m(ariable)75 1538 y(to)i(another.)28 b(Also,)19 b(in)g(mac)o(hines)g (with)g(a)f(segmen)o(ted)g(address)g(space,)h(addresses)g(are)f(not)f (unique)75 1594 y(and)f(address)g(arithmetic)g(has)g(some)f(p)q (eculiar)j(prop)q(erties.)k(Th)o(us,)16 b(the)g(use)g(of)f Fq(addresses)p Fv(,)g(that)g(is,)75 1651 y(displacemen)o(ts)i(relativ)o (e)e(to)g(the)g(start)f(address)h Fi(MPI)p 1008 1651 13 2 v 15 w(BOTTOM)p Fv(,)f(has)h(to)g(b)q(e)h(restricted.)166 1707 y(V)l(ariables)g(b)q(elong)f(to)f(the)g(same)g Fq(sequen)o(tial)k (storage)c Fv(if)h(they)f(b)q(elong)i(to)e(the)g(same)g(arra)o(y)l(,)g (to)75 1764 y(the)k(same)g Fw(COMMON)g Fv(blo)q(c)o(k)h(in)g(F)l (ortran,)e(or)h(to)f(the)h(same)g(structure)f(in)i(C.)f(V)l(alid)h (addresses)f(are)75 1820 y(de\014ned)f(recursiv)o(ely)f(as)f(follo)o (ws:)131 1926 y(1.)22 b(The)g(function)g Fw(MPI)p 558 1926 14 2 v 16 w(ADDRESS)h Fv(returns)e(a)h(v)m(alid)h(address,)h(when) e(passed)g(as)f(argumen)o(t)g(a)189 1983 y(v)m(ariable)16 b(of)f(the)g(calling)i(program.)131 2077 y(2.)22 b(The)d Fw(buf)h Fv(argumen)o(t)f(of)g(a)g(comm)o(unication)h(function)g(ev)m (aluates)h(to)d(a)h(v)m(alid)i(address,)f(when)189 2133 y(passed)15 b(as)g(argumen)o(t)f(a)h(v)m(ariable)i(of)e(the)g(calling)i (program.)131 2227 y(3.)22 b(If)15 b Fw(v)g Fv(is)h(a)f(v)m(alid)i (address,)d(and)i Fw(i)f Fv(is)g(an)h(in)o(teger,)f(then)g Fw(v+i)g Fv(is)h(a)f(v)m(alid)i(address,)e(pro)o(vided)h Fw(v)f Fv(and)189 2283 y Fw(v+i)g Fv(are)g(in)h(the)f(same)g(sequen)o (tial)h(storage.)131 2377 y(4.)22 b(If)15 b Fw(v)g Fv(is)h(a)f(v)m (alid)i(address)e(then)g Fi(MPI)p 806 2377 13 2 v 15 w(BOTTOM)e(+)h(v)i Fv(is)f(a)g(v)m(alid)i(address.)166 2483 y(A)22 b(correct)g(program)g(uses)g(only)h(v)m(alid)i(addresses)d (to)g(iden)o(tify)i(the)e(lo)q(cations)h(of)f(en)o(tries)h(in)75 2540 y(comm)o(unication)16 b(bu\013ers.)k(F)l(urthermore,)14 b(if)h Fw(u)h Fv(and)f Fw(v)g Fv(are)g(t)o(w)o(o)f(v)m(alid)j (addresses,)e(then)g(the)g(\(in)o(teger\))75 2596 y(di\013erence)j Fw(u)g(-)g(v)f Fv(can)g(b)q(e)h(computed)g(only)f(if)h(b)q(oth)g Fw(u)f Fv(and)h Fw(v)f Fv(are)g(in)h(the)f(same)g(sequen)o(tial)i (storage.)75 2653 y(No)c(other)g(arithmetic)h(op)q(erations)f(can)g(b)q (e)h(meaningfully)h(executed)f(on)g(addresses.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 77 85 77 84 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fv(77)166 45 y(The)21 b(rules)h(ab)q(o)o(v)o(e)f(imp)q(ose)g(no) g(constrain)o(ts)g(on)g(the)g(use)g(of)g(deriv)o(ed)h(datat)o(yp)q(es,) f(as)g(long)g(as)75 102 y(they)d(are)f(used)h(to)f(de\014ne)h(a)g(comm) o(unication)g(bu\013er)f(that)g(is)h(wholly)h(con)o(tained)f(within)h (the)e(same)75 158 y(sequen)o(tial)23 b(storage.)38 b(Ho)o(w)o(ev)o (er,)22 b(the)g(construction)g(of)f(a)h(comm)o(unication)g(bu\013er)g (that)f(con)o(tains)75 214 y(v)m(ariables)f(that)f(are)g(not)g(within)h (the)f(same)g(sequen)o(tial)h(storage)e(m)o(ust)h(ob)q(ey)g(certain)h (restrictions.)75 271 y(Basically)l(,)g(a)e(comm)o(unication)h (bu\013er)f(with)g(v)m(ariables)i(that)d(are)h(not)g(within)h(the)f (same)g(sequen)o(tial)75 327 y(storage)g(can)h(b)q(e)h(used)g(only)g(b) o(y)f(sp)q(ecifying)i(in)f(the)g(comm)o(unication)g(call)g Fw(buf)g(=)g(MPI)p 1651 327 14 2 v 16 w(BOTTOM)p Fv(,)75 384 y Fw(count)i(=)g(1)p Fv(,)g(and)f(using)h(a)f Fw(datat)o(yp)q(e)i Fv(argumen)o(t)d(where)i(all)g(displacemen)o(ts)h(are)d(v)m(alid)j (\(absolute\))75 440 y(addresses.)189 547 y Fl(A)n(dvic)n(e)15 b(to)i(users.)43 b Fv(It)15 b(is)h(not)g(exp)q(ected)g(that)f Fw(MPI)g Fv(implemen)o(tations)i(will)g(b)q(e)f(able)h(to)e(detect)189 603 y(erroneous,)20 b(\\out)f(of)g(b)q(ound")h(displacemen)o(ts)h(|)f (unless)g(those)f(o)o(v)o(er\015o)o(w)g(the)g(user)h(address)189 659 y(space)15 b(|)g(since)g(the)g Fw(MPI)f Fv(call)i(ma)o(y)e(not)g (kno)o(w)h(the)f(exten)o(t)h(of)f(the)h(arra)o(ys)e(and)i(records)f(in) i(the)189 716 y(host)e(program.)19 b(\()p Fl(End)d(of)g(advic)n(e)g(to) h(users.)p Fv(\))189 822 y Fl(A)n(dvic)n(e)h(to)h(implementors.)57 b Fv(There)18 b(is)g(no)g(need)h(to)f(distinguish)i(\(absolute\))e (addresses)g(and)189 879 y(\(relativ)o(e\))12 b(displacemen)o(ts)i(on)e (a)g(mac)o(hine)h(with)g(con)o(tiguous)f(address)h(space:)18 b Fi(MPI)p 1638 879 13 2 v 15 w(BOTTOM)11 b Fv(is)189 935 y(zero,)j(and)g(b)q(oth)g(addresses)h(and)f(displacemen)o(ts)i(are) e(in)o(tegers.)19 b(On)c(mac)o(hines)g(where)g(the)f(dis-)189 992 y(tinction)d(is)g(required,)h(addresses)f(are)f(recognized)i(as)e (expressions)i(that)d(in)o(v)o(olv)o(e)i Fi(MPI)p 1668 992 V 15 w(BOTTOM)p Fv(.)189 1048 y(\()p Fl(End)k(of)i(advic)n(e)f(to)g (implementors.)p Fv(\))166 1154 y(Note)k(that)g(in)i(F)l(ortran,)e(F)l (ortran)g(INTEGERs)h(ma)o(y)f(b)q(e)h(to)q(o)f(small)h(to)f(con)o(tain) h(an)g(address)75 1211 y(\(e.g.,)d(32)g(bit)h(INTEGERs)g(on)f(a)h(mac)o (hine)g(with)g(64bit)g(p)q(oin)o(ters\).)30 b(Because)19 b(of)g(this,)g(in)h(F)l(ortran,)75 1267 y(implemen)o(tations)j(ma)o(y)e (restrict)h(the)g(use)g(of)f(absolute)i(addresses)f(to)f(only)h(part)f (of)h(the)g(pro)q(cess)75 1324 y(memory)l(,)14 b(and)h(restrict)g(the)g (use)g(of)f(relativ)o(e)i(displacemen)o(ts)g(to)e(subranges)h(of)g(the) f(pro)q(cess)i(memory)75 1380 y(where)f(they)h(are)f(constrained)g(b)o (y)h(the)f(size)h(of)f(F)l(ortran)f(INTEGERs.)75 1502 y Fj(3.12.7)49 b(Examples)75 1588 y Fv(The)15 b(follo)o(wing)h (examples)g(illustrate)h(the)e(use)g(of)g(deriv)o(ed)h(datat)o(yp)q (es.)75 1694 y Fq(Example)i(3.30)23 b Fv(Send)16 b(and)f(receiv)o(e)h (a)f(section)h(of)f(a)f(3D)h(arra)o(y)l(.)218 1800 y Ft(REAL)23 b(a\(100,100,100\),)f(e\(9,9,9\))218 1857 y(INTEGER)h(oneslice,)g(twoslice,)f(threeslice,)h(sizeofreal,)f (myrank,)h(ierr)218 1913 y(INTEGER)g(status\(MPI_STATUS_SIZE\))75 2026 y(C)143 b(extract)23 b(the)g(section)g(a\(1:17:2,)g(3:11,)g (2:10\))75 2082 y(C)143 b(and)23 b(store)h(it)f(in)h(e\(:,:,:\).)218 2195 y(CALL)f(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))218 2308 y(CALL)j(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 2421 y(C)119 b(create)23 b(datatype)g(for)g(a)h(1D)g(section)218 2478 y(CALL)f(MPI_TYPE_VECTOR\()f(9,)i(1,)f(2,)h(MPI_REAL,)f(oneslice,) f(ierr\))75 2591 y(C)119 b(create)23 b(datatype)g(for)g(a)h(2D)g (section)218 2647 y(CALL)f(MPI_TYPE_HVECTOR\(9,)f(1,)h(100*sizeofreal,) f(oneslice,)h(twoslice,)f(ierr\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 78 86 78 85 bop 75 -100 a Fv(78)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Ft(C)119 b(create)23 b(datatype)g(for)g(the)h(entire)f(section)218 102 y(CALL)g(MPI_TYPE_HVECTOR\()f(9,)i(1,)f(100*100*sizeofreal,)e (twoslice,)i(1,)767 158 y(threeslice,)f(ierr\))218 271 y(CALL)h(MPI_TYPE_COMMIT\()f(threeslice,)h(ierr\))218 327 y(CALL)g(MPI_SENDRECV\(a\(1,3,2\),)e(1,)j(threeslice,)e(myrank,)h (0,)h(e,)f(9*9*9,)648 384 y(MPI_REAL,)f(myrank,)h(0,)h(MPI_COMM_WORLD,) e(status,)h(ierr\))75 484 y Fq(Example)18 b(3.31)23 b Fv(Cop)o(y)14 b(the)h(\(strictly\))h(lo)o(w)o(er)e(triangular)i(part)e (of)h(a)g(matrix.)218 584 y Ft(REAL)23 b(a\(100,100\),)g(b\(100,100\)) 218 640 y(INTEGER)47 b(disp\(100\),)22 b(blocklen\(100\),)g(ltype,)h (myrank,)g(ierr)218 697 y(INTEGER)g(status\(MPI_STATUS_SIZE\))75 810 y(C)119 b(copy)23 b(lower)h(triangular)e(part)h(of)h(array)f(a)75 866 y(C)119 b(onto)23 b(lower)h(triangular)e(part)h(of)h(array)f(b)218 979 y(CALL)g(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))75 1092 y(C)119 b(compute)23 b(start)g(and)h(size)f(of)h(each)f(column)218 1149 y(DO)h(i=1,)f(100)266 1205 y(disp\(i\))g(=)h(100*\(i-1\))e(+)i(i) 266 1261 y(block\(i\))f(=)g(100-i)218 1318 y(END)h(DO)75 1431 y(C)119 b(create)23 b(datatype)g(for)g(lower)h(triangular)e(part) 218 1487 y(CALL)h(MPI_TYPE_INDEXED\()f(100,)h(block,)g(disp,)h (MPI_REAL,)e(ltype,)h(ierr\))218 1600 y(CALL)g(MPI_TYPE_COMMIT\(ltype,) e(ierr\))218 1657 y(CALL)i(MPI_SENDRECV\()g(a,)g(1,)h(ltype,)f(myrank,) g(0,)g(b,)h(1,)552 1713 y(ltype,)f(myrank,)g(0,)h(MPI_COMM_WORLD,)e (status,)h(ierr\))75 1813 y Fq(Example)18 b(3.32)23 b Fv(T)l(ransp)q(ose)15 b(a)g(matrix.)218 1913 y Ft(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 1970 y(INTEGER)g(row,)g(xpose,)g (sizeofreal,)g(myrank,)g(ierr)218 2026 y(INTEGER)g (status\(MPI_STATUS_SIZE\))75 2139 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 2252 y(CALL)g (MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))218 2365 y(CALL)j (MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 2478 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 2534 y(CALL)f(MPI_TYPE_VECTOR\()f(100,)h(1,)h(100,)f(MPI_REAL,)g(row,)g (ierr\))75 2647 y(C)119 b(create)23 b(datatype)g(for)g(matrix)g(in)h (row-major)f(order)218 2704 y(CALL)g(MPI_TYPE_HVECTOR\()f(100,)h(1,)h (sizeofreal,)e(row,)i(xpose,)f(ierr\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 79 87 79 86 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fv(79)218 102 y Ft(CALL)23 b(MPI_TYPE_COMMIT\()f(xpose,)h (ierr\))75 214 y(C)119 b(send)23 b(matrix)h(in)f(row-major)g(order)g (and)g(receive)g(in)h(column)f(major)g(order)218 271 y(CALL)g(MPI_SENDRECV\()g(a,)g(1,)h(xpose,)f(myrank,)g(0,)g(b,)h (100*100,)457 327 y(MPI_REAL,)e(myrank,)h(0,)h(MPI_COMM_WORLD,)e (status,)h(ierr\))75 430 y Fq(Example)18 b(3.33)23 b Fv(Another)15 b(approac)o(h)f(to)h(the)g(transp)q(ose)g(problem:)218 522 y Ft(REAL)23 b(a\(100,100\),)g(b\(100,100\))218 578 y(INTEGER)47 b(disp\(2\),)23 b(blocklen\(2\),)f(type\(2\),)h(row,)g (row1,)g(sizeofreal)218 635 y(INTEGER)47 b(myrank,)23 b(ierr)218 691 y(INTEGER)g(status\(MPI_STATUS_SIZE\))218 804 y(CALL)g(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\))75 917 y(C)119 b(transpose)23 b(matrix)g(a)h(onto)f(b)218 1030 y(CALL)g(MPI_TYPE_EXTENT\()f(MPI_REAL,)h(sizeofreal,)f(ierr\))75 1143 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row)218 1199 y(CALL)f(MPI_TYPE_VECTOR\()f(100,)h(1,)h(100,)f(MPI_REAL,)g(row,)g (ierr\))75 1312 y(C)119 b(create)23 b(datatype)g(for)g(one)h(row,)f (with)h(the)f(extent)g(of)h(one)f(real)g(number)218 1369 y(disp\(1\))g(=)h(0)218 1425 y(disp\(2\))f(=)h(sizeofreal)218 1482 y(type\(1\))47 b(=)24 b(row)218 1538 y(type\(2\))47 b(=)24 b(MPI_UB)218 1594 y(blocklen\(1\))46 b(=)24 b(1)218 1651 y(blocklen\(2\))46 b(=)24 b(1)218 1707 y(CALL)f(MPI_TYPE_STRUCT\() f(2,)i(blocklen,)e(disp,)i(type,)f(row1,)g(ierr\))218 1820 y(CALL)g(MPI_TYPE_COMMIT\()f(row1,)h(ierr\))75 1933 y(C)119 b(send)23 b(100)h(rows)f(and)h(receive)f(in)g(column)g(major)g (order)218 1990 y(CALL)g(MPI_SENDRECV\()g(a,)g(100,)g(row1,)h(myrank,)f (0,)g(b,)h(100*100,)457 2046 y(MPI_REAL,)e(myrank,)h(0,)h (MPI_COMM_WORLD,)e(status,)h(ierr\))75 2149 y Fq(Example)18 b(3.34)23 b Fv(W)l(e)15 b(manipulate)h(an)f(arra)o(y)f(of)h (structures.)75 2252 y Ft(struct)23 b(Partstruct)147 2308 y({)147 2365 y(int)95 b(class;)47 b(/*)23 b(particle)g(class)g(*/) 147 2421 y(double)g(d[6];)71 b(/*)23 b(particle)g(coordinates)f(*/)147 2478 y(char)71 b(b[7];)g(/*)23 b(some)g(additional)g(information)f(*/) 147 2534 y(};)75 2647 y(struct)h(Partstruct)94 b(particle[1000];)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 80 88 80 87 bop 75 -100 a Fv(80)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Ft(int)429 b(i,)24 b(dest,)f(rank;)75 102 y(MPI_Comm)118 b(comm;)75 271 y(/*)24 b(build)f(datatype)g(describing)f(structure)h(*/)75 384 y(MPI_Datatype)f(Particletype;)75 440 y(MPI_Datatype)g(type[3])h(=) h({MPI_INT,)e(MPI_DOUBLE,)h(MPI_CHAR};)75 497 y(int)238 b(blocklen[3])23 b(=)g({1,)h(6,)f(7};)75 553 y(MPI_Aint)118 b(disp[3];)75 610 y(int)238 b(base;)75 779 y(/*)24 b(compute)e (displacements)h(of)g(structure)g(components)f(*/)75 892 y(MPI_Address\()g(particle,)h(disp\);)75 948 y(MPI_Address\()f (particle[0].d,)g(disp+1\);)75 1005 y(MPI_Address\()g(particle[0].b,)g (disp+2\);)75 1061 y(base)h(=)h(disp[0];)75 1118 y(for)f(\(i=0;)h(i)f (<3;)h(i++\))f(disp[i])g(-=)h(base;)75 1231 y(MPI_Type_struct\()e(3,)h (blocklen,)g(disp,)g(type,)g(&Particletype\);)147 1344 y(/*)g(If)h(compiler)f(does)g(padding)g(in)g(mysterious)g(ways,)147 1400 y(the)g(following)g(may)g(be)h(safer)f(*/)75 1513 y(MPI_Datatype)f(type1[4])h(=)h({MPI_INT,)e(MPI_DOUBLE,)h(MPI_CHAR,)f (MPI_UB};)75 1569 y(int)238 b(blocklen1[4])22 b(=)i({1,)g(6,)f(7,)h (1};)75 1626 y(MPI_Aint)118 b(disp1[4];)75 1739 y(/*)24 b(compute)e(displacements)h(of)g(structure)g(components)f(*/)75 1852 y(MPI_Address\()g(particle,)h(disp1\);)75 1908 y(MPI_Address\()f (particle[0].d,)g(disp1+1\);)75 1965 y(MPI_Address\()g(particle[0].b,)g (disp1+2\);)75 2021 y(MPI_Address\()g(particle+1,)h(disp1+3\);)75 2077 y(base)g(=)h(disp1[0];)75 2134 y(for)f(\(i=0;)h(i)f(<4;)h(i++\))f (disp1[i])g(-=)g(base;)75 2247 y(/*)h(build)f(datatype)g(describing)f (structure)h(*/)75 2360 y(MPI_Type_struct\()f(4,)h(blocklen1,)g(disp1,) g(type1,)g(&Particletype\);)409 2529 y(/*)h(4.1:)266 2586 y(send)f(the)h(entire)f(array)g(*/)75 2698 y(MPI_Type_commit\()f (&Particletype\);)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 81 89 81 88 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fv(81)75 45 y Ft(MPI_Send\()23 b(particle,)f(1000,)h (Particletype,)g(dest,)g(tag,)g(comm\);)409 214 y(/*)h(4.2:)266 271 y(send)f(only)h(the)f(entries)g(of)g(class)h(zero)f(particles,)266 327 y(preceded)g(by)g(the)h(number)f(of)g(such)h(entries)f(*/)75 440 y(MPI_Datatype)f(Zparticles;)70 b(/*)24 b(datatype)f(describing)f (all)i(particles)791 497 y(with)f(class)g(zero)h(\(needs)f(to)g(be)h (recomputed)791 553 y(if)f(classes)g(change\))g(*/)75 610 y(MPI_Datatype)f(Ztype;)75 723 y(MPI_Aint)118 b(zdisp[1000];)75 779 y(int)23 b(zblock[1000],)g(j,)g(k;)75 835 y(int)g(zzblock[2])g(=)h ({1,1};)75 892 y(MPI_Aint)118 b(zzdisp[2];)75 948 y(MPI_Datatype)22 b(zztype[2];)75 1061 y(/*)i(compute)e(displacements)h(of)g(class)g (zero)h(particles)e(*/)75 1118 y(j)i(=)f(0;)75 1174 y(for\(i=0;)g(i)g (<)h(1000;)f(i++\))123 1231 y(if)g(\(particle[i].class==0\))194 1287 y({)194 1344 y(zdisp[j])g(=)h(i;)194 1400 y(zblock[j])f(=)h(1;)194 1456 y(j++;)194 1513 y(})75 1626 y(/*)g(create)f(datatype)f(for)i (class)f(zero)g(particles)47 b(*/)75 1682 y(MPI_Type_indexed\()22 b(j,)h(zblock,)g(zdisp,)g(Particletype,)f(&Zparticles\);)75 1795 y(/*)i(prepend)e(particle)h(count)g(*/)75 1852 y(MPI_Address\(&j,) f(zzdisp\);)75 1908 y(MPI_Address\(particle,)f(zzdisp+1\);)75 1965 y(zztype[0])i(=)g(MPI_INT;)75 2021 y(zztype[1])g(=)g(Zparticles;) 75 2077 y(MPI_Type_struct\(2,)e(zzblock,)i(zzdisp,)g(zztype,)g (&Ztype\);)75 2190 y(MPI_Type_commit\()f(&Ztype\);)75 2247 y(MPI_Send\()h(MPI_BOTTOM,)f(1,)i(Ztype,)f(dest,)g(tag,)g(comm\);) 242 2416 y(/*)h(A)f(probably)g(more)g(efficient)g(way)h(of)f(defining)g (Zparticles)f(*/)75 2529 y(/*)i(consecutive)e(particles)h(with)g(index) g(zero)g(are)h(handled)f(as)g(one)h(block)f(*/)75 2586 y(j=0;)75 2642 y(for)g(\(i=0;)h(i)f(<)h(1000;)f(i++\))123 2698 y(if)g(\(particle[i].index==0\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 82 90 82 89 bop 75 -100 a Fv(82)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Ft({)170 102 y(for)24 b(\(k=i+1;)f(\(k)g(<)h(1000\)&&\(particle[k].index)d(==)i (0\))h(;)f(k++\);)170 158 y(zdisp[j])g(=)h(i;)170 214 y(zblock[j])f(=)h(k-i;)170 271 y(j++;)170 327 y(i)g(=)g(k;)170 384 y(})75 440 y(MPI_Type_indexed\()e(j,)h(zblock,)g(zdisp,)g (Particletype,)f(&Zparticles\);)457 610 y(/*)h(4.3:)314 666 y(send)g(the)g(first)h(two)f(coordinates)f(of)i(all)f(entries)g(*/) 75 779 y(MPI_Datatype)f(Allpairs;)142 b(/*)24 b(datatype)f(for)g(all)g (pairs)h(of)f(coordinates)g(*/)75 892 y(MPI_Aint)g(sizeofentry;)75 1005 y(MPI_Type_extent\()f(Particletype,)g(&sizeofentry\);)194 1118 y(/*)i(sizeofentry)e(can)i(also)f(be)h(computed)e(by)i (subtracting)e(the)i(address)266 1174 y(of)f(particle[0])g(from)g(the)h (address)f(of)g(particle[1])f(*/)75 1287 y(MPI_Type_hvector\()g(1000,)h (2,)g(sizeofentry,)g(MPI_DOUBLE,)f(&Allpairs\);)75 1344 y(MPI_Type_commit\()g(&Allpairs\);)75 1400 y(MPI_Send\()h (particle[0].d,)f(1,)h(Allpairs,)g(dest,)g(tag,)g(comm\);)218 1513 y(/*)h(an)f(alternative)g(solution)f(to)i(4.3)g(*/)75 1626 y(MPI_Datatype)e(Onepair;)71 b(/*)23 b(datatype)g(for)h(one)f (pair)g(of)h(coordinates,)e(with)695 1682 y(the)i(extent)f(of)h(one)f (particle)g(entry)g(*/)75 1739 y(MPI_Aint)g(disp2[3];)75 1795 y(MPI_Datatype)f(type2[3])h(=)h({MPI_LB,)e(MPI_DOUBLE,)h(MPI_UB};) 75 1852 y(int)g(blocklen2[3])g(=)g({1,)h(2,)f(1};)75 1965 y(MPI_Address\()f(particle,)h(disp2\);)75 2021 y(MPI_Address\()f (particle[0].d,)g(disp2+1\);)75 2077 y(MPI_Address\()g(particle+1,)h (disp2+2\);)75 2134 y(base)g(=)h(disp2[0];)75 2190 y(for)f(\(i=0;)h (i<2;)f(i++\))g(disp2[i])g(-=)h(base;)75 2303 y(MPI_Type_struct\()e(3,) h(blocklen2,)g(disp2,)g(type2,)g(&Onepair\);)75 2360 y(MPI_Type_commit\()f(&Onepair\);)75 2416 y(MPI_Send\()h (particle[0].d,)f(1000,)h(Onepair,)g(dest,)g(tag,)g(comm\);)75 2560 y Fq(Example)18 b(3.35)23 b Fv(The)15 b(same)g(manipulations)i(as) e(in)h(the)g(previous)g(example,)g(but)f(use)h(absolute)g(ad-)75 2616 y(dresses)f(in)h(datat)o(yp)q(es.)75 2704 y Ft(struct)23 b(Partstruct)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 83 91 83 90 bop 75 -100 a Fm(3.12.)29 b(DERIVED)15 b(D)o(A)l(T)l(A)l(TYPES) 1124 b Fv(83)147 45 y Ft({)147 102 y(int)23 b(class;)147 158 y(double)g(d[6];)147 214 y(char)g(b[7];)147 271 y(};)75 384 y(struct)g(Partstruct)g(particle[1000];)337 497 y(/*)h(build)f (datatype)g(describing)g(first)g(array)g(entry)g(*/)75 610 y(MPI_Datatype)f(Particletype;)75 666 y(MPI_Datatype)g(type[3])h(=) h({MPI_INT,)e(MPI_DOUBLE,)h(MPI_CHAR};)75 723 y(int)238 b(block[3])23 b(=)h({1,)f(6,)h(7};)75 779 y(MPI_Aint)118 b(disp[3];)75 892 y(MPI_Address\()22 b(particle,)h(disp\);)75 948 y(MPI_Address\()f(particle[0].d,)g(disp+1\);)75 1005 y(MPI_Address\()g(particle[0].b,)g(disp+2\);)75 1061 y(MPI_Type_struct\()g(3,)h(block,)g(disp,)g(type,)h(&Particletype\);)75 1174 y(/*)g(Particletype)e(describes)g(first)i(array)f(entry)g(--)h (using)f(absolute)147 1231 y(addresses)f(*/)505 1344 y(/*)h(5.1:)361 1400 y(send)h(the)f(entire)g(array)g(*/)75 1513 y(MPI_Type_commit\()f(&Particletype\);)75 1569 y(MPI_Send\()h (MPI_BOTTOM,)f(1000,)h(Particletype,)f(dest,)h(tag,)h(comm\);)481 1739 y(/*)f(5.2:)290 1795 y(send)g(the)h(entries)e(of)i(class)f(zero,) 290 1852 y(preceded)g(by)g(the)h(number)f(of)g(such)h(entries)e(*/)75 1965 y(MPI_Datatype)g(Zparticles,)h(Ztype;)75 2077 y(MPI_Aint)g (zdisp[1000])75 2134 y(int)g(zblock[1000],)g(i,)g(j,)h(k;)75 2190 y(int)f(zzblock[2])g(=)h({1,1};)75 2247 y(MPI_Datatype)e (zztype[2];)75 2303 y(MPI_Aint)118 b(zzdisp[2];)75 2416 y(j=0;)75 2473 y(for)23 b(\(i=0;)h(i)f(<)h(1000;)f(i++\))123 2529 y(if)g(\(particle[i].index==0\))170 2586 y({)170 2642 y(for)h(\(k=i+1;)f(\(k)g(<)h(1000\)&&\(particle[k].index)d(=)i (0\))h(;)g(k++\);)170 2698 y(zdisp[j])f(=)h(i;)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 84 92 84 91 bop 75 -100 a Fv(84)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Ft(zblock[j])23 b(=)h(k-i;)170 102 y(j++;)170 158 y(i)g(=)g(k;)170 214 y(})75 271 y(MPI_Type_indexed\()e(j,)h(zblock,)g(zdisp,)g (Particletype,)f(&Zparticles\);)75 327 y(/*)i(Zparticles)e(describe)h (particles)f(with)i(class)f(zero,)g(using)147 384 y(their)g(absolute)g (addresses*/)75 497 y(/*)h(prepend)e(particle)h(count)g(*/)75 553 y(MPI_Address\(&j,)f(zzdisp\);)75 610 y(zzdisp[1])h(=)g (MPI_BOTTOM;)75 666 y(zztype[0])g(=)g(MPI_INT;)75 723 y(zztype[1])g(=)g(Zparticles;)75 779 y(MPI_Type_struct\(2,)e(zzblock,)i (zzdisp,)g(zztype,)g(&Ztype\);)75 892 y(MPI_Type_commit\()f(&Ztype\);) 75 948 y(MPI_Send\()h(MPI_BOTTOM,)f(1,)i(Ztype,)f(dest,)g(tag,)g (comm\);)75 1111 y Fq(Example)18 b(3.36)23 b Fv(Handling)16 b(of)f(unions.)75 1217 y Ft(union)23 b({)147 1274 y(int)118 b(ival;)147 1330 y(float)70 b(fval;)218 1387 y(})24 b(u[1000])75 1500 y(int)119 b(utype;)75 1613 y(/*)24 b(All)f(entries)g(of)h(u)f (have)h(identical)e(type;)h(variable)147 1669 y(utype)g(keeps)g(track)g (of)h(their)f(current)g(type)g(*/)75 1782 y(MPI_Datatype)70 b(type[2];)75 1838 y(int)286 b(blocklen[2])22 b(=)i({1,1};)75 1895 y(MPI_Aint)166 b(disp[2];)75 1951 y(MPI_Datatype)70 b(mpi_utype[2];)75 2008 y(MPI_Aint)166 b(i,j;)75 2121 y(/*)24 b(compute)e(an)i(MPI)g(datatype)e(for)i(each)f(possible)g (union)g(type;)147 2177 y(assume)g(values)g(are)g(left-aligned)f(in)i (union)f(storage.)g(*/)75 2290 y(MPI_Address\()f(u,)i(&i\);)75 2346 y(MPI_Address\()e(u+1,)i(&j\);)75 2403 y(disp[0])f(=)h(0;)f (disp[1])g(=)h(j-i;)75 2459 y(type[1])f(=)h(MPI_UB;)75 2572 y(type[0])f(=)h(MPI_INT;)75 2629 y(MPI_Type_struct\(2,)d (blocklen,)i(disp,)g(type,)g(&mpi_utype[0]\);)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 85 93 85 92 bop 75 -100 a Fm(3.13.)34 b(P)l(A)o(CK)15 b(AND)g(UNP)l(A)o(CK) 1169 b Fv(85)75 45 y Ft(type[0])23 b(=)h(MPI_FLOAT;)75 102 y(MPI_Type_struct\(2,)d(blocklen,)i(disp,)g(type,)g (&mpi_utype[1]\);)75 214 y(for\(i=0;)g(i<2;)g(i++\))g (MPI_Type_commit\(&mpi_utype[i])o(\);)75 327 y(/*)h(actual)f (communication)f(*/)75 440 y(MPI_Send\(u,)g(1000,)i(mpi_utype[utype],)d (dest,)i(tag,)h(comm\);)75 584 y Fn(3.13)59 b(P)n(ack)20 b(and)f(unpack)75 685 y Fv(Some)c(existing)i(comm)o(unication)e (libraries)i(pro)o(vide)f(pac)o(k/unpac)o(k)f(functions)h(for)f (sending)h(noncon-)75 741 y(tiguous)c(data.)19 b(In)13 b(these,)f(the)h(user)f(explicitly)j(pac)o(ks)d(data)g(in)o(to)g(a)g (con)o(tiguous)h(bu\013er)f(b)q(efore)h(sending)75 798 y(it,)h(and)h(unpac)o(ks)g(it)f(from)g(a)g(con)o(tiguous)h(bu\013er)f (after)g(receiving)i(it.)k(Deriv)o(ed)14 b(datat)o(yp)q(es,)g(whic)o(h) h(are)75 854 y(describ)q(ed)g(in)e(Section)h(3.12,)e(allo)o(w)h(one,)g (in)h(most)e(cases,)g(to)h(a)o(v)o(oid)f(explicit)j(pac)o(king)e(and)g (unpac)o(king.)75 911 y(The)19 b(user)h(sp)q(eci\014es)h(the)e(la)o(y)o (out)g(of)g(the)g(data)f(to)h(b)q(e)h(sen)o(t)f(or)g(receiv)o(ed,)i (and)e(the)g(comm)o(unication)75 967 y(library)h(directly)g(accesses)f (a)f(noncon)o(tiguous)h(bu\013er.)31 b(The)19 b(pac)o(k/unpac)o(k)g (routines)g(are)f(pro)o(vided)75 1024 y(for)d(compatibilit)o(y)i(with)f (previous)h(libraries.)22 b(Also,)16 b(they)g(pro)o(vide)g(some)f (functionalit)o(y)i(that)e(is)h(not)75 1080 y(otherwise)f(a)o(v)m (ailable)i(in)f Fw(MPI)p Fv(.)e(F)l(or)g(instance,)i(a)e(message)h(can) g(b)q(e)h(receiv)o(ed)g(in)g(sev)o(eral)f(parts,)f(where)75 1137 y(the)19 b(receiv)o(e)h(op)q(eration)g(done)f(on)h(a)f(later)g (part)f(ma)o(y)h(dep)q(end)i(on)e(the)g(con)o(ten)o(t)g(of)g(a)f (former)h(part.)75 1193 y(Another)f(use)h(is)f(that)g(outgoing)g (messages)f(ma)o(y)h(b)q(e)h(explicitly)i(bu\013ered)d(in)h(user)g (supplied)h(space,)75 1250 y(th)o(us)f(o)o(v)o(erriding)h(the)f(system) g(bu\013ering)g(p)q(olicy)l(.)34 b(Finally)l(,)22 b(the)d(a)o(v)m (ailabilit)o(y)i(of)e(pac)o(k)g(and)h(unpac)o(k)75 1306 y(op)q(erations)14 b(facilitates)h(the)f(dev)o(elopmen)o(t)h(of)e (additional)j(comm)o(unication)f(libraries)g(la)o(y)o(ered)f(on)g(top) 75 1362 y(of)h Fw(MPI)p Fv(.)75 1513 y Fw(MPI)p 160 1513 14 2 v 16 w(P)l(A)o(CK\(inbuf,)g(incount,)i(datat)o(yp)q(e,)f(outbuf,)g (outsize,)g(p)q(osition,)g(comm\))117 1591 y Fk(IN)155 b Fw(inbuf)493 b Fk(input)14 b(bu\013er)h(start)f(\(c)o(hoice\))117 1666 y(IN)155 b Fw(incount)448 b Fk(n)o(um)o(b)q(er)13 b(of)h(input)f(data)h(items)f(\(in)o(teger\))117 1741 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(input)g(data)f(item)g(\(handle\))117 1816 y(OUT)108 b Fw(outbuf)465 b Fk(output)14 b(bu\013er)h(start)g(\(c) o(hoice\))117 1891 y(IN)155 b Fw(outsize)457 b Fk(output)14 b(bu\013er)h(size,)f(in)g(b)o(ytes)g(\(in)o(teger\))117 1966 y(INOUT)62 b Fw(p)q(osition)439 b Fk(curren)o(t)16 b(p)q(osition)d(in)g(bu\013er,)i(in)e(b)o(ytes)i(\(in)o(teger\))117 2041 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(for)j(pac)o(k)o (ed)g(message)g(\(handle\))75 2166 y Ft(int)23 b(MPI)p 245 2166 15 2 v 17 w(Pack\(void*)g(inbuf,)g(int)g(incount,)g(MPI)p 1074 2166 V 17 w(Datatype)g(datatype,)f(void)h(*outbuf,)393 2222 y(int)h(outsize,)e(int)i(*position,)e(MPI)p 1136 2222 V 17 w(Comm)h(comm\))75 2309 y(MPI)p 150 2309 V 17 w(PACK\(INBUF,)f(INCOUNT,)h(DATATYPE,)g(OUTBUF,)f(OUTSIZE,)h (POSITION,)g(COMM,)g(IERROR\))170 2365 y()g(INBUF\(*\),)g (OUTBUF\(*\))170 2422 y(INTEGER)g(INCOUNT,)g(DATATYPE,)g(OUTSIZE,)g (POSITION,)f(COMM,)h(IERROR)166 2508 y Fv(P)o(ac)o(ks)11 b(the)i(message)e(in)i(the)g(send)f(bu\013er)h(sp)q(eci\014ed)h(b)o(y)e Fw(inbuf,)i(incount,)g(datat)o(yp)q(e)g Fv(in)o(to)e(the)g(bu\013er)75 2565 y(space)j(sp)q(eci\014ed)i(b)o(y)e Fw(outbuf)h Fv(and)f Fw(outcount)p Fv(.)22 b(The)15 b(input)h(bu\013er)e(can)h(b)q(e)h(an)o (y)e(comm)o(unication)i(bu\013er)75 2621 y(allo)o(w)o(ed)k(in)g Fw(MPI)p 383 2621 14 2 v 16 w(SEND)p Fv(.)g(The)g(output)f(bu\013er)h (is)g(a)f(con)o(tiguous)g(storage)g(area)g(con)o(taining)h Fw(outsize)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 86 94 86 93 bop 75 -100 a Fv(86)608 b Fm(CHAPTER)15 b(3.)35 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Fv(b)o(ytes,)e(starting) f(at)h(the)g(address)h Fw(outbuf)g Fv(\(length)g(is)g(coun)o(ted)f(in)h Fw(b)o(ytes)p Fv(,)h(not)d(elemen)o(ts,)i(as)f(if)h(it)f(w)o(ere)75 102 y(a)g(comm)o(unication)h(bu\013er)f(for)g(a)f(message)h(of)g(t)o (yp)q(e)g Fw(MPI)p 1081 102 14 2 v 16 w(P)l(A)o(CKED)p Fv(\).)166 158 y(The)21 b(input)g(v)m(alue)h(of)e Fw(p)q(osition)h Fv(is)g(the)g(\014rst)f(lo)q(cation)h(in)g(the)g(output)f(bu\013er)h (to)e(b)q(e)j(used)f(for)75 214 y(pac)o(king.)27 b Fw(p)q(osition)19 b Fv(is)f(incremen)o(ted)g(b)o(y)g(the)f(size)h(of)f(the)h(pac)o(k)o (ed)f(message,)g(and)h(the)g(output)f(v)m(alue)75 271 y(of)e Fw(p)q(osition)i Fv(is)f(the)g(\014rst)f(lo)q(cation)i(in)f(the) g(output)f(bu\013er)h(follo)o(wing)h(the)e(lo)q(cations)i(o)q(ccupied)g (b)o(y)f(the)75 327 y(pac)o(k)o(ed)g(message.)k(The)c Fw(comm)e Fv(argumen)o(t)g(is)i(the)g(comm)o(unicator)f(that)g(will)i (b)q(e)g(subsequen)o(tly)f(used)75 384 y(for)f(sending)h(the)f(pac)o(k) o(ed)h(message.)75 535 y Fw(MPI)p 160 535 V 16 w(UNP)l(A)o(CK\(inbuf,)g (insize,)f(p)q(osition,)h(outbuf,)g(outcount,)h(datat)o(yp)q(e,)g (comm\))117 612 y Fk(IN)155 b Fw(inbuf)493 b Fk(input)14 b(bu\013er)h(start)f(\(c)o(hoice\))117 684 y(IN)155 b Fw(insize)485 b Fk(size)15 b(of)e(input)h(bu\013er,)g(in)g(b)o(ytes)g (\(in)o(teger\))117 757 y(INOUT)62 b Fw(p)q(osition)439 b Fk(curren)o(t)16 b(p)q(osition)d(in)g(b)o(ytes)i(\(in)o(teger\))117 829 y(OUT)108 b Fw(outbuf)465 b Fk(output)14 b(bu\013er)h(start)g(\(c)o (hoice\))117 902 y(IN)155 b Fw(outcount)420 b Fk(n)o(um)o(b)q(er)13 b(of)h(items)f(to)g(b)q(e)i(unpac)o(k)o(ed)f(\(in)o(teger\))117 974 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(of)g(eac)o(h)g(output)g(data)g(item)e(\(handle\))117 1046 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(for)j(pac)o(k)o (ed)g(message)g(\(handle\))75 1171 y Ft(int)23 b(MPI)p 245 1171 15 2 v 17 w(Unpack\(void*)f(inbuf,)h(int)h(insize,)f(int)g (*position,)g(void)g(*outbuf,)393 1227 y(int)h(outcount,)e(MPI)p 802 1227 V 17 w(Datatype)h(datatype,)f(MPI)p 1344 1227 V 17 w(Comm)i(comm\))75 1314 y(MPI)p 150 1314 V 17 w(UNPACK\(INBUF,)e (INSIZE,)h(POSITION,)f(OUTBUF,)h(OUTCOUNT,)g(DATATYPE,)g(COMM,)393 1370 y(IERROR\))170 1427 y()g(INBUF\(*\),)g(OUTBUF\(*\))170 1483 y(INTEGER)g(INSIZE,)g(POSITION,)g(OUTCOUNT,)g(DATATYPE,)f(COMM,)h (IERROR)166 1570 y Fv(Unpac)o(ks)15 b(a)g(message)f(in)o(to)h(the)g (receiv)o(e)h(bu\013er)f(sp)q(eci\014ed)i(b)o(y)e Fw(outbuf,)h (outcount,)g(datat)o(yp)q(e)h Fv(from)75 1626 y(the)d(bu\013er)g(space) g(sp)q(eci\014ed)i(b)o(y)e Fw(inbuf)i Fv(and)e Fw(insize)p Fv(.)20 b(The)14 b(output)g(bu\013er)g(can)g(b)q(e)h(an)o(y)e(comm)o (unication)75 1682 y(bu\013er)h(allo)o(w)o(ed)g(in)g Fw(MPI)p 500 1682 14 2 v 16 w(RECV)p Fv(.)g(The)g(input)h(bu\013er)e (is)h(a)g(con)o(tiguous)g(storage)e(area)h(con)o(taining)h Fw(insize)75 1739 y Fv(b)o(ytes,)f(starting)g(at)g(address)h Fw(inbuf)p Fv(.)21 b(The)13 b(input)i(v)m(alue)g(of)e Fw(p)q(osition)i Fv(is)f(the)g(\014rst)f(lo)q(cation)i(in)f(the)g (input)-2034 b Fx(>)75 1795 y Fv(bu\013er)18 b(o)q(ccupied)h(b)o(y)f (the)f(pac)o(k)o(ed)h(message.)27 b Fw(p)q(osition)18 b Fv(is)g(incremen)o(ted)h(b)o(y)f(the)g(size)g(of)f(the)h(pac)o(k)o (ed)75 1852 y(message,)f(so)g(that)f(the)h(output)g(v)m(alue)i(of)d Fw(p)q(osition)j Fv(is)e(the)h(\014rst)e(lo)q(cation)i(in)g(the)f (input)i(bu\013er)e(after)-2036 b Fx(?)75 1908 y Fv(the)14 b(lo)q(cations)g(o)q(ccupied)i(b)o(y)e(the)g(message)f(that)g(w)o(as)g (unpac)o(k)o(ed.)20 b Fw(comm)12 b Fv(is)i(the)g(comm)o(unicator)f (used)75 1965 y(to)i(receiv)o(e)h(the)f(pac)o(k)o(ed)g(message.)189 2052 y Fl(A)n(dvic)n(e)21 b(to)i(users.)83 b Fv(Note)22 b(the)g(di\013erence)h(b)q(et)o(w)o(een)g Fw(MPI)p 1280 2052 V 16 w(RECV)g Fv(and)f Fw(MPI)p 1611 2052 V 16 w(UNP)l(A)o(CK)p Fv(:)g(in)189 2108 y Fw(MPI)p 274 2108 V 15 w(RECV)p Fv(,)g(the)g Fw(count)h Fv(argumen)o(t)e(sp)q(eci\014es)i(the)f(maxim)o (um)f(n)o(um)o(b)q(er)h(of)f(items)h(that)f(can)189 2165 y(b)q(e)g(receiv)o(ed.)39 b(The)22 b(actual)f(n)o(um)o(b)q(er)h(of)e (items)i(receiv)o(ed)g(is)g(determined)g(b)o(y)f(the)h(length)f(of)189 2221 y(the)h(incoming)h(message.)40 b(In)23 b Fw(MPI)p 837 2221 V 15 w(UNP)l(A)o(CK)p Fv(,)f(the)g Fw(count)i Fv(argumen)o(t)d(sp)q(eci\014es)j(the)e(actual)189 2278 y(n)o(um)o(b)q(er)c(of)g(items)h(that)f(are)g(unpac)o(k)o(ed;)i(the)f (\\size")g(of)f(the)g(corresp)q(onding)i(message)e(is)h(the)189 2334 y(incremen)o(t)c(in)g Fw(p)q(osition)p Fv(.)21 b(The)14 b(reason)g(for)g(this)g(c)o(hange)h(is)f(that)g(the)g(\\incoming)h (message)f(size")189 2391 y(is)19 b(not)g(predetermined)i(since)g(the)e (user)g(decides)i(ho)o(w)e(m)o(uc)o(h)g(to)g(unpac)o(k;)i(nor)e(is)h (it)g(easy)f(to)189 2447 y(determine)c(the)g(\\message)f(size")i(from)e (the)h(n)o(um)o(b)q(er)g(of)f(items)h(to)g(b)q(e)g(unpac)o(k)o(ed.)20 b(In)c(fact,)e(in)h(a)189 2504 y(heterogeneous)i(system,)f(this)i(n)o (um)o(b)q(er)f(ma)o(y)f(not)h(b)q(e)g(determined)h Fl(a)g(priori)p Fv(.)26 b(\()p Fl(End)17 b(of)h(advic)n(e)189 2560 y(to)e(users.)p Fv(\))166 2647 y(T)l(o)g(understand)h(the)f(b)q(eha)o(vior)h(of)e(pac)o (k)h(and)h(unpac)o(k,)f(it)h(is)f(con)o(v)o(enien)o(t)h(to)f(think)h (of)e(the)i(data)75 2704 y(part)11 b(of)g(a)g(message)g(as)g(b)q(eing)i (the)e(sequence)i(obtained)f(b)o(y)g(concatenating)f(the)h(successiv)o (e)h(v)m(alues)f(sen)o(t)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 87 95 87 94 bop 75 -100 a Fm(3.13.)34 b(P)l(A)o(CK)15 b(AND)g(UNP)l(A)o(CK) 1169 b Fv(87)75 45 y(in)16 b(that)f(message.)20 b(The)c(pac)o(k)f(op)q (eration)h(stores)f(this)g(sequence)i(in)f(the)g(bu\013er)f(space,)h (as)f(if)h(sending)75 102 y(the)d(message)g(to)g(that)f(bu\013er.)19 b(The)14 b(unpac)o(k)f(op)q(eration)h(retriev)o(es)f(this)h(sequence)g (from)f(bu\013er)g(space,)75 158 y(as)h(if)h(receiving)h(a)f(message)f (from)g(that)g(bu\013er.)19 b(\(It)c(is)g(helpful)h(to)e(think)i(of)e (in)o(ternal)h(F)l(ortran)f(\014les)h(or)75 214 y Fw(sscanf)h Fv(in)g(C,)f(for)g(a)g(similar)h(function.\))166 271 y(Sev)o(eral)h(messages)g(can)g(b)q(e)h(successiv)o(ely)g(pac)o(k)o(ed) f(in)o(to)g(one)h Fq(pac)o(king)i(unit)p Fv(.)26 b(This)17 b(is)h(e\013ected)75 327 y(b)o(y)c(sev)o(eral)h(successiv)o(e)h Fq(related)f Fv(calls)g(to)f Fw(MPI)p 909 327 14 2 v 16 w(P)l(A)o(CK)p Fv(,)g(where)g(the)h(\014rst)f(call)h(pro)o(vides)g Fw(p)q(osition)h(=)f(0)p Fv(,)75 384 y(and)i(eac)o(h)g(successiv)o(e)h (call)g(inputs)g(the)f(v)m(alue)i(of)d Fw(p)q(osition)i Fv(that)f(w)o(as)f(output)h(b)o(y)f(the)h(previous)h(call,)75 440 y(and)g(the)g(same)f(v)m(alues)i(for)e Fw(outbuf,)i(outcount)h Fv(and)e Fw(comm)p Fv(.)25 b(This)18 b(pac)o(king)g(unit)h(no)o(w)e (con)o(tains)h(the)75 497 y(equiv)m(alen)o(t)h(information)f(that)e(w)o (ould)i(ha)o(v)o(e)f(b)q(een)h(stored)f(in)h(a)f(message)g(b)o(y)h(one) f(send)h(call)h(with)e(a)75 553 y(send)f(bu\013er)f(that)f(is)i(the)f (\\concatenation")g(of)g(the)g(individual)k(send)d(bu\013ers.)166 610 y(A)g(pac)o(king)g(unit)g(can)g(b)q(e)h(sen)o(t)e(using)i(t)o(yp)q (e)f Fw(MPI)p 1029 610 V 15 w(P)l(A)o(CKED)p Fv(.)g(An)o(y)f(p)q(oin)o (t)i(to)e(p)q(oin)o(t)h(or)f(collectiv)o(e)75 666 y(comm)o(unication)g (function)h(can)f(b)q(e)g(used)g(to)f(mo)o(v)o(e)g(the)h(sequence)g(of) g(b)o(ytes)f(that)g(forms)g(the)g(pac)o(king)75 723 y(unit)i(from)e (one)h(pro)q(cess)g(to)f(another.)19 b(This)d(pac)o(king)f(unit)h(can)f (no)o(w)f(b)q(e)i(receiv)o(ed)g(using)g(an)o(y)e(receiv)o(e)75 779 y(op)q(eration,)j(with)g(an)o(y)f(datat)o(yp)q(e:)22 b(the)16 b(t)o(yp)q(e)h(matc)o(hing)g(rules)g(are)f(relaxed)h(for)f (messages)g(sen)o(t)h(with)75 835 y(t)o(yp)q(e)e Fw(MPI)p 262 835 V 16 w(P)l(A)o(CKED)p Fv(.)166 892 y(A)e(message)g(sen)o(t)g (with)h(an)o(y)e(t)o(yp)q(e)i(\(including)h Fw(MPI)p 1062 892 V 16 w(P)l(A)o(CKED)p Fv(\))e(can)g(b)q(e)h(receiv)o(ed)h (using)f(the)f(t)o(yp)q(e)75 948 y Fw(MPI)p 160 948 V 16 w(P)l(A)o(CKED)p Fv(.)h(Suc)o(h)i(a)f(message)g(can)g(then)h(b)q(e)g (unpac)o(k)o(ed)f(b)o(y)h(calls)g(to)e Fw(MPI)p 1452 948 V 16 w(UNP)l(A)o(CK)p Fv(.)166 1005 y(A)19 b(pac)o(king)h(unit)g (\(or)f(a)g(message)g(created)g(b)o(y)g(a)g(regular,)h(\\t)o(yp)q(ed")f (send\))h(can)f(b)q(e)h(unpac)o(k)o(ed)75 1061 y(in)o(to)j(sev)o(eral)g (successiv)o(e)h(messages.)42 b(This)23 b(is)h(e\013ected)f(b)o(y)f (sev)o(eral)h(successiv)o(e)h(related)g(calls)g(to)75 1118 y Fw(MPI)p 160 1118 V 16 w(UNP)l(A)o(CK)p Fv(,)19 b(where)h(the)f(\014rst)g(call)i(pro)o(vides)f Fw(p)q(osition)g(=)g(0)p Fv(,)g(and)f(eac)o(h)h(successiv)o(e)g(call)h(inputs)75 1174 y(the)d(v)m(alue)h(of)e Fw(p)q(osition)i Fv(that)e(w)o(as)g (output)h(b)o(y)f(the)h(previous)h(call,)g(and)f(the)g(same)f(v)m (alues)i(for)e Fw(inbuf,)75 1231 y(insize)f Fv(and)f Fw(comm)p Fv(.)166 1287 y(The)22 b(concatenation)f(of)g(t)o(w)o(o)g (pac)o(king)h(units)g(is)g(not)f(necessarily)i(a)e(pac)o(king)h(unit;)j (nor)c(is)h(a)75 1344 y(substring)17 b(of)g(a)f(pac)o(king)h(unit)h (necessarily)g(a)f(pac)o(king)g(unit.)25 b(Th)o(us,)17 b(one)g(cannot)g(concatenate)f(t)o(w)o(o)75 1400 y(pac)o(king)11 b(units)g(and)g(then)g(unpac)o(k)g(the)f(result)h(as)g(one)f(pac)o (king)h(unit;)i(nor)d(can)h(one)f(unpac)o(k)h(a)g(substring)75 1456 y(of)f(a)g(pac)o(king)h(unit)g(as)f(a)g(separate)g(pac)o(king)h (unit.)19 b(Eac)o(h)11 b(pac)o(king)f(unit,)i(that)e(w)o(as)g(created)g (b)o(y)g(a)h(related)75 1513 y(sequence)17 b(of)e(pac)o(k)h(calls,)g (or)f(b)o(y)h(a)f(regular)h(send,)g(m)o(ust)f(b)q(e)h(unpac)o(k)o(ed)h (as)e(a)g(unit,)i(b)o(y)e(a)h(sequence)g(of)75 1569 y(related)g(unpac)o (k)f(calls.)189 1676 y Fl(R)n(ationale.)74 b Fv(The)21 b(restriction)h(on)f(\\atomic")f(pac)o(king)h(and)g(unpac)o(king)h(of)f (pac)o(king)g(units)189 1732 y(allo)o(ws)13 b(the)f(implemen)o(tation)j (to)d(add)h(at)f(the)h(head)g(of)f(pac)o(king)h(units)h(additional)g (information,)189 1789 y(suc)o(h)i(as)g(a)g(description)h(of)f(the)g (sender)h(arc)o(hitecture)f(\(to)f(b)q(e)i(used)g(for)e(t)o(yp)q(e)h (con)o(v)o(ersion,)g(in)h(a)189 1845 y(heterogeneous)e(en)o(vironmen)o (t\))g(\()p Fl(End)h(of)g(r)n(ationale.)p Fv(\))166 1951 y(The)21 b(follo)o(wing)g(call)g(allo)o(ws)g(the)g(user)f(to)g(\014nd)h (out)f(ho)o(w)g(m)o(uc)o(h)h(space)f(is)h(needed)h(to)e(pac)o(k)g(a)75 2008 y(message)15 b(and,)g(th)o(us,)f(manage)h(space)h(allo)q(cation)g (for)e(bu\013ers.)75 2159 y Fw(MPI)p 160 2159 V 16 w(P)l(A)o(CK)p 291 2159 V 16 w(SIZE\(incount,)i(datat)o(yp)q(e,)g(comm,)d(size\))117 2236 y Fk(IN)155 b Fw(incount)448 b Fk(coun)o(t)14 b(argumen)o(t)f(to)h (pac)o(king)f(call)g(\(in)o(teger\))117 2311 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(datat)o(yp)q(e)14 b(argumen)o(t)f(to)h (pac)o(king)f(call)g(\(handle\))117 2386 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(argumen)o(t)i(to)h(pac)o(king)f(call)g (\(handle\))117 2461 y(OUT)108 b Fw(size)519 b Fk(upp)q(er)16 b(b)q(ound)f(on)g(size)g(of)g(pac)o(k)o(ed)g(message,)f(in)h(b)o(ytes)g (\(in-)905 2518 y(teger\))75 2642 y Ft(int)23 b(MPI)p 245 2642 15 2 v 17 w(Pack)p 358 2642 V 17 w(size\(int)g(incount,)f(MPI) p 876 2642 V 17 w(Datatype)h(datatype,)g(MPI)p 1419 2642 V 16 w(Comm)h(comm,)393 2699 y(int)g(*size\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 88 96 88 95 bop 75 -100 a Fv(88)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(PACK)p 263 45 V 16 w(SIZE\(INCOUNT,)22 b(DATATYPE,)h(COMM,)g(SIZE,)g(IERROR\))170 102 y(INTEGER)g(INCOUNT,)g (DATATYPE,)g(COMM,)g(SIZE,)g(IERROR)166 188 y Fv(A)12 b(call)h(to)f Fw(MPI)p 430 188 14 2 v 16 w(P)l(A)o(CK)p 561 188 V 16 w(SIZE\(incount,)h(datat)o(yp)q(e,)h(comm,)d(size\))h Fv(returns)g(in)h Fw(size)f Fv(an)g(upp)q(er)h(b)q(ound)75 244 y(on)h(the)g(incremen)o(t)g(in)h Fw(p)q(osition)g Fv(that)e(is)i(e\013ected)f(b)o(y)f(a)h(call)h(to)e Fw(MPI)p 1263 244 V 16 w(P)l(A)o(CK\(inbuf,)h(incount,)i(datat)o(yp)q(e,)75 301 y(outbuf,)g(outcount,)h(p)q(osition,)f(comm\))p Fv(.)189 402 y Fl(R)n(ationale.)43 b Fv(The)16 b(call)h(returns)f(an)g(upp)q(er) h(b)q(ound,)f(rather)f(than)h(an)g(exact)f(b)q(ound,)i(since)g(the)189 458 y(exact)e(amoun)o(t)g(of)g(space)g(needed)i(to)e(pac)o(k)g(the)h (message)f(ma)o(y)g(dep)q(end)i(on)e(the)h(con)o(text)f(\(e.g.,)189 515 y(\014rst)f(message)h(pac)o(k)o(ed)g(in)h(a)f(pac)o(king)h(unit)g (ma)o(y)e(tak)o(e)h(more)g(space\).)20 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fv(\))75 616 y Fq(Example)h(3.37)23 b Fv(An)15 b(example)h(using)g Fw(MPI)p 863 616 V 16 w(P)l(A)o(CK)p Fv(.)75 706 y Ft(int)23 b(position,)g(i,)h(j,)f(a[2];)75 762 y(char)g(buff[1000];)75 875 y(....)75 988 y (MPI_Comm_rank\(MPI_COMM_WOR)o(LD,)e(&myrank\);)75 1045 y(if)j(\(myrank)e(==)i(0\))75 1101 y({)147 1158 y(/)f(*)h(SENDER)f (CODE)g(*/)123 1270 y(position)f(=)i(0;)123 1327 y(MPI_Pack\(&i,)e(1,)i (MPI_INT,)e(buff,)h(1000,)h(&position,)e(MPI_COMM_WORLD\);)123 1383 y(MPI_Pack\(&j,)g(1,)i(MPI_INT,)e(buff,)h(1000,)h(&position,)e (MPI_COMM_WORLD\);)123 1440 y(MPI_Send\()g(buff,)i(position,)e (MPI_PACKED,)h(1,)g(0,)h(MPI_COMM_WORLD\);)75 1496 y(})75 1553 y(else)47 b(/*)24 b(RECEIVER)f(CODE)g(*/)123 1609 y(MPI_Recv\()f(a,)i(2,)g(MPI_INT,)e(0,)i(0,)f(MPI_COMM_WORLD\))75 1722 y(})75 1823 y Fq(Example)18 b(3.38)23 b Fv(A)15 b(elab)q(orate)g(example.)75 1913 y Ft(int)23 b(position,)g(i;)75 1970 y(float)g(a[1000];)75 2026 y(char)g(buff[1000])75 2139 y(....)75 2252 y(MPI_Comm_rank\(MPI_Comm_wor)o(ld,)e(&myrank\);)75 2308 y(if)j(\(myrank)e(==)i(0\))75 2365 y({)123 2421 y(/)f(*)h(SENDER)f(CODE)h(*/)123 2534 y(int)f(len[2];)123 2591 y(MPI_Aint)f(disp[2];)123 2647 y(MPI_Datatype)g(type[2],)h (newtype;)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 89 97 89 96 bop 75 -100 a Fm(3.13.)29 b(P)l(A)o(CK)15 b(AND)g(UNP)l(A)o(CK) 1174 b Fv(89)123 45 y Ft(/*)23 b(build)g(datatype)g(for)h(i)f(followed) g(by)h(a[0]...a[i-1])e(*/)123 158 y(len[0])h(=)g(1;)123 214 y(len[1])g(=)g(i;)123 271 y(MPI_Address\()f(&i,)h(disp\);)123 327 y(MPI_Address\()f(a,)i(disp+1\);)123 384 y(type[0])f(=)g(MPI_INT;) 123 440 y(type[1])g(=)g(MPI_FLOAT;)123 497 y(MPI_Type_struct\()e(2,)j (len,)f(disp,)g(type,)h(&newtype\);)123 553 y(MPI_Type_commit\()d (&newtype\);)123 666 y(/*)i(Pack)h(i)f(followed)g(by)h(a[0]...a[i-1]*/) 123 779 y(position)e(=)i(0;)123 835 y(MPI_Pack\()e(MPI_BOTTOM,)h(1,)g (newtype,)g(buff,)g(1000,)g(&position,)g(MPI_COMM_WORLD\);)123 948 y(/*)g(Send)h(*/)123 1061 y(MPI_Send\()e(buff,)i(position,)e (MPI_PACKED,)h(1,)g(0,)361 1118 y(MPI_COMM_WORLD\))75 1231 y(/*)h(*****)147 1287 y(One)f(can)g(replace)g(the)h(last)f(three)g (lines)h(with)147 1344 y(MPI_Send\()e(MPI_BOTTOM,)h(1,)g(newtype,)g(1,) h(0,)f(MPI_COMM_WORLD\);)147 1400 y(*****)g(*/)75 1456 y(})75 1513 y(else)g(/*)h(myrank)f(==)h(1)f(*/)75 1569 y({)147 1626 y(/*)g(RECEIVER)g(CODE)g(*/)123 1739 y(MPI_Status)f (status;)123 1852 y(/*)h(Receive)g(*/)123 1965 y(MPI_Recv\()f(buff,)i (1000,)f(MPI_PACKED,)f(0,)i(0,)f(&status\);)123 2077 y(/*)g(Unpack)g(i)h(*/)99 2190 y(position)f(=)g(0;)99 2247 y(MPI_Unpack\(buff,)f(1000,)h(&position,)f(&i,)i(1,)f(MPI_INT,)g (MPI_COMM_WORLD\);)99 2360 y(/*)g(Unpack)g(a[0]...a[i-1])g(*/)99 2416 y(MPI_Unpack\(buff,)f(1000,)h(&position,)f(a,)i(i,)f(MPI_FLOAT,)g (MPI_COMM_WORLD\);)75 2473 y(})75 2564 y Fq(Example)18 b(3.39)23 b Fv(Eac)o(h)16 b(pro)q(cess)g(sends)h(a)f(coun)o(t,)f(follo) o(w)o(ed)i(b)o(y)f(coun)o(t)g(c)o(haracters)g(to)f(the)h(ro)q(ot;)g (the)75 2620 y(ro)q(ot)e(concatenate)h(all)h(c)o(haracters)f(in)o(to)g (one)g(string.)75 2704 y Ft(int)23 b(count,)g(gsize,)h(counts[64],)e (totalcount,)g(k1,)i(k2,)f(k,)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 90 98 90 97 bop 75 -100 a Fv(90)613 b Fm(CHAPTER)16 b(3.)29 b(POINT-TO-POINT)17 b(COMMUNICA)l(TION)170 45 y Ft(displs[64],)23 b(position,)f(concat_pos;)75 102 y(char)h(chr[100],)g(*lbuf,)g(*rbuf,)g (*cbuf;)75 158 y(...)75 214 y(MPI_Comm_size\(comm,)e(&gsize\);)75 271 y(MPI_Comm_rank\(comm,)g(&myrank\);)218 384 y(/*)j(allocate)f (local)g(pack)g(buffer)g(*/)75 440 y(MPI_Pack_size\(1,)f(MPI_INT,)g (comm,)i(&k1\);)75 497 y(MPI_Pack_size\(count,)d(MPI_CHAR,)i(&k2\);)75 553 y(k)h(=)f(k1+k2;)75 610 y(lbuf)g(=)h(\(char)f(*\)malloc\(k\);)218 723 y(/*)h(pack)f(count,)g(followed)g(by)g(count)h(characters)e(*/)75 779 y(position)h(=)g(0;)75 835 y(MPI_Pack\(&count,)f(1,)h(MPI_INT,)g (lbuf,)g(k,)h(&position,)e(comm\);)75 892 y(MPI_Pack\(chr,)g(count,)h (MPI_CHAR,)g(&lbuf,)g(k,)g(&position,)g(comm\);)75 1005 y(if)h(\(myrank)e(!=)i(root\))218 1061 y(/*)g(gather)f(at)g(root)h (sizes)f(of)g(all)h(packed)f(messages)g(*/)147 1118 y(MPI_Gather\()f (&position,)g(1,)i(MPI_INT,)f(NULL,)g(NULL,)385 1174 y(NULL,)g(root,)h(comm\);)218 1287 y(/*)g(gather)f(at)g(root)h(packed)f (messages)g(*/)123 1344 y(MPI_Gatherv\()f(&buf,)h(position,)g (MPI_PACKED,)f(NULL,)337 1400 y(NULL,)i(NULL,)f(NULL,)g(root,)g (comm\);)75 1513 y(else)g({)72 b(/*)23 b(root)h(code)f(*/)218 1569 y(/*)h(gather)f(sizes)g(of)h(all)f(packed)g(messages)g(*/)147 1626 y(MPI_Gather\()f(&position,)g(1,)i(MPI_INT,)f(counts,)g(1,)385 1682 y(MPI_INT,)g(root,)g(comm\);)218 1795 y(/*)h(gather)f(all)g (packed)g(messages)g(*/)123 1852 y(displs[0])f(=)i(0;)123 1908 y(for)f(\(i=1;)g(i)h(<)g(gsize;)f(i++\))170 1965 y(displs[i])g(=)h(displs[i-1])e(+)i(counts[i-1];)123 2021 y(totalcount)e(=)i(dipls[gsize-1])e(+)i(counts[gsize-1];)123 2077 y(rbuf)f(=)h(\(char)f(*\)malloc\(totalcount\);)123 2134 y(cbuf)g(=)h(\(char)f(*\)malloc\(totalcount\);)123 2190 y(MPI_Gatherv\()f(lbuf,)h(position,)g(MPI_PACKED,)f(rbuf,)337 2247 y(counts,)h(displs,)g(MPI_PACKED,)g(root,)g(comm\);)218 2360 y(/*)h(unpack)f(all)g(messages)g(and)g(concatenate)g(strings)g(*/) 123 2416 y(concat_pos)f(=)i(0;)123 2473 y(for)f(\(i=0;)g(i)h(<)g (gsize;)f(i++\))g({)170 2529 y(position)g(=)h(0;)170 2586 y(MPI_Unpack\()f(rbuf+displs[i],)f(totalcount-displs[i],)314 2642 y(&position,)g(&count,)h(1,)h(MPI_INT,)e(comm\);)170 2698 y(MPI_Unpack\()h(rbuf+displs[i],)f(totalcount-displs[i],)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 91 99 91 98 bop 75 -100 a Fm(3.13.)34 b(P)l(A)o(CK)15 b(AND)g(UNP)l(A)o(CK) 1169 b Fv(91)314 45 y Ft(&position,)22 b(cbuf+concat_pos,)g(count,)h (MPI_CHAR,)f(comm\);)170 102 y(concat_pos)h(+=)h(count;)170 158 y(})123 214 y(cbuf[concat_pos])d(=)j(`\\0';)123 271 y(})-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 92 100 92 99 bop 75 356 a Fo(Chapter)34 b(4)75 564 y Fr(Collecti)q(v)m(e)42 b(Comm)m(unication)75 805 y Fn(4.1)59 b(Intro)r(duction)18 b(and)i(Overview)75 906 y Fv(Collectiv)o(e)e(comm)o(unication)g(is)f (de\014ned)h(as)e(comm)o(unication)i(that)e(in)o(v)o(olv)o(es)h(a)f (group)h(of)f(pro)q(cesses.)75 963 y(The)f(functions)h(of)f(this)h(t)o (yp)q(e)f(pro)o(vided)h(b)o(y)f Fw(MPI)g Fv(are)g(the)g(follo)o(wing:) 143 1056 y Fx(\017)23 b Fv(Barrier)15 b(sync)o(hronization)h(across)f (all)h(group)f(mem)o(b)q(ers)g(\(Sec.)20 b(4.3\).)143 1150 y Fx(\017)j Fv(Broadcast)15 b(from)h(one)h(mem)o(b)q(er)g(to)f (all)h(mem)o(b)q(ers)g(of)f(a)h(group)f(\(Sec.)25 b(4.4\).)d(This)c(is) f(sho)o(wn)f(in)189 1207 y(\014gure)f(4.1.)143 1301 y Fx(\017)23 b Fv(Gather)18 b(data)h(from)f(all)j(group)e(mem)o(b)q(ers)g (to)g(one)g(mem)o(b)q(er)g(\(Sec.)33 b(4.5\).)e(This)19 b(is)h(sho)o(wn)f(in)189 1357 y(\014gure)c(4.1.)143 1451 y Fx(\017)23 b Fv(Scatter)16 b(data)g(from)h(one)g(mem)o(b)q(er)g(to)f (all)i(mem)o(b)q(ers)g(of)e(a)h(group)g(\(Sec.)25 b(4.6\).)f(This)18 b(is)f(sho)o(wn)189 1507 y(in)f(\014gure)f(4.1.)143 1601 y Fx(\017)23 b Fv(A)16 b(v)m(ariation)i(on)e(Gather)g(where)h(all)h (mem)o(b)q(ers)f(of)f(the)h(group)f(receiv)o(e)i(the)f(result)g(\(Sec.) 25 b(4.7\).)189 1658 y(This)15 b(is)h(sho)o(wn)f(as)g(\\allgather")g (in)h(\014gure)f(4.1.)143 1751 y Fx(\017)23 b Fv(Scatter/Gather)10 b(data)h(from)h(all)g(mem)o(b)q(ers)g(to)g(all)g(mem)o(b)q(ers)g(of)g (a)g(group)f(\(also)h(called)h(complete)189 1808 y(exc)o(hange)i(or)g (all-to-all\))h(\(Sec.)k(4.8\).)f(This)d(is)g(sho)o(wn)e(as)h (\\alltoall")h(in)g(\014gure)g(4.1.)143 1902 y Fx(\017)23 b Fv(Global)15 b(reduction)h(op)q(erations)g(suc)o(h)f(as)g(sum,)g (max,)f(min,)i(or)e(user-de\014ned)j(functions,)f(where)189 1958 y(the)11 b(result)h(is)g(returned)g(to)e(all)j(group)e(mem)o(b)q (ers)g(and)h(a)f(v)m(ariation)h(where)f(the)h(result)g(is)g(returned) 189 2015 y(to)i(only)i(one)f(mem)o(b)q(er)h(\(Sec.)k(4.9\).)143 2108 y Fx(\017)j Fv(A)15 b(com)o(bined)h(reduction)g(and)g(scatter)e (op)q(eration)h(\(Sec.)21 b(4.10\).)143 2202 y Fx(\017)i Fv(Scan)15 b(across)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group)g(\(also)g (called)h(pre\014x\))g(\(Sec.)k(4.11\).)166 2296 y(A)e(collectiv)o(e)i (op)q(eration)e(is)g(executed)h(b)o(y)f(ha)o(ving)g(all)h(pro)q(cesses) f(in)h(the)f(group)g(call)h(the)f(com-)75 2352 y(m)o(unication)f (routine,)f(with)g(matc)o(hing)g(argumen)o(ts.)21 b(The)16 b(syn)o(tax)f(and)h(seman)o(tics)g(of)f(the)h(collectiv)o(e)75 2409 y(op)q(erations)e(are)g(de\014ned)i(to)e(b)q(e)g(consisten)o(t)h (with)f(the)h(syn)o(tax)e(and)i(seman)o(tics)f(of)g(the)g(p)q(oin)o (t-to-p)q(oin)o(t)75 2465 y(op)q(erations.)22 b(Th)o(us,)16 b(general)g(datat)o(yp)q(es)g(are)f(allo)o(w)o(ed)i(and)f(m)o(ust)f (matc)o(h)g(b)q(et)o(w)o(een)i(sending)g(and)f(re-)75 2522 y(ceiving)i(pro)q(cesses)e(as)g(sp)q(eci\014ed)i(in)f(Chapter)32 b(3.)23 b(One)17 b(of)e(the)i(k)o(ey)f(argumen)o(ts)f(is)h(a)g(comm)o (unicator)75 2578 y(that)f(de\014nes)i(the)f(group)g(of)g (participating)h(pro)q(cesses)f(and)g(pro)o(vides)h(a)e(con)o(text)h (for)f(the)h(op)q(eration.)75 2635 y(Sev)o(eral)f(collectiv)o(e)h (routines)f(suc)o(h)g(as)f(broadcast)g(and)h(gather)e(ha)o(v)o(e)i(a)f (single)i(originating)f(or)f(receiv-)75 2691 y(ing)i(pro)q(cess.)k(Suc) o(h)15 b(pro)q(cesses)h(are)f(called)h(the)f Fl(r)n(o)n(ot)p Fv(.)20 b(Some)15 b(argumen)o(ts)f(in)i(the)f(collectiv)o(e)i (functions)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 93 101 93 100 bop 75 -100 a Fm(4.1.)34 b(INTR)o(ODUCTION)16 b(AND)f(O)o(VER)-5 b(VIEW)890 b Fv(93)75 2444 y @beginspecial @setspecial %%BeginDocument: coll-fig1.ps % Cookbook Example Program from First Printing, Revised 7 Jan 1985 % Program: Drawing Arrows Number: 8 %----------------------------------------------------------------------------- % /arrowdict 13 dict def % Local storage for the procedure % ``arrow.'' /arrow % The procedure ``arrow'' adds an { arrowdict begin % arrow shape to the current path. /headlength exch def % It takes seven arguments: the x /halfheadthickness exch 2 div def % and y coordinates of the tail /halfthickness exch 2 div def % (imagine that a line has been /tipy exch def /tipx exch def % drawn down the center of the /taily exch def /tailx exch def % arrow from the tip to the tail, % then x and y lie on this line), % the x and y coordinates of the % tip of the arrow, the thickness % of the arrow in the tail % portion, the thickness of the % arrow at the widest part of the % arrowhead and the length of the % arrowhead. /dx tipx tailx sub def % Compute the differences in x and /dy tipy taily sub def % y for the tip and tail. These /arrowlength dx dx mul dy dy mul add % will be used to compute the sqrt def % length of the arrow and to /angle dy dx atan def % compute the angle of direction % that the arrow is facing with % respect to the current user % coordinate system origin. /base arrowlength headlength sub def % Compute where the base of the % arrowhead will be. /savematrix matrix currentmatrix def % Save the current user coordinate % system. We are using the same % strategy to localize the effect % of transformations as was used % in the program to draw an % ellipse. tailx taily translate % Translate to the starting point % of the tail. angle rotate % Rotate the x-axis to correspond % with the center line of the % arrow. 0 halfthickness neg moveto % Add the arrow shape to the % current path. base halfthickness neg lineto base halfheadthickness neg lineto arrowlength 0 lineto base halfheadthickness lineto base halfthickness lineto 0 halfthickness lineto closepath savematrix setmatrix % Restore the current user % coordinate system. end } def /Box { /height exch def /length exch def length 0 rlineto 0 height rlineto length neg 0 rlineto closepath } def /Gdict 200 dict def /Grid { Gdict begin /ny exch def /nx exch def /dely exch def /delx exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def 0 1 ny { pop x y moveto lenx 0 rlineto stroke /y y dely add def} for /y ypos def /x xpos def 0 1 nx { pop x y moveto 0 leny rlineto stroke /x x delx add def} for end } def /GLdict 300 dict def /GridLabels { GLdict begin /shift exch def /raise exch def /yoff exch def /xoff exch def /p1 exch def /p2 exch def /ny exch def /nx exch def /dely exch def /delx exch def /Darray exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def /dx3 delx 3 div def /dy3 dely 3 div def /ix -1 def /iy ny 1 sub def Darray{ aload pop /Subc exch def /Text exch def /ix ix 1 add def ix nx ge { /ix 0 def /iy iy 1 sub def} if /x xpos delx ix 0.5 add mul add /Helvetica findfont p1 scalefont setfont Text stringwidth pop /Helvetica findfont p2 scalefont setfont Subc stringwidth pop add xoff add 2 div sub shift add def /y ypos dely iy 0.5 add mul add raise add def x y moveto /Helvetica findfont p1 scalefont Text show xoff yoff rmoveto /Helvetica findfont p2 scalefont Subc show } forall end clear } def 2 setlinecap 6.5 72 mul 320 sub 2 div 0 translate 0 300 moveto 20 20 6 6 Grid 0 300 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 380 200 380 12 24 18 arrow stroke 200 330 140 330 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (scatter) dup stringwidth pop 170 exch 2 div sub 400 moveto show (gather) dup stringwidth pop 170 exch 2 div sub 350 moveto show 220 300 moveto 20 20 6 6 Grid 220 300 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(1)] [()()] [()()] [()()] [()()] [()()] [(A)(2)] [()()] [()()] [()()] [()()] [()()] [(A)(3)] [()()] [()()] [()()] [()()] [()()] [(A)(4)] [()()] [()()] [()()] [()()] [()()] [(A)(5)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 0 moveto 20 20 6 6 Grid 0 0 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [(B)(0)] [(B)(1)] [(B)(2)] [(B)(3)] [(B)(4)] [(B)(5)] [(C)(0)] [(C)(1)] [(C)(2)] [(C)(3)] [(C)(4)] [(C)(5)] [(D)(0)] [(D)(1)] [(D)(2)] [(D)(3)] [(D)(4)] [(D)(5)] [(E)(0)] [(E)(1)] [(E)(2)] [(E)(3)] [(E)(4)] [(E)(5)] [(F)(0)] [(F)(1)] [(F)(2)] [(F)(3)] [(F)(4)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 220 0 moveto 20 20 6 6 Grid 220 0 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(1)] [(B)(1)] [(C)(1)] [(D)(1)] [(E)(1)] [(F)(1)] [(A)(2)] [(B)(2)] [(C)(2)] [(D)(2)] [(E)(2)] [(F)(2)] [(A)(3)] [(B)(3)] [(C)(3)] [(D)(3)] [(E)(3)] [(F)(3)] [(A)(4)] [(B)(4)] [(C)(4)] [(D)(4)] [(E)(4)] [(F)(4)] [(A)(5)] [(B)(5)] [(C)(5)] [(D)(5)] [(E)(5)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 60 200 60 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (alltoall) dup stringwidth pop 170 exch 2 div sub 77 moveto show 0 150 moveto 20 20 6 6 Grid 0 150 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(B)(0)] [()()] [()()] [()()] [()()] [()()] [(C)(0)] [()()] [()()] [()()] [()()] [()()] [(D)(0)] [()()] [()()] [()()] [()()] [()()] [(E)(0)] [()()] [()()] [()()] [()()] [()()] [(F)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 200 200 200 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (allgather) dup stringwidth pop 170 exch 2 div sub 220 moveto show 220 150 moveto 20 20 6 6 Grid 220 150 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 450 moveto 20 20 6 6 Grid 0 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 510 200 510 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont 0 575 moveto (data) show (data) stringwidth pop 4 add 575 4 add (data) stringwidth pop 33 add 575 4 add 1 4 5 arrow fill (broadcast) dup stringwidth pop 170 exch 2 div sub 527 moveto show gsave 0 570 (processes) stringwidth pop sub translate 90 rotate 0 5 moveto (processes) show -4 8 -33 8 1 4 5 arrow fill grestore 220 450 moveto 20 20 6 6 Grid 220 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels showpage %%EndDocument @endspecial 98 x(Figure)15 b(4.1:)k(Collectiv)o(e)e(mo)o(v)o(e)d (functions)i(illustrated)h(for)e(a)f(group)h(of)g(six)h(pro)q(cesses.)k (In)c(eac)o(h)f(case,)75 2598 y(eac)o(h)f(ro)o(w)e(of)i(b)q(o)o(xes)f (represen)o(ts)h(data)f(lo)q(cations)i(in)f(one)g(pro)q(cess.)20 b(Th)o(us,)13 b(in)i(the)e(broadcast,)g(initially)75 2655 y(just)i(the)g(\014rst)g(pro)q(cess)g(con)o(tains)g(the)g(data)g Fp(A)889 2662 y Fg(0)909 2655 y Fv(,)f(but)h(after)g(the)g(broadcast)f (all)i(pro)q(cesses)g(con)o(tain)f(it.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 94 102 94 101 bop 75 -100 a Fv(94)703 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fv(are)i(sp)q(eci\014ed)i (as)e(\\signi\014can)o(t)h(only)f(at)g(ro)q(ot,")f(and)i(are)e(ignored) i(for)f(all)h(participan)o(ts)f(except)h(the)75 102 y(ro)q(ot.)f(The)12 b(reader)g(is)g(referred)g(to)f(Chapter)h(3)g(for)f(information)h (concerning)h(comm)o(unication)f(bu\013ers,)75 158 y(general)19 b(datat)o(yp)q(es)f(and)h(t)o(yp)q(e)g(matc)o(hing)g(rules,)h(and)f(to) f(Chapter)h(5)f(for)h(information)g(on)f(ho)o(w)h(to)75 214 y(de\014ne)d(groups)f(and)h(create)f(comm)o(unicators.)166 271 y(The)d(t)o(yp)q(e-matc)o(hing)h(conditions)g(for)e(the)i (collectiv)o(e)g(op)q(erations)g(are)e(more)h(strict)g(than)g(the)g (cor-)75 327 y(resp)q(onding)j(conditions)g(b)q(et)o(w)o(een)f(sender)h (and)f(receiv)o(er)h(in)f(p)q(oin)o(t-to-p)q(oin)o(t.)20 b(Namely)l(,)15 b(for)e(collectiv)o(e)75 384 y(op)q(erations,)18 b(the)g(amoun)o(t)f(of)g(data)g(sen)o(t)g(m)o(ust)g(exactly)h(matc)o(h) f(the)h(amoun)o(t)f(of)g(data)g(sp)q(eci\014ed)j(b)o(y)75 440 y(the)15 b(receiv)o(er.)20 b(Distinct)c(t)o(yp)q(e)f(maps)f(\(the)h (la)o(y)o(out)f(in)i(memory)l(,)e(see)h(Sec.)20 b(3.12\))13 b(b)q(et)o(w)o(een)i(sender)h(and)75 497 y(receiv)o(er)g(are)f(still)h (allo)o(w)o(ed.)166 553 y(Collectiv)o(e)g(routine)f(calls)h(can)f (\(but)f(are)g(not)h(required)g(to\))f(return)h(as)f(so)q(on)h(as)f (their)h(participa-)75 610 y(tion)d(in)h(the)f(collectiv)o(e)i(comm)o (unication)e(is)h(complete.)19 b(The)12 b(completion)h(of)f(a)f(call)i (indicates)h(that)d(the)75 666 y(caller)k(is)g(no)o(w)f(free)g(to)g (access)g(lo)q(cations)h(in)g(the)g(comm)o(unication)g(bu\013er.)k(It) 14 b(do)q(es)h(not)f(indicate)i(that)75 723 y(other)c(pro)q(cesses)h (in)h(the)f(group)f(ha)o(v)o(e)h(completed)g(or)f(ev)o(en)i(started)d (the)i(op)q(eration)g(\(unless)h(otherwise)75 779 y(indicated)k(in)f (the)f(description)i(of)e(the)g(op)q(eration\).)23 b(Th)o(us,)16 b(a)g(collectiv)o(e)i(comm)o(unication)f(call)h(ma)o(y)l(,)75 835 y(or)c(ma)o(y)h(not,)f(ha)o(v)o(e)h(the)g(e\013ect)f(of)h(sync)o (hronizing)h(all)g(calling)h(pro)q(cesses.)j(This)c(statemen)o(t)e (excludes,)75 892 y(of)h(course,)g(the)g(barrier)g(function.)166 948 y(Collectiv)o(e)24 b(comm)o(unication)f(calls)g(ma)o(y)f(use)g(the) g(same)g(comm)o(unicators)g(as)g(p)q(oin)o(t-to-p)q(oin)o(t)75 1005 y(comm)o(unication;)16 b Fw(MPI)f Fv(guaran)o(tees)g(that)g (messages)g(generated)h(on)g(b)q(ehalf)h(of)e(collectiv)o(e)i(comm)o (uni-)75 1061 y(cation)c(calls)h(will)g(not)f(b)q(e)g(confused)h(with)f (messages)f(generated)h(b)o(y)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication.)75 1118 y(A)h(more)g(detailed)i(discussion)g(of)d(correct)h (use)h(of)e(collectiv)o(e)j(routines)f(is)g(found)f(in)h(Sec.)21 b(4.12.)189 1195 y Fl(R)n(ationale.)50 b Fv(The)18 b(equal-data)f (restriction)g(\(on)g(t)o(yp)q(e)g(matc)o(hing\))g(w)o(as)f(made)h(so)f (as)h(to)f(a)o(v)o(oid)189 1252 y(the)e(complexit)o(y)h(of)f(pro)o (viding)h(a)f(facilit)o(y)h(analogous)f(to)g(the)g(status)f(argumen)o (t)h(of)f Fw(MPI)p 1746 1252 14 2 v 16 w(RECV)189 1308 y Fv(for)f(disco)o(v)o(ering)j(the)e(amoun)o(t)g(of)g(data)f(sen)o(t.) 19 b(Some)14 b(of)f(the)g(collectiv)o(e)j(routines)d(w)o(ould)h (require)189 1365 y(an)h(arra)o(y)f(of)g(status)h(v)m(alues.)189 1436 y(The)e(statemen)o(ts)g(ab)q(out)g(sync)o(hronization)i(are)e (made)h(so)f(as)g(to)g(allo)o(w)h(a)f(v)m(ariet)o(y)h(of)f(implemen-) 189 1492 y(tations)h(of)h(the)g(collectiv)o(e)i(functions.)189 1563 y(The)c(collectiv)o(e)i(op)q(erations)e(do)g(not)g(accept)g(a)g (message)f(tag)g(argumen)o(t.)19 b(If)13 b(future)g(revisions)h(of)189 1619 y Fw(MPI)g Fv(de\014ne)j(non-blo)q(c)o(king)g(collectiv)o(e)g (functions,)f(then)f(tags)g(\(or)f(a)h(similar)i(mec)o(hanism\))e(will) 189 1676 y(need)i(to)g(b)q(e)g(added)h(so)f(as)f(to)h(allo)o(w)g(the)g (dis-am)o(biguation)h(of)e(m)o(ultiple,)j(p)q(ending,)g(collectiv)o(e) 189 1732 y(op)q(erations.)h(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fv(\))189 1810 y Fl(A)n(dvic)n(e)h(to)h(users.)59 b Fv(It)19 b(is)f(dangerous)h(to)e(rely)i(on)f(sync)o(hronization)i (side-e\013ects)f(of)f(the)g(col-)189 1866 y(lectiv)o(e)j(op)q (erations)f(for)f(program)g(correctness.)34 b(F)l(or)20 b(example,)h(ev)o(en)g(though)f(a)f(particular)189 1923 y(implemen)o(tation)i(ma)o(y)e(pro)o(vide)i(a)e(broadcast)g(routine)i (with)f(a)g(side-e\013ect)h(of)e(sync)o(hroniza-)189 1979 y(tion,)14 b(the)g(standard)g(do)q(es)h(not)f(require)h(this,)g (and)f(a)g(program)f(that)h(relies)h(on)g(this)f(will)i(not)e(b)q(e)189 2036 y(p)q(ortable.)189 2107 y(On)d(the)g(other)f(hand,)i(a)f(correct,) g(p)q(ortable)g(program)f(m)o(ust)g(allo)o(w)h(for)f(the)h(fact)g(that) f(a)g(collectiv)o(e)189 2163 y(call)j Fl(may)f Fv(b)q(e)g(sync)o (hronizing.)21 b(Though)11 b(one)h(cannot)g(rely)g(on)g(an)o(y)g(sync)o (hronization)g(side-e\013ect,)189 2220 y(one)17 b(m)o(ust)f(program)f (so)i(as)f(to)g(allo)o(w)h(it.)25 b(These)18 b(issues)f(are)g (discussed)h(further)f(in)g(Sec.)26 b(4.12.)189 2276 y(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fv(\))189 2353 y Fl(A)n(dvic)n(e)21 b(to)h(implementors.)77 b Fv(While)22 b(v)o(endors)g(ma)o(y)e(write)i(optimized)g(collectiv)o(e)i(routines) 189 2410 y(matc)o(hed)c(to)f(their)i(arc)o(hitectures,)g(a)f(complete)h (library)f(of)g(the)g(collectiv)o(e)i(comm)o(unication)189 2466 y(routines)d(can)h(b)q(e)f(written)h(en)o(tirely)g(using)g(the)f Fw(MPI)f Fv(p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)g(func-)189 2523 y(tions)11 b(and)g(a)f(few)h(auxiliary)i(functions.)19 b(If)11 b(implemen)o(ting)i(on)e(top)f(of)h(p)q(oin)o(t-to-p)q(oin)o (t,)h(a)f(hidden,)189 2579 y(sp)q(ecial)j(comm)o(unicator)e(m)o(ust)g (b)q(e)h(created)g(for)f(the)g(collectiv)o(e)i(op)q(eration)f(so)f(as)g (to)g(a)o(v)o(oid)h(in)o(ter-)189 2636 y(ference)j(with)f(an)o(y)g (on-going)h(p)q(oin)o(t-to-p)q(oin)o(t)f(comm)o(unication)h(at)f(the)g (time)h(of)f(the)g(collectiv)o(e)189 2692 y(call.)21 b(This)15 b(is)h(discussed)h(further)e(in)h(Sec.)k(4.12.)f(\()p Fl(End)d(of)g(advic)n(e)g(to)h(implementors.)p Fv(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 95 103 95 102 bop 75 -100 a Fm(4.2.)34 b(COMMUNICA)l(TOR)16 b(AR)o(GUMENT)957 b Fv(95)75 45 y Fn(4.2)59 b(Communicato)n(r)19 b(a)n(rgument)75 147 y Fv(The)c(k)o(ey)h(concept)f(of)g(the)g (collectiv)o(e)i(functions)f(is)g(to)f(ha)o(v)o(e)f(a)h(\\group")g(of)g (participating)h(pro)q(cesses.)75 203 y(The)d(routines)f(do)h(not)e(ha) o(v)o(e)h(a)g(group)g(iden)o(ti\014er)i(as)e(an)g(explicit)j(argumen)o (t.)j(Instead,)13 b(there)f(is)h(a)f(com-)75 259 y(m)o(unicator)k (argumen)o(t.)21 b(F)l(or)15 b(the)h(purp)q(oses)h(of)e(this)i(c)o (hapter,)e(a)h(comm)o(unicator)g(can)g(b)q(e)g(though)o(t)f(of)75 316 y(as)d(a)g(group)g(iden)o(ti\014er)h(link)o(ed)h(with)f(a)f(con)o (text.)18 b(An)12 b(in)o(ter-comm)o(unicator,)h(that)e(is,)i(a)f(comm)o (unicator)75 372 y(that)i(spans)i(t)o(w)o(o)e(groups,)g(is)i Fl(not)f Fv(allo)o(w)o(ed)g(as)g(an)g(argumen)o(t)g(to)f(a)h(collectiv) o(e)i(function.)75 516 y Fn(4.3)59 b(Ba)n(rrier)21 b(synchronization)75 712 y Fw(MPI)p 160 712 14 2 v 16 w(BARRIER\()16 b(comm)d(\))117 789 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))75 913 y Ft(int)23 b(MPI)p 245 913 15 2 v 17 w(Barrier\(MPI)p 526 913 V 16 w(Comm)g(comm)h(\))75 1000 y(MPI)p 150 1000 V 17 w(BARRIER\(COMM,)e(IERROR\))170 1056 y(INTEGER)h(COMM,)h(IERROR)166 1143 y Fw(MPI)p 251 1143 14 2 v 16 w(BARRIER)13 b Fv(blo)q(c)o(ks)g(the)f(caller)i(un)o (til)f(all)g(group)f(mem)o(b)q(ers)h(ha)o(v)o(e)e(called)j(it.)19 b(The)13 b(call)g(returns)75 1199 y(at)i(an)o(y)f(pro)q(cess)i(only)g (after)e(all)i(group)f(mem)o(b)q(ers)g(ha)o(v)o(e)g(en)o(tered)h(the)f (call.)75 1342 y Fn(4.4)59 b(Broadcast)90 1538 y Fw(MPI)p 175 1538 V 16 w(BCAST\()16 b(bu\013er,)f(count,)h(datat)o(yp)q(e,)g(ro) q(ot,)f(comm)e(\))117 1615 y Fk(INOUT)78 b Fw(bu\013er)462 b Fk(starting)14 b(address)h(of)f(bu\013er)g(\(c)o(hoice\))117 1691 y(IN)171 b Fw(count)466 b Fk(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e(bu\013er)i(\(in)o(teger\))117 1766 y(IN)171 b Fw(datat)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f (bu\013er)i(\(handle\))117 1841 y(IN)171 b Fw(ro)q(ot)492 b Fk(rank)14 b(of)f(broadcast)i(ro)q(ot)f(\(in)o(teger\))117 1916 y(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(\(handle\))75 2040 y Ft(int)23 b(MPI)p 245 2040 15 2 v 17 w(Bcast\(void*)g(buffer,)f (int)i(count,)f(MPI)p 1074 2040 V 17 w(Datatype)g(datatype,)f(int)i (root,)393 2097 y(MPI)p 468 2097 V 17 w(Comm)f(comm)h(\))75 2183 y(MPI)p 150 2183 V 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g (ROOT,)g(COMM,)g(IERROR\))170 2240 y()g(BUFFER\(*\))170 2296 y(INTEGER)g(COUNT,)g(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)166 2383 y Fw(MPI)p 251 2383 14 2 v 16 w(BCAST)17 b Fv(broadcasts)e(a)h (message)g(from)g(the)g(pro)q(cess)g(with)h(rank)f Fw(ro)q(ot)g Fv(to)g(all)h(pro)q(cesses)f(of)75 2439 y(the)h(group,)g(itself)h (included.)27 b(It)17 b(is)h(called)g(b)o(y)f(all)h(mem)o(b)q(ers)f(of) g(group)f(using)i(the)f(same)g(argumen)o(ts)75 2496 y(for)d Fw(comm,)f(ro)q(ot)p Fv(.)20 b(On)15 b(return,)g(the)g(con)o(ten)o(ts)f (of)h Fw(ro)q(ot)p Fv('s)f(comm)o(unication)h(bu\013er)g(has)g(b)q(een) h(copied)h(to)75 2552 y(all)f(pro)q(cesses.)166 2609 y(General,)24 b(deriv)o(ed)f(datat)o(yp)q(es)e(are)h(allo)o(w)o(ed)g (for)g Fw(datat)o(yp)q(e)p Fv(.)42 b(The)22 b(t)o(yp)q(e)g(signature)g (of)g Fw(count,)75 2665 y(datat)o(yp)q(e)16 b Fv(on)f(an)o(y)f(pro)q (cess)h(m)o(ust)f(b)q(e)h(equal)g(to)f(the)h(t)o(yp)q(e)g(signature)g (of)f Fw(count,)h(datat)o(yp)q(e)i Fv(at)c(the)i(ro)q(ot.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 96 104 96 103 bop 75 -100 a Fv(96)703 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fv(This)g(implies)i(that)d (the)h(amoun)o(t)e(of)i(data)f(sen)o(t)g(m)o(ust)g(b)q(e)h(equal)h(to)e (the)h(amoun)o(t)e(receiv)o(ed,)j(pairwise)75 102 y(b)q(et)o(w)o(een)i (eac)o(h)h(pro)q(cess)f(and)g(the)g(ro)q(ot.)31 b Fw(MPI)p 909 102 14 2 v 16 w(BCAST)20 b Fv(and)f(all)h(other)f(data-mo)o(v)o (emen)o(t)e(collectiv)o(e)75 158 y(routines)j(mak)o(e)g(this)g (restriction.)34 b(Distinct)21 b(t)o(yp)q(e)f(maps)g(b)q(et)o(w)o(een)g (sender)g(and)g(receiv)o(er)h(are)f(still)75 214 y(allo)o(w)o(ed.)75 336 y Fj(4.4.1)49 b(Example)17 b(using)g(MPI)p 622 336 15 2 v 18 w(BCAST)75 422 y Fq(Example)h(4.1)k Fv(Broadcast)15 b(100)f(in)o(ts)h(from)g(pro)q(cess)g Ft(0)g Fv(to)g(ev)o(ery)g(pro)q (cess)g(in)i(the)e(group.)170 528 y Ft(MPI_Comm)23 b(comm;)170 585 y(int)h(array[100];)170 641 y(int)g(root=0;)170 698 y(...)170 754 y(MPI_Bcast\()f(array,)g(100,)g(MPI_INT,)g(root,)g (comm\);)75 860 y Fv(As)15 b(in)h(man)o(y)f(of)g(our)g(example)h(co)q (de)g(fragmen)o(ts,)e(w)o(e)h(assume)g(that)g(some)g(of)g(the)g(v)m (ariables)i(\(suc)o(h)e(as)75 917 y Ft(comm)g Fv(in)h(the)f(ab)q(o)o(v) o(e\))f(ha)o(v)o(e)h(b)q(een)i(assigned)e(appropriate)h(v)m(alues.)75 1060 y Fn(4.5)59 b(Gather)75 1256 y Fw(MPI)p 160 1256 14 2 v 16 w(GA)l(THER\()16 b(sendbuf,)h(sendcount,)h(sendt)o(yp)q(e,)f (recvbuf,)f(recvcount,)g(recvt)o(yp)q(e,)g(ro)q(ot,)e(comm\))117 1333 y Fk(IN)171 b Fw(sendbuf)428 b Fk(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 1408 y(IN)171 b Fw(sendcount)383 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h (send)h(bu\013er)f(\(in)o(teger\))117 1483 y(IN)171 b Fw(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er) g(elemen)o(ts)f(\(handle\))117 1559 y(OUT)124 b Fw(recvbuf)434 b Fk(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i (signi\014can)o(t)d(only)g(at)905 1615 y(ro)q(ot\))117 1690 y(IN)171 b Fw(recvcount)389 b Fk(n)o(um)o(b)q(er)11 b(of)g(elemen)o(ts)g(for)g(an)o(y)g(single)g(receiv)o(e)i(\(in)o (teger,)f(sig-)905 1747 y(ni\014can)o(t)i(only)f(at)h(ro)q(ot\))117 1822 y(IN)171 b Fw(recvt)o(yp)q(e)414 b Fk(data)16 b(t)o(yp)q(e)g(of)f (recv)i(bu\013er)g(elemen)o(ts)f(\(signi\014can)o(t)f(only)g(at)905 1878 y(ro)q(ot\))f(\(handle\))117 1953 y(IN)171 b Fw(ro)q(ot)492 b Fk(rank)14 b(of)f(receiving)i(pro)q(cess)g(\(in)o(teger\))117 2029 y(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(\(handle\))75 2153 y Ft(int)23 b(MPI)p 245 2153 15 2 v 17 w(Gather\(void*)f(sendbuf,) h(int)h(sendcount,)e(MPI)p 1217 2153 V 17 w(Datatype)h(sendtype,)393 2209 y(void*)g(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 2209 V 17 w(Datatype)h(recvtype,)f(int)i(root,)393 2266 y(MPI)p 468 2266 V 17 w(Comm)f(comm\))75 2352 y(MPI)p 150 2352 V 17 w(GATHER\(SENDBUF,)f(SENDCOUNT,)g(SENDTYPE,)h(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 2409 y(ROOT,)g(COMM,)h(IERROR\))170 2465 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 2522 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g (COMM,)g(IERROR)166 2608 y Fv(Eac)o(h)16 b(pro)q(cess)h(\(ro)q(ot)f (pro)q(cess)h(included\))h(sends)g(the)e(con)o(ten)o(ts)g(of)g(its)h (send)h(bu\013er)e(to)g(the)h(ro)q(ot)75 2665 y(pro)q(cess.)i(The)11 b(ro)q(ot)f(pro)q(cess)i(receiv)o(es)g(the)f(messages)f(and)i(stores)e (them)h(in)h(rank)f(order.)18 b(The)11 b(outcome)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 97 105 97 104 bop 75 -100 a Fm(4.5.)34 b(GA)l(THER)1452 b Fv(97)75 45 y(is)14 b Fl(as)h(if)f Fv(eac)o(h)g(of)f(the)h Ft(n)g Fv(pro)q(cesses)g(in)h(the)f(group)f(\(including)k(the)d(ro)q(ot)f(pro) q(cess\))g(had)h(executed)h(a)f(call)75 102 y(to)189 195 y Ft(MPI)p 264 195 15 2 v 16 w(Send)p Fv(\()p Ft(sendbuf)p Fp(;)8 b Ft(sendcount)p Fp(;)f Ft(se)o(ndtype)p Fp(;)g Ft(roo)o(t)p Fp(;)h(:)o(::)o Fv(\))p Fp(;)75 289 y Fv(and)15 b(the)h(ro)q(ot)e(had)h(executed)i Ft(n)e Fv(calls)h(to)189 383 y Ft(MPI)p 264 383 V 16 w(Recv)p Fv(\()p Ft(recvbuf)9 b Fv(+)h Ft(i)g Fx(\001)g Ft(recvcount)f Fx(\001)g Ft(extent)p Fv(\()p Ft(recvtype)p Fv(\))p Fp(;)f Ft(recvc)o(ount)p Fp(;)f Ft(recv)o(type)p Fp(;)g Ft(i)p Fp(;)g(::)o(:)p Fv(\))p Fp(;)75 477 y Fv(where)15 b Ft(extent\(recvtype\))e Fv(is)j(the)f(t)o(yp)q(e)h(exten)o(t)f(obtained)g(from)g(a)g(call)h(to) f Ft(MPI)p 1515 477 V 16 w(Type)p 1627 477 V 17 w(extent\(\))p Fv(.)166 533 y(An)j(alternativ)o(e)h(description)h(is)e(that)g(the)g Ft(n)g Fv(messages)g(sen)o(t)g(b)o(y)g(the)g(pro)q(cesses)h(in)g(the)f (group)75 590 y(are)c(concatenated)h(in)g(rank)f(order,)g(and)h(the)g (resulting)g(message)f(is)h(receiv)o(ed)h(b)o(y)e(the)h(ro)q(ot)f(as)g (if)h(b)o(y)f(a)75 646 y(call)i(to)f Fw(MPI)p 299 646 14 2 v 16 w(RECV\(recvbuf,)h(recvcount)p Fx(\001)p Fw(n,)g(recvt)o(yp)q (e,)g(...\))p Fv(.)166 703 y(The)f(receiv)o(e)h(bu\013er)g(is)f (ignored)h(for)f(all)h(non-ro)q(ot)f(pro)q(cesses.)166 759 y(General,)i(deriv)o(ed)h(datat)o(yp)q(es)e(are)g(allo)o(w)o(ed)h (for)f(b)q(oth)h Fw(sendt)o(yp)q(e)i Fv(and)e Fw(recvt)o(yp)q(e)p Fv(.)25 b(The)17 b(t)o(yp)q(e)g(sig-)75 816 y(nature)c(of)g Fw(sendcount,)j(sendt)o(yp)q(e)f Fv(on)e(pro)q(cess)h Ft(i)e Fv(m)o(ust)h(b)q(e)h(equal)g(to)e(the)h(t)o(yp)q(e)g(signature)h (of)e Fw(recvcount,)75 872 y(recvt)o(yp)q(e)h Fv(at)e(the)i(ro)q(ot.)18 b(This)12 b(implies)i(that)e(the)g(amoun)o(t)f(of)h(data)f(sen)o(t)h(m) o(ust)g(b)q(e)g(equal)h(to)f(the)g(amoun)o(t)75 928 y(of)j(data)g (receiv)o(ed,)i(pairwise)f(b)q(et)o(w)o(een)g(eac)o(h)g(pro)q(cess)g (and)g(the)g(ro)q(ot.)k(Distinct)c(t)o(yp)q(e)g(maps)g(b)q(et)o(w)o (een)75 985 y(sender)g(and)f(receiv)o(er)h(are)f(still)i(allo)o(w)o (ed.)166 1041 y(All)e(argumen)o(ts)f(to)f(the)h(function)h(are)f (signi\014can)o(t)h(on)f(pro)q(cess)h Fw(ro)q(ot)p Fv(,)e(while)j(on)e (other)g(pro)q(cesses,)75 1098 y(only)19 b(argumen)o(ts)f Fw(sendbuf,)k(sendcount,)g(sendt)o(yp)q(e,)g(ro)q(ot,)d(comm)e Fv(are)i(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 1154 y Fw(ro)q(ot)c Fv(and)g Fw(comm)f Fv(m)o(ust)g(ha)o(v)o(e)h(iden)o (tical)i(v)m(alues)f(on)f(all)i(pro)q(cesses.)166 1211 y(The)f(sp)q(eci\014cation)h(of)f(coun)o(ts)f(and)h(t)o(yp)q(es)f (should)i(not)e(cause)h(an)o(y)g(lo)q(cation)g(on)g(the)f(ro)q(ot)g(to) g(b)q(e)75 1267 y(written)g(more)g(than)g(once.)20 b(Suc)o(h)c(a)f (call)h(is)g(erroneous.)166 1324 y(Note)e(that)f(the)h Fw(recvcount)i Fv(argumen)o(t)d(at)h(the)g(ro)q(ot)f(indicates)j(the)e (n)o(um)o(b)q(er)h(of)e(items)i(it)f(receiv)o(es)75 1380 y(from)g Fl(e)n(ach)i Fv(pro)q(cess,)f(not)f(the)i(total)e(n)o(um)o(b)q (er)i(of)f(items)g(it)h(receiv)o(es.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 98 106 98 105 bop 75 -100 a Fv(98)703 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(GA)l(THERV\()24 b(sendbuf,)j(sendcount,)g(sendt)o(yp)q (e,)g(recvbuf,)e(recvcounts,)h(displs,)g(recvt)o(yp)q(e,)g(ro)q(ot,)75 102 y(comm\))117 179 y Fk(IN)171 b Fw(sendbuf)428 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 254 y(IN)171 b Fw(sendcount)383 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 329 y(IN)171 b Fw(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f (send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 404 y(OUT)124 b Fw(recvbuf)434 b Fk(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o (hoice,)i(signi\014can)o(t)d(only)g(at)905 461 y(ro)q(ot\))117 536 y(IN)171 b Fw(recvcounts)372 b Fk(in)o(teger)10 b(arra)o(y)f(\(of)g (length)g(group)g(size\))h(con)o(taining)e(the)i(n)o(um-)905 592 y(b)q(er)20 b(of)e(elemen)o(ts)h(that)f(are)i(receiv)o(ed)g(from)d (eac)o(h)i(pro)q(cess)905 649 y(\(signi\014can)o(t)14 b(only)f(at)h(ro)q(ot\))117 724 y(IN)171 b Fw(displs)469 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Fk(sp)q(eci\014es)905 780 y(the)h(displacemen)o (t)e(relativ)o(e)g(to)h Fi(recvbuf)i Fk(at)d(whic)o(h)h(to)g(place)905 837 y(the)h(incoming)e(data)h(from)f(pro)q(cess)k Fb(i)d Fk(\(signi\014can)o(t)g(only)g(at)905 893 y(ro)q(ot\))117 968 y(IN)171 b Fw(recvt)o(yp)q(e)414 b Fk(data)16 b(t)o(yp)q(e)g(of)f (recv)i(bu\013er)g(elemen)o(ts)f(\(signi\014can)o(t)f(only)g(at)905 1025 y(ro)q(ot\))f(\(handle\))117 1100 y(IN)171 b Fw(ro)q(ot)492 b Fk(rank)14 b(of)f(receiving)i(pro)q(cess)g(\(in)o(teger\))117 1175 y(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(\(handle\))75 1299 y Ft(int)23 b(MPI)p 245 1299 15 2 v 17 w(Gatherv\(void*)f (sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 1299 V 17 w(Datatype)h (sendtype,)393 1356 y(void*)g(recvbuf,)g(int)h(*recvcounts,)e(int)h (*displs,)393 1412 y(MPI)p 468 1412 V 17 w(Datatype)g(recvtype,)f(int)i (root,)f(MPI)p 1249 1412 V 17 w(Comm)g(comm\))75 1499 y(MPI)p 150 1499 V 17 w(GATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)g (RECVBUF,)f(RECVCOUNTS,)h(DISPLS,)393 1555 y(RECVTYPE,)g(ROOT,)g(COMM,) g(IERROR\))170 1612 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1668 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g (DISPLS\(*\),)h(RECVTYPE,)g(ROOT,)170 1725 y(COMM,)h(IERROR)166 1811 y Fw(MPI)p 251 1811 14 2 v 16 w(GA)l(THERV)15 b Fv(extends)g(the)f(functionalit)o(y)h(of)e Fw(MPI)p 1126 1811 V 16 w(GA)l(THER)i Fv(b)o(y)f(allo)o(wing)h(a)f(v)m(arying)g(coun) o(t)75 1868 y(of)i(data)g(from)g(eac)o(h)h(pro)q(cess,)g(since)h Fw(recvcounts)g Fv(is)f(no)o(w)f(an)h(arra)o(y)l(.)23 b(It)17 b(also)f(allo)o(ws)h(more)f(\015exibili)q(t)o(y)75 1924 y(as)f(to)f(where)i(the)f(data)g(is)g(placed)i(on)e(the)g(ro)q (ot,)f(b)o(y)h(pro)o(viding)h(the)g(new)f(argumen)o(t,)f Fw(displs)p Fv(.)166 1980 y(The)i(outcome)g(is)h Fl(as)g(if)f Fv(eac)o(h)h(pro)q(cess,)f(including)j(the)e(ro)q(ot)e(pro)q(cess,)i (sends)f(a)g(message)g(to)g(the)75 2037 y(ro)q(ot,)189 2131 y Ft(MPI)p 264 2131 15 2 v 16 w(Send)p Fv(\()p Ft(sendbuf)p Fp(;)8 b Ft(sendcount)p Fp(;)f Ft(se)o(ndtype)p Fp(;)g Ft(roo)o(t)p Fp(;)h(:)o(::)o Fv(\))p Fp(;)75 2225 y Fv(and)15 b(the)h(ro)q(ot)e(executes)i Ft(n)f Fv(receiv)o(es,)189 2318 y Ft(MPI)p 264 2318 V 16 w(Recv)p Fv(\()p Ft(recvbuf)9 b Fv(+)h Ft(disp)p Fv([)p Ft(i)p Fv(])f Fx(\001)h Ft(extent)p Fv(\()p Ft(recvtype)p Fv(\))p Fp(;)d Ft(recvc)o(ounts)p Fv([)o Ft(i)p Fv(])p Fp(;)g Ft(recvt)o(ype)p Fp(;)g Ft(i)p Fp(;)g(:::)o Fv(\))p Fp(:)166 2412 y Fv(Messages)14 b(are)g(placed)i (in)f(the)g(receiv)o(e)g(bu\013er)g(of)f(the)g(ro)q(ot)g(pro)q(cess)h (in)g(rank)f(order,)g(that)g(is,)h(the)75 2469 y(data)e(sen)o(t)g(from) f(pro)q(cess)i Ft(j)f Fv(is)g(placed)i(in)f(the)f Ft(j)p Fv(th)g(p)q(ortion)h(of)f(the)g(receiv)o(e)h(bu\013er)g Fw(recvbuf)f Fv(on)h(pro)q(cess)75 2525 y Fw(ro)q(ot)p Fv(.)19 b(The)13 b Ft(j)p Fv(th)g(p)q(ortion)h(of)f Fw(recvbuf)h Fv(b)q(egins)g(at)f(o\013set)f Fw(displs[j])j Fv(elemen)o(ts)f(\(in)g (terms)e(of)h Fw(recvt)o(yp)q(e)p Fv(\))h(in)o(to)75 2582 y Fw(recvbuf)p Fv(.)166 2638 y(The)h(receiv)o(e)h(bu\013er)g(is)f (ignored)h(for)f(all)h(non-ro)q(ot)f(pro)q(cesses.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 99 107 99 106 bop 75 -100 a Fm(4.5.)29 b(GA)l(THER)1457 b Fv(99)458 4 y 16577003 6465030 0 0 33943388 13419479 startTexFig 458 4 a %%BeginDocument: mycoll-fig2.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 369.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth % Polyline n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr % Polyline n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr % Polyline n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr % Polyline n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr % Polyline n 89 244 m 119 294 l gs 0.50 setgray fill gr gs col-1 s gr n 116.599 286.111 m 119.000 294.000 l 113.169 288.169 l gs 2 setlinejoin col-1 s gr % Polyline n 189 244 m 189 294 l gs 0.50 setgray fill gr gs col-1 s gr n 191.000 286.000 m 189.000 294.000 l 187.000 286.000 l gs 2 setlinejoin col-1 s gr % Polyline n 289 244 m 259 294 l gs 0.50 setgray fill gr gs col-1 s gr n 264.831 288.169 m 259.000 294.000 l 261.401 286.111 l gs 2 setlinejoin col-1 s gr % Polyline n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 409 m gs 1 -1 scale (rbuf) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr $F2psEnd %%EndDocument endTexFig 186 511 a Fv(Figure)16 b(4.2:)j(The)c(ro)q(ot)g(pro)q(cess)g(gathers)f (100)h Ft(int)p Fv(s)f(from)h(eac)o(h)g(pro)q(cess)h(in)g(the)f(group.) 166 644 y(The)i(t)o(yp)q(e)f(signature)h(implied)i(b)o(y)d Fw(sendcount,)j(sendt)o(yp)q(e)g Fv(on)e(pro)q(cess)g Ft(i)f Fv(m)o(ust)g(b)q(e)h(equal)g(to)f(the)75 701 y(t)o(yp)q(e)f (signature)h(implied)i(b)o(y)d Fw(recvcounts[i],)i(recvt)o(yp)q(e)f Fv(at)f(the)g(ro)q(ot.)20 b(This)c(implies)i(that)c(the)i(amoun)o(t)75 757 y(of)g(data)f(sen)o(t)h(m)o(ust)g(b)q(e)h(equal)g(to)e(the)i(amoun) o(t)e(of)h(data)f(receiv)o(ed,)j(pairwise)f(b)q(et)o(w)o(een)f(eac)o(h) h(pro)q(cess)75 814 y(and)12 b(the)g(ro)q(ot.)17 b(Distinct)c(t)o(yp)q (e)f(maps)f(b)q(et)o(w)o(een)h(sender)g(and)g(receiv)o(er)g(are)g (still)h(allo)o(w)o(ed,)f(as)g(illustrated)75 870 y(in)k(Example)g (4.6.)166 927 y(All)f(argumen)o(ts)f(to)f(the)h(function)h(are)f (signi\014can)o(t)h(on)f(pro)q(cess)h Fw(ro)q(ot)p Fv(,)e(while)j(on)e (other)g(pro)q(cesses,)75 983 y(only)19 b(argumen)o(ts)f Fw(sendbuf,)k(sendcount,)g(sendt)o(yp)q(e,)g(ro)q(ot,)d(comm)e Fv(are)i(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 1040 y Fw(ro)q(ot)c Fv(and)g Fw(comm)f Fv(m)o(ust)g(ha)o(v)o(e)h(iden)o (tical)i(v)m(alues)f(on)f(all)i(pro)q(cesses.)166 1096 y(The)d(sp)q(eci\014cation)i(of)d(coun)o(ts,)h(t)o(yp)q(es,)g(and)g (displacemen)o(ts)h(should)g(not)f(cause)g(an)o(y)f(lo)q(cation)i(on)75 1153 y(the)g(ro)q(ot)g(to)f(b)q(e)i(written)f(more)g(than)g(once.)20 b(Suc)o(h)c(a)f(call)h(is)g(erroneous.)75 1274 y Fj(4.5.1)49 b(Examples)16 b(using)h(MPI)p 640 1274 15 2 v 18 w(GA)l(THER,)g(MPI)p 963 1274 V 18 w(GA)l(THERV)75 1360 y Fq(Example)h(4.2)k Fv(Gather)15 b(100)f(in)o(ts)i(from)e(ev)o(ery)h(pro)q(cess)h(in)g (group)e(to)h(ro)q(ot.)k(See)d(\014gure)f(4.2.)170 1466 y Ft(MPI_Comm)23 b(comm;)170 1523 y(int)h(gsize,sendarray[100];)170 1579 y(int)g(root,)f(*rbuf;)170 1636 y(...)170 1692 y(MPI_Comm_size\()f (comm,)i(&gsize\);)170 1749 y(rbuf)g(=)f(\(int)h (*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 1805 y(MPI_Gather\()f (sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(100,)f(MPI_INT,)g(root,)g (comm\);)75 1911 y Fq(Example)18 b(4.3)k Fv(Previous)15 b(example)g(mo)q(di\014ed)g({)f(only)h(the)f(ro)q(ot)f(allo)q(cates)i (memory)e(for)h(the)g(receiv)o(e)75 1968 y(bu\013er.)170 2074 y Ft(MPI_Comm)23 b(comm;)170 2130 y(int)h(gsize,sendarray[100];) 170 2187 y(int)g(root,)f(myrank,)g(*rbuf;)170 2243 y(...)170 2300 y(MPI_Comm_rank\()f(comm,)i(myrank\);)170 2356 y(if)g(\()g(myrank) f(==)g(root\))h({)242 2413 y(MPI_Comm_size\()e(comm,)h(&gsize\);)242 2469 y(rbuf)g(=)h(\(int)f(*\)malloc\(gsize*100*sizeof\(int)o(\)\);)242 2526 y(})170 2582 y(MPI_Gather\()g(sendarray,)f(100,)i(MPI_INT,)e (rbuf,)i(100,)f(MPI_INT,)g(root,)g(comm\);)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 100 108 100 107 bop 75 -100 a Fv(100)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fq(Example)i(4.4)k Fv(Do)d(the)h(same)f(as)g(the)h(previous)g(example,)h(but)f(use)g(a)f (deriv)o(ed)i(datat)o(yp)q(e.)32 b(Note)75 102 y(that)17 b(the)h(t)o(yp)q(e)g(cannot)g(b)q(e)h(the)f(en)o(tire)g(set)g(of)g Ft(gsize*100)k(int)p Fv(s)c(since)h(t)o(yp)q(e)f(matc)o(hing)g(is)h (de\014ned)75 158 y(pairwise)d(b)q(et)o(w)o(een)g(the)f(ro)q(ot)f(and)i (eac)o(h)f(pro)q(cess)g(in)h(the)g(gather.)170 264 y Ft(MPI_Comm)23 b(comm;)170 320 y(int)h(gsize,sendarray[100];)170 376 y(int)g(root,)f(*rbuf;)170 433 y(MPI_Datatype)g(rtype;)170 489 y(...)170 546 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 602 y(MPI_Type_contiguous\()e(100,)h(MPI_INT,)g(&rtype)g(\);)170 659 y(MPI_Type_commit\()f(&rtype)h(\);)170 715 y(rbuf)h(=)f(\(int)h (*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 772 y(MPI_Gather\()f (sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(1,)f(rtype,)g(root,)g(comm\);) 75 877 y Fq(Example)18 b(4.5)k Fv(No)o(w)d(ha)o(v)o(e)g(eac)o(h)g(pro)q (cess)h(send)f(100)g(in)o(ts)g(to)g(ro)q(ot,)g(but)g(place)h(eac)o(h)g (set)f(\(of)f(100\))75 934 y Fl(stride)f Fv(in)o(ts)f(apart)g(at)g (receiving)i(end.)25 b(Use)17 b Fw(MPI)p 945 934 14 2 v 16 w(GA)l(THERV)h Fv(and)f(the)f Fw(displs)j Fv(argumen)o(t)c(to)h (ac)o(hiev)o(e)75 990 y(this)g(e\013ect.)j(Assume)d Fp(str)q(ide)c Fx(\025)h Fv(100.)19 b(See)d(\014gure)f(4.3.)170 1096 y Ft(MPI_Comm)23 b(comm;)170 1152 y(int)h(gsize,sendarray[100];)170 1209 y(int)g(root,)f(*rbuf,)g(stride;)170 1265 y(int)h (*displs,i,*rcounts;)170 1378 y(...)170 1491 y(MPI_Comm_size\()e(comm,) i(&gsize\);)170 1547 y(rbuf)g(=)f(\(int)h(*\)malloc\(gsize*stride*siz)o (eof\(int)o(\)\);)170 1604 y(displs)f(=)h(\(int)g (*\)malloc\(gsize*sizeof\(i)o(nt\)\);)170 1660 y(rcounts)f(=)h(\(int)f (*\)malloc\(gsize*sizeof\(int\)\);)170 1717 y(for)h(\(i=0;)f(i)23 b(SENDBUF\(*\),)g(RECVBUF\(*\))170 102 y(INTEGER)g(SENDCOUNT,)g (SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR)166 188 y Fw(MPI)p 251 188 14 2 v 16 w(SCA)l(TTER)16 b Fv(is)g(the)f(in)o (v)o(erse)h(op)q(eration)f(to)g Fw(MPI)p 1097 188 V 15 w(GA)l(THER)p Fv(.)166 244 y(The)g(outcome)g(is)h Fl(as)g(if)f Fv(the)g(ro)q(ot)g(executed)h Ft(n)f Fv(send)h(op)q(erations,)189 320 y Ft(MPI)p 264 320 15 2 v 16 w(Send)p Fv(\()p Ft(sendbuf)9 b Fv(+)h Ft(i)g Fx(\001)g Ft(sendcount)f Fx(\001)g Ft(extent)p Fv(\()p Ft(sendtype)p Fv(\))p Fp(;)f Ft(sendc)o(ount)p Fp(;)f Ft(send)o(type)p Fp(;)g Ft(i)p Fp(;)g(::)o(:)p Fv(\))p Fp(;)75 395 y Fv(and)15 b(eac)o(h)h(pro)q(cess)f(executed)h(a)f (receiv)o(e,)189 471 y Ft(MPI)p 264 471 V 16 w(Recv)p Fv(\()p Ft(recvbuf)p Fp(;)8 b Ft(recvcount)p Fp(;)f Ft(re)o(cvtype)p Fp(;)g Ft(i)p Fp(;)g(::)o(:)p Fv(\))o Fp(:)166 546 y Fv(An)20 b(alternativ)o(e)g(description)h(is)g(that)e(the)h(ro)q(ot)f (sends)h(a)f(message)h(with)g Fw(MPI)p 1594 546 14 2 v 16 w(Send\(sendbuf,)75 602 y(sendcount)p Fx(\001)p Fw(n,)f(sendt)o(yp)q(e,)h(...\))p Fv(.)i(This)17 b(message)f(is)h (split)h(in)o(to)e Ft(n)g Fv(equal)i(segmen)o(ts,)e(the)g Fp(i)p Fv(th)g(segmen)o(t)g(is)75 659 y(sen)o(t)f(to)g(the)g Fp(i)p Fv(th)g(pro)q(cess)g(in)h(the)f(group,)g(and)g(eac)o(h)g(pro)q (cess)h(receiv)o(es)g(this)g(message)e(as)h(ab)q(o)o(v)o(e.)166 715 y(The)g(send)h(bu\013er)f(is)h(ignored)g(for)e(all)i(non-ro)q(ot)f (pro)q(cesses.)166 772 y(The)j(t)o(yp)q(e)f(signature)h(asso)q(ciated)g (with)f Fw(sendcount,)k(sendt)o(yp)q(e)f Fv(at)d(the)h(ro)q(ot)e(m)o (ust)h(b)q(e)h(equal)h(to)75 828 y(the)f(t)o(yp)q(e)g(signature)g(asso) q(ciated)g(with)g Fw(recvcount,)h(recvt)o(yp)q(e)g Fv(at)e(all)i(pro)q (cesses)f(\(ho)o(w)o(ev)o(er,)f(the)h(t)o(yp)q(e)75 885 y(maps)f(ma)o(y)g(b)q(e)h(di\013eren)o(t\).)27 b(This)18 b(implies)i(that)d(the)g(amoun)o(t)g(of)g(data)g(sen)o(t)g(m)o(ust)g(b) q(e)i(equal)f(to)f(the)75 941 y(amoun)o(t)f(of)f(data)h(receiv)o(ed,)i (pairwise)f(b)q(et)o(w)o(een)f(eac)o(h)h(pro)q(cess)f(and)h(the)f(ro)q (ot.)23 b(Distinct)17 b(t)o(yp)q(e)f(maps)75 998 y(b)q(et)o(w)o(een)g (sender)f(and)h(receiv)o(er)g(are)e(still)j(allo)o(w)o(ed.)166 1054 y(All)e(argumen)o(ts)f(to)f(the)h(function)h(are)f(signi\014can)o (t)h(on)f(pro)q(cess)h Fw(ro)q(ot)p Fv(,)e(while)j(on)e(other)g(pro)q (cesses,)75 1110 y(only)f(argumen)o(ts)f Fw(recvbuf,)i(recvcount,)g (recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)f Fv(are)h(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fw(ro)q(ot)75 1167 y Fv(and)j Fw(comm)f Fv(m)o(ust)g(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)g(on)e (all)h(pro)q(cesses.)166 1223 y(The)g(sp)q(eci\014cation)h(of)f(coun)o (ts)f(and)h(t)o(yp)q(es)f(should)i(not)e(cause)h(an)o(y)g(lo)q(cation)g (on)g(the)f(ro)q(ot)g(to)g(b)q(e)75 1280 y(read)g(more)g(than)g(once.) 189 1360 y Fl(R)n(ationale.)79 b Fv(Though)22 b(not)g(needed,)i(the)e (last)g(restriction)g(is)g(imp)q(osed)h(so)f(as)f(to)g(ac)o(hiev)o(e) 189 1417 y(symmetry)16 b(with)h Fw(MPI)p 593 1417 V 16 w(GA)l(THER)p Fv(,)h(where)f(the)g(corresp)q(onding)h(restriction)g (\(a)e(m)o(ultiple-write)189 1473 y(restriction\))f(is)h(necessary)l(.) k(\()p Fl(End)c(of)g(r)n(ationale.)p Fv(\))75 1648 y Fw(MPI)p 160 1648 V 16 w(SCA)l(TTERV\()21 b(sendbuf,)j(sendcounts)q(,)g (displs,)f(sendt)o(yp)q(e,)h(recvbuf,)f(recvcount,)f(recvt)o(yp)q(e,)h (ro)q(ot,)75 1705 y(comm\))117 1782 y Fk(IN)171 b Fw(sendbuf)428 b Fk(address)13 b(of)d(send)i(bu\013er)h(\(c)o(hoice,)f(signi\014can)o (t)e(only)h(at)g(ro)q(ot\))117 1853 y(IN)171 b Fw(sendcounts)366 b Fk(in)o(teger)10 b(arra)o(y)f(\(of)g(length)g(group)g(size\))h(sp)q (ecifying)g(the)g(n)o(um-)905 1910 y(b)q(er)15 b(of)e(elemen)o(ts)h(to) g(send)h(to)e(eac)o(h)i(pro)q(cessor)117 1981 y(IN)171 b Fw(displs)469 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g (size\).)20 b(En)o(try)15 b Fb(i)f Fk(sp)q(eci\014es)905 2038 y(the)20 b(displacemen)o(t)e(\(relativ)o(e)h(to)f Fi(sendbuf)j Fk(from)c(whic)o(h)i(to)905 2094 y(tak)o(e)14 b(the)h(outgoing)d(data)i(to)g(pro)q(cess)h Fb(i)117 2165 y Fk(IN)171 b Fw(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 2237 y(OUT)124 b Fw(recvbuf)434 b Fk(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 2308 y(IN)171 b Fw(recvcount)389 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 2380 y(IN)171 b Fw(recvt)o(yp)q(e)414 b Fk(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e (\(handle\))117 2451 y(IN)171 b Fw(ro)q(ot)492 b Fk(rank)14 b(of)f(sending)h(pro)q(cess)i(\(in)o(teger\))117 2523 y(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(\(handle\))75 2647 y Ft(int)23 b(MPI)p 245 2647 15 2 v 17 w(Scatterv\(void*)f (sendbuf,)h(int)g(*sendcounts,)g(int)g(*displs,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)g(sendtype,)f(void*)i(recvbuf,)e(int)i (recvcount,)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 107 115 107 114 bop 75 -100 a Fm(4.6.)29 b(SCA)l(TTER)1413 b Fv(107)393 45 y Ft(MPI)p 468 45 15 2 v 17 w(Datatype)23 b(recvtype,)f(int)i(root,)f(MPI)p 1249 45 V 17 w(Comm)g(comm\))75 132 y(MPI)p 150 132 V 17 w(SCATTERV\(SENDBUF,)e(SENDCOUNTS,)i(DISPLS,)g (SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)393 188 y(RECVTYPE,)g(ROOT,)g(COMM,) g(IERROR\))170 244 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 301 y(INTEGER)g(SENDCOUNTS\(*\),)f(DISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNT,)h(RECVTYPE,)g(ROOT,)170 357 y(COMM,)h(IERROR)166 444 y Fw(MPI)p 251 444 14 2 v 16 w(SCA)l(TTERV)16 b Fv(is)g(the)f(in)o (v)o(erse)h(op)q(eration)f(to)g Fw(MPI)p 1127 444 V 16 w(GA)l(THERV)p Fv(.)166 500 y Fw(MPI)p 251 500 V 16 w(SCA)l(TTERV)24 b Fv(extends)f(the)g(functionalit)o(y)g(of)g Fw(MPI)p 1193 500 V 15 w(SCA)l(TTER)h Fv(b)o(y)f(allo)o(wing)g(a)f(v)m(arying)75 557 y(coun)o(t)17 b(of)h(data)f(to)g(b)q(e)h(sen)o(t)f(to)g(eac)o(h)h (pro)q(cess,)g(since)h Fw(sendcounts)i Fv(is)d(no)o(w)f(an)g(arra)o(y)l (.)27 b(It)17 b(also)h(allo)o(ws)75 613 y(more)k(\015exibilit)o(y)i(as) e(to)f(where)h(the)g(data)g(is)g(tak)o(en)g(from)f(on)h(the)g(ro)q(ot,) h(b)o(y)f(pro)o(viding)h(the)f(new)75 670 y(argumen)o(t,)14 b Fw(displs)p Fv(.)166 726 y(The)h(outcome)g(is)h(as)f(if)g(the)h(ro)q (ot)e(executed)i Ft(n)f Fv(send)h(op)q(erations,)189 820 y Ft(MPI)p 264 820 15 2 v 16 w(Send)p Fv(\()p Ft(sendbuf)9 b Fv(+)h Ft(displs)p Fv([)p Ft(i)p Fv(])f Fx(\001)g Ft(extent)p Fv(\()p Ft(sendtype)p Fv(\))p Fp(;)f Ft(sen)o(dcounts)p Fv([)o Ft(i)p Fv(])p Fp(;)f Ft(sen)o(dtype)p Fp(;)g Ft(i)p Fp(;)g(::)o(:)p Fv(\))p Fp(;)75 914 y Fv(and)15 b(eac)o(h)h(pro)q(cess) f(executed)h(a)f(receiv)o(e,)189 1008 y Ft(MPI)p 264 1008 V 16 w(Recv)p Fv(\()p Ft(recvbuf)p Fp(;)8 b Ft(recvcount)p Fp(;)f Ft(re)o(cvtype)p Fp(;)g Ft(i)p Fp(;)g(::)o(:)p Fv(\))o Fp(:)166 1101 y Fv(The)15 b(send)h(bu\013er)f(is)h(ignored)g (for)e(all)i(non-ro)q(ot)f(pro)q(cesses.)166 1158 y(The)h(t)o(yp)q(e)g (signature)g(implied)i(b)o(y)e Fw(sendcount[i],)j(sendt)o(yp)q(e)g Fv(at)c(the)h(ro)q(ot)f(m)o(ust)g(b)q(e)i(equal)g(to)e(the)75 1214 y(t)o(yp)q(e)f(signature)h(implied)i(b)o(y)d Fw(recvcount,)i (recvt)o(yp)q(e)f Fv(at)f(pro)q(cess)g Ft(i)g Fv(\(ho)o(w)o(ev)o(er,)f (the)i(t)o(yp)q(e)f(maps)h(ma)o(y)e(b)q(e)75 1271 y(di\013eren)o(t\).) 20 b(This)c(implies)h(that)e(the)g(amoun)o(t)f(of)h(data)g(sen)o(t)g(m) o(ust)f(b)q(e)i(equal)g(to)f(the)g(amoun)o(t)g(of)g(data)75 1327 y(receiv)o(ed,)j(pairwise)f(b)q(et)o(w)o(een)g(eac)o(h)g(pro)q (cess)g(and)g(the)g(ro)q(ot.)23 b(Distinct)17 b(t)o(yp)q(e)g(maps)f(b)q (et)o(w)o(een)h(sender)75 1384 y(and)e(receiv)o(er)h(are)f(still)i (allo)o(w)o(ed.)166 1440 y(All)e(argumen)o(ts)f(to)f(the)h(function)h (are)f(signi\014can)o(t)h(on)f(pro)q(cess)h Fw(ro)q(ot)p Fv(,)e(while)j(on)e(other)g(pro)q(cesses,)75 1497 y(only)f(argumen)o (ts)f Fw(recvbuf,)i(recvcount,)g(recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)f Fv(are)h(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fw(ro)q(ot)75 1553 y Fv(and)j Fw(comm)f Fv(m)o(ust)g(ha)o(v)o(e)h(iden) o(tical)i(v)m(alues)g(on)e(all)h(pro)q(cesses.)166 1609 y(The)e(sp)q(eci\014cation)i(of)d(coun)o(ts,)h(t)o(yp)q(es,)g(and)g (displacemen)o(ts)h(should)g(not)f(cause)g(an)o(y)f(lo)q(cation)i(on)75 1666 y(the)g(ro)q(ot)g(to)f(b)q(e)i(read)f(more)g(than)g(once.)75 1788 y Fj(4.6.1)49 b(Examples)16 b(using)h(MPI)p 640 1788 V 18 w(SCA)l(TTER,)h(MPI)p 988 1788 V 18 w(SCA)l(TTERV)75 1873 y Fq(Example)g(4.11)23 b Fv(The)15 b(rev)o(erse)g(of)f(Example)i (4.2.)j(Scatter)14 b(sets)h(of)g(100)f(in)o(ts)h(from)f(the)h(ro)q(ot)f (to)h(eac)o(h)75 1930 y(pro)q(cess)g(in)h(the)g(group.)j(See)d (\014gure)g(4.7.)170 2036 y Ft(MPI_Comm)23 b(comm;)170 2093 y(int)h(gsize,*sendbuf;)170 2149 y(int)g(root,)f(rbuf[100];)170 2206 y(...)170 2262 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 2318 y(sendbuf)f(=)h(\(int)f(*\)malloc\(gsize*100*sizeof\(int)o(\)\);) 170 2375 y(...)170 2431 y(MPI_Scatter\()g(sendbuf,)f(100,)i(MPI_INT,)f (rbuf,)g(100,)g(MPI_INT,)g(root,)g(comm\);)75 2587 y Fq(Example)18 b(4.12)23 b Fv(The)16 b(rev)o(erse)h(of)f(Example)i(4.5.) 23 b(The)17 b(ro)q(ot)f(pro)q(cess)h(scatters)f(sets)g(of)h(100)f(in)o (ts)h(to)75 2644 y(the)d(other)g(pro)q(cesses,)g(but)g(the)g(sets)g(of) g(100)f(are)h Fl(stride)f Fv(in)o(ts)i(apart)e(in)i(the)f(sending)h (bu\013er.)20 b(Requires)75 2700 y(use)c(of)e Fw(MPI)p 290 2700 14 2 v 16 w(SCA)l(TTERV)p Fv(.)i(Assume)g Fp(str)q(ide)c Fx(\025)h Fv(100.)19 b(See)d(\014gure)g(4.8.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 108 116 108 115 bop 75 -100 a Fv(108)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)458 4 y 16577003 6630800 0 0 33943388 13682606 startTexFig 458 4 a %%BeginDocument: mycoll-fig7.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 373.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth % Polyline n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr % Polyline n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr % Polyline n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr % Polyline n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 414 m gs 1 -1 scale (sendbuf) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr % Polyline n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr % Polyline n 119 299 m 89 244 l gs col-1 s gr n 91.075 251.981 m 89.000 244.000 l 94.587 250.065 l gs 2 setlinejoin col-1 s gr % Polyline n 189 299 m 189 244 l gs col-1 s gr n 187.000 252.000 m 189.000 244.000 l 191.000 252.000 l gs 2 setlinejoin col-1 s gr % Polyline n 249 299 m 289 244 l gs col-1 s gr n 282.677 249.294 m 289.000 244.000 l 285.912 251.646 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr $F2psEnd %%EndDocument endTexFig 138 522 a Fv(Figure)g(4.7:)i(The)e(ro)q(ot)e(pro)q(cess)i(scatters)e (sets)h(of)g(100)f Ft(int)p Fv(s)h(to)f(eac)o(h)i(pro)q(cess)f(in)h (the)f(group.)458 580 y 16577003 6796570 0 0 33943388 14011514 startTexFig 458 580 a %%BeginDocument: mycoll-fig8.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 378.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth % Polyline n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr % Polyline n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr % Polyline n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 239 359 m 239 319 l 179 319 l 179 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 194 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 319 359 m 319 319 l 259 319 l 259 359 l clp gs 0.50 setgray fill gr gs col-1 s gr % Polyline n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 314 m gs 1 -1 scale (100) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 99 419 m gs 1 -1 scale (sendbuf) col-1 show gr % Polyline n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr % Polyline n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 187.000 371.000 m 179.000 369.000 l 187.000 367.000 l gs 2 setlinejoin col-1 s gr % Polyline n 179 369 m 259 369 l gs col-1 s gr n 251.000 367.000 m 259.000 369.000 l 251.000 371.000 l gs 2 setlinejoin col-1 s gr % Polyline n 119 299 m 84 244 l gs col-1 s gr n 86.608 251.823 m 84.000 244.000 l 89.982 249.676 l gs 2 setlinejoin col-1 s gr % Polyline n 204 304 m 189 244 l gs col-1 s gr n 189.000 252.246 m 189.000 244.000 l 192.881 251.276 l gs 2 setlinejoin col-1 s gr % Polyline n 284 304 m 289 244 l gs col-1 s gr n 286.343 251.806 m 289.000 244.000 l 290.329 252.138 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 199 394 m gs 1 -1 scale (stride) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 1109 a Fv(Figure)g(4.8:)j(The)d(ro)q(ot)e(pro)q(cess)i(scatters)f (sets)g(of)g(100)f Ft(int)p Fv(s,)h(mo)o(ving)g(b)o(y)h Ft(stride)e Fv(in)o(ts)i(from)f(send)h(to)75 1165 y(send)h(in)g(the)f (scatter.)170 1290 y Ft(MPI_Comm)23 b(comm;)170 1346 y(int)h(gsize,*sendbuf;)170 1403 y(int)g(root,)f(rbuf[100],)g(i,)g (*displs,)g(*scounts;)170 1515 y(...)170 1628 y(MPI_Comm_size\()f (comm,)i(&gsize\);)170 1685 y(sendbuf)f(=)h(\(int)f (*\)malloc\(gsize*stride*sizeof\()o(int\)\);)170 1741 y(...)170 1798 y(displs)g(=)h(\(int)g(*\)malloc\(gsize*sizeof\(i)o (nt\)\);)170 1854 y(scounts)f(=)h(\(int)f (*\)malloc\(gsize*sizeof\(int\)\);)170 1911 y(for)h(\(i=0;)f(i)g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1319 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)166 1405 y Fw(MPI)p 251 1405 14 2 v 16 w(ALLGA)l(THER)16 b Fv(can)g(b)q(e)h(though)o(t)e(of)g(as)h Fw(MPI)p 1047 1405 V 15 w(GA)l(THER)p Fv(,)h(but)f(where)g(all)h(pro)q(cesses)f (receiv)o(e)75 1462 y(the)e(result,)f(instead)i(of)e(just)g(the)g(ro)q (ot.)19 b(The)13 b Ft(j)p Fv(th)h(blo)q(c)o(k)g(of)f(data)g(sen)o(t)g (from)g(eac)o(h)g(pro)q(cess)h(is)g(receiv)o(ed)75 1518 y(b)o(y)h(ev)o(ery)g(pro)q(cess)h(and)f(placed)h(in)g(the)g Ft(j)p Fv(th)f(blo)q(c)o(k)h(of)e(the)i(bu\013er)f Fw(recvbuf)p Fv(.)166 1575 y(The)g(t)o(yp)q(e)g(signature)g(asso)q(ciated)g(with)h Fw(sendcount,)h(sendt)o(yp)q(e)p Fv(,)h(at)c(a)h(pro)q(cess)g(m)o(ust)f (b)q(e)i(equal)g(to)75 1631 y(the)f(t)o(yp)q(e)h(signature)f(asso)q (ciated)g(with)h Fw(recvcount,)g(recvt)o(yp)q(e)g Fv(at)f(an)o(y)f (other)h(pro)q(cess.)166 1687 y(The)f(outcome)g(of)g(a)g(call)h(to)f Fw(MPI)p 749 1687 V 16 w(ALLGA)l(THER\(...\))19 b Fv(is)c(as)f(if)g (all)h(pro)q(cesses)g(executed)g Ft(n)f Fv(calls)i(to)147 1781 y Ft(MPI_GATHER\(sendbuf,send)o(count,s)o(endtype,)o(recvbuf)o (,recvco)o(unt,)1244 1838 y(recvtype,root,comm\),)75 1932 y Fv(for)11 b Ft(root)23 b(=)h(0)g(,)f(...,)h(n-1)p Fv(.)18 b(The)12 b(rules)g(for)f(correct)g(usage)g(of)h Fw(MPI)p 1289 1932 V 15 w(ALLGA)l(THER)h Fv(are)e(easily)i(found)75 1988 y(from)h(the)i(corresp)q(onding)g(rules)g(for)e Fw(MPI)p 816 1988 V 16 w(GA)l(THER)p Fv(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 111 119 111 118 bop 75 -100 a Fm(4.7.)29 b(GA)l(THER-TO-ALL)1246 b Fv(111)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(ALLGA)l(THERV\()11 b(sendbuf,)i(sendcount,)h(sendt)o(yp)q(e,)f(recvbuf,)f(recvcounts,)g (displs,)g(recvt)o(yp)q(e,)g(comm\))117 179 y Fk(IN)171 b Fw(sendbuf)428 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h (\(c)o(hoice\))117 254 y(IN)171 b Fw(sendcount)383 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in) o(teger\))117 329 y(IN)171 b Fw(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 404 y(OUT)124 b Fw(recvbuf)434 b Fk(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 479 y(IN)171 b Fw(recvcounts)372 b Fk(in)o(teger)10 b(arra)o(y)f(\(of)g (length)g(group)g(size\))h(con)o(taining)e(the)i(n)o(um-)905 536 y(b)q(er)15 b(of)e(elemen)o(ts)h(that)g(are)g(receiv)o(ed)h(from)e (eac)o(h)h(pro)q(cess)117 611 y(IN)171 b Fw(displs)469 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(i)f Fk(sp)q(eci\014es)905 667 y(the)d(displacemen)o (t)e(\(relativ)o(e)g(to)h Fi(recvbuf)p Fk(\))i(at)d(whic)o(h)h(to)g (place)905 724 y(the)15 b(incoming)c(data)j(from)e(pro)q(cess)k Fb(i)117 799 y Fk(IN)171 b Fw(recvt)o(yp)q(e)414 b Fk(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e(\(handle\)) 117 874 y(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(\(handle\))75 999 y Ft(int)23 b(MPI)p 245 999 15 2 v 17 w(Allgatherv\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 999 V 16 w(Datatype)g(sendtype,)393 1055 y(void*)g(recvbuf,)g(int) h(*recvcounts,)e(int)h(*displs,)393 1111 y(MPI)p 468 1111 V 17 w(Datatype)g(recvtype,)f(MPI)p 1010 1111 V 17 w(Comm)h(comm\))75 1198 y(MPI)p 150 1198 V 17 w (ALLGATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h (RECVCOUNTS,)f(DISPLS,)393 1254 y(RECVTYPE,)h(COMM,)g(IERROR\))170 1311 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1367 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g(DISPLS\(*\),)h (RECVTYPE,)g(COMM,)170 1424 y(IERROR)166 1510 y Fw(MPI)p 251 1510 14 2 v 16 w(ALLGA)l(THERV)11 b Fv(can)g(b)q(e)g(though)o(t)e (of)h(as)g Fw(MPI)p 1044 1510 V 16 w(GA)l(THERV)p Fv(,)h(but)g(where)f (all)h(pro)q(cesses)g(receiv)o(e)75 1567 y(the)j(result,)f(instead)i (of)e(just)g(the)g(ro)q(ot.)19 b(The)13 b Ft(j)p Fv(th)h(blo)q(c)o(k)g (of)f(data)g(sen)o(t)g(from)g(eac)o(h)g(pro)q(cess)h(is)g(receiv)o(ed) 75 1623 y(b)o(y)i(ev)o(ery)h(pro)q(cess)g(and)f(placed)i(in)f(the)g Ft(j)p Fv(th)f(blo)q(c)o(k)h(of)f(the)h(bu\013er)f Fw(recvbuf)p Fv(.)25 b(These)17 b(blo)q(c)o(ks)g(need)g(not)75 1680 y(all)f(b)q(e)g(the)f(same)g(size.)166 1736 y(The)g(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)g Fw(sendcount,)j(sendt)o(yp)q(e)p Fv(,)f(at)d(pro)q(cess)h Ft(j)g Fv(m)o(ust)f(b)q(e)i(equal)g(to)75 1792 y(the)f(t)o(yp)q(e)h(signature)f(asso)q(ciated)g(with)h Fw(recvcounts[j],)h(recvt)o(yp)q(e)f Fv(at)e(an)o(y)h(other)g(pro)q (cess.)166 1849 y(The)g(outcome)g(is)h(as)f(if)g(all)h(pro)q(cesses)g (executed)g(calls)g(to)170 1943 y Ft(MPI_GATHERV\(sendbuf,sendcount)o (,sendtyp)o(e,recvb)o(uf,recv)o(counts,d)o(ispls,)1292 1999 y(recvtype,root,comm\),)75 2093 y Fv(for)k Ft(root)j(=)h(0)f(,)h (...,)f(n-1)p Fv(.)35 b(The)20 b(rules)h(for)e(correct)h(usage)g(of)g Fw(MPI)p 1365 2093 V 16 w(ALLGA)l(THERV)h Fv(are)f(easily)75 2149 y(found)c(from)e(the)h(corresp)q(onding)h(rules)g(for)f Fw(MPI)p 944 2149 V 16 w(GA)l(THERV)p Fv(.)75 2271 y Fj(4.7.1)49 b(Examples)16 b(using)h(MPI)p 640 2271 15 2 v 18 w(ALLGA)l(THER,)h(MPI)p 1048 2271 V 18 w(ALLGA)l(THERV)75 2357 y Fq(Example)g(4.14)23 b Fv(The)18 b(all-gather)h(v)o(ersion)g(of) f(Example)i(4.2.)29 b(Using)19 b Fw(MPI)p 1434 2357 14 2 v 16 w(ALLGA)l(THER)p Fv(,)g(w)o(e)f(will)75 2413 y(gather)d(100)f (in)o(ts)h(from)g(ev)o(ery)g(pro)q(cess)g(in)h(the)g(group)e(to)h(ev)o (ery)g(pro)q(cess.)170 2520 y Ft(MPI_Comm)23 b(comm;)170 2576 y(int)h(gsize,sendarray[100];)170 2633 y(int)g(*rbuf;)170 2689 y(...)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 112 120 112 119 bop 75 -100 a Fv(112)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)170 45 y Ft(MPI_Comm_size\()22 b(comm,)i(&gsize\);)170 102 y(rbuf)g(=)f(\(int)h (*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 158 y(MPI_Allgather\()e (sendarray,)h(100,)g(MPI_INT,)g(rbuf,)g(100,)g(MPI_INT,)g(comm\);)166 264 y Fv(After)15 b(the)g(call,)h(ev)o(ery)f(pro)q(cess)h(has)f(the)g (group-wide)h(concatenation)f(of)g(the)g(sets)g(of)g(data.)75 407 y Fn(4.8)59 b(All-to-All)20 b(Scatter/Gather)75 603 y Fw(MPI)p 160 603 14 2 v 16 w(ALL)l(TO)o(ALL\(sendbuf,)c(sendcount,)i (sendt)o(yp)q(e,)f(recvbuf,)f(recvcount,)g(recvt)o(yp)q(e,)g(comm\))117 681 y Fk(IN)171 b Fw(sendbuf)428 b Fk(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 756 y(IN)171 b Fw(sendcount)383 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(to)f(eac)o(h)g (pro)q(cess)i(\(in)o(teger\))117 831 y(IN)171 b Fw(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 906 y(OUT)124 b Fw(recvbuf)434 b Fk(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 981 y(IN)171 b Fw(recvcount)389 b Fk(n)o(um)o(b)q(er)17 b(of)f(elemen)o(ts) h(receiv)o(ed)i(from)c(an)o(y)i(pro)q(cess)i(\(in)o(te-)905 1038 y(ger\))117 1113 y(IN)171 b Fw(recvt)o(yp)q(e)414 b Fk(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e (\(handle\))117 1188 y(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(\(handle\))75 1312 y Ft(int)23 b(MPI)p 245 1312 15 2 v 17 w(Alltoall\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1312 V 17 w(Datatype)f(sendtype,)393 1369 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1369 V 17 w(Datatype)h(recvtype,)393 1425 y(MPI)p 468 1425 V 17 w(Comm)g(comm\))75 1512 y(MPI)p 150 1512 V 17 w(ALLTOALL\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,) h(RECVCOUNT,)g(RECVTYPE,)393 1568 y(COMM,)g(IERROR\))170 1625 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1681 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)166 1768 y Fw(MPI)p 251 1768 14 2 v 16 w(ALL)l(TO)o(ALL)18 b Fv(is)h(an)g(extension)h(of)e Fw(MPI)p 962 1768 V 16 w(ALLGA)l(THER)i Fv(to)e(the)h(case)g(where)g(eac)o(h)g(pro)q(cess)75 1824 y(sends)e(distinct)g(data)f(to)g(eac)o(h)g(of)g(the)g(receiv)o (ers.)24 b(The)17 b Ft(j)p Fv(th)f(blo)q(c)o(k)h(sen)o(t)f(from)f(pro)q (cess)i Ft(i)f Fv(is)h(receiv)o(ed)75 1880 y(b)o(y)e(pro)q(cess)h Ft(j)f Fv(and)g(is)h(placed)g(in)g(the)f Ft(i)p Fv(th)g(blo)q(c)o(k)h (of)f Fw(recvbuf)p Fv(.)166 1937 y(The)g(t)o(yp)q(e)g(signature)g(asso) q(ciated)g(with)h Fw(sendcount,)h(sendt)o(yp)q(e)p Fv(,)h(at)c(a)h(pro) q(cess)g(m)o(ust)f(b)q(e)i(equal)g(to)75 1993 y(the)i(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)g Fw(recvcount,)i(recvt)o(yp)q(e)f Fv(at)e(an)o(y)h(other)f(pro)q(cess.)28 b(This)19 b(implies)75 2050 y(that)10 b(the)h(amoun)o(t)g(of)f(data)h(sen)o(t)f(m)o(ust)h(b)q (e)h(equal)f(to)g(the)g(amoun)o(t)f(of)h(data)f(receiv)o(ed,)j (pairwise)f(b)q(et)o(w)o(een)75 2106 y(ev)o(ery)j(pair)h(of)e(pro)q (cesses.)21 b(As)15 b(usual,)g(ho)o(w)o(ev)o(er,)f(the)i(t)o(yp)q(e)f (maps)g(ma)o(y)f(b)q(e)i(di\013eren)o(t.)166 2163 y(The)d(outcome)g(is) h(as)e(if)i(eac)o(h)f(pro)q(cess)g(executed)h(a)f(send)h(to)e(eac)o(h)h (pro)q(cess)h(\(itself)f(included\))j(with)75 2219 y(a)f(call)h(to,)189 2313 y Ft(MPI)p 264 2313 15 2 v 16 w(Send)p Fv(\()p Ft(sendbuf)9 b Fv(+)h Ft(i)g Fx(\001)g Ft(sendcount)f Fx(\001)g Ft(extent)p Fv(\()p Ft(sendtype)p Fv(\))p Fp(;)f Ft(sendc)o(ount)p Fp(;)f Ft(send)o(type)p Fp(;)g Ft(i)p Fp(;)g(::)o(:)p Fv(\))p Fp(;)75 2407 y Fv(and)15 b(a)g(receiv)o(e)h(from)f(ev)o(ery)g (other)g(pro)q(cess)g(with)h(a)f(call)h(to,)189 2501 y Ft(MPI)p 264 2501 V 16 w(Recv)p Fv(\()p Ft(recvbuf)9 b Fv(+)h Ft(i)g Fx(\001)g Ft(recvcount)f Fx(\001)g Ft(extent)p Fv(\()p Ft(recvtype)p Fv(\))p Fp(;)f Ft(recvc)o(ount)p Fp(;)f Ft(i)p Fp(;)g(::)o(:)p Fv(\))p Fp(:)166 2594 y Fv(All)14 b(argumen)o(ts)d(on)h(all)i(pro)q(cesses)f(are)f (signi\014can)o(t.)20 b(The)12 b(argumen)o(t)g Fw(comm)e Fv(m)o(ust)i(ha)o(v)o(e)g(iden)o(tical)75 2651 y(v)m(alues)k(on)f(all)i (pro)q(cesses.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 113 121 113 120 bop 75 -100 a Fm(4.8.)34 b(ALL-TO-ALL)18 b(SCA)l(TTER/GA)l (THER)892 b Fv(113)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V\(sendbuf,)13 b(sendcounts)q(,)h(sdispls,)g (sendt)o(yp)q(e,)g(recvbuf,)e(recvcounts,)i(rdispls,)e(recvt)o(yp)q(e,) 75 102 y(comm\))117 179 y Fk(IN)171 b Fw(sendbuf)428 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 252 y(IN)171 b Fw(sendcounts)366 b Fk(in)o(teger)20 b(arra)o(y)f(equal) h(to)f(the)h(group)g(size)g(sp)q(ecifying)f(the)905 309 y(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(to)h(send)h(to)f(eac)o(h)g (pro)q(cessor)117 382 y(IN)171 b Fw(sdispls)452 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fb(j)f Fk(sp)q(eci\014es)905 439 y(the)20 b(displacemen)o(t)e (\(relativ)o(e)h(to)f Fi(sendbuf)j Fk(from)c(whic)o(h)i(to)905 495 y(tak)o(e)14 b(the)h(outgoing)d(data)i(destined)h(for)e(pro)q(cess) j Fb(j)117 568 y Fk(IN)171 b Fw(sendt)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 642 y(OUT)124 b Fw(recvbuf)434 b Fk(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 715 y(IN)171 b Fw(recvcounts)372 b Fk(in)o(teger)20 b(arra)o(y)f(equal)h (to)f(the)h(group)g(size)g(sp)q(ecifying)f(the)905 772 y(n)o(um)o(b)q(er)g(of)f(elemen)o(ts)h(that)g(can)h(b)q(e)g(receiv)o (ed)g(from)d(eac)o(h)905 828 y(pro)q(cessor)117 902 y(IN)171 b Fw(rdispls)453 b Fk(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group) g(size\).)20 b(En)o(try)15 b Fb(i)f Fk(sp)q(eci\014es)905 958 y(the)f(displacemen)o(t)e(\(relativ)o(e)g(to)h Fi(recvbuf)i Fk(at)d(whic)o(h)h(to)g(place)905 1015 y(the)j(incoming)c(data)j(from)e (pro)q(cess)k Fb(i)117 1088 y Fk(IN)171 b Fw(recvt)o(yp)q(e)414 b Fk(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e (\(handle\))117 1162 y(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(\(handle\))75 1286 y Ft(int)23 b(MPI)p 245 1286 15 2 v 17 w(Alltoallv\(void*)f(sendbuf,)h(int)g(*sendcounts,)f(int)i (*sdispls,)393 1342 y(MPI)p 468 1342 V 17 w(Datatype)f(sendtype,)f (void*)i(recvbuf,)e(int)i(*recvcounts,)393 1399 y(int)g(*rdispls,)e (MPI)p 802 1399 V 17 w(Datatype)h(recvtype,)f(MPI)p 1344 1399 V 17 w(Comm)i(comm\))75 1485 y(MPI)p 150 1485 V 17 w(ALLTOALLV\(SENDBUF,)d(SENDCOUNTS,)i(SDISPLS,)f(SENDTYPE,)h (RECVBUF,)g(RECVCOUNTS,)393 1542 y(RDISPLS,)g(RECVTYPE,)g(COMM,)g (IERROR\))170 1598 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1655 y(INTEGER)g(SENDCOUNTS\(*\),)f(SDISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNTS\(*\),)g(RDISPLS\(*\),)170 1711 y(RECVTYPE,)h(COMM,)g (IERROR)166 1798 y Fw(MPI)p 251 1798 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V)12 b Fv(adds)g(\015exibili)q(t)o(y)j(to)c Fw(MPI)p 959 1798 V 16 w(ALL)l(TO)o(ALL)h Fv(in)h(that)f(the)h(lo)q(cation)g(of) f(data)g(for)g(the)75 1854 y(send)j(is)f(sp)q(eci\014ed)j(b)o(y)d Fw(sdispls)i Fv(and)e(the)g(lo)q(cation)h(of)f(the)g(placemen)o(t)h(of) f(the)g(data)g(on)g(the)g(receiv)o(e)h(side)75 1911 y(is)h(sp)q (eci\014ed)h(b)o(y)e Fw(rdispls)p Fv(.)166 1967 y(The)k Ft(j)p Fv(th)f(blo)q(c)o(k)i(sen)o(t)e(from)g(pro)q(cess)h Ft(i)g Fv(is)g(receiv)o(ed)h(b)o(y)e(pro)q(cess)h Ft(j)g Fv(and)g(is)g(placed)h(in)f(the)g Ft(i)p Fv(th)75 2023 y(blo)q(c)o(k)d(of)f Fw(recvbuf)p Fv(.)20 b(These)c(blo)q(c)o(ks)g (need)g(not)f(all)h(ha)o(v)o(e)f(the)g(same)g(size.)166 2080 y(The)i(t)o(yp)q(e)h(signature)f(asso)q(ciated)h(with)f Fw(sendcount[)q(j],)j(sendt)o(yp)q(e)g Fv(at)d(pro)q(cess)g Ft(i)g Fv(m)o(ust)g(b)q(e)h(equal)75 2136 y(to)d(the)i(t)o(yp)q(e)f (signature)g(asso)q(ciated)g(with)h Fw(recvcount[i],)g(recvt)o(yp)q(e)g Fv(at)e(pro)q(cess)i Ft(j)p Fv(.)22 b(This)17 b(implies)h(that)75 2193 y(the)f(amoun)o(t)g(of)g(data)g(sen)o(t)g(m)o(ust)f(b)q(e)i(equal) g(to)f(the)h(amoun)o(t)e(of)h(data)g(receiv)o(ed,)h(pairwise)h(b)q(et)o (w)o(een)75 2249 y(ev)o(ery)c(pair)h(of)e(pro)q(cesses.)21 b(Distinct)16 b(t)o(yp)q(e)f(maps)g(b)q(et)o(w)o(een)g(sender)h(and)f (receiv)o(er)h(are)f(still)i(allo)o(w)o(ed.)166 2306 y(The)e(outcome)g(is)h(as)f(if)g(eac)o(h)h(pro)q(cess)f(sen)o(t)g(a)g (message)g(to)f(ev)o(ery)h(other)g(pro)q(cess)h(with,)189 2391 y Ft(MPI)p 264 2391 15 2 v 16 w(Send)p Fv(\()p Ft(sendbuf)9 b Fv(+)h Ft(displs)p Fv([)p Ft(i)p Fv(])f Fx(\001)g Ft(extent)p Fv(\()p Ft(sendtype)p Fv(\))p Fp(;)f Ft(sen)o(dcounts)p Fv([)o Ft(i)p Fv(])p Fp(;)f Ft(sen)o(dtype)p Fp(;)g Ft(i)p Fp(;)g(::)o(:)p Fv(\))p Fp(;)75 2476 y Fv(and)15 b(receiv)o(ed)i(a)e (message)f(from)h(ev)o(ery)g(other)g(pro)q(cess)g(with)h(a)f(call)h(to) 189 2562 y Ft(MPI)p 264 2562 V 16 w(Recv)p Fv(\()p Ft(recvbuf)9 b Fv(+)h Ft(displs)p Fv([)p Ft(i)p Fv(])f Fx(\001)g Ft(extent)p Fv(\()p Ft(recvtype)p Fv(\))p Fp(;)f Ft(rec)o(vcounts)p Fv([)o Ft(i)p Fv(])p Fp(;)f Ft(rec)o(vtype)p Fp(;)g Ft(i)p Fp(;)g(::)o(:)p Fv(\))p Fp(:)166 2647 y Fv(All)14 b(argumen)o(ts)d(on)h (all)i(pro)q(cesses)f(are)f(signi\014can)o(t.)20 b(The)12 b(argumen)o(t)g Fw(comm)e Fv(m)o(ust)i(ha)o(v)o(e)g(iden)o(tical)75 2704 y(v)m(alues)k(on)f(all)i(pro)q(cesses.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 114 122 114 121 bop 75 -100 a Fv(114)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)189 45 y Fl(R)n(ationale.)60 b Fv(The)19 b(de\014nitions)h(of)e Fw(MPI)p 911 45 14 2 v 16 w(ALL)l(TO)o(ALL)g Fv(and)h Fw(MPI)p 1338 45 V 16 w(ALL)l(TO)o(ALL)-5 b(V)18 b Fv(giv)o(e)h(as)f(m)o(uc)o(h)189 102 y(\015exibilit)o(y)e(as)d(one)h(w)o(ould)g(ac)o(hiev)o(e)g(b)o(y)f (sp)q(ecifying)j Ft(n)d Fv(indep)q(enden)o(t,)j(p)q(oin)o(t-to-p)q(oin) o(t)e(comm)o(uni-)189 158 y(cations,)i(with)h(t)o(w)o(o)f(exceptions:) 23 b(all)18 b(messages)e(use)h(the)g(same)f(datat)o(yp)q(e,)g(and)h (messages)f(are)189 214 y(scattered)e(from)h(\(or)f(gathered)h(to\))f (sequen)o(tial)j(storage.)h(\()p Fl(End)e(of)g(r)n(ationale.)p Fv(\))189 318 y Fl(A)n(dvic)n(e)k(to)h(implementors.)70 b Fv(Although)21 b(the)f(discussion)i(of)e(collectiv)o(e)i(comm)o (unication)f(in)189 375 y(terms)f(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(op) q(eration)g(implies)i(that)d(eac)o(h)g(message)g(is)i(transferred)e (directly)189 431 y(from)i(sender)h(to)f(receiv)o(er,)j(implemen)o (tations)f(ma)o(y)e(use)h(a)g(tree)f(comm)o(unication)i(pattern.)189 488 y(Messages)12 b(can)h(b)q(e)h(forw)o(arded)e(b)o(y)h(in)o (termediate)h(no)q(des)f(where)h(they)f(are)f(split)j(\(for)d (scatter\))f(or)189 544 y(concatenated)k(\(for)f(gather\),)g(if)i(this) f(is)h(more)f(e\016cien)o(t.)20 b(\()p Fl(End)c(of)g(advic)n(e)g(to)h (implementors.)p Fv(\))75 687 y Fn(4.9)59 b(Global)20 b(Reduction)e(Op)r(erations)75 788 y Fv(The)e(functions)g(in)g(this)g (section)g(p)q(erform)f(a)g(global)h(reduce)g(op)q(eration)g(\(suc)o(h) f(as)g(sum,)g(max,)g(logical)75 845 y(AND,)d(etc.\))19 b(across)12 b(all)i(the)e(mem)o(b)q(ers)h(of)f(a)h(group.)18 b(The)13 b(reduction)h(op)q(eration)f(can)g(b)q(e)g(either)h(one)e(of) 75 901 y(a)j(prede\014ned)i(list)f(of)f(op)q(erations,)g(or)g(a)g (user-de\014ned)j(op)q(eration.)i(The)c(global)g(reduction)g(functions) 75 958 y(come)h(in)h(sev)o(eral)g(\015a)o(v)o(ors:)23 b(a)17 b(reduce)h(that)f(returns)g(the)g(result)h(of)f(the)g(reduction) i(at)d(one)i(no)q(de,)g(an)75 1014 y(all-reduce)i(that)e(returns)g (this)g(result)h(at)e(all)i(no)q(des,)g(and)g(a)f(scan)g(\(parallel)h (pre\014x\))f(op)q(eration.)29 b(In)75 1071 y(addition,)15 b(a)f(reduce-scatter)g(op)q(eration)h(com)o(bines)g(the)f(functionalit) o(y)h(of)f(a)g(reduce)h(and)f(of)g(a)g(scatter)75 1127 y(op)q(eration.)75 1248 y Fj(4.9.1)49 b(Reduce)75 1429 y Fw(MPI)p 160 1429 V 16 w(REDUCE\()15 b(sendbuf,)j(recvbuf,)d(count,)h (datat)o(yp)q(e,)h(op,)e(ro)q(ot,)f(comm\))117 1506 y Fk(IN)171 b Fw(sendbuf)428 b Fk(address)15 b(of)f(send)g(bu\013er)h (\(c)o(hoice\))117 1581 y(OUT)124 b Fw(recvbuf)434 b Fk(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i (signi\014can)o(t)d(only)g(at)905 1637 y(ro)q(ot\))117 1712 y(IN)171 b Fw(count)466 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1787 y(IN)171 b Fw(datat)o(yp)q(e)408 b Fk(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(send)i(bu\013er)g(\(handle\))117 1861 y(IN)171 b Fw(op)525 b Fk(reduce)16 b(op)q(eration)e(\(handle\)) 117 1936 y(IN)171 b Fw(ro)q(ot)492 b Fk(rank)14 b(of)f(ro)q(ot)h(pro)q (cess)i(\(in)o(teger\))117 2011 y(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(\(handle\))75 2135 y Ft(int)23 b(MPI)p 245 2135 15 2 v 17 w(Reduce\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)h(count,)393 2192 y(MPI)p 468 2192 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2192 V 17 w(Op)i(op,)f(int)h (root,)f(MPI)p 1505 2192 V 17 w(Comm)g(comm\))75 2278 y(MPI)p 150 2278 V 17 w(REDUCE\(SENDBUF,)f(RECVBUF,)g(COUNT,)h (DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR\))170 2335 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2391 y(INTEGER)g(COUNT,)g (DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR)166 2478 y Fw(MPI)p 251 2478 14 2 v 16 w(REDUCE)c Fv(com)o(bines)g(the)g(elemen)o (ts)g(pro)o(vided)g(in)g(the)g(input)g(bu\013er)g(of)f(eac)o(h)g(pro)q (cess)h(in)75 2534 y(the)d(group,)g(using)h(the)f(op)q(eration)g Fw(op)p Fv(,)g(and)h(returns)f(the)g(com)o(bined)h(v)m(alue)g(in)g(the) f(output)g(bu\013er)g(of)75 2591 y(the)j(pro)q(cess)f(with)h(rank)f Fw(ro)q(ot)p Fv(.)30 b(The)19 b(input)g(bu\013er)g(is)g(de\014ned)h(b)o (y)e(the)h(argumen)o(ts)e Fw(sendbuf)p Fv(,)22 b Fw(count)75 2647 y Fv(and)d Fw(datat)o(yp)q(e)p Fv(;)i(the)d(output)h(bu\013er)f (is)h(de\014ned)h(b)o(y)e(the)g(argumen)o(ts)g Fw(recvbuf)p Fv(,)h Fw(count)h Fv(and)f Fw(datat)o(yp)q(e)p Fv(;)75 2704 y(b)q(oth)e(ha)o(v)o(e)f(the)h(same)g(n)o(um)o(b)q(er)g(of)f (elemen)o(ts,)h(with)h(the)e(same)h(t)o(yp)q(e.)24 b(The)17 b(routine)h(is)f(called)h(b)o(y)f(all)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 115 123 115 122 bop 75 -100 a Fm(4.9.)34 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)814 b Fv(115)75 45 y(group)13 b(mem)o(b)q(ers)f(using)i(the)f (same)f(argumen)o(ts)g(for)h Fw(count,)h(datat)o(yp)q(e,)g(op,)f(ro)q (ot)g Fv(and)g Fw(comm)p Fv(.)k(Th)o(us,)c(all)75 102 y(pro)q(cesses)i(pro)o(vide)f(input)i(bu\013ers)e(and)g(output)g (bu\013ers)h(of)e(the)i(same)f(length,)h(with)f(elemen)o(ts)h(of)f(the) 75 158 y(same)g(t)o(yp)q(e.)20 b(Eac)o(h)14 b(pro)q(cess)h(can)f(pro)o (vide)h(one)g(elemen)o(t,)g(or)f(a)g(sequence)h(of)g(elemen)o(ts,)f(in) i(whic)o(h)f(case)75 214 y(the)d(com)o(bine)h(op)q(eration)g(is)f (executed)h(elemen)o(t-wise)h(on)e(eac)o(h)g(en)o(try)g(of)g(the)g (sequence.)20 b(F)l(or)11 b(example,)75 271 y(if)h(the)f(op)q(eration)h (is)g Fi(MPI)p 506 271 13 2 v 14 w(MAX)f Fv(and)g(the)h(send)g (bu\013er)f(con)o(tains)h(t)o(w)o(o)e(elemen)o(ts)i(that)e(are)h (\015oating)g(p)q(oin)o(t)75 327 y(n)o(um)o(b)q(ers)h(\()p Fw(count)h Fv(=)f(2)g(and)g Fw(datat)o(yp)q(e)i Fv(=)e Fw(MPI)p 867 327 14 2 v 16 w(FLO)o(A)l(T)p Fv(\),)f(then)h Fw(recvbuf)s Fv(\()p Fw(1)p Fv(\))h(=)g(global)8 b(max\()p Fw(sendbuf)s Fv(\()p Fw(1)p Fv(\)\))75 384 y(and)15 b Fw(recvbuf)s Fv(\()p Fw(2)p Fv(\))e(=)g(global)8 b(max\()p Fw(sendbuf)s Fv(\()p Fw(2)p Fv(\)\).)166 440 y(Sec.)31 b(4.9.2,)17 b(lists)j(the)e(set)h(of)f(prede\014ned)i(op)q(erations)f (pro)o(vided)g(b)o(y)g Fw(MPI)p Fv(.)f(That)g(section)h(also)75 497 y(en)o(umerates)d(the)g(datat)o(yp)q(es)f(eac)o(h)i(op)q(eration)f (can)g(b)q(e)h(applied)h(to.)k(In)16 b(addition,)h(users)f(ma)o(y)g (de\014ne)75 553 y(their)i(o)o(wn)e(op)q(erations)i(that)e(can)h(b)q(e) h(o)o(v)o(erloaded)f(to)g(op)q(erate)f(on)i(sev)o(eral)f(datat)o(yp)q (es,)f(either)i(basic)75 610 y(or)d(deriv)o(ed.)21 b(This)15 b(is)h(further)f(explained)i(in)f(Sec.)21 b(4.9.4.)166 666 y(The)11 b(op)q(eration)g Fw(op)h Fv(is)f(alw)o(a)o(ys)f(assumed)h (to)g(b)q(e)g(asso)q(ciativ)o(e.)19 b(All)13 b(prede\014ned)f(op)q (erations)f(are)g(also)75 723 y(assumed)f(to)g(b)q(e)h(comm)o(utativ)o (e.)17 b(Users)10 b(ma)o(y)g(de\014ne)h(op)q(erations)g(that)e(are)h (assumed)h(to)e(b)q(e)i(asso)q(ciativ)o(e,)75 779 y(but)i(not)f(comm)o (utativ)o(e.)19 b(The)13 b(\\canonical")g(ev)m(aluation)h(order)e(of)h (a)f(reduction)i(is)f(determined)h(b)o(y)f(the)75 835 y(ranks)18 b(of)f(the)h(pro)q(cesses)g(in)h(the)f(group.)28 b(Ho)o(w)o(ev)o(er,)17 b(the)h(implemen)o(tation)h(can)f(tak)o(e)f(adv) m(an)o(tage)g(of)75 892 y(asso)q(ciativit)o(y)l(,)d(or)f(asso)q (ciativit)o(y)g(and)h(comm)o(utativit)o(y)f(in)h(order)f(to)f(c)o (hange)i(the)f(order)g(of)g(ev)m(aluation.)75 948 y(This)j(ma)o(y)e(c)o (hange)h(the)g(result)g(of)g(the)g(reduction)h(for)e(op)q(erations)i (that)e(are)h(not)f(strictly)i(asso)q(ciativ)o(e)75 1005 y(and)f(comm)o(utativ)o(e,)f(suc)o(h)i(as)f(\015oating)g(p)q(oin)o(t)h (addition.)189 1111 y Fl(A)n(dvic)n(e)h(to)i(implementors.)53 b Fv(It)17 b(is)h(strongly)f(recommended)i(that)d Fw(MPI)p 1480 1111 V 16 w(REDUCE)i Fv(b)q(e)g(imple-)189 1168 y(men)o(ted)e(so)g(that)g(the)g(same)h(result)f(b)q(e)i(obtained)f (whenev)o(er)f(the)h(function)g(is)g(applied)i(on)d(the)189 1224 y(same)f(argumen)o(ts,)f(app)q(earing)i(in)g(the)f(same)g(order.) 20 b(Note)14 b(that)h(this)h(ma)o(y)e(prev)o(en)o(t)h(optimiza-)189 1280 y(tions)k(that)f(tak)o(e)h(adv)m(an)o(tage)f(of)h(the)g(ph)o (ysical)h(lo)q(cation)g(of)f(pro)q(cessors.)31 b(\()p Fl(End)19 b(of)h(advic)n(e)g(to)189 1337 y(implementors.)p Fv(\))166 1443 y(The)c Fw(datat)o(yp)q(e)i Fv(argumen)o(t)e(of)f Fw(MPI)p 783 1443 V 16 w(REDUCE)i Fv(m)o(ust)f(b)q(e)g(compatible)i (with)e Fw(op)p Fv(.)23 b(Prede\014ned)18 b(op-)75 1500 y(erators)h(w)o(ork)g(only)i(with)f(the)g Fw(MPI)g Fv(t)o(yp)q(es)g (listed)h(in)g(Sec.)35 b(4.9.2)18 b(and)j(Sec.)35 b(4.9.3.)d (User-de\014ned)75 1556 y(op)q(erators)14 b(ma)o(y)g(op)q(erate)h(on)g (general,)g(deriv)o(ed)h(datat)o(yp)q(es.)k(In)15 b(this)h(case,)e(eac) o(h)h(argumen)o(t)g(that)f(the)75 1613 y(reduce)21 b(op)q(eration)g(is) g(applied)h(to)e(is)h(one)f(elemen)o(t)h(describ)q(ed)i(b)o(y)d(suc)o (h)h(a)f(datat)o(yp)q(e,)g(whic)o(h)i(ma)o(y)75 1669 y(con)o(tain)15 b(sev)o(eral)h(basic)g(v)m(alues.)21 b(This)15 b(is)h(further)f(explained)i(in)f(Section)h(4.9.4.)75 1791 y Fj(4.9.2)49 b(Prede\014ned)15 b(reduce)g(op)q(erations)75 1877 y Fv(The)k(follo)o(wing)h(prede\014ned)h(op)q(erations)e(are)f (supplied)k(for)c Fw(MPI)p 1238 1877 V 16 w(REDUCE)i Fv(and)f(related)g(functions)75 1933 y Fw(MPI)p 160 1933 V 16 w(ALLREDUCE)p Fv(,)14 b Fw(MPI)p 541 1933 V 15 w(REDUCE)p 732 1933 V 17 w(SCA)l(TTER)p Fv(,)h(and)f Fw(MPI)p 1144 1933 V 16 w(SCAN)p Fv(.)g(These)g(op)q(erations)g(are)g(in)o(v)o(ok)o (ed)75 1989 y(b)o(y)h(placing)i(the)e(follo)o(wing)h(in)g Fw(op)p Fv(.)117 2150 y Fi(Name)729 b Fv(Meaning)117 2258 y Fi(MPI)p 194 2258 13 2 v 14 w(MAX)649 b Fv(maxim)o(um)117 2313 y Fi(MPI)p 194 2313 V 14 w(MIN)664 b Fv(minim)o(um)117 2367 y Fi(MPI)p 194 2367 V 14 w(SUM)653 b Fv(sum)117 2422 y Fi(MPI)p 194 2422 V 14 w(PROD)626 b Fv(pro)q(duct)117 2476 y Fi(MPI)p 194 2476 V 14 w(LAND)632 b Fv(logical)16 b(and)117 2530 y Fi(MPI)p 194 2530 V 14 w(BAND)626 b Fv(bit-wise)16 b(and)117 2585 y Fi(MPI)p 194 2585 V 14 w(LOR)661 b Fv(logical)16 b(or)117 2639 y Fi(MPI)p 194 2639 V 14 w(BOR)655 b Fv(bit-wise)16 b(or)117 2693 y Fi(MPI)p 194 2693 V 14 w(LX)o(OR)634 b Fv(logical)16 b(xor)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 116 124 116 123 bop 75 -100 a Fv(116)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)117 45 y Fi(MPI)p 194 45 13 2 v 14 w(BX)o(OR)628 b Fv(bit-wise)16 b(xor)117 97 y Fi(MPI)p 194 97 V 14 w(MAXLOC)569 b Fv(max)14 b(v)m(alue)j(and)e (lo)q(cation)117 150 y Fi(MPI)p 194 150 V 14 w(MINLOC)584 b Fv(min)16 b(v)m(alue)g(and)f(lo)q(cation)166 233 y(The)22 b(t)o(w)o(o)e(op)q(erations)h Fi(MPI)p 662 233 V 14 w(MINLOC)h Fv(and)f Fi(MPI)p 1024 233 V 14 w(MAXLOC)g Fv(are)g(discussed)i (separately)e(in)i(Sec.)75 289 y(4.9.3.)g(F)l(or)17 b(the)g(other)f (prede\014ned)j(op)q(erations,)e(w)o(e)g(en)o(umerate)f(b)q(elo)o(w)i (the)f(allo)o(w)o(ed)g(com)o(binations)75 346 y(of)e Fw(op)g Fv(and)g Fw(datat)o(yp)q(e)i Fv(argumen)o(ts.)i(First,)14 b(de\014ne)j(groups)d(of)h Fw(MPI)g Fv(basic)h(datat)o(yp)q(es)e(in)i (the)f(follo)o(wing)75 402 y(w)o(a)o(y)l(.)117 562 y Fi(C)e(integer:)661 b Fw(MPI)p 1032 562 14 2 v 16 w(INT,)124 b(MPI)p 1343 562 V 15 w(LONG,)h(MPI)p 1698 562 V 16 w(SHORT,)905 619 y(MPI)p 990 619 V 16 w(UNSIGNED)p 1229 619 V 17 w(SHORT,)135 b(MPI)p 1626 619 V 16 w(UNSIGNED,)905 675 y(MPI)p 990 675 V 16 w(UNSIGNED)p 1229 675 V 17 w(LONG)117 727 y Fi(F)o(o)o(rtran)13 b(integer:)561 b Fw(MPI)p 1032 727 V 16 w(INTEGER)117 780 y Fi(Floating)13 b(p)q(oint:)574 b Fw(MPI)p 1032 780 V 16 w(FLO)o(A)l(T,)82 b(MPI)p 1365 780 V 16 w(DOUBLE,)h(MPI)p 1738 780 V 16 w(REAL,)905 836 y(MPI)p 990 836 V 16 w(DOUBLE)p 1185 836 V 17 w(PRECISION,)16 b(MPI)p 1543 836 V 15 w(LONG)p 1678 836 V 17 w(DOUBLE)117 889 y Fi(Logical:)696 b Fw(MPI)p 1032 889 V 16 w(LOGICAL)117 941 y Fi(Complex:)668 b Fw(MPI)p 1032 941 V 16 w(COMPLEX)117 993 y Fi(Byte:)738 b Fw(MPI)p 1032 993 V 16 w(BYTE)166 1076 y Fv(No)o(w,)14 b(the)h(v)m(alid)i(datat)o(yp)q(es)e(for)f(eac)o (h)i(option)f(is)h(sp)q(eci\014ed)h(b)q(elo)o(w.)117 1236 y Fi(Op)778 b Fv(Allo)o(w)o(ed)16 b(T)o(yp)q(es)-160 1224 y Fx(>)117 1341 y Fi(MPI)p 194 1341 13 2 v 14 w(MAX,)d(MPI)p 400 1341 V 14 w(MIN)458 b(C)13 b(integer,)i(F)o(o)o(rtran)e(integer,)i (Floating)e(p)q(oint)-1819 b Fx(?)117 1393 y Fi(MPI)p 194 1393 V 14 w(SUM,)13 b(MPI)p 396 1393 V 14 w(PROD)424 b(C)13 b(integer,)i(F)o(o)o(rtran)e(integer,)i(Floating)e(p)q(oint,)h (Complex)117 1446 y(MPI)p 194 1446 V 14 w(LAND,)g(MPI)p 418 1446 V 14 w(LOR,)f(MPI)p 612 1446 V 15 w(LX)o(OR)215 b(C)13 b(integer,)i(Logical)117 1498 y(MPI)p 194 1498 V 14 w(BAND,)e(MPI)p 423 1498 V 14 w(BOR,)g(MPI)p 623 1498 V 14 w(BX)o(OR)199 b(C)13 b(integer,)i(F)o(o)o(rtran)e(integer,)i (Byte)75 1616 y Fq(Example)j(4.15)23 b Fv(A)10 b(routine)i(that)e (computes)h(the)g(dot)f(pro)q(duct)i(of)e(t)o(w)o(o)g(v)o(ectors)g (that)g(are)g(distributed)75 1673 y(across)k(a)h(group)g(of)g(pro)q (cesses)h(and)f(returns)g(the)g(answ)o(er)g(at)g(no)q(de)g(zero.)75 1765 y Ft(SUBROUTINE)22 b(PAR_BLAS1\(m,)h(a,)g(b,)h(c,)f(comm\))75 1821 y(REAL)g(a\(m\),)g(b\(m\))167 b(!)24 b(local)f(slice)g(of)h(array) 75 1878 y(REAL)f(c)382 b(!)24 b(result)f(\(at)g(node)h(zero\))75 1934 y(REAL)f(sum)75 1991 y(INTEGER)g(m,)g(comm,)h(i,)f(ierr)75 2104 y(!)h(local)f(sum)75 2160 y(sum)g(=)h(0.0)75 2216 y(DO)g(i)f(=)h(1,)g(m)147 2273 y(sum)f(=)h(sum)f(+)h(a\(i\)*b\(i\))75 2329 y(END)f(DO)75 2442 y(!)h(global)f(sum)75 2499 y(CALL)g (MPI_REDUCE\(sum,)f(c,)i(1,)f(MPI_REAL,)g(MPI_SUM,)g(0,)g(comm,)g (ierr\))75 2555 y(RETURN)75 2647 y Fq(Example)18 b(4.16)23 b Fv(A)c(routine)h(that)f(computes)h(the)f(pro)q(duct)h(of)g(a)f(v)o (ector)g(and)h(an)f(arra)o(y)f(that)h(are)75 2704 y(distributed)e (across)d(a)h(group)g(of)g(pro)q(cesses)g(and)h(returns)f(the)g(answ)o (er)g(at)f(no)q(de)i(zero.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 117 125 117 124 bop 75 -100 a Fm(4.9.)34 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)814 b Fv(117)75 45 y Ft(SUBROUTINE)22 b(PAR_BLAS2\(m,)h(n,)g(a,) h(b,)f(c,)h(comm\))75 102 y(REAL)f(a\(m\),)g(b\(m,n\))95 b(!)24 b(local)f(slice)g(of)h(array)75 158 y(REAL)f(c\(n\))286 b(!)24 b(result)75 214 y(REAL)f(sum\(n\))75 271 y(INTEGER)g(n,)g(comm,) h(i,)f(j,)h(ierr)75 384 y(!)g(local)f(sum)75 440 y(DO)h(j=)f(1,)h(n)123 497 y(sum\(j\))f(=)g(0.0)123 553 y(DO)g(i)h(=)g(1,)f(m)170 610 y(sum\(j\))g(=)h(sum\(j\))f(+)h(a\(i\)*b\(i,j\))123 666 y(END)f(DO)75 723 y(END)g(DO)75 835 y(!)h(global)f(sum)75 892 y(CALL)g(MPI_REDUCE\(sum,)f(c,)i(n,)f(MPI_REAL,)g(MPI_SUM,)g(0,)g (comm,)g(ierr\))75 1005 y(!)h(return)f(result)g(at)g(node)h(zero)f (\(and)g(garbage)g(at)h(the)f(other)g(nodes\))75 1061 y(RETURN)75 1183 y Fj(4.9.3)49 b(MINLOC)18 b(and)e(MAXLOC)75 1269 y Fv(The)d(op)q(erator)e Fi(MPI)p 423 1269 13 2 v 15 w(MINLOC)h Fv(is)h(used)g(to)f(compute)g(a)h(global)g(minim)o(um)g (and)g(also)f(an)h(index)h(attac)o(hed)75 1325 y(to)e(the)g(minim)o(um) i(v)m(alue.)20 b Fi(MPI)p 615 1325 V 14 w(MAXLOC)12 b Fv(similarly)i(computes)f(a)f(global)h(maxim)o(um)f(and)h(index.)20 b(One)75 1382 y(application)13 b(of)e(these)h(is)g(to)f(compute)g(a)g (global)h(minim)o(um)h(\(maxim)o(um\))e(and)g(the)h(rank)f(of)g(the)h (pro)q(cess)75 1438 y(con)o(taining)k(this)g(v)m(alue.)166 1495 y(The)f(op)q(eration)h(that)e(de\014nes)i Fi(MPI)p 787 1495 V 15 w(MAXLOC)e Fv(is:)189 1559 y Fe( )242 1603 y Fp(u)247 1659 y(i)289 1559 y Fe(!)332 1631 y Fx(\016)365 1559 y Fe( )418 1603 y Fp(v)420 1659 y(j)463 1559 y Fe(!)508 1631 y Fv(=)556 1559 y Fe( )610 1603 y Fp(w)614 1659 y(k)664 1559 y Fe(!)75 1768 y Fv(where)189 1861 y Fp(w)f Fv(=)g(max\()p Fp(u;)8 b(v)r Fv(\))75 1955 y(and)189 2095 y Fp(k)13 b Fv(=)274 1997 y Fe(8)274 2034 y(>)274 2046 y(<)274 2121 y(>)274 2134 y(:)332 2039 y Fp(i)194 b Fv(if)15 b Fp(u)e(>)g(v)332 2095 y Fv(min)q(\()p Fp(i;)8 b(j)s Fv(\))39 b(if)15 b Fp(u)e Fv(=)g Fp(v)332 2152 y(j)191 b Fv(if)15 b Fp(u)e(<)g(v)166 2244 y Fi(MPI)p 243 2244 V 14 w(MINLOC)i Fv(is)h(de\014ned)g(similarly:)189 2309 y Fe( )242 2352 y Fp(u)247 2409 y(i)289 2309 y Fe(!)332 2381 y Fx(\016)365 2309 y Fe( )418 2352 y Fp(v)420 2409 y(j)463 2309 y Fe(!)508 2381 y Fv(=)556 2309 y Fe( )610 2352 y Fp(w)614 2409 y(k)664 2309 y Fe(!)75 2517 y Fv(where)189 2611 y Fp(w)d Fv(=)g(min)q(\()p Fp(u;)8 b(v)r Fv(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 118 126 118 125 bop 75 -100 a Fv(118)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fv(and)189 185 y Fp(k)d Fv(=)274 87 y Fe(8)274 124 y(>)274 136 y(<)274 211 y(>)274 224 y(:)332 129 y Fp(i)194 b Fv(if)15 b Fp(u)e(<)g(v)332 185 y Fv(min)q(\()p Fp(i;)8 b(j)s Fv(\))39 b(if)15 b Fp(u)e Fv(=)g Fp(v)332 242 y(j)191 b Fv(if)15 b Fp(u)e(>)g(v)166 334 y Fv(Both)g(op)q(erations)g(are)g(asso)q(ciativ)o(e)h(and)f(comm)o (utativ)o(e.)19 b(Note)12 b(that)h(if)h Fi(MPI)p 1491 334 13 2 v 14 w(MAXLOC)e Fv(is)i(applied)75 390 y(to)k(reduce)i(a)e (sequence)i(of)e(pairs)h(\()p Fp(u)729 397 y Fg(0)748 390 y Fp(;)8 b Fv(0\))p Fp(;)g Fv(\()p Fp(u)875 397 y Fg(1)893 390 y Fp(;)g Fv(1\))p Fp(;)g(:)g(:)f(:)t(;)h Fv(\()p Fp(u)1099 397 y Ff(n)p Fh(\000)p Fg(1)1167 390 y Fp(;)g(n)j Fx(\000)i Fv(1\),)19 b(then)g(the)f(v)m(alue)i(returned)f (is)75 447 y(\()p Fp(u;)8 b(r)q Fv(\),)15 b(where)i Fp(u)f Fv(=)f(max)518 454 y Ff(i)540 447 y Fp(u)566 454 y Ff(i)597 447 y Fv(and)i Fp(r)g Fv(is)g(the)g(index)i(of)d(the)h(\014rst)f (global)i(maxim)o(um)f(in)g(the)g(sequence.)75 503 y(Th)o(us,)12 b(if)h(eac)o(h)f(pro)q(cess)g(supplies)i(a)e(v)m(alue)h(and)f(its)h (rank)e(within)j(the)e(group,)g(then)g(a)g(reduce)h(op)q(eration)75 560 y(with)18 b Fw(op)g Fv(=)g Fi(MPI)p 375 560 V 14 w(MAXLOC)f Fv(will)i(return)f(the)g(maxim)o(um)f(v)m(alue)i(and)f(the)g (rank)f(of)h(the)f(\014rst)h(pro)q(cess)75 616 y(with)g(that)f(v)m (alue.)29 b(Similarly)l(,)21 b Fi(MPI)p 708 616 V 14 w(MINLOC)d Fv(can)g(b)q(e)g(used)h(to)e(return)h(a)f(minim)o(um)i(and)f (its)g(index.)75 673 y(More)10 b(generally)l(,)i Fi(MPI)p 463 673 V 14 w(MINLOC)e Fv(computes)h(a)f Fl(lexic)n(o)n(gr)n(aphic)h (minimum)p Fv(,)g(where)g(elemen)o(ts)g(are)f(ordered)75 729 y(according)k(to)g(the)g(\014rst)g(comp)q(onen)o(t)g(of)f(eac)o(h)h (pair,)h(and)f(ties)g(are)g(resolv)o(ed)g(according)h(to)e(the)h (second)75 786 y(comp)q(onen)o(t.)166 842 y(The)i(reduce)h(op)q (eration)f(is)h(de\014ned)g(to)e(op)q(erate)h(on)g(argumen)o(ts)f(that) g(consist)i(of)e(a)h(pair:)22 b(v)m(alue)75 899 y(and)12 b(index.)20 b(F)l(or)12 b(b)q(oth)g(F)l(ortran)f(and)h(C,)g(t)o(yp)q (es)g(are)g(pro)o(vided)h(to)e(describ)q(e)j(the)e(pair.)19 b(The)13 b(p)q(oten)o(tially)75 955 y(mixed-t)o(yp)q(e)g(nature)e(of)g (suc)o(h)h(argumen)o(ts)f(is)h(a)f(problem)i(in)f(F)l(ortran.)17 b(The)12 b(problem)h(is)f(circum)o(v)o(en)o(ted,)75 1011 y(for)k(F)l(ortran,)g(b)o(y)h(ha)o(ving)h(the)f Fw(MPI)p Fv(-pro)o(vided)g(t)o(yp)q(e)g(consist)g(of)g(a)g(pair)g(of)g(the)g (same)f(t)o(yp)q(e)h(as)g(v)m(alue,)75 1068 y(and)j(co)q(ercing)g(the)g (index)g(to)f(this)h(t)o(yp)q(e)f(also.)33 b(In)20 b(C,)f(the)g Fw(MPI)p Fv(-pro)o(vided)h(pair)g(t)o(yp)q(e)f(has)h(distinct)75 1124 y(t)o(yp)q(es)15 b(and)h(the)f(index)h(is)g(an)f Fw(int)p Fv(.)166 1181 y(In)f(order)g(to)f(use)h Fi(MPI)p 547 1181 V 14 w(MINLOC)f Fv(and)h Fi(MPI)p 893 1181 V 15 w(MAXLOC)e Fv(in)j(a)e(reduce)i(op)q(eration,)f(one)f(m)o(ust)g(pro) o(vide)75 1237 y(a)19 b Fw(datat)o(yp)q(e)j Fv(argumen)o(t)d(that)g (represen)o(ts)h(a)f(pair)h(\(v)m(alue)h(and)f(index\).)34 b Fw(MPI)19 b Fv(pro)o(vides)i(sev)o(en)f(suc)o(h)75 1294 y(prede\014ned)14 b(datat)o(yp)q(es.)19 b(The)13 b(op)q(erations)g Fi(MPI)p 905 1294 V 14 w(MAXLOC)f Fv(and)h Fi(MPI)p 1264 1294 V 14 w(MINLOC)f Fv(can)h(b)q(e)h(used)f(with)g(eac)o (h)75 1350 y(of)i(the)g(follo)o(wing)h(datat)o(yp)q(es.)-764 b Fx(>)117 1454 y Fi(F)o(o)o(rtran:)117 1508 y(Name)729 b Fv(Description)117 1563 y Fi(MPI)p 194 1563 V 14 w(2REAL)618 b Fv(pair)15 b(of)g Ft(REAL)p Fv(s)117 1617 y Fi(MPI)p 194 1617 V 14 w(2DOUBLE)p 394 1617 V 14 w(PRECISION)328 b Fv(pair)15 b(of)g Ft(DOUBLE)23 b(PRECISION)14 b Fv(v)m(ariables)117 1671 y Fi(MPI)p 194 1671 V 14 w(2INTEGER)546 b Fv(pair)15 b(of)g Ft(INTEGER)p Fv(s)-160 1802 y Fx(?)117 1858 y Fi(C:)117 1912 y(Name)729 b Fv(Description)117 1967 y Fi(MPI)p 194 1967 V 14 w(FLO)o(A)m(T)p 337 1967 V 14 w(INT)529 b Fw(\015oat)15 b Fv(and)h Fw(int)117 2021 y Fi(MPI)p 194 2021 V 14 w(DOUBLE)p 373 2021 V 14 w(INT)493 b Fw(double)16 b Fv(and)g Fw(int)117 2076 y Fi(MPI)p 194 2076 V 14 w(LONG)p 318 2076 V 15 w(INT)547 b Fw(long)15 b Fv(and)g Fw(int)117 2130 y Fi(MPI)p 194 2130 V 14 w(2INT)651 b Fv(pair)15 b(of)g Fw(int)117 2184 y Fi(MPI)p 194 2184 V 14 w(SHORT)p 346 2184 V 15 w(INT)519 b Fw(sho)o(rt)15 b Fv(and)h Fw(int)117 2239 y Fi(MPI)p 194 2239 V 14 w(LONG)p 318 2239 V 15 w(DOUBLE)p 498 2239 V 14 w(INT)368 b Fw(long)15 b(double)h Fv(and)f Fw(int)166 2322 y Fv(The)g(datat)o(yp)q(e)g Fi(MPI)p 527 2322 V 14 w(2REAL)g Fv(is)h Fl(as)g(if)f Fv(de\014ned)h(b)o(y)g(the)f(follo)o(wing)h(\(see)f(Section)h(3.12\).) 75 2428 y Ft(MPI_TYPE_CONTIGUOUS\(2,)21 b(MPI_REAL,)h(MPI_2REAL\))166 2534 y Fv(Similar)17 b(statemen)o(ts)d(apply)i(for)e Fi(MPI)p 823 2534 V 15 w(2INTEGER)p Fv(,)f Fi(MPI)p 1134 2534 V 15 w(2DOUBLE)p 1335 2534 V 13 w(PRECISION)p Fv(,)g(and)i Fi(MPI)p 1750 2534 V 15 w(2INT)p Fv(.)166 2591 y(The)f(datat)o(yp)q(e)g Fi(MPI)p 525 2591 V 14 w(FLO)o(A)m(T)p 668 2591 V 14 w(INT)g Fv(is)h Fl(as)g(if)e Fv(de\014ned)j(b)o(y)e(the)g(follo)o(wing) h(sequence)g(of)e(instructions.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 119 127 119 126 bop 75 -100 a Fm(4.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)819 b Fv(119)75 45 y Ft(type[0])23 b(=)h(MPI_FLOAT)75 102 y(type[1])f(=)h(MPI_INT)75 158 y(disp[0])f(=)h(0)75 214 y(disp[1])f(=)h(sizeof\(float\))75 271 y(block[0])f(=)g(1)75 327 y(block[1])g(=)g(1)75 384 y(MPI_TYPE_STRUCT\(2,)e(block,)i(disp,)h (type,)f(MPI_FLOAT_INT\))75 477 y Fv(Similar)17 b(statemen)o(ts)d (apply)i(for)e Fi(MPI)p 732 477 13 2 v 15 w(LONG)p 857 477 V 15 w(INT)h Fv(and)g Fi(MPI)p 1119 477 V 15 w(DOUBLE)p 1299 477 V 14 w(INT)p Fv(.)75 582 y Fq(Example)j(4.17)23 b Fv(Eac)o(h)12 b(pro)q(cess)g(has)g(an)g(arra)o(y)f(of)h(30)g Ft(double)p Fv(s,)f(in)i(C.)f(F)l(or)g(eac)o(h)g(of)g(the)g(30)g(lo)q (cations,)75 638 y(compute)j(the)h(v)m(alue)g(and)f(rank)g(of)g(the)g (pro)q(cess)h(con)o(taining)g(the)f(largest)g(v)m(alue.)170 743 y Ft(...)170 800 y(/*)24 b(each)f(process)g(has)h(an)f(array)g(of)h (30)g(double:)f(ain[30])194 856 y(*/)170 913 y(double)g(ain[30],)g (aout[30];)170 969 y(int)48 b(ind[30];)170 1026 y(struct)23 b({)266 1082 y(double)g(val;)266 1139 y(int)71 b(rank;)170 1195 y(})24 b(in[30],)f(out[30];)170 1251 y(int)h(i,)f(myrank,)g(root;) 170 1364 y(MPI_Comm_rank\(MPI_COMM_WORLD,)d(&myrank\);)170 1421 y(for)k(\(i=0;)f(i<30;)g(++i\))h({)266 1477 y(in[i].val)f(=)g (ain[i];)266 1534 y(in[i].rank)f(=)i(myrank;)170 1590 y(})170 1647 y(MPI_Reduce\()f(in,)g(out,)h(30,)f(MPI_DOUBLE_INT,)f (MPI_MAXLOC,)g(root,)h(comm)h(\);)170 1703 y(/*)g(At)g(this)f(point,)g (the)g(answer)g(resides)g(on)h(process)f(root)194 1760 y(*/)170 1816 y(if)h(\(myrank)f(==)h(root\))f({)266 1872 y(/*)g(read)h(ranks)f(out)290 1929 y(*/)266 1985 y(for)g(\(i=0;)h (i<30;)f(++i\))g({)361 2042 y(aout[i])g(=)h(out[i].val;)361 2098 y(ind[i])f(=)h(out[i].rank;)266 2155 y(})170 2211 y(})75 2316 y Fq(Example)18 b(4.18)23 b Fv(Same)15 b(example,)g(in)h(F) l(ortran.)170 2421 y Ft(...)170 2478 y(!)24 b(each)f(process)g(has)h (an)f(array)h(of)f(30)h(double:)f(ain\(30\))170 2591 y(DOUBLE)g(PRECISION)g(ain\(30\),)g(aout\(30\))170 2647 y(INTEGER)g(ind\(30\);)170 2704 y(DOUBLE)g(PRECISION)g(in\(2,30\),)g (out\(2,30\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 120 128 120 127 bop 75 -100 a Fv(120)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)170 45 y Ft(INTEGER)23 b(i,)h(myrank,)f(root,)g(ierr;)170 158 y (MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\);)170 214 y(DO)k(I=1,)f(30) 266 271 y(in\(1,i\))g(=)h(ain\(i\))266 327 y(in\(2,i\))f(=)h(myrank)94 b(!)24 b(myrank)f(is)h(coerced)e(to)i(a)g(double)170 384 y(END)g(DO)170 497 y(MPI_REDUCE\()f(in,)g(out,)h(30,)f (MPI_2DOUBLE_PRECISION,)e(MPI_MAXLOC,)h(root,)1459 553 y(comm,)h(ierr)h(\);)170 610 y(!)g(At)g(this)f(point,)g(the)h(answer)f (resides)g(on)g(process)g(root)170 723 y(IF)h(\(myrank)f(.EQ.)g(root\)) g(THEN)266 779 y(!)h(read)f(ranks)g(out)266 835 y(DO)g(I=)h(1,)g(30)361 892 y(aout\(i\))f(=)h(out\(1,i\))361 948 y(ind\(i\))f(=)h(out\(2,i\))47 b(!)23 b(rank)h(is)f(coerced)g(back)h(to)f(an)h(integer)266 1005 y(END)f(DO)170 1061 y(END)h(IF)75 1168 y Fq(Example)18 b(4.19)23 b Fv(Eac)o(h)16 b(pro)q(cess)h(has)f(a)h(non-empt)o(y)f(arra) o(y)g(of)g(v)m(alues.)25 b(Find)18 b(the)f(minim)o(um)g(global)75 1224 y(v)m(alue,)f(the)f(rank)g(of)g(the)g(pro)q(cess)h(that)e(holds)i (it)g(and)f(its)g(index)i(on)e(this)h(pro)q(cess.)75 1330 y Ft(#define)47 b(LEN)71 b(1000)75 1443 y(float)23 b(val[LEN];)190 b(/*)23 b(local)h(array)f(of)g(values)g(*/)75 1500 y(int)g(count;)310 b(/*)23 b(local)h(number)f(of)g(values)g(*/)75 1556 y(int)g(myrank,)g(minrank,)g(minindex;)75 1613 y(float)g(minval;) 75 1725 y(struct)g({)170 1782 y(float)h(value;)170 1838 y(int)72 b(index;)75 1895 y(})24 b(in,)f(out;)170 2008 y(/*)h(local)f(minloc)g(*/)75 2064 y(in.value)g(=)g(val[0];)75 2121 y(in.index)g(=)g(0;)75 2177 y(for)g(\(i=1;)h(i)f(<)h(count;)f (i++\))170 2234 y(if)h(\(in.value)f(>)g(val[i]\))g({)266 2290 y(in.value)g(=)g(val[i];)266 2346 y(in.index)g(=)g(i;)170 2403 y(})170 2516 y(/*)h(global)f(minloc)g(*/)75 2572 y(MPI_Comm_rank\(MPI_COMM_WOR)o(LD,)e(&myrank\);)75 2629 y(in.index)i(=)g(myrank*LEN)g(+)h(in.index;)75 2685 y(MPI_Reduce\()e (in,)i(out,)f(1,)h(MPI_FLOAT_INT,)e(MPI_MINLOC,)g(root,)h(comm)h(\);) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 121 129 121 128 bop 75 -100 a Fm(4.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)819 b Fv(121)170 45 y Ft(/*)24 b(At)g(this)f(point,)g(the)g (answer)g(resides)g(on)h(process)f(root)194 102 y(*/)75 158 y(if)h(\(myrank)e(==)i(root\))f({)170 214 y(/*)h(read)f(answer)g (out)194 271 y(*/)170 327 y(minval)g(=)h(out.value;)170 384 y(minrank)f(=)h(out.index)f(/)g(LEN;)170 440 y(minindex)g(=)h (out.index)f(\045)g(LEN;)75 497 y(})189 603 y Fl(R)n(ationale.)84 b Fv(The)23 b(de\014nition)h(of)e Fi(MPI)p 921 603 13 2 v 14 w(MINLOC)h Fv(and)f Fi(MPI)p 1285 603 V 15 w(MAXLOC)f Fv(giv)o(en)i(here)g(has)g(the)189 659 y(adv)m(an)o(tage)14 b(that)h(it)h(do)q(es)f(not)g(require)h(an)o(y)f(sp)q(ecial-case)j (handling)f(of)e(these)g(t)o(w)o(o)f(op)q(erations:)189 716 y(they)i(are)g(handled)i(lik)o(e)g(an)o(y)e(other)g(reduce)i(op)q (eration.)24 b(A)16 b(programmer)g(can)g(pro)o(vide)h(his)g(or)189 772 y(her)e(o)o(wn)g(de\014nition)i(of)f Fi(MPI)p 694 772 V 14 w(MAXLOC)e Fv(and)i Fi(MPI)p 1058 772 V 14 w(MINLOC)p Fv(,)f(if)h(so)f(desired.)22 b(The)16 b(disadv)m(an)o(tage)189 829 y(is)d(that)f(v)m(alues)h(and)g(indices)i(ha)o(v)o(e)d(to)g(b)q(e)h (\014rst)f(in)o(terlea)o(v)o(ed,)i(and)e(that)g(indices)j(and)e(v)m (alues)g(ha)o(v)o(e)189 885 y(to)h(b)q(e)i(co)q(erced)g(to)f(the)g (same)g(t)o(yp)q(e,)g(in)h(F)l(ortran.)i(\()p Fl(End)e(of)g(r)n (ationale.)p Fv(\))75 1007 y Fj(4.9.4)49 b(User-De\014ned)15 b(Op)q(erations)75 1187 y Fw(MPI)p 160 1187 14 2 v 16 w(OP)p 238 1187 V 17 w(CREA)l(TE\()h(function,)g(commute,)e(op\))117 1265 y Fk(IN)155 b Fw(function)434 b Fk(user)15 b(de\014ned)g(function) f(\(function\))117 1340 y(IN)155 b Fw(commute)411 b Fb(true)13 b Fk(if)g(comm)o(utativ)o(e;)e Fb(false)i Fk(otherwise.)117 1415 y(OUT)108 b Fw(op)541 b Fk(op)q(eration)14 b(\(handle\))75 1539 y Ft(int)23 b(MPI)p 245 1539 15 2 v 17 w(Op)p 310 1539 V 17 w(create\(MPI)p 567 1539 V 16 w(User)p 679 1539 V 17 w(function)f(*function,)h(int)g(commute,)g(MPI)p 1555 1539 V 17 w(Op)g(*op\))75 1626 y(MPI)p 150 1626 V 17 w(OP)p 215 1626 V 17 w(CREATE\()g(FUNCTION,)f(COMMUTE,)h(OP,)g (IERROR\))170 1682 y(EXTERNAL)g(FUNCTION)170 1739 y(LOGICAL)g(COMMUTE) 170 1795 y(INTEGER)g(OP,)h(IERROR)166 1882 y Fw(MPI)p 251 1882 14 2 v 16 w(OP)p 329 1882 V 17 w(CREA)l(TE)h Fv(binds)h(a)f(user-de\014ned)h(global)f(op)q(eration)g(to)f(an)h Fw(op)g Fv(handle)h(that)e(can)75 1938 y(subsequen)o(tly)e(b)q(e)f (used)g(in)g Fw(MPI)p 669 1938 V 16 w(REDUCE)p Fv(,)g Fw(MPI)p 977 1938 V 15 w(ALLREDUCE)p Fv(,)g Fw(MPI)p 1364 1938 V 15 w(REDUCE)p 1555 1938 V 18 w(SCA)l(TTER)p Fv(,)g(and)75 1994 y Fw(MPI)p 160 1994 V 16 w(SCAN)p Fv(.)c(The)g(user-de\014ned)i(op)q(eration)e(is)g(assumed)g(to)g(b)q(e) g(asso)q(ciativ)o(e.)25 b(If)18 b Fw(commute)e Fv(=)h Ft(true)p Fv(,)75 2051 y(then)k(the)g(op)q(eration)g(should)h(b)q(e)g (b)q(oth)e(comm)o(utativ)o(e)h(and)g(asso)q(ciativ)o(e.)37 b(If)21 b Fw(commute)f Fv(=)h Ft(false)p Fv(,)75 2107 y(then)d(the)f(order)h(of)f(op)q(erands)h(is)g(\014xed)g(and)f(is)i (de\014ned)f(to)f(b)q(e)h(in)h(ascending,)g(pro)q(cess)e(rank)h(order,) 61 b Fx(>)75 2164 y Fv(b)q(eginning)17 b(with)e(pro)q(cess)g(zero.)20 b(The)15 b(order)g(of)f(ev)m(aluation)i(can)f(b)q(e)h(c)o(hanged,)f (talking)g(adv)m(an)o(tage)f(of)75 2220 y(the)i(asso)q(ciativit)o(y)g (of)g(the)g(op)q(eration.)22 b(If)16 b Fw(commute)f Fv(=)h Ft(true)g Fv(then)g(the)g(order)g(of)f(ev)m(aluation)i(can)f(b)q(e)75 2277 y(c)o(hanged,)f(taking)g(adv)m(an)o(tage)g(of)g(comm)o(utativit)o (y)f(and)i(asso)q(ciativit)o(y)l(.)619 b Fx(?)166 2333 y Fw(function)20 b Fv(is)g(the)e(user-de\014ned)j(function,)f(whic)o(h) g(m)o(ust)e(ha)o(v)o(e)g(the)h(follo)o(wing)h(four)e(argumen)o(ts:)75 2390 y Fw(invec,)d(inoutvec,)i(len)e Fv(and)h Fw(datat)o(yp)q(e)p Fv(.)166 2446 y(The)f(ANSI-C)h(protot)o(yp)q(e)f(for)f(the)i(function)g (is)f(the)h(follo)o(wing.)75 2540 y Ft(typedef)23 b(void)g (MPI_User_function\()f(void)h(*invec,)g(void)g(*inoutvec,)g(int)g (*len,)1220 2596 y(MPI_Datatype)g(*datatype\);)166 2690 y Fv(The)15 b(F)l(ortran)f(declaration)i(of)f(the)g(user-de\014ned)i (function)f(app)q(ears)g(b)q(elo)o(w.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 122 130 122 129 bop 75 -100 a Fv(122)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(FUNCTION)23 b(USER_FUNCTION\()f(INVEC\(*\),)g(INOUTVEC\(*\),)h(LEN,)g(TYPE\))75 102 y()g(INVEC\(LEN\),)f(INOUTVEC\(LEN\))99 158 y(INTEGER)h(LEN,)g(TYPE)166 252 y Fv(The)18 b Fw(datat)o(yp)q(e)h Fv(argumen)o(t)e(is)h(a)f(handle)i(to)e(the)h(data)e(t)o(yp)q(e)i(that) f(w)o(as)g(passed)h(in)o(to)f(the)h(call)h(to)75 308 y Fw(MPI)p 160 308 14 2 v 16 w(REDUCE)p Fv(.)f(The)g(user)g(reduce)h (function)g(should)g(b)q(e)f(written)g(suc)o(h)h(that)e(the)h(follo)o (wing)h(holds:)75 365 y(Let)h Fw(u[0],)g(...)32 b(,)20 b(u[len-1])g Fv(b)q(e)g(the)g Fw(len)g Fv(elemen)o(ts)g(in)g(the)g (comm)o(unication)g(bu\013er)f(describ)q(ed)j(b)o(y)d(the)75 421 y(argumen)o(ts)14 b Fw(invec,)i(len)f Fv(and)g Fw(datat)o(yp)q(e)i Fv(when)f(the)f(function)h(is)f(in)o(v)o(ok)o(ed;)g(let)h Fw(v[0],)e(...)19 b(,)c(v[len-1])h Fv(b)q(e)f Fw(len)75 478 y Fv(elemen)o(ts)e(in)g(the)f(comm)o(unication)h(bu\013er)f (describ)q(ed)i(b)o(y)e(the)g(argumen)o(ts)f Fw(inoutvec,)j(len)e Fv(and)g Fw(datat)o(yp)q(e)75 534 y Fv(when)17 b(the)g(function)g(is)h (in)o(v)o(ok)o(ed;)f(let)g Fw(w[0],)g(...)23 b(,)17 b(w[len-1])g Fv(b)q(e)g Fw(len)h Fv(elemen)o(ts)f(in)g(the)g(comm)o(unication)75 591 y(bu\013er)i(describ)q(ed)j(b)o(y)d(the)h(argumen)o(ts)e Fw(inoutvec,)k(len)e Fv(and)g Fw(datat)o(yp)q(e)h Fv(when)f(the)f (function)h(returns;)75 647 y(then)d Fw(w[i])f(=)h(u[i])p Fx(\016)p Fw(v[i])p Fv(,)g(for)f Fw(i=0)g(,)g(...)23 b(,)16 b(len-1)p Fv(,)h(where)g Fx(\016)f Fv(is)h(the)f(reduce)h(op)q (eration)g(that)f(the)g(function)75 703 y(computes.)166 760 y(Informally)l(,)j(w)o(e)e(can)g(think)h(of)f Fw(invec)h Fv(and)g Fw(inoutvec)h Fv(as)e(arra)o(ys)f(of)h Fw(len)h Fv(elemen)o(ts)g(that)e Fw(function)75 816 y Fv(is)i(com)o(bining.)28 b(The)18 b(result)g(of)f(the)h(reduction)h(o)o(v)o(er-writes)e(v)m (alues)h(in)h Fw(inoutvec)p Fv(,)g(hence)g(the)e(name.)75 873 y(Eac)o(h)g(in)o(v)o(o)q(cation)g(of)f(the)h(function)h(results)f (in)h(the)f(p)q(oin)o(t)o(wise)h(ev)m(aluation)g(of)e(the)h(reduce)h (op)q(erator)75 929 y(on)h Fw(len)h Fv(elemen)o(ts:)29 b(I.e,)21 b(the)e(function)h(returns)g(in)g Fw(inoutvec[i])h Fv(the)e(v)m(alue)i Fw(invec)p Fv([)p Fw(i)p Fv(])12 b Fx(\016)h Fw(inoutvec)p Fv([)p Fw(i)p Fv(],)21 b(for)75 986 y Fw(i)12 b Fv(=)h Fw(0)p Fp(;)8 b(:)g(:)g(:)d(;)j Fw(count)j Fx(\000)f Fw(1)p Fv(,)15 b(where)g Fx(\016)g Fv(is)h(the)f(com)o(bining)h(op)q(eration)g(computed)f(b)o(y)g(the)h (function.)189 1092 y Fl(R)n(ationale.)47 b Fv(The)16 b(len)i(argumen)o(t)d(allo)o(ws)i Fw(MPI)p 1033 1092 V 16 w(REDUCE)g Fv(to)f(a)o(v)o(oid)g(calling)i(the)e(function)i(for) 189 1148 y(eac)o(h)13 b(elemen)o(t)h(in)g(the)f(input)h(bu\013er.)19 b(Rather,)13 b(the)h(system)e(can)h(c)o(ho)q(ose)h(to)e(apply)i(the)f (function)189 1205 y(to)k(c)o(h)o(unks)i(of)f(input.)30 b(In)19 b(C,)e(it)i(is)g(passed)f(in)h(as)f(a)g(reference)h(for)f (reasons)g(of)f(compatibilit)o(y)189 1261 y(with)e(F)l(ortran.)189 1337 y(By)e(in)o(ternally)h(comparing)f(the)g(v)m(alue)h(of)e(the)h Fw(datat)o(yp)q(e)h Fv(argumen)o(t)e(to)g(kno)o(wn,)h(global)g (handles,)189 1393 y(it)h(is)h(p)q(ossible)h(to)e(o)o(v)o(erload)g(the) g(use)h(of)f(a)g(single)h(user-de\014ned)i(function)e(for)e(sev)o (eral,)i(di\013eren)o(t)189 1449 y(data)f(t)o(yp)q(es.)20 b(\()p Fl(End)c(of)g(r)n(ationale.)p Fv(\))166 1556 y(General)e(datat)o (yp)q(es)f(ma)o(y)g(b)q(e)i(passed)f(to)f(the)g(user)h(function.)21 b(Ho)o(w)o(ev)o(er,)12 b(use)i(of)g(datat)o(yp)q(es)f(that)75 1612 y(are)i(not)g(con)o(tiguous)g(is)h(lik)o(ely)h(to)d(lead)i(to)f (ine\016ciencies.)166 1669 y(No)g Fw(MPI)g Fv(comm)o(unication)i (function)f(ma)o(y)f(b)q(e)h(called)h(inside)h(the)d(user)h(function.) 22 b Fw(MPI)p 1708 1669 V 16 w(ABORT)75 1725 y Fv(ma)o(y)14 b(b)q(e)i(called)h(inside)g(the)e(function)h(in)h(case)e(of)g(an)g (error.)189 1831 y Fl(A)n(dvic)n(e)g(to)i(users.)42 b Fv(Supp)q(ose)17 b(one)e(de\014nes)i(a)e(library)i(of)e(user-de\014ned) i(reduce)f(functions)h(that)189 1888 y(are)10 b(o)o(v)o(erloaded:)17 b(the)11 b Fw(datat)o(yp)q(e)h Fv(argumen)o(t)d(is)i(used)g(to)f (select)h(the)g(righ)o(t)f(execution)i(path)e(at)g(eac)o(h)189 1944 y(in)o(v)o(o)q(cation,)15 b(according)h(to)e(the)i(t)o(yp)q(es)f (of)g(the)g(op)q(erands.)21 b(The)15 b(user-de\014ned)i(reduce)f (function)189 2001 y(cannot)11 b(\\deco)q(de")h(the)f Fw(datat)o(yp)q(e)i Fv(argumen)o(t)d(that)h(it)h(is)f(passed,)h(and)g (cannot)f(iden)o(tify)l(,)i(b)o(y)e(itself,)189 2057 y(the)18 b(corresp)q(ondence)h(b)q(et)o(w)o(een)f(the)h(datat)o(yp)q(e) e(handles)i(and)g(the)f(datat)o(yp)q(e)f(they)h(represen)o(t.)189 2114 y(This)i(corresp)q(ondence)h(w)o(as)d(established)j(when)g(the)e (datat)o(yp)q(es)g(w)o(ere)g(created.)33 b(Before)20 b(the)189 2170 y(library)g(is)h(used,)h(a)d(library)i(initialization)i (pream)o(ble)e(m)o(ust)e(b)q(e)i(executed.)36 b(This)20 b(pream)o(ble)189 2227 y(co)q(de)14 b(will)i(de\014ne)f(the)e(datat)o (yp)q(es)h(that)f(are)g(used)i(b)o(y)f(the)g(library)l(,)g(and)g(store) f(handles)j(to)d(these)189 2283 y(datat)o(yp)q(es)i(in)h(global,)g (static)f(v)m(ariables)i(that)e(are)g(shared)h(b)o(y)f(the)h(user)g(co) q(de)g(and)g(the)f(library)189 2339 y(co)q(de.)189 2415 y(The)f(F)l(ortran)f(v)o(ersion)i(of)e Fw(MPI)p 734 2415 V 16 w(REDUCE)i Fv(will)h(in)o(v)o(ok)o(e)e(a)g(user-de\014ned)i (reduce)f(function)g(using)189 2471 y(the)g(F)l(ortran)g(calling)i(con) o(v)o(en)o(tions)f(and)g(will)h(pass)f(a)f(F)l(ortran-t)o(yp)q(e)g (datat)o(yp)q(e)g(argumen)o(t;)g(the)189 2527 y(C)d(v)o(ersion)h(will)h (use)f(C)g(calling)h(con)o(v)o(en)o(tion)f(and)g(the)f(C)h(represen)o (tation)f(of)h(a)f(datat)o(yp)q(e)g(handle.)189 2584 y(Users)i(who)h(plan)h(to)e(mix)h(languages)g(should)h(de\014ne)g (their)g(reduction)f(functions)h(accordingly)l(.)189 2640 y(\()p Fl(End)f(of)i(advic)n(e)f(to)g(users.)p Fv(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 123 131 123 130 bop 75 -100 a Fm(4.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)819 b Fv(123)189 45 y Fl(A)n(dvic)n(e)15 b(to)h(implementors.)39 b Fv(W)l(e)15 b(outline)h(b)q(elo)o(w)f(a)g(naiv)o(e)g(and)g (ine\016cien)o(t)h(implemen)o(tation)g(of)189 102 y Fw(MPI)p 274 102 14 2 v 15 w(REDUCE)p Fv(.)189 227 y Ft(if)23 b(\(rank)g(>)h(0\))g({)284 283 y(RECV\(tempbuf,)e(count,)h(datatype,)g (rank-1,...\))284 339 y(User_reduce\()f(tempbuf,)h(sendbuf,)g(count,)g (datatype\))189 396 y(})189 452 y(if)g(\(rank)g(<)h(groupsize-1\))e({) 284 509 y(SEND\()h(sendbuf,)g(count,)g(datatype,)g(rank+1,)g(...\))189 565 y(})189 622 y(/*)g(answer)g(now)h(resides)f(in)g(process)g (groupsize-1)f(...)i(now)f(send)h(to)f(root)212 678 y(*/)189 735 y(if)g(\(rank)g(==)h(groupsize-1\))e({)284 791 y(SEND\()h(sendbuf,) g(count,)g(datatype,)g(root,)g(...\))189 848 y(})189 904 y(if)g(\(rank)g(==)h(root\))f({)284 960 y(RECV\(recvbuf,)f(count,)h (datatype,)g(groupsize-1,...\))189 1017 y(})189 1142 y Fv(The)16 b(reduction)h(computation)e(pro)q(ceeds,)i(sequen)o(tially) l(,)h(from)d(pro)q(cess)h Ft(0)f Fv(to)h(pro)q(cess)g Ft(group-)189 1198 y(size-1)p Fv(.)i(This)13 b(order)g(is)g(c)o(hosen)g (so)f(as)h(to)f(resp)q(ect)h(the)g(order)f(of)h(a)f(p)q(ossibly)i (non-comm)o(utativ)o(e)189 1255 y(op)q(erator)g(de\014ned)j(b)o(y)f (the)g(function)g Ft(User)p 949 1255 15 2 v 17 w(reduce\(\))p Fv(.)j(A)d(more)f(e\016cien)o(t)i(implemen)o(tation)f(is)189 1311 y(ac)o(hiev)o(ed)f(b)o(y)g(taking)g(adv)m(an)o(tage)g(of)f(asso)q (ciativit)o(y)h(and)h(using)f(a)g(logarithmic)h(tree)e(reduction.)189 1368 y(Comm)o(utativit)o(y)e(can)i(b)q(e)g(used)g(to)f(adv)m(an)o (tage,)g(for)g(those)g(cases)g(in)i(whic)o(h)f(the)g Fw(commute)e Fv(argu-)189 1424 y(men)o(t)h(to)g Fw(MPI)p 441 1424 14 2 v 16 w(OP)p 519 1424 V 17 w(CREA)l(TE)i Fv(is)f(true.)20 b(Also,)14 b(the)g(amoun)o(t)f(of)g(temp)q(orary)g (bu\013er)h(required)h(can)189 1481 y(b)q(e)i(reduced,)i(and)e(comm)o (unication)h(can)f(b)q(e)h(pip)q(elined)j(with)c(computation,)g(b)o(y)g (transferring)189 1537 y(and)e(reducing)h(the)g(elemen)o(ts)g(in)g(c)o (h)o(unks)f(of)g(size)h Ft(len)f Fp(<)p Ft(count)p Fv(.)189 1612 y(The)21 b(prede\014ned)h(reduce)g(op)q(erations)f(can)g(b)q(e)h (implemen)o(ted)g(as)f(a)f(library)i(of)e(user-de\014ned)189 1669 y(op)q(erations.)g(Ho)o(w)o(ev)o(er,)14 b(b)q(etter)h(p)q (erformance)g(migh)o(t)g(b)q(e)h(ac)o(hiev)o(ed)g(if)g Fw(MPI)p 1521 1669 V 16 w(REDUCE)g Fv(handles)189 1725 y(these)f(functions)h(as)f(a)g(sp)q(ecial)i(case.)j(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fv(\))75 1926 y Fw(MPI)p 160 1926 V 16 w(OP)p 238 1926 V 17 w(FREE\()f(op\))117 2003 y Fk(IN)155 b Fw(op)541 b Fk(op)q(eration)14 b(\(handle\))75 2127 y Ft(int)23 b(MPI)p 245 2127 15 2 v 17 w(op)p 310 2127 V 17 w(free\()g(MPI)p 542 2127 V 17 w(Op)h(*op\))75 2214 y(MPI)p 150 2214 V 17 w(OP)p 215 2214 V 17 w(FREE\()f(OP,)g (IERROR\))170 2270 y(INTEGER)g(OP,)h(IERROR)166 2357 y Fv(Marks)12 b(a)h(user-de\014ned)i(reduction)f(op)q(eration)g(for)f (deallo)q(cation)i(and)e(sets)g Fw(op)g Fv(to)g Fi(MPI)p 1675 2357 13 2 v 14 w(OP)p 1747 2357 V 14 w(NULL)p Fv(.)75 2477 y Fw(Example)i(of)g(User-de\014ned)i(Reduce)75 2563 y Fv(It)e(is)h(time)f(for)g(an)g(example)h(of)f(user-de\014ned)i (reduction.)75 2669 y Fq(Example)h(4.20)23 b Fv(Compute)14 b(the)i(pro)q(duct)f(of)g(an)g(arra)o(y)f(of)h(complex)h(n)o(um)o(b)q (ers,)f(in)h(C.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 124 132 124 131 bop 75 -100 a Fv(124)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(typedef)23 b(struct)g({)170 102 y(double)g(real,imag;)75 158 y(})h(Complex;)75 271 y(/*)g(the)f(user-defined)f(function)99 327 y(*/)75 384 y(void)h(myProd\()g(Complex)g(*in,)g(Complex)g(*inout,)g(int)h (*len,)f(MPI_Datatype)f(*dptr)h(\))75 440 y({)170 497 y(int)h(i;)170 553 y(Complex)f(c;)170 666 y(for)h(\(i=0;)f(i<)h(*len;)f (++i\))g({)266 723 y(c.real)g(=)h(inout->real*in->real)d(-)528 779 y(inout->imag*in->imag;)266 835 y(c.imag)i(=)h (inout->real*in->imag)d(+)528 892 y(inout->imag*in->real;)266 948 y(*inout)i(=)h(c;)266 1005 y(in++;)f(inout++;)170 1061 y(})75 1118 y(})75 1231 y(/*)h(and,)f(to)g(call)h(it...)99 1287 y(*/)75 1344 y(...)170 1456 y(/*)g(each)f(process)g(has)h(an)f (array)g(of)h(100)g(Complexes)194 1513 y(*/)170 1569 y(Complex)f(a[100],)g(answer[100];)170 1626 y(MPI_Op)g(myOp;)170 1682 y(MPI_Datatype)g(ctype;)170 1795 y(/*)h(explain)f(to)h(MPI)f(how)g (type)h(Complex)f(is)g(defined)194 1852 y(*/)170 1908 y(MPI_Type_contiguous\()f(2,)h(MPI_DOUBLE,)g(&ctype)g(\);)170 1965 y(MPI_Type_commit\()f(&ctype)h(\);)170 2021 y(/*)h(create)f(the)h (complex-product)d(user-op)194 2077 y(*/)170 2134 y(MPI_Op_create\()h (myProd,)h(True,)g(&myOp)h(\);)170 2247 y(MPI_Reduce\()f(a,)g(answer,)g (100,)h(ctype,)f(myOp,)g(root,)g(comm)g(\);)170 2360 y(/*)h(At)g(this)f(point,)g(the)g(answer,)g(which)h(consists)e(of)i (100)f(Complexes,)194 2416 y(*)h(resides)f(on)h(process)e(root)194 2473 y(*/)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 125 133 125 132 bop 75 -100 a Fm(4.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l (TIONS)819 b Fv(125)75 45 y Fj(4.9.5)49 b(All-Reduce)75 131 y Fw(MPI)17 b Fv(includes)j(v)m(arian)o(ts)e(of)f(eac)o(h)h(of)f (the)h(reduce)g(op)q(erations)g(where)g(the)g(result)g(is)g(returned)g (to)f(all)75 187 y(pro)q(cesses)i(in)h(the)f(group.)32 b Fw(MPI)18 b Fv(requires)i(that)e(all)i(pro)q(cesses)g(participating)g (in)g(these)f(op)q(erations)75 244 y(receiv)o(e)d(iden)o(tical)h (results.)75 395 y Fw(MPI)p 160 395 14 2 v 16 w(ALLREDUCE\()e(sendbuf,) i(recvbuf,)f(count,)g(datat)o(yp)q(e,)g(op,)f(comm\))117 472 y Fk(IN)155 b Fw(sendbuf)444 b Fk(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 547 y(OUT)108 b Fw(recvbuf)450 b Fk(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er) g(\(c)o(hoice\))117 622 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 697 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(send)i(bu\013er)g(\(handle\))117 772 y(IN)155 b Fw(op)541 b Fk(op)q(eration)14 b(\(handle\))117 847 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))75 972 y Ft(int)23 b(MPI)p 245 972 15 2 v 17 w(Allreduce\(void*)f (sendbuf,)h(void*)g(recvbuf,)g(int)g(count,)393 1028 y(MPI)p 468 1028 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 1028 V 17 w(Op)i(op,)f(MPI)p 1266 1028 V 17 w(Comm)g(comm\))75 1115 y(MPI)p 150 1115 V 17 w(ALLREDUCE\(SENDBUF,)e(RECVBUF,)i(COUNT,)g (DATATYPE,)g(OP,)g(COMM,)g(IERROR\))170 1171 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1227 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 1314 y Fv(Same)18 b(as)g Fw(MPI)p 434 1314 14 2 v 15 w(REDUCE)h Fv(except)g(that)e(the)h(result)g(app)q(ears)h(in) f(the)g(receiv)o(e)h(bu\013er)f(of)g(all)h(the)75 1370 y(group)c(mem)o(b)q(ers.)189 1476 y Fl(A)n(dvic)n(e)20 b(to)h(implementors.)71 b Fv(The)21 b(all-reduce)h(op)q(erations)f(can) f(b)q(e)i(implemen)o(ted)g(as)e(a)g(re-)189 1533 y(duce,)d(follo)o(w)o (ed)h(b)o(y)e(a)h(broadcast.)24 b(Ho)o(w)o(ev)o(er,)16 b(a)g(direct)i(implemen)o(tation)g(can)f(lead)h(to)e(b)q(etter)189 1589 y(p)q(erformance.)k(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g (implementors.)p Fv(\))75 1695 y Fq(Example)i(4.21)23 b Fv(A)c(routine)h(that)f(computes)h(the)f(pro)q(duct)h(of)g(a)f(v)o (ector)g(and)h(an)f(arra)o(y)f(that)h(are)75 1751 y(distributed)13 b(across)d(a)h(group)g(of)g(pro)q(cesses)h(and)f(returns)g(the)h(answ)o (er)e(at)h(all)h(no)q(des)g(\(see)f(also)g(Example)75 1808 y(4.16\).)75 1913 y Ft(SUBROUTINE)22 b(PAR_BLAS2\(m,)h(n,)g(a,)h (b,)f(c,)h(comm\))75 1970 y(REAL)f(a\(m\),)g(b\(m,n\))95 b(!)24 b(local)f(slice)g(of)h(array)75 2026 y(REAL)f(c\(n\))286 b(!)24 b(result)75 2083 y(REAL)f(sum\(n\))75 2139 y(INTEGER)g(n,)g (comm,)h(i,)f(j,)h(ierr)75 2252 y(!)g(local)f(sum)75 2308 y(DO)h(j=)f(1,)h(n)123 2365 y(sum\(j\))f(=)g(0.0)123 2421 y(DO)g(i)h(=)g(1,)f(m)170 2478 y(sum\(j\))g(=)h(sum\(j\))f(+)h (a\(i\)*b\(i,j\))123 2534 y(END)f(DO)75 2591 y(END)g(DO)75 2704 y(!)h(global)f(sum)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 126 134 126 133 bop 75 -100 a Fv(126)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(CALL)23 b(MPI_ALLREDUCE\(sum,)f(c,)h(n,)h(MPI_REAL,)f(MPI_SUM,)f(0,)i(comm,)f (ierr\))75 158 y(!)h(return)f(result)g(at)g(all)h(nodes)75 214 y(RETURN)75 358 y Fn(4.10)59 b(Reduce-Scatte)o(r)75 459 y Fw(MPI)16 b Fv(includes)k(v)m(arian)o(ts)d(of)f(eac)o(h)h(of)g (the)g(reduce)h(op)q(erations)f(where)g(the)g(result)h(is)f(scattered)g (to)f(all)75 516 y(pro)q(cesses)g(in)g(the)f(group)g(on)g(return.)75 667 y Fw(MPI)p 160 667 14 2 v 16 w(REDUCE)p 352 667 V 17 w(SCA)l(TTER\()h(sendbuf,)h(recvbuf,)f(recvcounts,)g(datat)o(yp)q (e,)h(op,)e(comm\))117 744 y Fk(IN)155 b Fw(sendbuf)444 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 819 y(OUT)108 b Fw(recvbuf)450 b Fk(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 894 y(IN)155 b Fw(recvcounts)388 b Fk(in)o(teger)14 b(arra)o(y)f(sp)q(ecifying)g (the)h(n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g(in)f(re-)905 950 y(sult)i(distributed)h(to)f(eac)o(h)h(pro)q(cess.)21 b(Arra)o(y)14 b(m)o(ust)f(b)q(e)i(iden-)905 1007 y(tical)e(on)h(all)f (calling)f(pro)q(cesses.)117 1082 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 1157 y(IN)155 b Fw(op)541 b Fk(op)q(eration)14 b(\(handle\))117 1232 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))75 1357 y Ft(int)23 b(MPI)p 245 1357 15 2 v 17 w(Reduce)p 406 1357 V 17 w(scatter\(void*)f(sendbuf,)g(void*)i (recvbuf,)e(int)i(*recvcounts,)393 1413 y(MPI)p 468 1413 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 1413 V 17 w(Op)i(op,)f(MPI)p 1266 1413 V 17 w(Comm)g(comm\))75 1500 y(MPI)p 150 1500 V 17 w(REDUCE)p 311 1500 V 16 w(SCATTER\(SENDBUF,)f(RECVBUF,)h (RECVCOUNTS,)f(DATATYPE,)h(OP,)g(COMM,)393 1556 y(IERROR\))170 1613 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1669 y(INTEGER)g(RECVCOUNTS\(*\),)f(DATATYPE,)h(OP,)g(COMM,)h(IERROR)166 1756 y Fw(MPI)p 251 1756 14 2 v 16 w(REDUCE)p 443 1756 V 17 w(SCA)l(TTER)f Fv(\014rst)e(do)q(es)h(an)g(elemen)o(t-wise)h (reduction)g(on)e(v)o(ector)g(of)h Fw(count)j Fv(=)75 1780 y Fe(P)119 1825 y Fi(i)138 1812 y Fw(recvcounts)p Fv([)p Fw(i)p Fv(])18 b(elemen)o(ts)g(in)f(the)g(send)h(bu\013er)f (de\014ned)h(b)o(y)f Fw(sendbuf,)i(count)g Fv(and)e Fw(datat)o(yp)q(e)p Fv(.)26 b(Next,)75 1868 y(the)19 b(resulting)h(v)o(ector)e(of)g (results)h(is)h(split)g(in)o(to)e Ft(n)h Fv(disjoin)o(t)h(segmen)o(ts,) e(where)h Ft(n)g Fv(is)g(the)g(n)o(um)o(b)q(er)g(of)75 1925 y(mem)o(b)q(ers)c(in)h(the)f(group.)k(Segmen)o(t)c Ft(i)g Fv(con)o(tains)g Fw(recvcounts[i])h Fv(elemen)o(ts.)21 b(The)15 b Ft(i)p Fv(th)f(segmen)o(t)h(is)g(sen)o(t)75 1981 y(to)g(pro)q(cess)g Ft(i)g Fv(and)g(stored)g(in)h(the)f(receiv)o (e)i(bu\013er)e(de\014ned)h(b)o(y)f Fw(recvbuf,)h(recvcounts[i])h Fv(and)e Fw(datat)o(yp)q(e)p Fv(.)189 2088 y Fl(A)n(dvic)n(e)d(to)h (implementors.)37 b Fv(The)12 b Fw(MPI)p 875 2088 V 16 w(REDUCE)p 1067 2088 V 17 w(SCA)l(TTER)h Fv(routine)f(is)g (functionally)i(equiv)m(a-)189 2144 y(len)o(t)c(to:)17 b(A)10 b Fw(MPI)p 473 2144 V 16 w(REDUCE)h Fv(op)q(eration)g(function)g (with)g Fw(count)g Fv(equal)g(to)f(the)g(sum)g(of)g Fw(recvcounts[i]) 189 2201 y Fv(follo)o(w)o(ed)17 b(b)o(y)g Fw(MPI)p 516 2201 V 16 w(SCA)l(TTERV)h Fv(with)f Fw(sendcounts)k Fv(equal)c(to)g Fw(recvcounts)p Fv(.)27 b(Ho)o(w)o(ev)o(er,)16 b(a)g(direct)189 2257 y(implemen)o(tation)g(ma)o(y)f(run)g(faster.)k(\()p Fl(End)d(of)g(advic)n(e)g(to)h(implementors.)p Fv(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 127 135 127 134 bop 75 -100 a Fm(4.11.)34 b(SCAN)1477 b Fv(127)75 45 y Fn(4.11)59 b(Scan)75 241 y Fw(MPI)p 160 241 14 2 v 16 w(SCAN\()15 b(sendbuf,)j(recvbuf,)d(count,)h(datat)o(yp)q(e,)h (op,)e(comm)e(\))117 318 y Fk(IN)155 b Fw(sendbuf)444 b Fk(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 393 y(OUT)108 b Fw(recvbuf)450 b Fk(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 469 y(IN)155 b Fw(count)482 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input) g(bu\013er)g(\(in)o(teger\))117 544 y(IN)155 b Fw(datat)o(yp)q(e)424 b Fk(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 619 y(IN)155 b Fw(op)541 b Fk(op)q(eration)14 b(\(handle\))117 694 y(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))75 818 y Ft(int)23 b(MPI)p 245 818 15 2 v 17 w(Scan\(void*)g(sendbuf,)f(void*)i(recvbuf,)e(int)i(count,)393 875 y(MPI)p 468 875 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 875 V 17 w(Op)i(op,)f(MPI)p 1266 875 V 17 w(Comm)g(comm)h(\))75 961 y(MPI)p 150 961 V 17 w(SCAN\(SENDBUF,)e(RECVBUF,)h(COUNT,)g (DATATYPE,)f(OP,)i(COMM,)f(IERROR\))170 1018 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1074 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 1161 y Fw(MPI)p 251 1161 14 2 v 16 w(SCAN)15 b Fv(is)h(used)f(to)g(p)q(erform)f(a)h(pre\014x)g(reduction)h(on)f (data)f(distributed)i(across)f(the)g(group.)75 1217 y(The)k(op)q (eration)g(returns,)h(in)f(the)g(receiv)o(e)h(bu\013er)f(of)f(the)h (pro)q(cess)g(with)h(rank)e Ft(i)p Fv(,)i(the)f(reduction)g(of)75 1274 y(the)h(v)m(alues)h(in)g(the)f(send)h(bu\013ers)e(of)h(pro)q (cesses)g(with)h(ranks)e Ft(0,...,i)g Fv(\(inclusiv)o(e\).)36 b(The)20 b(t)o(yp)q(e)g(of)75 1330 y(op)q(erations)c(supp)q(orted,)h (their)g(seman)o(tics,)f(and)g(the)h(constrain)o(ts)f(on)g(send)h(and)f (receiv)o(e)h(bu\013ers)f(are)75 1386 y(as)f(for)f Fw(MPI)p 285 1386 V 16 w(REDUCE)p Fv(.)189 1493 y Fl(R)n(ationale.)38 b Fv(W)l(e)13 b(ha)o(v)o(e)g(de\014ned)h(an)f(inclusiv)o(e)i(scan,)e (that)f(is,)i(the)f(pre\014x)g(reduction)h(on)f(pro)q(cess)189 1549 y Ft(i)k Fv(includes)j(the)e(data)f(from)f(pro)q(cess)i Ft(i)p Fv(.)27 b(An)18 b(alternativ)o(e)g(is)g(to)f(de\014ne)i(scan)e (in)i(an)e(exclusiv)o(e)189 1606 y(manner,)11 b(where)f(the)h(result)f (on)h Ft(i)f Fv(only)h(includes)h(data)e(up)h(to)e Ft(i-1)p Fv(.)18 b(Both)10 b(de\014nitions)i(are)e(useful.)189 1662 y(The)k(latter)g(has)g(some)g(adv)m(an)o(tages:)19 b(the)c(inclusiv)o(e)h(scan)f(can)f(alw)o(a)o(ys)g(b)q(e)h(computed)f (from)g(the)189 1719 y(exclusiv)o(e)k(scan)f(with)g(no)g(additional)h (comm)o(unication;)g(for)f(non-in)o(v)o(ertible)i(op)q(erations)e(suc)o (h)189 1775 y(as)f(max)g(and)g(min,)i(comm)o(unication)f(is)g(required) g(to)f(compute)h(the)f(exclusiv)o(e)j(scan)d(from)g(the)189 1831 y(inclusiv)o(e)i(scan.)23 b(There)16 b(is,)g(ho)o(w)o(ev)o(er,)f (a)h(complication)i(with)e(exclusiv)o(e)i(scan)e(since)h(one)g(m)o(ust) 189 1888 y(de\014ne)g(the)f(\\unit")g(elemen)o(t)h(for)e(the)h (reduction)h(in)g(this)f(case.)22 b(That)15 b(is,)i(one)f(m)o(ust)f (explicitly)189 1944 y(sa)o(y)21 b(what)h(o)q(ccurs)g(for)g(pro)q(cess) g Ft(0)p Fv(.)41 b(This)23 b(w)o(as)e(though)o(t)g(to)h(b)q(e)h (complex)g(for)e(user-de\014ned)189 2001 y(op)q(erations)15 b(and)g(hence,)h(the)g(exclusiv)o(e)g(scan)g(w)o(as)e(dropp)q(ed.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fv(\))75 2123 y Fj(4.11.1)49 b(Example)17 b(using)g(MPI)p 646 2123 15 2 v 18 w(SCAN)75 2208 y Fq(Example)h(4.22)23 b Fv(This)17 b(example)h(uses)f(a)f(user-de\014ned)j(op)q(eration)e(to)f(pro)q(duce) i(a)e Fl(se)n(gmente)n(d)g(sc)n(an)p Fv(.)75 2265 y(A)k(segmen)o(ted)g (scan)g(tak)o(es,)f(as)h(input,)h(a)f(set)f(of)h(v)m(alues)h(and)f(a)f (set)h(of)f(logicals,)j(and)e(the)g(logicals)75 2321 y(delineate)d(the)e(v)m(arious)h(segmen)o(ts)f(of)f(the)i(scan.)k(F)l (or)14 b(example:)321 2419 y Fp(v)r(al)q(ues)66 b(v)540 2426 y Fg(1)650 2419 y Fp(v)672 2426 y Fg(2)781 2419 y Fp(v)803 2426 y Fg(3)913 2419 y Fp(v)935 2426 y Fg(4)1143 2419 y Fp(v)1165 2426 y Fg(5)1323 2419 y Fp(v)1345 2426 y Fg(6)1455 2419 y Fp(v)1477 2426 y Fg(7)1587 2419 y Fp(v)1609 2426 y Fg(8)321 2476 y Fp(l)q(og)r(ical)q(s)49 b Fv(0)109 b(0)g(1)g(1)206 b(1)158 b(0)109 b(0)g(1)321 2532 y Fp(r)q(esul)q(t)76 b(v)540 2539 y Fg(1)601 2532 y Fp(v)623 2539 y Fg(1)653 2532 y Fv(+)10 b Fp(v)720 2539 y Fg(2)781 2532 y Fp(v)803 2539 y Fg(3)865 2532 y Fp(v)887 2539 y Fg(3)917 2532 y Fv(+)g Fp(v)984 2539 y Fg(4)1045 2532 y Fp(v)1067 2539 y Fg(3)1097 2532 y Fv(+)h Fp(v)1165 2539 y Fg(4)1194 2532 y Fv(+)g Fp(v)1262 2539 y Fg(5)1323 2532 y Fp(v)1345 2539 y Fg(6)1406 2532 y Fp(v)1428 2539 y Fg(6)1458 2532 y Fv(+)g Fp(v)1526 2539 y Fg(7)1587 2532 y Fp(v)1609 2539 y Fg(8)166 2631 y Fv(The)k(op)q(erator)g(that)f(pro)q(duces)i(this)g(e\013ect)f(is,)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 128 136 128 135 bop 75 -100 a Fv(128)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)711 51 y Fe( )764 95 y Fp(u)769 151 y(i)811 51 y Fe(!)854 123 y Fx(\016)887 51 y Fe( )940 95 y Fp(v)942 151 y(j)985 51 y Fe(!)1030 123 y Fv(=)1078 51 y Fe( )1132 95 y Fp(w)1138 151 y(j)1186 51 y Fe(!)1227 123 y Fp(;)166 234 y Fv(where,)735 369 y Fp(w)d Fv(=)829 297 y Fe(\()883 340 y Fp(u)d Fv(+)h Fp(v)43 b Fv(if)16 b Fp(i)c Fv(=)h Fp(j)883 397 y(v)125 b Fv(if)16 b Fp(i)c Fx(6)p Fv(=)h Fp(j)1203 369 y(:)166 480 y Fv(Note)20 b(that)g(this)h(is)g(a)f(non-comm)o(utativ)o(e)g(op)q (erator.)36 b(C)20 b(co)q(de)h(that)f(implemen)o(ts)i(it)e(is)h(giv)o (en)75 537 y(b)q(elo)o(w.)75 638 y Ft(typedef)i(struct)g({)170 694 y(double)g(val;)170 751 y(int)h(log;)75 807 y(})g(SegScanPair;)75 920 y(/*)g(the)f(user-defined)f(function)99 977 y(*/)75 1033 y(void)h(segScan\()g(SegScanPair)f(*in,)i(SegScanPair)e(*inout,)h (int)g(*len,)75 1090 y(MPI_Datatype)f(*dptr)h(\))75 1146 y({)170 1202 y(int)h(i;)170 1259 y(SegScanPair)f(c;)170 1372 y(for)h(\(i=0;)f(i<)h(*len;)f(++i\))g({)266 1428 y(if)g(\()h(in->log)f(==)h(inout->log)e(\))361 1485 y(c.val)i(=)f (in->val)g(+)h(inout->val;)266 1541 y(else)361 1598 y(c.val)g(=)f (inout->val;)266 1654 y(c.log)g(=)h(inout->log;)266 1711 y(*inout)f(=)h(c;)266 1767 y(in++;)f(inout++;)170 1823 y(})75 1880 y(})166 1981 y Fv(Note)c(that)f(the)i Fw(inout)g Fv(argumen)o(t)e(to)h(the)g(user-de\014ned)i(function)f(corresp)q(onds) g(to)f(the)g(righ)o(t-)75 2038 y(hand)13 b(op)q(erand)g(of)g(the)f(op)q (erator.)19 b(When)13 b(using)g(this)g(op)q(erator,)f(w)o(e)h(m)o(ust)f (b)q(e)h(careful)h(to)e(sp)q(ecify)i(that)75 2094 y(it)h(is)h(non-comm) o(utativ)o(e,)f(as)f(in)i(the)g(follo)o(wing.)170 2195 y Ft(int)24 b(i,base;)170 2252 y(SeqScanPair)47 b(a,)23 b(answer;)170 2308 y(MPI_Op)167 b(myOp;)170 2365 y(MPI_Datatype)23 b(type[2])g(=)g({MPI_DOUBLE,)g(MPI_INT};)170 2421 y(MPI_Aint)119 b(disp[2];)170 2478 y(int)239 b(blocklen[2])22 b(=)i({)f(1,)h(1};)170 2534 y(MPI_Datatype)f(sspair;)170 2647 y(/*)h(explain)f(to)h(MPI)f(how) g(type)h(SegScanPair)e(is)i(defined)194 2704 y(*/)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 129 137 129 136 bop 75 -100 a Fm(4.12.)29 b(CORRECTNESS)1262 b Fv(129)170 45 y Ft(MPI_Address\()23 b(a,)g(disp\);)170 102 y(MPI_Address\()g(a.log,)g(disp+1\);)170 158 y(base)h(=)f(disp[0];) 170 214 y(for)h(\(i=0;)f(i<2;)g(++i\))h(disp[i])f(-=)g(base;)170 271 y(MPI_Type_struct\()f(2,)i(blocklen,)e(disp,)i(type,)f(&sspair)g (\);)170 327 y(MPI_Type_commit\()f(&sspair)h(\);)170 384 y(/*)h(create)f(the)h(segmented-scan)d(user-op)194 440 y(*/)170 497 y(MPI_Op_create\()h(segScan,)h(False,)g(&myOp)g(\);) 170 553 y(...)170 610 y(MPI_Scan\()g(a,)h(answer,)f(1,)g(sspair,)g (myOp,)g(root,)g(comm)h(\);)75 753 y Fn(4.12)59 b(Co)n(rrectness)75 854 y Fv(A)11 b(correct,)f(p)q(ortable)h(program)f(m)o(ust)g(in)o(v)o (ok)o(e)g(collectiv)o(e)i(comm)o(unications)g(so)e(that)g(deadlo)q(c)o (k)h(will)h(not)75 911 y(o)q(ccur,)h(whether)g(collectiv)o(e)h(comm)o (unications)g(are)e(sync)o(hronizing)i(or)e(not.)19 b(The)13 b(follo)o(wing)g(examples)75 967 y(illustrate)j(dangerous)f(use)h(of)f (collectiv)o(e)i(routines.)75 1074 y Fq(Example)h(4.23)23 b Fv(The)15 b(follo)o(wing)h(is)g(erroneous.)75 1180 y Ft(switch\(rank\))22 b({)170 1236 y(case)i(0:)266 1293 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h(comm\);)266 1349 y(MPI_Bcast\(buf2,)e(count,)h(type,)g(1,)h(comm\);)266 1406 y(break;)170 1462 y(case)g(1:)266 1519 y(MPI_Bcast\(buf2,)e (count,)h(type,)g(1,)h(comm\);)266 1575 y(MPI_Bcast\(buf1,)e(count,)h (type,)g(0,)h(comm\);)266 1631 y(break;)75 1688 y(})166 1794 y Fv(W)l(e)19 b(assume)h(that)f(the)g(group)g(of)g Fw(comm)f Fv(is)i Fx(f)p Fv(0,1)p Fx(g)p Fv(.)31 b(Tw)o(o)19 b(pro)q(cesses)g(execute)i(t)o(w)o(o)d(broadcast)75 1851 y(op)q(erations)d(in)h(rev)o(erse)f(order.)20 b(If)15 b(the)h(op)q(eration)f(is)h(sync)o(hronizing)h(then)e(a)g(deadlo)q(c)o (k)h(will)h(o)q(ccur.)166 1907 y(Collectiv)o(e)h(op)q(erations)e(m)o (ust)g(b)q(e)g(executed)i(in)f(the)f(same)g(order)g(at)f(all)j(mem)o(b) q(ers)e(of)g(the)g(com-)75 1964 y(m)o(unication)g(group.)75 2070 y Fq(Example)i(4.24)23 b Fv(The)15 b(follo)o(wing)h(is)g (erroneous.)75 2176 y Ft(switch\(rank\))22 b({)170 2233 y(case)i(0:)266 2289 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h (comm0\);)266 2345 y(MPI_Bcast\(buf2,)e(count,)h(type,)g(2,)h(comm2\);) 266 2402 y(break;)170 2458 y(case)g(1:)266 2515 y(MPI_Bcast\(buf1,)e (count,)h(type,)g(1,)h(comm1\);)266 2571 y(MPI_Bcast\(buf2,)e(count,)h (type,)g(0,)h(comm0\);)266 2628 y(break;)170 2684 y(case)g(2:)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 130 138 130 137 bop 75 -100 a Fv(130)685 b Fm(CHAPTER)15 b(4.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)266 45 y Ft(MPI_Bcast\(buf1,)22 b(count,)h(type,)g(2,)h(comm2\);)266 102 y(MPI_Bcast\(buf2,)e(count,)h (type,)g(1,)h(comm1\);)266 158 y(break;)75 214 y(})166 319 y Fv(Assume)13 b(that)f(the)i(group)e(of)h Fw(comm0)e Fv(is)i Fx(f)p Fv(0,1)p Fx(g)p Fv(,)f(of)h Fw(comm1)d Fv(is)k Fx(f)p Fv(1,)e(2)p Fx(g)h Fv(and)g(of)g Fw(comm2)e Fv(is)i Fx(f)p Fv(2,0)p Fx(g)p Fv(.)18 b(If)75 375 y(the)12 b(broadcast)g(is)h(a)f(sync)o(hronizing)i(op)q(eration,)e(then)h(there) g(is)f(a)g(cyclic)j(dep)q(endency:)20 b(the)13 b(broadcast)75 432 y(in)i Fw(comm2)d Fv(completes)j(only)f(after)g(the)g(broadcast)f (in)i Fw(comm0)p Fv(;)d(the)i(broadcast)f(in)i Fw(comm0)d Fv(completes)75 488 y(only)18 b(after)e(the)h(broadcast)g(in)h Fw(comm1)p Fv(;)d(and)i(the)h(broadcast)e(in)i Fw(comm1)d Fv(completes)j(only)f(after)g(the)75 545 y(broadcast)d(in)i Fw(comm2)p Fv(.)i(Th)o(us,)d(the)g(co)q(de)h(will)h(deadlo)q(c)o(k.)166 601 y(Collectiv)o(e)c(op)q(erations)f(m)o(ust)f(b)q(e)i(executed)g(in)f (an)g(order)g(so)f(that)g(no)h(cyclic)i(dep)q(endences)g(o)q(ccur.)75 705 y Fq(Example)k(4.25)23 b Fv(The)15 b(follo)o(wing)h(is)g (erroneous.)75 810 y Ft(switch\(rank\))22 b({)170 866 y(case)i(0:)266 923 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h(comm\);) 266 979 y(MPI_Send\(buf2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)266 1036 y(break;)170 1092 y(case)h(1:)266 1149 y(MPI_Recv\(buf2,)e(count,) h(type,)g(0,)h(tag,)f(comm,)g(status\);)266 1205 y(MPI_Bcast\(buf1,)f (count,)h(type,)g(0,)h(comm\);)266 1261 y(break;)75 1318 y(})166 1422 y Fv(Pro)q(cess)15 b(zero)f(executes)i(a)e(broadcast,)g (follo)o(w)o(ed)h(b)o(y)g(a)f(blo)q(c)o(king)j(send)e(op)q(eration.)20 b(Pro)q(cess)15 b(one)75 1479 y(\014rst)21 b(executes)g(a)g(blo)q(c)o (king)h(receiv)o(e)g(that)e(matc)o(hes)h(the)g(send,)h(follo)o(w)o(ed)g (b)o(y)e(broadcast)h(call)h(that)75 1535 y(matc)o(hes)14 b(the)h(broadcast)e(of)i(pro)q(cess)f(zero.)20 b(This)15 b(program)e(ma)o(y)h(deadlo)q(c)o(k.)21 b(The)15 b(broadcast)e(call)j (on)75 1592 y(pro)q(cess)g(zero)f Fl(may)g Fv(blo)q(c)o(k)h(un)o(til)h (pro)q(cess)e(one)h(executes)f(the)h(matc)o(hing)f(broadcast)f(call,)i (so)f(that)g(the)75 1648 y(send)i(is)h(not)e(executed.)26 b(Pro)q(cess)17 b(one)g(will)h(de\014nitely)h(blo)q(c)o(k)f(on)e(the)h (receiv)o(e)h(and)f(so,)f(in)i(this)f(case,)75 1705 y(nev)o(er)e (executes)h(the)f(broadcast.)166 1761 y(The)j(relativ)o(e)g(order)f(of) g(execution)h(of)f(collectiv)o(e)i(op)q(erations)f(and)g(p)q(oin)o (t-to-p)q(oin)o(t)g(op)q(erations)75 1817 y(should)f(b)q(e)h(suc)o(h,)e (so)g(that)g(ev)o(en)h(if)g(the)g(collectiv)o(e)h(op)q(erations)e(and)h (the)g(p)q(oin)o(t-to-p)q(oin)o(t)g(op)q(erations)75 1874 y(are)e(sync)o(hronizing,)h(no)f(deadlo)q(c)o(k)i(will)f(o)q (ccur.)75 1978 y Fq(Example)i(4.26)23 b Fv(A)15 b(correct,)f(but)h (non-deterministic)j(program.)75 2083 y Ft(switch\(rank\))k({)170 2139 y(case)i(0:)266 2195 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h (comm\);)266 2252 y(MPI_Send\(buf2,)e(count,)h(type,)g(1,)h(tag,)f (comm\);)266 2308 y(break;)170 2365 y(case)h(1:)266 2421 y(MPI_Recv\(buf2,)e(count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h(comm,)g (status\);)266 2478 y(MPI_Bcast\(buf1,)f(count,)h(type,)g(0,)h(comm\);) 266 2534 y(MPI_Recv\(buf2,)e(count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h (comm,)g(status\);)266 2591 y(break;)170 2647 y(case)h(2:)266 2704 y(MPI_Send\(buf2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 131 139 131 138 bop 75 -100 a Fm(4.12.)34 b(CORRECTNESS)1257 b Fv(131)383 4 y 18945146 16482269 0 0 29733355 25918013 startTexFig 383 4 a %%BeginDocument: coll-matchings.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -50 0 translate -40.0 436.0 translate 0.900 -0.900 scale 0.500 setlinewidth % Polyline n 469 134 m 334 134 l gs col-1 s gr n 342.000 136.000 m 334.000 134.000 l 342.000 132.000 l gs 2 setlinejoin col-1 s gr % Polyline n 154 399 m 284 399 l gs col-1 s gr n 276.000 397.000 m 284.000 399.000 l 276.000 401.000 l gs 2 setlinejoin col-1 s gr % Polyline n 469 454 m 329 454 l gs col-1 s gr n 337.000 456.000 m 329.000 454.000 l 337.000 452.000 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 279 59 m gs 1 -1 scale (First Execution) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 274 299 m gs 1 -1 scale (Second Execution) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 139 94 m gs 1 -1 scale (0) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 319 94 m gs 1 -1 scale (1) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 509 94 m gs 1 -1 scale (2) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 194 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 484 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 194 m gs 1 -1 scale (send) col-1 show gr % Polyline n 159 189 m 279 189 l gs col-1 s gr n 271.000 187.000 m 279.000 189.000 l 271.000 191.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 16.00 scalefont setfont 294 139 m gs 1 -1 scale (recv) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 44 94 m gs 1 -1 scale (process:) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 484 139 m gs 1 -1 scale (send) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 199 184 m gs 1 -1 scale (match) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 384 129 m gs 1 -1 scale (match) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 429 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 404 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 459 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 479 459 m gs 1 -1 scale (send) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 479 484 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 404 m gs 1 -1 scale (send) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 379 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 194 394 m gs 1 -1 scale (match) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 379 449 m gs 1 -1 scale (match) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 1146 a Fv(Figure)13 b(4.10:)k(A)c(race)f(condition)i(causes)f (non-deterministic)i(matc)o(hing)d(of)g(sends)i(and)e(receiv)o(es.)20 b(One)75 1202 y(cannot)15 b(rely)h(on)f(sync)o(hronization)h(from)e(a)h (broadcast)g(to)f(mak)o(e)h(the)g(program)f(deterministic.)266 1335 y Ft(MPI_Bcast\(buf1,)22 b(count,)h(type,)g(0,)h(comm\);)266 1392 y(break;)75 1448 y(})166 1555 y Fv(All)19 b(three)f(pro)q(cesses)g (participate)h(in)g(a)e(broadcast.)28 b(Pro)q(cess)17 b(0)h(sends)g(a)g(message)f(to)h(pro)q(cess)75 1611 y(1)h(after)g(the)h (broadcast,)f(and)h(pro)q(cess)f(2)h(sends)g(a)f(message)g(to)g(pro)q (cess)h(1)f(b)q(efore)g(the)h(broadcast.)75 1667 y(Pro)q(cess)15 b(1)g(receiv)o(es)h(b)q(efore)g(and)f(after)f(the)i(broadcast,)e(with)h (a)g(wildcard)i(source)e(argumen)o(t.)166 1724 y(Tw)o(o)10 b(p)q(ossible)j(executions)g(of)e(this)g(program,)g(with)h(di\013eren)o (t)f(matc)o(hings)g(of)g(sends)h(and)f(receiv)o(es,)75 1780 y(are)k(illustrated)i(in)f(\014gure)g(4.10.)j(Note)c(that)g(the)g (second)h(execution)g(has)g(the)f(p)q(eculiar)i(e\013ect)f(that)e(a)75 1837 y(send)k(executed)g(after)f(the)h(broadcast)e(is)i(receiv)o(ed)h (at)e(another)g(no)q(de)h(b)q(efore)g(the)f(broadcast.)26 b(This)75 1893 y(example)15 b(illustrates)g(the)g(fact)f(that)f(one)i (should)g(not)f(rely)h(on)f(collectiv)o(e)i(comm)o(unication)f (functions)75 1950 y(to)h(ha)o(v)o(e)g(particular)h(sync)o(hronization) h(e\013ects.)23 b(A)17 b(program)f(that)f(w)o(orks)h(correctly)h(only)g (when)g(the)75 2006 y(\014rst)e(execution)h(o)q(ccurs)g(\(only)f(when)h (broadcast)e(is)i(sync)o(hronizing\))g(is)g(erroneous.)166 2112 y(Finally)l(,)h(in)g(m)o(ultithreaded)g(implemen)o(tations,)g(one) f(can)g(ha)o(v)o(e)g(more)g(than)f(one,)h(concurren)o(tly)75 2169 y(executing,)e(collectiv)o(e)h(comm)o(unication)f(call)h(at)d(a)h (pro)q(cess.)20 b(In)13 b(these)h(situations,)f(it)h(is)g(the)f(user's) g(re-)75 2225 y(sp)q(onsibilit)o(y)i(to)d(ensure)h(that)f(the)h(same)f (comm)o(unicator)h(is)g(not)f(used)h(concurren)o(tly)h(b)o(y)e(t)o(w)o (o)g(di\013eren)o(t)75 2282 y(collectiv)o(e)17 b(comm)o(unication)f (calls)g(at)f(the)g(same)g(pro)q(cess.)189 2388 y Fl(A)n(dvic)n(e)e(to) h(implementors.)39 b Fv(Assume)13 b(that)f(broadcast)g(is)h(implemen)o (ted)i(using)e(p)q(oin)o(t-to-p)q(oin)o(t)189 2445 y Fw(MPI)h Fv(comm)o(unication.)21 b(Supp)q(ose)16 b(the)f(follo)o(wing)h (t)o(w)o(o)e(rules)i(are)f(follo)o(w)o(ed.)231 2538 y(1.)22 b(All)16 b(receiv)o(es)g(sp)q(ecify)h(their)e(source)h(explicitly)i (\(no)c(wildcards\).)231 2611 y(2.)22 b(Eac)o(h)13 b(pro)q(cess)g (sends)h(all)g(messages)f(that)f(p)q(ertain)i(to)f(one)g(collectiv)o(e) i(call)f(b)q(efore)g(sending)289 2668 y(an)o(y)g(message)h(that)g(p)q (ertain)h(to)e(a)h(subsequen)o(t)h(collectiv)o(e)h(call.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 132 140 132 139 bop 75 -100 a Fv(132)680 b Fm(CHAPTER)15 b(4.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)189 45 y Fv(Then,)h(messages)f(b)q (elonging)j(to)d(successiv)o(e)i(broadcasts)e(cannot)g(b)q(e)i (confused,)f(as)f(the)h(order)189 102 y(of)d(p)q(oin)o(t-to-p)q(oin)o (t)i(messages)f(is)h(preserv)o(ed.)189 177 y(It)g(is)h(the)g(implemen)o (tor's)g(resp)q(onsibilit)o(y)i(to)d(ensure)i(that)e(p)q(oin)o(t-to-p)q (oin)o(t)h(messages)f(are)g(not)189 233 y(confused)d(with)g(collectiv)o (e)h(messages.)19 b(One)13 b(w)o(a)o(y)f(to)g(accomplish)i(this)f(is,)g (whenev)o(er)g(a)f(comm)o(u-)189 290 y(nicator)j(is)g(created,)g(to)g (also)g(create)g(a)g(\\hidden)h(comm)o(unicator")f(for)f(collectiv)o(e) j(comm)o(unica-)189 346 y(tion.)j(One)15 b(could)h(ac)o(hiev)o(e)f(a)f (similar)i(e\013ect)e(more)g(c)o(heaply)l(,)i(for)e(example,)h(b)o(y)g (using)g(a)f(hidden)189 403 y(tag)f(or)h(con)o(text)g(bit)h(to)f (indicate)i(whether)f(the)f(comm)o(unicator)g(is)h(used)g(for)f(p)q (oin)o(t-to-p)q(oin)o(t)h(or)189 459 y(collectiv)o(e)i(comm)o (unication.)j(\()p Fl(End)c(of)g(advic)n(e)g(to)h(implementors.)p Fv(\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 133 141 133 140 bop 75 356 a Fo(Chapter)34 b(5)75 564 y Fr(Groups,)40 b(Con)m(texts,)g(and)75 689 y(Comm)m(unicators)75 929 y Fn(5.1)59 b(Intro)r(duction)75 1031 y Fv(This)20 b(c)o(hapter)g(in)o (tro)q(duces)h Fw(MPI)e Fv(features)g(that)h(supp)q(ort)g(the)f(dev)o (elopmen)o(t)i(of)e(parallel)j(libraries.)75 1087 y(P)o(arallel)15 b(libraries)h(are)f(needed)g(to)f(encapsulate)i(the)e(distracting)h (complications)h(inheren)o(t)f(in)h(paral-)75 1144 y(lel)i(implemen)o (tations)g(of)f(k)o(ey)g(algorithms.)24 b(They)18 b(help)g(to)e(ensure) h(consisten)o(t)g(correctness)g(of)g(suc)o(h)75 1200 y(pro)q(cedures,)22 b(and)e(pro)o(vide)h(a)f(\\higher)h(lev)o(el")g(of) f(p)q(ortabilit)o(y)h(than)f Fw(MPI)g Fv(itself)h(can)f(pro)o(vide.)36 b(As)75 1257 y(suc)o(h,)23 b(libraries)g(prev)o(en)o(t)e(eac)o(h)g (programmer)f(from)h(rep)q(eating)h(the)g(w)o(ork)e(of)h(de\014ning)i (consisten)o(t)75 1313 y(data)14 b(structures,)f(data)h(la)o(y)o(outs,) g(and)g(metho)q(ds)g(that)g(implemen)o(t)i(k)o(ey)e(algorithms)g(\(suc) o(h)g(as)g(matrix)75 1369 y(op)q(erations\).)24 b(Since)18 b(the)e(b)q(est)h(libraries)h(come)e(with)h(sev)o(eral)g(v)m(ariations) g(on)f(parallel)i(systems)e(\(dif-)75 1426 y(feren)o(t)f(data)g(la)o(y) o(outs,)g(di\013eren)o(t)g(strategies)g(dep)q(ending)j(on)e(the)f(size) i(of)e(the)g(system)h(or)f(problem,)g(or)75 1482 y(t)o(yp)q(e)g(of)g (\015oating)g(p)q(oin)o(t\),)g(this)h(to)q(o)e(needs)i(to)f(b)q(e)h (hidden)h(from)d(the)h(user.)166 1539 y(W)l(e)h(refer)f(the)g(reader)h (to)f([26)o(])g(and)h([3)o(])f(for)g(further)g(information)h(on)g (writing)g(libraries)h(in)f Fw(MPI)p Fv(,)75 1595 y(using)g(the)f (features)g(describ)q(ed)i(in)f(this)g(c)o(hapter.)75 1717 y Fj(5.1.1)49 b(F)o(eatures)14 b(Needed)i(to)g(Supp)q(o)o(rt)g (Lib)o(ra)o(ries)75 1803 y Fv(The)f(k)o(ey)h(features)e(needed)j(to)d (supp)q(ort)i(the)f(creation)g(of)g(robust)g(parallel)i(libraries)f (are)f(as)g(follo)o(ws:)143 1897 y Fx(\017)23 b Fv(Safe)18 b(comm)o(unication)i(space,)g(that)e(guaran)o(tees)g(that)g(libraries)i (can)f(comm)o(unicate)g(as)g(they)189 1953 y(need)d(to,)e(without)h (con\015icting)i(with)e(comm)o(unication)h(extraneous)f(to)g(the)g (library)l(,)143 2047 y Fx(\017)23 b Fv(Group)14 b(scop)q(e)h(for)g (collectiv)o(e)h(op)q(erations,)f(that)f(allo)o(w)h(libraries)h(to)e(a) o(v)o(oid)h(unnecessarily)i(syn-)189 2103 y(c)o(hronizing)f(unin)o(v)o (olv)o(ed)h(pro)q(cesses)e(\(p)q(oten)o(tially)i(running)f(unrelated)g (co)q(de\),)143 2197 y Fx(\017)23 b Fv(Abstract)15 b(pro)q(cess)h (naming)h(to)e(allo)o(w)h(libraries)i(to)d(describ)q(e)j(their)e(comm)o (unication)h(in)g(terms)189 2254 y(suitable)f(to)f(their)g(o)o(wn)g (data)g(structures)f(and)i(algorithms,)143 2347 y Fx(\017)23 b Fv(The)14 b(abilit)o(y)i(to)e(\\adorn")f(a)h(set)h(of)f(comm)o (unicating)h(pro)q(cesses)g(with)f(additional)i(user-de\014ned)189 2404 y(attributes,)23 b(suc)o(h)f(as)g(extra)g(collectiv)o(e)i(op)q (erations.)41 b(This)23 b(mec)o(hanism)f(should)i(pro)o(vide)e(a)189 2460 y(means)15 b(for)f(the)h(user)g(or)g(library)h(writer)f (e\013ectiv)o(ely)h(to)e(extend)i(a)f(message-passing)g(notation.)75 2554 y(In)f(addition,)h(a)f(uni\014ed)h(mec)o(hanism)g(or)e(ob)s(ject)g (is)i(needed)g(for)e(con)o(v)o(enien)o(tly)i(denoting)f(comm)o(unica-) 75 2611 y(tion)g(con)o(text,)f(the)h(group)f(of)g(comm)o(unicating)h (pro)q(cesses,)g(to)f(house)h(abstract)f(pro)q(cess)h(naming,)g(and)75 2667 y(to)h(store)f(adornmen)o(ts.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 134 142 134 141 bop 75 -100 a Fv(134)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fj(5.1.2)49 b(MPI's)16 b(Supp)q(o)o(rt)g(fo)o(r)g(Lib)o(ra)o(ries) 75 131 y Fv(The)f(corresp)q(onding)h(concepts)g(that)e Fw(MPI)h Fv(pro)o(vides,)g(sp)q(eci\014cally)j(to)c(supp)q(ort)h (robust)g(libraries,)h(are)75 187 y(as)f(follo)o(ws:)143 281 y Fx(\017)23 b Fq(Con)o(texts)14 b Fv(of)h(comm)o(unication,)143 375 y Fx(\017)23 b Fq(Groups)14 b Fv(of)h(pro)q(cesses,)143 469 y Fx(\017)23 b Fq(Virtual)18 b(top)q(ologies)p Fv(,)143 563 y Fx(\017)23 b Fq(A)o(ttribute)17 b(cac)o(hing)p Fv(,)143 656 y Fx(\017)23 b Fq(Comm)o(unicators)p Fv(.)75 750 y Fq(Comm)o(unicators)18 b Fv(\(see)g([16)o(,)f(24,)h(27)o(]\))g (encapsulate)h(all)g(of)f(these)g(ideas)h(in)g(order)f(to)f(pro)o(vide) i(the)75 807 y(appropriate)h(scop)q(e)h(for)f(all)h(comm)o(unication)g (op)q(erations)f(in)h Fw(MPI)p Fv(.)e(Comm)o(unicators)h(are)g(divided) 75 863 y(in)o(to)15 b(t)o(w)o(o)f(kinds:)21 b(in)o(tra-comm)o (unicators)15 b(for)f(op)q(erations)h(within)i(a)e(single)h(group)f(of) g(pro)q(cesses,)g(and)75 920 y(in)o(ter-comm)o(unicators,)g(for)f(p)q (oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)g(b)q(et)o(w)o(een)g(t)o(w)o (o)e(groups)g(of)h(pro)q(cesses.)75 1040 y Fw(Caching.)46 b Fv(Comm)o(unicators)17 b(\(see)g(b)q(elo)o(w\))i(pro)o(vide)f(a)f (\\cac)o(hing")h(mec)o(hanism)h(that)e(allo)o(ws)h(one)g(to)75 1096 y(asso)q(ciate)h(new)g(attributes)g(with)g(comm)o(unicators,)g(on) g(a)g(par)g(with)g Fw(MPI)f Fv(built-in)k(features.)30 b(This)75 1153 y(can)15 b(b)q(e)h(used)g(b)o(y)f(adv)m(anced)h(users)f (to)f(adorn)h(comm)o(unicators)g(further,)f(and)i(b)o(y)f Fw(MPI)f Fv(to)h(implemen)o(t)75 1209 y(some)20 b(comm)o(unicator)g (functions.)35 b(F)l(or)19 b(example,)j(the)f(virtual-top)q(ology)f (functions)h(describ)q(ed)h(in)75 1266 y(Chapter)15 b(6)g(are)g(lik)o (ely)i(to)d(b)q(e)i(supp)q(orted)g(this)f(w)o(a)o(y)l(.)75 1386 y Fw(Groups.)46 b Fv(Groups)16 b(de\014ne)h(an)f(ordered)g (collection)i(of)e(pro)q(cesses,)g(eac)o(h)g(with)h(a)f(rank,)f(and)i (it)f(is)h(this)75 1442 y(group)d(that)g(de\014nes)i(the)f(lo)o(w-lev)o (el)h(names)e(for)g(in)o(ter-pro)q(cess)i(comm)o(unication)f(\(ranks)f (are)g(used)i(for)75 1498 y(sending)k(and)g(receiving\).)33 b(Th)o(us,)20 b(groups)e(de\014ne)j(a)d(scop)q(e)i(for)f(pro)q(cess)g (names)g(in)h(p)q(oin)o(t-to-p)q(oin)o(t)75 1555 y(comm)o(unication.)i (In)16 b(addition,)h(groups)e(de\014ne)i(the)e(scop)q(e)i(of)e (collectiv)o(e)i(op)q(erations.)k(Groups)16 b(ma)o(y)75 1611 y(b)q(e)k(manipulated)h(separately)f(from)e(comm)o(unicators)h(in) i Fw(MPI)p Fv(,)d(but)i(only)g(comm)o(unicators)f(can)g(b)q(e)75 1668 y(used)d(in)g(comm)o(unication)g(op)q(erations.)75 1788 y Fw(Intra-communicato)o(rs.)44 b Fv(The)18 b(most)f(commonly)g (used)i(means)e(for)g(message)g(passing)h(in)h Fw(MPI)e Fv(is)h(via)75 1844 y(in)o(tra-comm)o(unicators.)41 b(In)o(tra-comm)o (unicators)22 b(con)o(tain)g(an)h(instance)g(of)f(a)g(group,)i(con)o (texts)d(of)75 1901 y(comm)o(unication)g(for)e(b)q(oth)h(p)q(oin)o (t-to-p)q(oin)o(t)h(and)f(collectiv)o(e)i(comm)o(unication,)g(and)e (the)g(abilit)o(y)i(to)75 1957 y(include)c(virtual)d(top)q(ology)g(and) h(other)e(attributes.)20 b(These)c(features)f(w)o(ork)f(as)h(follo)o (ws:)143 2051 y Fx(\017)23 b Fq(Con)o(texts)13 b Fv(pro)o(vide)i(the)g (abilit)o(y)h(to)d(ha)o(v)o(e)h(separate)g(safe)g(\\univ)o(erses")h(of) f(message)g(passing)h(in)189 2108 y Fw(MPI)p Fv(.)f(A)h(con)o(text)g (is)h(akin)f(to)g(an)g(additional)i(tag)d(that)h(di\013eren)o(tiates)g (messages.)20 b(The)15 b(system)189 2164 y(manages)20 b(this)h(di\013eren)o(tiation)g(pro)q(cess.)36 b(The)21 b(use)g(of)f(separate)g(comm)o(unication)i(con)o(texts)189 2220 y(b)o(y)13 b(distinct)i(libraries)g(\(or)e(distinct)i(library)f (in)o(v)o(o)q(cations\))g(insulates)h(comm)o(unication)f(in)o(ternal) 189 2277 y(to)h(the)h(library)h(execution)g(from)e(external)h(comm)o (unication.)23 b(This)17 b(allo)o(ws)f(the)g(in)o(v)o(o)q(cation)g(of) 189 2333 y(the)e(library)g(ev)o(en)h(if)f(there)g(are)g(p)q(ending)i (comm)o(unications)e(on)g(\\other")f(comm)o(unicators,)h(and)189 2390 y(a)o(v)o(oids)f(the)h(need)g(to)f(sync)o(hronize)i(en)o(try)e(or) g(exit)i(in)o(to)e(library)i(co)q(de.)20 b(P)o(ending)14 b(p)q(oin)o(t-to-p)q(oin)o(t)189 2446 y(comm)o(unications)j(are)g(also) f(guaran)o(teed)h(not)f(to)g(in)o(terfere)i(with)f(collectiv)o(e)h (comm)o(unications)189 2503 y(within)e(a)f(single)h(comm)o(unicator.) 143 2597 y Fx(\017)23 b Fq(Groups)12 b Fv(de\014ne)i(the)e(participan)o (ts)i(in)f(the)g(comm)o(unication)g(\(see)g(ab)q(o)o(v)o(e\))f(of)g(a)h (comm)o(unicator.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 135 143 135 142 bop 75 -100 a Fm(5.1.)34 b(INTR)o(ODUCTION)1256 b Fv(135)143 45 y Fx(\017)23 b Fv(A)14 b Fq(virtual)j(top)q(ology)f Fv(de\014nes)f(a)f(sp)q(ecial)i(mapping)f(of)f(the)h(ranks)f(in)h(a)f (group)g(to)f(and)i(from)e(a)189 102 y(top)q(ology)l(.)19 b(Sp)q(ecial)f(constructors)c(for)g(comm)o(unicators)h(are)g(de\014ned) h(in)g(c)o(hapter)f(6)g(to)f(pro)o(vide)189 158 y(this)f(feature.)19 b(In)o(tra-comm)o(unicators)12 b(as)h(describ)q(ed)i(in)f(this)f(c)o (hapter)g(do)g(not)f(ha)o(v)o(e)h(top)q(ologies.)143 252 y Fx(\017)23 b Fq(A)o(ttributes)14 b Fv(de\014ne)h(the)g(lo)q(cal)g (information)g(that)f(the)g(user)h(or)f(library)h(has)f(added)h(to)f(a) g(com-)189 308 y(m)o(unicator)h(for)f(later)h(reference.)189 415 y Fl(A)n(dvic)n(e)f(to)i(users.)40 b Fv(The)15 b(curren)o(t)f (practice)i(in)f(man)o(y)f(comm)o(unication)i(libraries)g(is)f(that)f (there)189 471 y(is)k(a)f(unique,)i(prede\014ned)g(comm)o(unication)g (univ)o(erse)f(that)f(includes)j(all)e(pro)q(cesses)g(a)o(v)m(ailable) 189 527 y(when)f(the)f(parallel)i(program)d(is)i(initiated;)i(the)d (pro)q(cesses)h(are)f(assigned)h(consecutiv)o(e)h(ranks.)189 584 y(P)o(articipan)o(ts)d(in)h(a)g(p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o (unication)g(are)g(iden)o(ti\014ed)h(b)o(y)f(their)g(rank;)f(a)g (collec-)189 640 y(tiv)o(e)i(comm)o(unication)h(\(suc)o(h)g(as)f (broadcast\))f(alw)o(a)o(ys)g(in)o(v)o(olv)o(es)i(all)h(pro)q(cesses.) 26 b(This)18 b(practice)189 697 y(can)e(b)q(e)g(follo)o(w)o(ed)h(in)g Fw(MPI)e Fv(b)o(y)h(using)h(the)f(prede\014ned)i(comm)o(unicator)d Fw(MPI)p 1529 697 14 2 v 16 w(COMM)p 1687 697 V 16 w(W)o(ORLD)p Fv(.)189 753 y Fl(Users)i(who)h(ar)n(e)g(satis\014e)n(d)f(with)h(this)g (pr)n(actic)n(e)f(c)n(an)h(plug)g(in)f Fw(MPI)p 1351 753 V 16 w(COMM)p 1509 753 V 16 w(W)o(ORLD)h Fl(wher)n(ever)189 810 y(a)f(c)n(ommunic)n(ator)g(ar)n(gument)g(is)f(r)n(e)n(quir)n(e)n (d,)g(and)h(c)n(an)f(c)n(onse)n(quently)f(disr)n(e)n(gar)n(d)h(the)h(r) n(est)f(of)h(this)189 866 y(chapter.)k Fv(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fv(\))75 986 y Fw(Inter-communicato)o (rs.)44 b Fv(The)18 b(discussion)h(has)f(dealt)g(so)f(far)g(with)g Fq(in)o(tra-comm)o(unication)p Fv(:)27 b(com-)75 1043 y(m)o(unication)21 b(within)h(a)e(group.)36 b Fw(MPI)19 b Fv(also)i(supp)q(orts)f Fq(in)o(ter-comm)o(unication)p Fv(:)32 b(comm)o(unication)75 1099 y(b)q(et)o(w)o(een)18 b(t)o(w)o(o)f(non-o)o(v)o(erlapping)i(groups.)28 b(When)18 b(an)g(application)i(is)e(built)h(b)o(y)f(comp)q(osing)h(sev)o(eral)75 1156 y(parallel)e(mo)q(dules,)g(it)f(is)h(con)o(v)o(enien)o(t)f(to)f (allo)o(w)h(one)g(mo)q(dule)h(to)e(comm)o(unicate)i(with)f(another)f (using)75 1212 y(lo)q(cal)22 b(ranks)e(for)f(addressing)i(within)h(the) f(second)f(mo)q(dule.)37 b(This)21 b(is)g(esp)q(ecially)i(con)o(v)o (enien)o(t)e(in)g(a)75 1269 y(clien)o(t-serv)o(er)e(computing)f (paradigm,)g(where)g(either)h(clien)o(t)g(or)e(serv)o(er)g(are)h (parallel.)29 b(The)18 b(supp)q(ort)75 1325 y(of)f(in)o(ter-comm)o (unication)h(also)f(pro)o(vides)g(a)g(mec)o(hanism)g(for)f(the)i (extension)f(of)g Fw(MPI)f Fv(to)g(a)h(dynamic)75 1381 y(mo)q(del)h(where)g(not)f(all)i(pro)q(cesses)f(are)f(preallo)q(cated)h (at)f(initialization)k(time.)27 b(In)18 b(suc)o(h)g(a)f(situation,)75 1438 y(it)h(b)q(ecomes)g(necessary)g(to)g(supp)q(ort)f(comm)o (unication)i(across)e(\\univ)o(erses.")28 b(In)o(ter-comm)o(unication) 75 1494 y(is)17 b(supp)q(orted)f(b)o(y)g(ob)s(jects)g(called)h Fq(in)o(ter-comm)o(unicators)p Fv(.)23 b(These)16 b(ob)s(jects)g(bind)h (t)o(w)o(o)e(groups)g(to-)75 1551 y(gether)c(with)h(comm)o(unication)g (con)o(texts)e(shared)i(b)o(y)f(b)q(oth)g(groups.)18 b(F)l(or)11 b(in)o(ter-comm)o(unicators,)h(these)75 1607 y(features)j(w)o(ork)f(as)h(follo)o(ws:)143 1701 y Fx(\017)23 b Fq(Con)o(texts)16 b Fv(pro)o(vide)i(the)f(abilit)o(y)i(to)d(ha)o(v)o (e)h(a)g(separate)g(safe)f(\\univ)o(erse")i(of)f(message)g(passing)189 1758 y(b)q(et)o(w)o(een)k(the)f(t)o(w)o(o)g(groups.)36 b(A)21 b(send)g(in)g(the)g(lo)q(cal)h(group)e(is)i(alw)o(a)o(ys)e(a)g (receiv)o(e)i(in)f(the)g(re-)189 1814 y(mote)d(group,)g(and)h(vice)h(v) o(ersa.)29 b(The)19 b(system)f(manages)g(this)h(di\013eren)o(tiation)g (pro)q(cess.)31 b(The)189 1870 y(use)19 b(of)f(separate)g(comm)o (unication)i(con)o(texts)e(b)o(y)g(distinct)i(libraries)h(\(or)d (distinct)h(library)h(in-)189 1927 y(v)o(o)q(cations\))f(insulates)i (comm)o(unication)g(in)o(ternal)f(to)g(the)g(library)g(execution)h (from)f(external)189 1983 y(comm)o(unication.)29 b(This)19 b(allo)o(ws)f(the)g(in)o(v)o(o)q(cation)h(of)f(the)g(library)h(ev)o(en) f(if)h(there)f(are)g(p)q(ending)189 2040 y(comm)o(unications)c(on)f (\\other")f(comm)o(unicators,)h(and)h(a)o(v)o(oids)f(the)g(need)h(to)f (sync)o(hronize)h(en)o(try)189 2096 y(or)k(exit)h(in)o(to)g(library)h (co)q(de.)32 b(There)19 b(is)g(no)g(general-purp)q(ose)i(collectiv)o(e) f(comm)o(unication)g(on)189 2153 y(in)o(ter-comm)o(unicators,)d(so)g (con)o(texts)f(are)h(used)h(just)e(to)h(isolate)h(p)q(oin)o(t-to-p)q (oin)o(t)f(comm)o(unica-)189 2209 y(tion.)143 2303 y Fx(\017)23 b Fv(A)c(lo)q(cal)i(and)e(remote)g(group)g(sp)q(ecify)i(the) f(recipien)o(ts)h(and)e(destinations)h(for)f(an)h(in)o(ter-com-)189 2359 y(m)o(unicator.)143 2453 y Fx(\017)j Fv(Virtual)16 b(top)q(ology)e(is)i(unde\014ned)h(for)e(an)g(in)o(ter-comm)o (unicator.)143 2547 y Fx(\017)23 b Fv(As)16 b(b)q(efore,)g(attributes)g (cac)o(he)h(de\014nes)g(the)f(lo)q(cal)i(information)e(that)g(the)g (user)g(or)g(library)h(has)189 2604 y(added)e(to)g(a)g(comm)o(unicator) g(for)f(later)i(reference.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 136 144 136 143 bop 75 -100 a Fv(136)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)166 45 y Fw(MPI)i Fv(pro)o(vides)i(mec)o(hanisms)g(for)e(creating)h(and)h (manipulating)g(in)o(ter-comm)o(unicators.)29 b(They)75 102 y(are)18 b(used)h(for)f(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)g(in)g(an)f(related)h(manner)f(to)g(in)o(tra-comm)o (unicators.)75 158 y(Users)j(who)g(do)g(not)g(need)h(in)o(ter-comm)o (unication)h(in)f(their)f(applications)i(can)e(safely)h(ignore)g(this) 75 214 y(extension.)k(Users)18 b(who)e(need)i(collectiv)o(e)h(op)q (erations)f(via)f(in)o(ter-comm)o(unicators)g(m)o(ust)g(la)o(y)o(er)g (it)g(on)75 271 y(top)e(of)h Fw(MPI)p Fv(.)f(Users)g(who)h(require)h (in)o(ter-comm)o(unication)g(b)q(et)o(w)o(een)f(o)o(v)o(erlapping)g (groups)g(m)o(ust)f(also)75 327 y(la)o(y)o(er)g(this)h(capabilit)o(y)g (on)f(top)g(of)g Fw(MPI)p Fv(.)75 471 y Fn(5.2)59 b(Basic)19 b(Concepts)75 572 y Fv(In)d(this)f(section,)h(w)o(e)f(turn)g(to)f(a)h (more)g(formal)g(de\014nition)i(of)e(the)g(concepts)h(in)o(tro)q(duced) g(ab)q(o)o(v)o(e.)75 694 y Fj(5.2.1)49 b(Groups)75 780 y Fv(A)24 b Fq(group)h Fv(is)g(an)f(ordered)h(set)f(of)g(pro)q(cess)g (iden)o(ti\014ers)i(\(henceforth)f(pro)q(cesses\);)j(pro)q(cesses)d (are)75 836 y(implemen)o(tation-dep)q(endent)e(ob)s(jects.)36 b(Eac)o(h)20 b(pro)q(cess)i(in)f(a)g(group)f(is)h(asso)q(ciated)g(with) h(an)e(in)o(te-)75 893 y(ger)e Fq(rank)p Fv(.)28 b(Ranks)18 b(are)g(con)o(tiguous)g(and)g(start)f(from)g(zero.)28 b(Groups)18 b(are)f(represen)o(ted)i(b)o(y)f(opaque)75 949 y Fq(group)g(ob)s(jects)p Fv(,)f(and)f(hence)h(cannot)f(b)q(e)h (directly)g(transferred)f(from)f(one)i(pro)q(cess)f(to)g(another.)22 b(A)75 1005 y(group)13 b(is)h(used)g(within)g(a)f(comm)o(unicator)g(to) g(describ)q(e)i(the)e(participan)o(ts)h(in)g(a)f(comm)o(unication)h (\\uni-)75 1062 y(v)o(erse")d(and)i(to)e(rank)g(suc)o(h)i(participan)o (ts)f(\(th)o(us)f(giving)i(them)f(unique)h(names)f(within)h(that)f (\\univ)o(erse")75 1118 y(of)j(comm)o(unication\).)166 1175 y(There)20 b(is)f(a)h(sp)q(ecial)h(pre-de\014ned)g(group:)28 b Fi(MPI)p 1017 1175 13 2 v 14 w(GROUP)p 1173 1175 V 13 w(EMPTY)p Fv(,)19 b(whic)o(h)h(is)g(a)f(group)g(with)h(no)75 1231 y(mem)o(b)q(ers.)33 b(The)20 b(prede\014ned)h(constan)o(t)d Fi(MPI)p 884 1231 V 15 w(GROUP)p 1041 1231 V 13 w(NULL)i Fv(is)g(the)g(v)m(alue)g(used)h(for)d(in)o(v)m(alid)k(group)75 1288 y(handles.)189 1394 y Fl(A)n(dvic)n(e)c(to)i(users.)62 b Fi(MPI)p 650 1394 V 14 w(GROUP)p 806 1394 V 13 w(EMPTY)p Fv(,)18 b(whic)o(h)i(is)f(a)g(v)m(alid)h(handle)g(to)e(an)h(empt)o(y)g (group,)189 1450 y(should)h(not)f(b)q(e)h(confused)h(with)e Fi(MPI)p 863 1450 V 15 w(GROUP)p 1020 1450 V 13 w(NULL)p Fv(,)h(whic)o(h)g(in)g(turn)g(is)g(an)f(in)o(v)m(alid)j(handle.)189 1507 y(The)e(former)f(ma)o(y)g(b)q(e)h(used)g(as)g(an)f(argumen)o(t)g (to)h(group)f(op)q(erations;)j(the)e(latter,)g(whic)o(h)g(is)189 1563 y(returned)15 b(when)h(a)f(group)g(is)g(freed,)h(in)g(not)e(a)h(v) m(alid)i(argumen)o(t.)i(\()p Fl(End)d(of)g(advic)n(e)g(to)h(users.)p Fv(\))189 1670 y Fl(A)n(dvic)n(e)g(to)h(implementors.)53 b Fv(A)18 b(group)f(ma)o(y)f(b)q(e)i(represen)o(ted)g(b)o(y)f(a)g (virtual-to-real)i(pro)q(cess-)189 1726 y(address-translation)11 b(table.)19 b(Eac)o(h)11 b(comm)o(unicator)f(ob)s(ject)h(\(see)g(b)q (elo)o(w\))g(w)o(ould)g(ha)o(v)o(e)g(a)f(p)q(oin)o(ter)189 1782 y(to)k(suc)o(h)i(a)f(table.)189 1858 y(Simple)j(implemen)o (tations)g(of)e Fw(MPI)g Fv(will)i(en)o(umerate)e(groups,)g(suc)o(h)h (as)f(in)i(a)e(table.)24 b(Ho)o(w)o(ev)o(er,)189 1914 y(more)11 b(adv)m(anced)i(data)e(structures)g(mak)o(e)g(sense)i(in)f (order)g(to)f(impro)o(v)o(e)g(scalabilit)o(y)j(and)e(memory)189 1971 y(usage)j(with)g(large)h(n)o(um)o(b)q(ers)f(of)g(pro)q(cesses.)21 b(Suc)o(h)16 b(implemen)o(tations)g(are)f(p)q(ossible)i(with)f Fw(MPI)p Fv(.)189 2027 y(\()p Fl(End)f(of)i(advic)n(e)f(to)g (implementors.)p Fv(\))75 2149 y Fj(5.2.2)49 b(Contexts)75 2235 y Fv(A)18 b Fq(con)o(text)g Fv(is)h(a)f(prop)q(ert)o(y)f(of)h (comm)o(unicators)g(\(de\014ned)h(next\))f(that)f(allo)o(ws)h (partitioning)h(of)f(the)75 2291 y(comm)o(unication)13 b(space.)19 b(A)12 b(message)g(sen)o(t)g(in)h(one)f(con)o(text)g (cannot)g(b)q(e)h(receiv)o(ed)g(in)g(another)f(con)o(text.)75 2347 y(F)l(urthermore,)17 b(where)g(p)q(ermitted,)h(collectiv)o(e)h(op) q(erations)e(are)g(indep)q(enden)o(t)i(of)e(p)q(ending)i(p)q(oin)o (t-to-)75 2404 y(p)q(oin)o(t)g(op)q(erations.)28 b(Con)o(texts)17 b(are)h(not)f(explicit)k Fw(MPI)c Fv(ob)s(jects;)h(they)g(app)q(ear)h (only)f(as)g(part)f(of)h(the)75 2460 y(realization)e(of)f(comm)o (unicators)g(\(b)q(elo)o(w\).)189 2567 y Fl(A)n(dvic)n(e)h(to)i (implementors.)48 b Fv(Distinct)17 b(comm)o(unicators)f(in)h(the)g (same)f(pro)q(cess)h(ha)o(v)o(e)f(distinct)189 2623 y(con)o(texts.)31 b(A)20 b(con)o(text)f(is)g(essen)o(tially)i(a)e(system-managed)g(tag)g (\(or)f(tags\))g(needed)j(to)d(mak)o(e)189 2680 y(a)i(comm)o(unicator)h (safe)g(for)f(p)q(oin)o(t-to-p)q(oin)o(t)i(and)f Fw(MPI)p Fv(-de\014ned)h(collectiv)o(e)h(comm)o(unication.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 137 145 137 144 bop 75 -100 a Fm(5.2.)34 b(BASIC)16 b(CONCEPTS)1215 b Fv(137)189 45 y(Safet)o(y)18 b(means)h(that)f(collectiv)o(e)j(and)e (p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o(unication)h(within)g(one)f(comm)o (u-)189 102 y(nicator)d(do)g(not)g(in)o(terfere,)h(and)g(that)e(comm)o (unication)i(o)o(v)o(er)f(distinct)h(comm)o(unicators)f(don't)189 158 y(in)o(terfere.)189 230 y(A)c(p)q(ossible)i(implemen)o(tation)f (for)e(a)h(con)o(text)g(is)g(as)g(a)g(supplemen)o(tal)h(tag)f(attac)o (hed)f(to)g(messages)189 287 y(on)j(send)g(and)h(matc)o(hed)f(on)g (receiv)o(e.)20 b(Eac)o(h)14 b(in)o(tra-comm)o(unicator)g(stores)g(the) g(v)m(alue)h(of)f(its)g(t)o(w)o(o)189 343 y(tags)i(\(one)i(for)f(p)q (oin)o(t-to-p)q(oin)o(t)h(and)g(one)g(for)f(collectiv)o(e)i(comm)o (unication\).)28 b(Comm)o(unicator-)189 400 y(generating)19 b(functions)i(use)f(a)f(collectiv)o(e)j(comm)o(unication)e(to)f(agree)g (on)h(a)f(new)h(group-wide)189 456 y(unique)c(con)o(text.)189 528 y(Analogously)l(,)c(in)f(in)o(ter-comm)o(unication)h(\(whic)o(h)f (is)g(strictly)g(p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o(unication\),)189 585 y(t)o(w)o(o)f(con)o(text)h(tags)g(are)h(stored)f(p)q(er)i(comm)o (unicator,)f(one)g(used)g(b)o(y)g(group)g(A)f(to)h(send)g(and)g(group) 189 641 y(B)j(to)g(receiv)o(e,)g(and)h(a)f(second)g(used)h(b)o(y)f (group)g(B)g(to)g(send)h(and)f(for)g(group)g(A)g(to)f(receiv)o(e.)189 713 y(Since)h(con)o(texts)f(are)f(not)h(explicit)i(ob)s(jects,)e(other) f(implemen)o(tations)j(are)d(also)h(p)q(ossible.)22 b(\()p Fl(End)189 770 y(of)16 b(advic)n(e)g(to)h(implementors.)p Fv(\))75 889 y Fj(5.2.3)49 b(Intra-Communicato)o(rs)75 975 y Fv(In)o(tra-comm)o(unicators)26 b(bring)i(together)e(the)h (concepts)h(of)e(group)h(and)g(con)o(text.)55 b(T)l(o)27 b(supp)q(ort)75 1031 y(implemen)o(tation-sp)q(eci\014c)15 b(optimizations,)e(and)f(application)h(top)q(ologies)g(\(de\014ned)f (in)h(the)f(next)g(c)o(hap-)75 1087 y(ter,)17 b(c)o(hapter)g(6\),)f (comm)o(unicators)h(ma)o(y)f(also)h(\\cac)o(he")f(additional)j (information)e(\(see)g(section)h(5.7\).)75 1144 y Fw(MPI)h Fv(comm)o(unication)g(op)q(erations)h(reference)f(comm)o(unicators)g (to)g(determine)h(the)f(scop)q(e)h(and)f(the)75 1200 y(\\comm)o(unication)d(univ)o(erse")g(in)g(whic)o(h)g(a)e(p)q(oin)o (t-to-p)q(oin)o(t)i(or)f(collectiv)o(e)i(op)q(eration)e(is)h(to)f(op)q (erate.)166 1257 y(Eac)o(h)g(comm)o(unicator)h(con)o(tains)g(a)f(group) g(of)h(v)m(alid)h(participan)o(ts;)f(this)g(group)g(alw)o(a)o(ys)f (includes)75 1313 y(the)k(lo)q(cal)h(pro)q(cess.)32 b(The)20 b(source)f(and)g(destination)h(of)f(a)f(message)h(is)h(iden)o(ti\014ed) h(b)o(y)e(pro)q(cess)g(rank)75 1370 y(within)d(that)f(group.)166 1426 y(F)l(or)10 b(collectiv)o(e)j(comm)o(unication,)f(the)f(in)o (tra-comm)o(unicator)g(sp)q(eci\014es)i(the)e(set)g(of)f(pro)q(cesses)i (that)75 1483 y(participate)g(in)g(the)g(collectiv)o(e)h(op)q(eration)e (\(and)h(their)f(order,)h(when)g(signi\014can)o(t\).)19 b(Th)o(us,)12 b(the)f(comm)o(u-)75 1539 y(nicator)16 b(restricts)g(the)g(\\spatial")g(scop)q(e)h(of)e(comm)o(unication,)i (and)f(pro)o(vides)h(mac)o(hine-indep)q(endent)75 1596 y(pro)q(cess)e(addressing)h(through)f(ranks.)166 1652 y(In)o(tra-comm)o(unicators)h(are)h(represen)o(ted)g(b)o(y)g(opaque)g Fq(in)o(tra-comm)o(unicator)k(ob)s(jects)p Fv(,)c(and)75 1708 y(hence)f(cannot)f(b)q(e)h(directly)h(transferred)d(from)h(one)g (pro)q(cess)h(to)e(another.)75 1827 y Fj(5.2.4)49 b(Prede\014ned)15 b(Intra-Communicato)o(rs)75 1913 y Fv(An)22 b(initial)i(in)o(tra-comm)o (unicator)e Fi(MPI)p 785 1913 13 2 v 14 w(COMM)p 929 1913 V 15 w(W)o(ORLD)f Fv(of)h(all)h(pro)q(cesses)f(the)g(lo)q(cal)h (pro)q(cess)f(can)75 1970 y(comm)o(unicate)d(with)g(after)f (initialization)k(\(itself)d(included\))j(is)d(de\014ned)h(once)f Fw(MPI)p 1578 1970 14 2 v 16 w(INIT)f Fv(has)h(b)q(een)75 2026 y(called.)i(In)13 b(addition,)h(the)e(comm)o(unicator)g Fi(MPI)p 911 2026 13 2 v 15 w(COMM)p 1056 2026 V 14 w(SELF)h Fv(is)g(pro)o(vided,)g(whic)o(h)h(includes)h(only)e(the)75 2083 y(pro)q(cess)i(itself.)166 2139 y(The)i(prede\014ned)i(constan)o (t)d Fi(MPI)p 745 2139 V 14 w(COMM)p 889 2139 V 15 w(NULL)i Fv(is)f(the)g(v)m(alue)h(used)g(for)e(in)o(v)m(alid)j(comm)o(unicator) 75 2195 y(handles.)166 2252 y(In)d(a)g(static-pro)q(cess-mo)q(del)h (implemen)o(tation)g(of)f Fw(MPI)p Fv(,)e(all)j(pro)q(cesses)g(that)e (participate)h(in)h(the)75 2308 y(computation)i(are)f(a)o(v)m(ailable)j (after)d Fw(MPI)g Fv(is)h(initialized.)34 b(F)l(or)18 b(this)h(case,)g Fi(MPI)p 1479 2308 V 15 w(COMM)p 1624 2308 V 14 w(W)o(ORLD)g Fv(is)g(a)75 2365 y(comm)o(unicator)g(of)g(all)i (pro)q(cesses)f(a)o(v)m(ailable)h(for)e(the)g(computation;)i(this)f (comm)o(unicator)g(has)f(the)75 2421 y(same)h(v)m(alue)h(in)g(all)f (pro)q(cesses.)35 b(In)21 b(an)e(implemen)o(tation)j(of)d Fw(MPI)h Fv(where)g(pro)q(cesses)g(can)g(dynami-)75 2478 y(cally)d(join)g(an)f Fw(MPI)g Fv(execution,)h(it)f(ma)o(y)g(b)q(e)g (the)h(case)f(that)f(a)h(pro)q(cess)g(starts)f(an)h Fw(MPI)g Fv(computation)75 2534 y(without)j(ha)o(ving)g(access)g(to)f(all)i (other)f(pro)q(cesses.)31 b(In)20 b(suc)o(h)f(situations,)h Fi(MPI)p 1479 2534 V 14 w(COMM)p 1623 2534 V 15 w(W)o(ORLD)f Fv(is)g(a)75 2591 y(comm)o(unicator)d(incorp)q(orating)h(all)h(pro)q (cesses)f(with)g(whic)o(h)g(the)g(joining)h(pro)q(cess)f(can)f (immediately)75 2647 y(comm)o(unicate.)28 b(Therefore,)17 b Fi(MPI)p 675 2647 V 15 w(COMM)p 820 2647 V 14 w(W)o(ORLD)h Fv(ma)o(y)f(sim)o(ultaneously)i(ha)o(v)o(e)e(di\013eren)o(t)h(v)m (alues)h(in)75 2704 y(di\013eren)o(t)c(pro)q(cesses.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 138 146 138 145 bop 75 -100 a Fv(138)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)166 45 y Fv(All)j Fw(MPI)d Fv(implemen)o(tations)j(are)e(required)h(to)f (pro)o(vide)h(the)f Fi(MPI)p 1315 45 13 2 v 15 w(COMM)p 1460 45 V 14 w(W)o(ORLD)g Fv(comm)o(unica-)75 102 y(tor.)34 b(It)21 b(cannot)f(b)q(e)h(deallo)q(cated)h(during)f(the)f(life)i(of)e (a)g(pro)q(cess.)36 b(The)20 b(group)g(corresp)q(onding)i(to)75 158 y(this)14 b(comm)o(unicator)g(do)q(es)g(not)g(app)q(ear)g(as)g(a)f (pre-de\014ned)j(constan)o(t,)d(but)h(it)h(ma)o(y)e(b)q(e)i(accessed)f (using)75 214 y Fw(MPI)p 160 214 14 2 v 16 w(COMM)p 318 214 V 16 w(GROUP)21 b Fv(\(see)f(b)q(elo)o(w\).)34 b Fw(MPI)19 b Fv(do)q(es)h(not)g(sp)q(ecify)h(the)f(corresp)q(ondence)h (b)q(et)o(w)o(een)f(the)75 271 y(pro)q(cess)15 b(rank)f(in)h Fi(MPI)p 466 271 13 2 v 15 w(COMM)p 611 271 V 14 w(W)o(ORLD)g Fv(and)f(its)h(\(mac)o(hine-dep)q(enden)o(t\))i(absolute)d(address.)20 b(Neither)75 327 y(do)q(es)14 b Fw(MPI)e Fv(sp)q(ecify)j(the)f (function)g(of)f(the)g(host)g(pro)q(cess,)h(if)g(an)o(y)l(.)19 b(Other)13 b(implemen)o(tation-dep)q(end)q(en)o(t,)75 384 y(prede\014ned)k(comm)o(unicators)e(ma)o(y)f(also)h(b)q(e)h(pro)o (vided.)75 527 y Fn(5.3)59 b(Group)20 b(Management)75 629 y Fv(This)h(section)g(describ)q(es)h(the)e(manipulation)i(of)e(pro) q(cess)h(groups)f(in)h Fw(MPI)p Fv(.)e(These)i(op)q(erations)f(are)75 685 y(lo)q(cal)c(and)g(their)f(execution)i(do)e(not)g(require)h(in)o (terpro)q(cess)f(comm)o(unication.)75 807 y Fj(5.3.1)49 b(Group)16 b(Accesso)o(rs)75 987 y Fw(MPI)p 160 987 14 2 v 16 w(GROUP)p 328 987 V 18 w(SIZE\(group,)e(size\))117 1064 y Fk(IN)155 b Fw(group)479 b Fk(group)14 b(\(handle\))117 1139 y(OUT)108 b Fw(size)519 b Fk(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g(group)g(\(in)o(teger\))75 1264 y Ft(int)23 b(MPI)p 245 1264 15 2 v 17 w(Group)p 382 1264 V 17 w(size\(MPI)p 591 1264 V 16 w(Group)g(group,)g(int)g (*size\))75 1350 y(MPI)p 150 1350 V 17 w(GROUP)p 287 1350 V 16 w(SIZE\(GROUP,)g(SIZE,)g(IERROR\))170 1407 y(INTEGER)g(GROUP,)g(SIZE,)h(IERROR)75 1588 y Fw(MPI)p 160 1588 14 2 v 16 w(GROUP)p 328 1588 V 18 w(RANK\(group,)14 b(rank\))117 1665 y Fk(IN)155 b Fw(group)479 b Fk(group)14 b(\(handle\))117 1740 y(OUT)108 b Fw(rank)504 b Fk(rank)50 b(of)g(the)h(calling)e(pro)q(cess)j(in)e(group,)59 b(or)905 1796 y Fi(MPI)p 982 1796 13 2 v 15 w(UNDEFINED)13 b Fk(if)h(the)h(pro)q (cess)i(is)d(not)h(a)f(mem)o(b)q(er)f(\(in-)905 1853 y(teger\))75 1977 y Ft(int)23 b(MPI)p 245 1977 15 2 v 17 w(Group)p 382 1977 V 17 w(rank\(MPI)p 591 1977 V 16 w(Group)g(group,)g(int)g(*rank\))75 2064 y(MPI)p 150 2064 V 17 w(GROUP)p 287 2064 V 16 w(RANK\(GROUP,)g(RANK,)g(IERROR\))170 2120 y(INTEGER)g(GROUP,)g(RANK,)h(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 139 147 139 146 bop 75 -100 a Fm(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b Fv(139)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(TRANSLA)l(TE)p 602 45 V 17 w(RANKS)16 b(\(group1,)e(n,)h (ranks1,)g(group2,)g(ranks2\))117 122 y Fk(IN)155 b Fw(group1)456 b Fk(group1)14 b(\(handle\))117 197 y(IN)155 b Fw(n)564 b Fk(n)o(um)o(b)q(er)11 b(of)h(ranks)g(in)23 b Fi(ranks1)13 b Fk(and)f Fi(ranks2)g Fk(arra)o(ys)g(\(in)o(teger\))117 273 y(IN)155 b Fw(ranks1)464 b Fk(arra)o(y)14 b(of)f(zero)i(or)f(more)f (v)n(alid)f(ranks)i(in)g(group1)117 348 y(IN)155 b Fw(group2)456 b Fk(group2)14 b(\(handle\))117 423 y(OUT)108 b Fw(ranks2)464 b Fk(arra)o(y)17 b(of)g(corresp)q(onding)h(ranks)f(in)g(group2,)g Fi(MPI)p 1736 423 13 2 v 14 w(UNDE-)905 479 y(FINED)c Fk(when)i(no)e(corresp)q(ondence)k(exists.)75 604 y Ft(int)23 b(MPI)p 245 604 15 2 v 17 w(Group)p 382 604 V 17 w(translate)p 615 604 V 16 w(ranks)g(\(MPI)p 870 604 V 16 w(Group)h(group1,)f(int)g (n,)h(int)f(*ranks1,)393 660 y(MPI)p 468 660 V 17 w(Group)g(group2,)g (int)g(*ranks2\))75 747 y(MPI)p 150 747 V 17 w(GROUP)p 287 747 V 16 w(TRANSLATE)p 519 747 V 16 w(RANKS\(GROUP1,)f(N,)i (RANKS1,)f(GROUP2,)g(RANKS2,)g(IERROR\))170 803 y(INTEGER)g(GROUP1,)g (N,)h(RANKS1\(*\),)e(GROUP2,)h(RANKS2\(*\),)g(IERROR)166 890 y Fv(This)12 b(function)g(is)g(imp)q(ortan)o(t)f(for)f(determining) j(the)e(relativ)o(e)h(n)o(um)o(b)q(ering)g(of)f(the)h(same)f(pro)q (cesses)75 946 y(in)h(t)o(w)o(o)e(di\013eren)o(t)h(groups.)18 b(F)l(or)11 b(instance,)h(if)g(one)f(kno)o(ws)f(the)i(ranks)e(of)h (certain)h(pro)q(cesses)f(in)h(the)f(group)75 1002 y(of)k Fi(MPI)p 204 1002 13 2 v 14 w(COMM)p 348 1002 V 15 w(W)o(ORLD)p Fv(,)f(one)i(migh)o(t)f(w)o(an)o(t)f(to)g(kno)o(w)h(their)h(ranks)f(in) h(a)f(subset)g(of)g(that)f(group.)75 1153 y Fw(MPI)p 160 1153 14 2 v 16 w(GROUP)p 328 1153 V 18 w(COMP)l(ARE\(group1,)h (group2,)g(result\))117 1231 y Fk(IN)155 b Fw(group1)456 b Fk(\014rst)15 b(group)f(\(handle\))117 1306 y(IN)155 b Fw(group2)456 b Fk(second)15 b(group)f(\(handle\))117 1381 y(OUT)108 b Fw(result)484 b Fk(result)15 b(\(in)o(teger\))75 1505 y Ft(int)23 b(MPI)p 245 1505 15 2 v 17 w(Group)p 382 1505 V 17 w(compare\(MPI)p 663 1505 V 15 w(Group)h(group1,MPI)p 1062 1505 V 15 w(Group)g(group2,)e(int)i(*result\))75 1592 y(MPI)p 150 1592 V 17 w(GROUP)p 287 1592 V 16 w(COMPARE\(GROUP1,)e (GROUP2,)h(RESULT,)g(IERROR\))170 1648 y(INTEGER)g(GROUP1,)g(GROUP2,)g (RESULT,)g(IERROR)75 1735 y Fi(MPI)p 152 1735 13 2 v 14 w(IDENT)12 b Fv(results)g(if)h(the)f(group)f(mem)o(b)q(ers)h(and)g (group)g(order)g(is)g(exactly)g(the)g(same)g(in)h(b)q(oth)f(groups.)75 1791 y(This)i(happ)q(ens)h(for)f(instance)g(if)h Fw(group1)e Fv(and)h Fw(group2)g Fv(are)f(the)h(same)g(handle.)21 b Fi(MPI)p 1520 1791 V 14 w(SIMILAR)13 b Fv(results)h(if)75 1848 y(the)f(group)f(mem)o(b)q(ers)h(are)f(the)h(same)f(but)h(the)g (order)f(is)h(di\013eren)o(t.)19 b Fi(MPI)p 1316 1848 V 15 w(UNEQUAL)11 b Fv(results)i(otherwise.)75 1969 y Fj(5.3.2)49 b(Group)16 b(Constructo)o(rs)75 2055 y Fv(Group)j (constructors)f(are)h(used)g(to)g(subset)g(and)g(sup)q(erset)h (existing)g(groups.)31 b(These)20 b(constructors)75 2112 y(construct)14 b(new)g(groups)g(from)f(existing)i(groups.)k(These)14 b(are)g(lo)q(cal)h(op)q(erations,)f(and)g(distinct)h(groups)75 2168 y(ma)o(y)20 b(b)q(e)h(de\014ned)h(on)e(di\013eren)o(t)g(pro)q (cesses;)j(a)e(pro)q(cess)f(ma)o(y)g(also)g(de\014ne)i(a)e(group)g (that)g(do)q(es)g(not)75 2225 y(include)i(itself.)33 b(Consisten)o(t)19 b(de\014nitions)i(are)e(required)h(when)g(groups)f (are)g(used)h(as)f(argumen)o(ts)f(in)75 2281 y(comm)o (unicator-building)23 b(functions.)39 b Fw(MPI)20 b Fv(do)q(es)h(not)g (pro)o(vide)h(a)e(mec)o(hanism)i(to)f(build)i(a)d(group)75 2337 y(from)j(scratc)o(h,)i(but)f(only)h(from)e(other,)i(previously)g (de\014ned)g(groups.)46 b(The)24 b(base)g(group,)h(up)q(on)75 2394 y(whic)o(h)d(all)g(other)e(groups)g(are)h(de\014ned,)i(is)f(the)f (group)f(asso)q(ciated)h(with)g(the)g(initial)i(comm)o(unica-)75 2450 y(tor)14 b Fw(MPI)p 233 2450 14 2 v 16 w(COMM)p 391 2450 V 17 w(W)o(ORLD)h Fv(\(accessible)h(through)f(the)h(function)g Fw(MPI)p 1318 2450 V 15 w(COMM)p 1475 2450 V 17 w(GROUP)p Fv(\).)189 2557 y Fl(R)n(ationale.)74 b Fv(In)21 b(what)g(follo)o(ws,)h (there)e(is)i(no)f(group)f(duplication)j(function)f(analogous)f(to)189 2613 y Fw(MPI)p 274 2613 V 15 w(COMM)p 431 2613 V 17 w(DUP)p Fv(,)16 b(de\014ned)j(later)e(in)g(this)h(c)o(hapter.)24 b(There)17 b(is)h(no)f(need)g(for)g(a)f(group)h(dupli-)189 2670 y(cator.)27 b(A)18 b(group,)g(once)g(created,)g(can)g(ha)o(v)o(e)g (sev)o(eral)g(references)g(to)g(it)g(b)o(y)g(making)g(copies)g(of)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 140 148 140 147 bop 75 -100 a Fv(140)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)189 45 y Fv(the)g(handle.)23 b(The)15 b(follo)o(wing)i(constructors)d (address)i(the)g(need)g(for)f(subsets)h(and)f(sup)q(ersets)h(of)189 102 y(existing)g(groups.)j(\()p Fl(End)d(of)g(r)n(ationale.)p Fv(\))189 208 y Fl(A)n(dvic)n(e)j(to)h(implementors.)65 b Fv(Eac)o(h)19 b(group)g(constructor)g(b)q(eha)o(v)o(es)h(as)e(if)i (it)g(returned)g(a)f(new)189 264 y(group)h(ob)s(ject.)35 b(When)20 b(this)h(new)g(group)f(is)h(a)f(cop)o(y)g(of)g(an)g(existing) i(group,)f(then)f(one)h(can)189 321 y(a)o(v)o(oid)16 b(creating)g(suc)o(h)h(new)g(ob)s(jects,)e(using)j(a)e(reference-coun)o (t)h(mec)o(hanism.)24 b(\()p Fl(End)17 b(of)g(advic)n(e)189 377 y(to)f(implementors.)p Fv(\))75 578 y Fw(MPI)p 160 578 14 2 v 16 w(COMM)p 318 578 V 16 w(GROUP\(comm,)e(group\))117 655 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 730 y(OUT)108 b Fw(group)479 b Fk(group)14 b(corresp)q(onding)h(to)f Fi(comm)e Fk(\(handle\))75 855 y Ft(int)23 b(MPI)p 245 855 15 2 v 17 w(Comm)p 358 855 V 17 w(group\(MPI)p 591 855 V 16 w(Comm)g(comm,)g(MPI)p 941 855 V 17 w(Group)g(*group\))75 941 y(MPI)p 150 941 V 17 w(COMM)p 263 941 V 16 w(GROUP\(COMM,)g(GROUP,)g(IERROR\))170 998 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)166 1084 y Fw(MPI)p 251 1084 14 2 v 16 w(COMM)p 409 1084 V 16 w(GROUP)17 b Fv(returns)e(in)h Fw(group)f Fv(a)g(handle)h(to)f(the)g(group)g(of)g Fw(comm)p Fv(.)75 1235 y Fw(MPI)p 160 1235 V 16 w(GROUP)p 328 1235 V 18 w(UNION\(group1,)f(group2,)h(newgroup\))117 1312 y Fk(IN)155 b Fw(group1)456 b Fk(\014rst)15 b(group)f(\(handle\)) 117 1387 y(IN)155 b Fw(group2)456 b Fk(second)15 b(group)f(\(handle\)) 117 1463 y(OUT)108 b Fw(newgroup)405 b Fk(union)14 b(group)f (\(handle\))75 1587 y Ft(int)23 b(MPI)p 245 1587 15 2 v 17 w(Group)p 382 1587 V 17 w(union\(MPI)p 615 1587 V 16 w(Group)g(group1,)g(MPI)p 1037 1587 V 17 w(Group)g(group2,)g(MPI)p 1460 1587 V 16 w(Group)h(*newgroup\))75 1673 y(MPI)p 150 1673 V 17 w(GROUP)p 287 1673 V 16 w(UNION\(GROUP1,)e(GROUP2,)h (NEWGROUP,)g(IERROR\))170 1730 y(INTEGER)g(GROUP1,)g(GROUP2,)g (NEWGROUP,)g(IERROR)75 1911 y Fw(MPI)p 160 1911 14 2 v 16 w(GROUP)p 328 1911 V 18 w(INTERSECTION\(group1,)15 b(group2,)f(newgroup\))117 1988 y Fk(IN)155 b Fw(group1)456 b Fk(\014rst)15 b(group)f(\(handle\))117 2063 y(IN)155 b Fw(group2)456 b Fk(second)15 b(group)f(\(handle\))117 2138 y(OUT)108 b Fw(newgroup)405 b Fk(in)o(tersection)15 b(group)f(\(handle\))75 2263 y Ft(int)23 b(MPI)p 245 2263 15 2 v 17 w(Group)p 382 2263 V 17 w(intersection\(MPI)p 783 2263 V 15 w(Group)g(group1,)g(MPI)p 1204 2263 V 17 w(Group)g(group2,)393 2319 y(MPI)p 468 2319 V 17 w(Group)g(*newgroup\)) 75 2406 y(MPI)p 150 2406 V 17 w(GROUP)p 287 2406 V 16 w(INTERSECTION\(GROUP1,)e(GROUP2,)i(NEWGROUP,)g(IERROR\))170 2462 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 141 149 141 148 bop 75 -100 a Fm(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b Fv(141)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(DIFFERENCE\(group1,)14 b(group2,)g(newgroup\))117 122 y Fk(IN)155 b Fw(group1)456 b Fk(\014rst)15 b(group)f(\(handle\)) 117 197 y(IN)155 b Fw(group2)456 b Fk(second)15 b(group)f(\(handle\)) 117 273 y(OUT)108 b Fw(newgroup)405 b Fk(di\013erence)16 b(group)e(\(handle\))75 397 y Ft(int)23 b(MPI)p 245 397 15 2 v 17 w(Group)p 382 397 V 17 w(difference\(MPI)p 735 397 V 15 w(Group)g(group1,)g(MPI)p 1156 397 V 17 w(Group)g(group2,)393 453 y(MPI)p 468 453 V 17 w(Group)g(*newgroup\))75 540 y(MPI)p 150 540 V 17 w(GROUP)p 287 540 V 16 w(DIFFERENCE\(GROUP1,)f (GROUP2,)h(NEWGROUP,)f(IERROR\))170 596 y(INTEGER)h(GROUP1,)g(GROUP2,)g (NEWGROUP,)g(IERROR)75 683 y Fv(The)15 b(set-lik)o(e)i(op)q(erations)e (are)g(de\014ned)i(as)d(follo)o(ws:)75 777 y Fq(union)24 b Fv(All)19 b(elemen)o(ts)g(of)f(the)h(\014rst)f(group)g(\()p Fw(group1)p Fv(\),)g(follo)o(w)o(ed)h(b)o(y)f(all)h(elemen)o(ts)g(of)f (second)h(group)189 833 y(\()p Fw(group2)p Fv(\))14 b(not)h(in)h (\014rst.)75 927 y Fq(in)o(tersect)23 b Fv(all)18 b(elemen)o(ts)h(of)e (the)g(\014rst)h(group)f(that)g(are)g(also)g(in)i(the)f(second)g (group,)f(ordered)h(as)f(in)189 983 y(\014rst)d(group.)75 1077 y Fq(di\013erence)22 b Fv(all)c(elemen)o(ts)f(of)g(the)g(\014rst)f (group)g(that)g(are)h(not)f(in)h(the)g(second)g(group,)g(ordered)g(as)f (in)189 1134 y(the)f(\014rst)g(group.)75 1227 y(Note)k(that)g(for)g (these)h(op)q(erations)g(the)f(order)h(of)f(pro)q(cesses)h(in)g(the)g (output)f(group)h(is)g(determined)75 1284 y(primarily)g(b)o(y)f(order)g (in)h(the)f(\014rst)g(group)f(\(if)h(p)q(ossible\))i(and)e(then,)h(if)f (necessary)l(,)h(b)o(y)f(order)g(in)h(the)75 1340 y(second)c(group.)j (Neither)d(union)h(nor)e(in)o(tersection)h(are)e(comm)o(utativ)o(e,)h (but)g(b)q(oth)g(are)g(asso)q(ciativ)o(e.)166 1397 y(The)g(new)h(group) f(can)g(b)q(e)h(empt)o(y)l(,)f(that)f(is,)h(equal)h(to)f Fi(MPI)p 1184 1397 13 2 v 14 w(GROUP)p 1340 1397 V 13 w(EMPTY)p Fv(.)75 1548 y Fw(MPI)p 160 1548 14 2 v 16 w(GROUP)p 328 1548 V 18 w(INCL\(group,)e(n,)j(ranks,)f(newgroup\))117 1625 y Fk(IN)155 b Fw(group)479 b Fk(group)14 b(\(handle\))117 1700 y(IN)155 b Fw(n)564 b Fk(n)o(um)o(b)q(er)16 b(of)h(elemen)o(ts)g (in)f(arra)o(y)h(ranks)g(\(and)g(size)h(of)e Fi(new-)905 1757 y(group)p Fk(\))f(\(in)o(teger\))117 1832 y(IN)155 b Fw(ranks)487 b Fk(ranks)13 b(of)f(pro)q(cesses)i(in)e Fi(group)h Fk(to)f(app)q(ear)h(in)f Fi(newgroup)i Fk(\(ar-)905 1888 y(ra)o(y)g(of)f(in)o(tegers\))117 1963 y(OUT)108 b Fw(newgroup)405 b Fk(new)12 b(group)f(deriv)o(ed)h(from)e(ab)q(o)o(v) o(e,)h(in)g(the)h(order)h(de\014ned)f(b)o(y)905 2020 y Fi(ranks)j Fk(\(handle\))75 2144 y Ft(int)23 b(MPI)p 245 2144 15 2 v 17 w(Group)p 382 2144 V 17 w(incl\(MPI)p 591 2144 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 2144 V 17 w(Group)h(*newgroup\))75 2231 y(MPI)p 150 2231 V 17 w(GROUP)p 287 2231 V 16 w(INCL\(GROUP,)g(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 2287 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)166 2374 y Fv(The)17 b(function)h Fw(MPI)p 526 2374 14 2 v 16 w(GROUP)p 694 2374 V 18 w(INCL)f Fv(creates)g(a)g(group)f Fw(newgroup)i Fv(that)f(consists)g(of)g(the)g Fw(n)h Fv(pro-)75 2430 y(cesses)e(in)h Fw(group)f Fv(with)h(ranks)e Fw(rank[0],)p Fp(:)8 b(:)g(:)n Fw(,)16 b(rank[n-1])p Fv(;)g(the)h(pro)q(cess)f(with)g(rank)g Fw(i)g Fv(in)h Fw(newgroup)f Fv(is)h(the)75 2487 y(pro)q(cess)h(with)f(rank)g Fw(ranks[i])h Fv(in)g Fw(group)p Fv(.)26 b(Eac)o(h)17 b(of)g(the)g Fw(n)h Fv(elemen)o(ts)g(of)f Fw(ranks)h Fv(m)o(ust)e(b)q(e)i(a)f(v)m(alid)i(rank)75 2543 y(in)h Fw(group)f Fv(and)g(all)h(elemen)o(ts)g(m)o(ust)f(b)q(e)g(distinct,)i (or)e(else)h(the)f(program)f(is)i(erroneous.)31 b(If)20 b Fw(n)f Fv(=)39 b(0,)75 2599 y(then)19 b Fw(newgroup)f Fv(is)h Fi(MPI)p 508 2599 13 2 v 14 w(GROUP)p 664 2599 V 13 w(EMPTY)p Fv(.)f(This)g(function)h(can,)g(for)f(instance,)h(b)q(e) g(used)g(to)e(reorder)75 2656 y(the)e(elemen)o(ts)h(of)f(a)g(group.)k (See)d(also)f Fw(MPI)p 832 2656 14 2 v 16 w(GROUP)p 1000 2656 V 18 w(COMP)l(ARE)p Fv(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 142 150 142 149 bop 75 -100 a Fv(142)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(GROUP)p 328 45 V 18 w(EX)o(CL\(group,)f(n,)h(ranks,)g(newgroup\))117 122 y Fk(IN)155 b Fw(group)479 b Fk(group)14 b(\(handle\))117 196 y(IN)155 b Fw(n)564 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f (in)h(arra)o(y)f(ranks)i(\(in)o(teger\))117 269 y(IN)155 b Fw(ranks)487 b Fk(arra)o(y)15 b(of)f(in)o(teger)i(ranks)f(in)g Fi(group)g Fk(not)g(to)g(app)q(ear)g(in)g Fi(new-)905 326 y(group)117 399 y Fk(OUT)108 b Fw(newgroup)405 b Fk(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i(preserving)h (the)f(order)905 455 y(de\014ned)d(b)o(y)28 b Fi(group)14 b Fk(\(handle\))75 580 y Ft(int)23 b(MPI)p 245 580 15 2 v 17 w(Group)p 382 580 V 17 w(excl\(MPI)p 591 580 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 580 V 17 w(Group)h(*newgroup\))75 666 y(MPI)p 150 666 V 17 w(GROUP)p 287 666 V 16 w(EXCL\(GROUP,)g(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 723 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)166 809 y Fv(The)12 b(function)h Fw(MPI)p 516 809 14 2 v 16 w(GROUP)p 684 809 V 17 w(EX)o(CL)f Fv(creates)g(a)f(group)h(of)g(pro)q(cesses)g Fw(newgroup)h Fv(that)e(is)h(obtained)75 866 y(b)o(y)k(deleting)i(from)e Fw(group)g Fv(those)g(pro)q(cesses)h(with)f(ranks)g Fw(ranks[0])h(,)p Fp(:)8 b(:)g(:)14 b Fw(ranks[n-1])p Fv(.)24 b(The)17 b(ordering)f(of)75 922 y(pro)q(cesses)f(in)h Fw(newgroup)g Fv(is)f(iden)o(tical)i(to)e(the)g(ordering)g(in)h Fw(group)p Fv(.)j(Eac)o(h)c(of)g(the)g Fw(n)g Fv(elemen)o(ts)h(of)e Fw(ranks)75 979 y Fv(m)o(ust)h(b)q(e)h(a)f(v)m(alid)i(rank)e(in)h Fw(group)f Fv(and)h(all)g(elemen)o(ts)g(m)o(ust)f(b)q(e)h(distinct;)g (otherwise,)f(the)g(program)f(is)75 1035 y(erroneous.)20 b(If)15 b Fw(n)h Fv(=)28 b(0,)14 b(then)i Fw(newgroup)g Fv(is)g(iden)o(tical)h(to)d Fw(group)p Fv(.)75 1186 y Fw(MPI)p 160 1186 V 16 w(GROUP)p 328 1186 V 18 w(RANGE)p 494 1186 V 17 w(INCL\(group,)g(n,)h(ranges,)g(newgroup\))117 1263 y Fk(IN)155 b Fw(group)479 b Fk(group)14 b(\(handle\))117 1336 y(IN)155 b Fw(n)564 b Fk(n)o(um)o(b)q(er)13 b(of)h(triplets)g(in)f (arra)o(y)28 b Fi(ranges)14 b Fk(\(in)o(teger\))117 1410 y(IN)155 b Fw(ranges)466 b Fk(an)11 b(arra)o(y)g(of)f(in)o(teger)i (triplets,)g(of)e(the)i(form)d(\(\014rst)j(rank,)f(last)905 1466 y(rank,)j(stride\))g(indicating)f(ranks)h(in)g Fi(group)g Fk(of)g(pro)q(cesses)i(to)905 1523 y(b)q(e)f(included)f(in)f Fi(newgroup)117 1596 y Fk(OUT)108 b Fw(newgroup)405 b Fk(new)12 b(group)f(deriv)o(ed)h(from)e(ab)q(o)o(v)o(e,)h(in)g(the)h (order)h(de\014ned)f(b)o(y)905 1653 y Fi(ranges)j Fk(\(handle\))75 1777 y Ft(int)23 b(MPI)p 245 1777 15 2 v 17 w(Group)p 382 1777 V 17 w(range)p 519 1777 V 16 w(incl\(MPI)p 727 1777 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(ranges[][3],)393 1834 y(MPI)p 468 1834 V 17 w(Group)g(*newgroup\))75 1920 y(MPI)p 150 1920 V 17 w(GROUP)p 287 1920 V 16 w(RANGE)p 423 1920 V 17 w(INCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1976 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(3,*\),)e(NEWGROUP,)h(IERROR)75 2063 y Fv(If)31 b Fw(ranges)15 b Fv(consist)g(of)g(the)h(triplets)189 2148 y(\()p Fp(f)5 b(ir)q(st)309 2155 y Fg(1)328 2148 y Fp(;)j(l)q(ast)425 2155 y Fg(1)444 2148 y Fp(;)g(str)q(ide)585 2155 y Fg(1)604 2148 y Fv(\))p Fp(;)g(:::;)g Fv(\()p Fp(f)d(irst)822 2155 y Ff(n)844 2148 y Fp(;)j(l)q(ast)941 2155 y Ff(n)964 2148 y Fp(;)g(str)q(ide)1105 2155 y Ff(n)1127 2148 y Fv(\))75 2233 y(then)16 b Fw(newgroup)f Fv(consists)h(of)f(the)g (sequence)h(of)f(pro)q(cesses)h(in)g Fw(group)f Fv(with)g(ranks)189 2337 y Fp(f)5 b(ir)q(st)291 2344 y Fg(1)311 2337 y Fp(;)j(f)d(ir)q(st) 434 2344 y Fg(1)463 2337 y Fv(+)10 b Fp(str)q(ide)628 2344 y Fg(1)648 2337 y Fp(;)e(:::;)g(f)d(ir)q(st)831 2344 y Fg(1)859 2337 y Fv(+)904 2277 y Fe(\026)933 2306 y Fp(l)q(ast)1009 2313 y Fg(1)1039 2306 y Fx(\000)11 b Fp(f)5 b(ir)q(st)1187 2313 y Fg(1)p 933 2326 274 2 v 1000 2368 a Fp(str)q(ide)1120 2375 y Fg(1)1212 2277 y Fe(\027)1243 2337 y Fp(str)q(ide)1363 2344 y Fg(1)1383 2337 y Fp(;)j(:::)189 2479 y(f)d(ir)q(st)291 2486 y Ff(n)314 2479 y Fp(;)j(f)d(ir)q(st)437 2486 y Ff(n)471 2479 y Fv(+)10 b Fp(str)q(ide)636 2486 y Ff(n)659 2479 y Fp(;)e(:::;)g(f)d(ir) q(st)842 2486 y Ff(n)874 2479 y Fv(+)919 2420 y Fe(\026)948 2448 y Fp(l)q(ast)1024 2455 y Ff(n)1058 2448 y Fx(\000)10 b Fp(f)5 b(ir)q(st)1205 2455 y Ff(n)p 948 2469 282 2 v 1017 2510 a Fp(str)q(ide)1137 2517 y Ff(n)1234 2420 y Fe(\027)1266 2479 y Fp(str)q(ide)1386 2486 y Ff(n)1409 2479 y Fp(:)166 2591 y Fv(Eac)o(h)17 b(computed)h(rank)f(m)o(ust)g(b)q (e)i(a)e(v)m(alid)i(rank)e(in)i Fw(group)e Fv(and)h(all)g(computed)g (ranks)f(m)o(ust)g(b)q(e)75 2647 y(distinct,)c(or)f(else)g(the)g (program)f(is)h(erroneous.)19 b(Note)11 b(that)g(w)o(e)h(ma)o(y)f(ha)o (v)o(e)h Fp(f)5 b(ir)q(st)1466 2654 y Ff(i)1493 2647 y Fp(>)13 b(l)q(ast)1617 2654 y Ff(i)1631 2647 y Fv(,)f(and)g Fp(str)q(ide)1861 2654 y Ff(i)75 2704 y Fv(ma)o(y)i(b)q(e)i(negativ)o (e,)f(but)h(cannot)f(b)q(e)g(zero.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 143 151 143 150 bop 75 -100 a Fm(5.3.)34 b(GR)o(OUP)15 b(MANA)o(GEMENT)1093 b Fv(143)166 45 y(The)18 b(functionalit)o(y)h(of)e(this)i(routine)f(is) g(sp)q(eci\014ed)i(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)e(expanding)i (the)f(arra)o(y)75 102 y(of)g(ranges)g(to)g(an)h(arra)o(y)e(of)h(the)h (included)i(ranks)d(and)h(passing)g(the)g(resulting)g(arra)o(y)e(of)i (ranks)f(and)75 158 y(other)e(argumen)o(ts)g(to)g Fw(MPI)p 560 158 14 2 v 16 w(GROUP)p 728 158 V 18 w(INCL)p Fv(.)g(A)g(call)i(to) e Fw(MPI)p 1148 158 V 16 w(GROUP)p 1316 158 V 18 w(INCL)g Fv(is)h(equiv)m(alen)o(t)h(to)e(a)h(call)75 214 y(to)e Fw(MPI)p 216 214 V 16 w(GROUP)p 384 214 V 18 w(RANGE)p 550 214 V 18 w(INCL)g Fv(with)i(eac)o(h)f(rank)g Fw(i)g Fv(in)h Fw(ranks)f Fv(replaced)i(b)o(y)e(the)g(triplet)h Ft(\(i,i,1\))e Fv(in)75 271 y(the)g(argumen)o(t)g Fw(ranges)p Fv(.)75 422 y Fw(MPI)p 160 422 V 16 w(GROUP)p 328 422 V 18 w(RANGE)p 494 422 V 17 w(EX)o(CL\(group,)g(n,)g(ranges,)g (newgroup\))117 499 y Fk(IN)155 b Fw(group)479 b Fk(group)14 b(\(handle\))117 574 y(IN)155 b Fw(n)564 b Fk(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(arra)o(y)f(ranks)i(\(in)o(teger\))117 649 y(IN)155 b Fw(ranges)466 b Fk(a)13 b(one-dimensional)d(arra)o(y)j (of)f(in)o(teger)h(triplets)g(of)f(the)i(form)905 706 y(\(\014rst)j(rank,)f(last)g(rank,)g(stride\),)h(indicating)e(the)i (ranks)f(in)905 762 y Fi(group)22 b Fk(of)f(pro)q(cesses)j(to)d(b)q(e)h (excluded)g(from)e(the)i(output)905 819 y(group)14 b Fi(newgroup)p Fk(.)117 894 y(OUT)108 b Fw(newgroup)405 b Fk(new)18 b(group)g(deriv)o(ed)g(from)e(ab)q(o)o(v)o(e,)i(preserving) h(the)f(order)905 950 y(in)c Fi(group)g Fk(\(handle\))75 1075 y Ft(int)23 b(MPI)p 245 1075 15 2 v 17 w(Group)p 382 1075 V 17 w(range)p 519 1075 V 16 w(excl\(MPI)p 727 1075 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(ranges[][3],)393 1131 y(MPI)p 468 1131 V 17 w(Group)g(*newgroup\))75 1218 y(MPI)p 150 1218 V 17 w(GROUP)p 287 1218 V 16 w(RANGE)p 423 1218 V 17 w(EXCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1274 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(3,*\),)e(NEWGROUP,)h(IERROR)75 1361 y Fv(Eac)o(h)12 b(computed)g(rank)g(m)o(ust)f(b)q(e)i(a)f(v)m (alid)h(rank)f(in)h Fw(group)f Fv(and)g(all)h(computed)f(ranks)g(m)o (ust)f(b)q(e)i(distinct,)75 1417 y(or)i(else)h(the)f(program)f(is)i (erroneous.)166 1473 y(The)i(functionalit)o(y)h(of)e(this)i(routine)f (is)g(sp)q(eci\014ed)i(to)d(b)q(e)i(equiv)m(alen)o(t)g(to)e(expanding)i (the)f(arra)o(y)75 1530 y(of)g(ranges)f(to)h(an)g(arra)o(y)f(of)h(the)g (excluded)i(ranks)e(and)g(passing)h(the)f(resulting)h(arra)o(y)e(of)h (ranks)g(and)75 1586 y(other)c(argumen)o(ts)g(to)g Fw(MPI)p 554 1586 14 2 v 16 w(GROUP)p 722 1586 V 18 w(EX)o(CL)p Fv(.)g(A)h(call)h(to)e Fw(MPI)p 1146 1586 V 16 w(GROUP)p 1314 1586 V 18 w(EX)o(CL)g Fv(is)i(equiv)m(alen)o(t)g(to)e(a)h(call)75 1643 y(to)g Fw(MPI)p 216 1643 V 15 w(GROUP)p 383 1643 V 18 w(RANGE)p 549 1643 V 18 w(EX)o(CL)g Fv(with)h(eac)o(h)f(rank)g Fw(i)g Fv(in)h Fw(ranks)g Fv(replaced)g(b)o(y)f(the)h(triplet)g Ft(\(i,i,1\))e Fv(in)75 1699 y(the)h(argumen)o(t)g Fw(ranges)p Fv(.)189 1806 y Fl(A)n(dvic)n(e)22 b(to)h(users.)86 b Fv(The)23 b(range)f(op)q(erations)h(do)f(not)h(explicitly)i(en)o (umerate)e(ranks,)h(and)189 1862 y(therefore)c(are)h(more)f(scalable)i (if)f(implemen)o(ted)i(e\016cien)o(tly)l(.)38 b(Hence,)23 b(w)o(e)d(recommend)i Fw(MPI)189 1918 y Fv(programmers)14 b(to)i(use)g(them)h(whenenev)o(er)g(p)q(ossible,)g(as)f(high-qualit)o (y)i(implemen)o(tations)f(will)189 1975 y(tak)o(e)d(adv)m(an)o(tage)h (of)g(this)g(fact.)k(\()p Fl(End)d(of)g(advic)n(e)h(to)f(users.)p Fv(\))189 2081 y Fl(A)n(dvic)n(e)h(to)h(implementors.)52 b Fv(The)17 b(range)g(op)q(erations)g(should)h(b)q(e)g(implemen)o(ted,) h(if)f(p)q(ossible,)189 2138 y(without)e(en)o(umerating)i(the)f(group)f (mem)o(b)q(ers,)h(in)h(order)e(to)g(obtain)i(b)q(etter)e(scalabilit)o (y)j(\(time)189 2194 y(and)c(space\).)20 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fv(\))75 2316 y Fj(5.3.3)49 b(Group)16 b(Destructo)o(rs)75 2496 y Fw(MPI)p 160 2496 V 16 w(GROUP)p 328 2496 V 18 w(FREE\(group\))117 2573 y Fk(INOUT)62 b Fw(group)479 b Fk(group)14 b(\(handle\))75 2698 y Ft(int)23 b(MPI)p 245 2698 15 2 v 17 w(Group)p 382 2698 V 17 w(free\(MPI)p 591 2698 V 16 w(Group)g(*group\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 144 152 144 151 bop 75 -100 a Fv(144)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(GROUP)p 287 45 V 16 w(FREE\(GROUP,)23 b(IERROR\))170 102 y(INTEGER)g(GROUP,)g(IERROR)166 188 y Fv(This)f(op)q(eration)h(marks)e(a)g(group)h(ob)s(ject)f(for)h (deallo)q(cation.)41 b(The)22 b(handle)i Fw(group)d Fv(is)i(set)f(to)75 244 y Fi(MPI)p 152 244 13 2 v 14 w(GROUP)p 308 244 V 13 w(NULL)17 b Fv(b)o(y)g(the)f(call.)24 b(An)o(y)17 b(on-going)f(op)q(eration)g(using)h(this)g(group)f(will)i(complete)f (nor-)75 301 y(mally)l(.)189 395 y Fl(A)n(dvic)n(e)c(to)h (implementors.)39 b Fv(One)13 b(can)h(k)o(eep)f(a)f(reference)i(coun)o (t)f(that)f(is)h(incremen)o(ted)i(for)d(eac)o(h)189 452 y(call)j(to)f Fw(MPI)p 411 452 14 2 v 16 w(COMM)p 569 452 V 16 w(CREA)l(TE)i Fv(and)e Fw(MPI)p 939 452 V 16 w(COMM)p 1097 452 V 17 w(DUP)p Fv(,)g(and)g(decremen)o(ted)i(for)d(eac) o(h)i(call)h(to)189 508 y Fw(MPI)p 274 508 V 15 w(GROUP)p 441 508 V 18 w(FREE)h Fv(or)g Fw(MPI)p 725 508 V 15 w(COMM)p 882 508 V 17 w(FREE)p Fv(;)g(the)g(group)f(ob)s(ject)g(is)i(ultimately) g(deallo)q(cated)189 565 y(when)d(the)h(reference)g(coun)o(t)f(drops)g (to)f(zero.)20 b(\()p Fl(End)c(of)g(advic)n(e)g(to)h(implementors.)p Fv(\))75 706 y Fn(5.4)59 b(Communicato)n(r)19 b(Management)75 808 y Fv(This)e(section)g(describ)q(es)h(the)f(manipulation)h(of)e (comm)o(unicators)g(in)h Fw(MPI)p Fv(.)f(Op)q(erations)h(that)f(access) 75 864 y(comm)o(unicators)f(are)g(lo)q(cal)h(and)f(their)h(execution)g (do)q(es)g(not)f(require)h(in)o(terpro)q(cess)g(comm)o(unication.)75 921 y(Op)q(erations)g(that)e(create)h(comm)o(unicators)f(are)h (collectiv)o(e)i(and)e(ma)o(y)f(require)i(in)o(terpro)q(cess)f(comm)o (u-)75 977 y(nication.)189 1072 y Fl(A)n(dvic)n(e)k(to)h(implementors.) 65 b Fv(High-qualit)o(y)21 b(implemen)o(tations)g(should)f(amortize)f (the)h(o)o(v)o(er-)189 1128 y(heads)15 b(asso)q(ciated)h(with)g(the)f (creation)h(of)e(comm)o(unicators)h(\(for)g(the)g(same)g(group,)g(or)g (subsets)189 1185 y(thereof)t(\))d(o)o(v)o(er)h(sev)o(eral)g(calls,)i (b)o(y)e(allo)q(cating)i(m)o(ultiple)g(con)o(texts)e(with)g(one)h (collectiv)o(e)h(comm)o(u-)189 1241 y(nication.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fv(\))75 1361 y Fj(5.4.1)49 b(Communicato)o(r)16 b(Accesso)o(rs)75 1447 y Fv(The)f(follo)o(wing)h(are)f(all)h(lo)q(cal)h(op)q(erations.)75 1598 y Fw(MPI)p 160 1598 V 16 w(COMM)p 318 1598 V 16 w(SIZE\(comm,)c(size\))117 1675 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 1749 y(OUT)108 b Fw(size)519 b Fk(n)o(um)o(b)q(er)13 b(of)h(pro)q(cesses)i(in)e(the)g (group)g(of)27 b Fi(comm)12 b Fk(\(in)o(teger\))75 1873 y Ft(int)23 b(MPI)p 245 1873 15 2 v 17 w(Comm)p 358 1873 V 17 w(size\(MPI)p 567 1873 V 16 w(Comm)g(comm,)g(int)h(*size\))75 1960 y(MPI)p 150 1960 V 17 w(COMM)p 263 1960 V 16 w(SIZE\(COMM,)f (SIZE,)g(IERROR\))170 2016 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)189 2140 y Fl(R)n(ationale.)51 b Fv(This)17 b(function)h(is)g(equiv)m(alen) o(t)g(to)f(accessing)h(the)f(comm)o(unicator's)f(group)h(with)189 2197 y Fw(MPI)p 274 2197 14 2 v 15 w(COMM)p 431 2197 V 17 w(GROUP)k Fv(\(see)g(ab)q(o)o(v)o(e\),)f(computing)h(the)f(size)i (using)f Fw(MPI)p 1494 2197 V 15 w(GROUP)p 1661 2197 V 18 w(SIZE)p Fv(,)f(and)189 2253 y(then)13 b(freeing)h(the)f(temp)q (orary)f(group)h(via)g Fw(MPI)p 1016 2253 V 16 w(GROUP)p 1184 2253 V 18 w(FREE)p Fv(.)g(Ho)o(w)o(ev)o(er,)f(this)i(function)g (is)f(so)189 2310 y(commonly)i(used,)g(that)g(this)h(shortcut)e(w)o(as) h(in)o(tro)q(duced.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fv(\))189 2404 y Fl(A)n(dvic)n(e)k(to)i(users.)80 b Fv(This)23 b(function)f(indicates)i(the)e(n)o(um)o(b)q(er)g(of)f(pro)q(cesses)i (in)o(v)o(olv)o(ed)g(in)f(a)189 2461 y(comm)o(unicator.)35 b(F)l(or)21 b Fi(MPI)p 684 2461 13 2 v 14 w(COMM)p 828 2461 V 14 w(W)o(ORLD)p Fv(,)g(it)g(indicates)h(the)e(total)g(n)o(um)o (b)q(er)h(of)g(pro)q(cesses)189 2517 y(a)o(v)m(ailable)c(\(for)f(this)g (v)o(ersion)h(of)f Fw(MPI)p Fv(,)f(there)h(is)h(no)f(standard)g(w)o(a)o (y)f(to)g(c)o(hange)i(the)f(n)o(um)o(b)q(er)g(of)189 2574 y(pro)q(cesses)f(once)h(initialization)i(has)d(tak)o(en)g (place\).)189 2647 y(This)20 b(call)h(is)f(often)g(used)g(with)h(the)f (next)g(call)h(to)e(determine)i(the)f(amoun)o(t)f(of)g(concurrency)189 2704 y(a)o(v)m(ailable)j(for)e(a)h(sp)q(eci\014c)i(library)e(or)g (program.)36 b(The)21 b(follo)o(wing)g(call,)i Fw(MPI)p 1580 2704 14 2 v 16 w(COMM)p 1738 2704 V 17 w(RANK)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 145 153 145 152 bop 75 -100 a Fm(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b Fv(145)189 45 y(indicates)15 b(the)f(rank)f(of)h (the)g(pro)q(cess)g(that)f(calls)i(it)f(in)h(the)f(range)f(from)g(0)8 b Fp(:)g(:)g(:)n Fw(size)p Fx(\000)p Fv(1,)14 b(where)g Fw(size)189 102 y Fv(is)h(the)h(return)f(v)m(alue)h(of)f Fw(MPI)p 706 102 14 2 v 16 w(COMM)p 864 102 V 16 w(SIZE)p Fv(.\()p Fl(End)g(of)i(advic)n(e)f(to)g(users.)p Fv(\))75 302 y Fw(MPI)p 160 302 V 16 w(COMM)p 318 302 V 16 w(RANK\(comm,)d (rank\))117 380 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 455 y(OUT)108 b Fw(rank)504 b Fk(rank)12 b(of)f(the)h(calling)e(pro)q(cess)j(in)f(group)f(of)23 b Fi(comm)9 b Fk(\(in)o(teger\))75 579 y Ft(int)23 b(MPI)p 245 579 15 2 v 17 w(Comm)p 358 579 V 17 w(rank\(MPI)p 567 579 V 16 w(Comm)g(comm,)g(int)h(*rank\))75 666 y(MPI)p 150 666 V 17 w(COMM)p 263 666 V 16 w(RANK\(COMM,)f(RANK,)g(IERROR\))170 722 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)1937 758 y Fx(>)189 858 y Fl(R)n(ationale.)51 b Fv(This)17 b(function)h(is)g(equiv)m(alen)o (t)g(to)f(accessing)h(the)f(comm)o(unicator's)f(group)h(with)189 915 y Fw(MPI)p 274 915 14 2 v 15 w(COMM)p 431 915 V 17 w(GROUP)e Fv(\(see)g(ab)q(o)o(v)o(e\),)e(computing)i(the)f(rank)g (using)h Fw(MPI)p 1469 915 V 16 w(GROUP)p 1637 915 V 18 w(RANK)p Fv(,)f(and)189 971 y(then)f(freeing)h(the)f(temp)q(orary)f (group)h(via)g Fw(MPI)p 1016 971 V 16 w(GROUP)p 1184 971 V 18 w(FREE)p Fv(.)g(Ho)o(w)o(ev)o(er,)f(this)i(function)g(is)f(so) 189 1028 y(commonly)i(used,)g(that)g(this)h(shortcut)e(w)o(as)h(in)o (tro)q(duced.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fv(\))1937 1089 y Fx(?)189 1184 y Fl(A)n(dvic)n(e)c(to)h(users.)38 b Fv(This)14 b(function)f(giv)o(es)g(the)g(rank)f(of)h(the)f(pro)q (cess)h(in)h(the)f(particular)g(comm)o(u-)189 1240 y(nicator's)h (group.)20 b(It)15 b(is)h(useful,)g(as)f(noted)g(ab)q(o)o(v)o(e,)f(in)i (conjunction)h(with)e Fw(MPI)p 1569 1240 V 16 w(COMM)p 1727 1240 V 16 w(SIZE)p Fv(.)189 1315 y(Man)o(y)e(programs)g(will)i(b)q (e)g(written)f(with)g(the)g(master-sla)o(v)o(e)f(mo)q(del,)i(where)f (one)g(pro)q(cess)g(\(suc)o(h)189 1372 y(as)19 b(the)g(rank-zero)h(pro) q(cess\))f(will)i(pla)o(y)f(a)f(sup)q(ervisory)h(role,)h(and)f(the)f (other)g(pro)q(cesses)h(will)189 1428 y(serv)o(e)g(as)f(compute)h(no)q (des.)35 b(In)21 b(this)g(framew)o(ork,)e(the)h(t)o(w)o(o)f(preceding)j (calls)f(are)e(useful)j(for)189 1485 y(determining)c(the)e(roles)h(of)f (the)h(v)m(arious)g(pro)q(cesses)g(of)f(a)g(comm)o(unicator.)24 b(\()p Fl(End)17 b(of)g(advic)n(e)h(to)189 1541 y(users.)p Fv(\))75 1742 y Fw(MPI)p 160 1742 V 16 w(COMM)p 318 1742 V 16 w(COMP)l(ARE\(comm1,)c(comm2,)f(result\))117 1819 y Fk(IN)155 b Fw(comm1)447 b Fk(\014rst)15 b(comm)o(unicator)c (\(handle\))117 1894 y(IN)155 b Fw(comm2)447 b Fk(second)15 b(comm)o(unicator)c(\(handle\))117 1969 y(OUT)108 b Fw(result)484 b Fk(result)15 b(\(in)o(teger\))75 2094 y Ft(int)23 b(MPI)p 245 2094 15 2 v 17 w(Comm)p 358 2094 V 17 w(compare\(MPI)p 639 2094 V 16 w(Comm)g(comm1,MPI)p 990 2094 V 16 w(Comm)g(comm2,)g(int) h(*result\))75 2180 y(MPI)p 150 2180 V 17 w(COMM)p 263 2180 V 16 w(COMPARE\(COMM1,)e(COMM2,)h(RESULT,)g(IERROR\))170 2237 y(INTEGER)g(COMM1,)g(COMM2,)g(RESULT,)g(IERROR)75 2323 y Fi(MPI)p 152 2323 13 2 v 14 w(IDENT)11 b Fv(results)g(if)h(and)f (only)g(if)h Fw(comm1)d Fv(and)i Fw(comm2)d Fv(are)j(handles)h(for)e (the)h(same)g(ob)s(ject)f(\(iden)o(tical)75 2380 y(groups)15 b(and)g(same)g(con)o(texts\).)j Fi(MPI)p 715 2380 V 15 w(CONGRUENT)c Fv(results)h(if)h(the)f(underlying)i(groups)d(are)h(iden) o(tical)75 2436 y(in)j(constituen)o(ts)f(and)h(rank)f(order;)h(these)f (comm)o(unicators)g(di\013er)h(only)f(b)o(y)h(con)o(text.)25 b Fi(MPI)p 1703 2436 V 15 w(SIMILAR)75 2493 y Fv(results)12 b(if)h(the)f(group)g(mem)o(b)q(ers)g(of)g(b)q(oth)g(comm)o(unicators)g (are)g(the)g(same)g(but)g(the)g(rank)g(order)g(di\013ers.)75 2549 y Fi(MPI)p 152 2549 V 14 w(UNEQUAL)i Fv(results)i(otherwise.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 146 154 146 153 bop 75 -100 a Fv(146)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fj(5.4.2)49 b(Communicato)o(r)16 b(Constructo)o(rs)75 131 y Fv(The)f(follo)o(wing)g(are)g(collectiv)o(e)h(functions)f(that)f (are)h(in)o(v)o(ok)o(ed)g(b)o(y)f(all)i(pro)q(cesses)f(in)g(the)g (group)f(asso)q(ci-)75 187 y(ated)h(with)h Fw(comm)p Fv(.)189 266 y Fl(R)n(ationale.)37 b Fv(Note)10 b(that)g(there)h(is)g (a)g(c)o(hic)o(k)o(en-and-egg)h(asp)q(ect)f(to)f Fw(MPI)g Fv(in)i(that)e(a)g(comm)o(unicator)189 323 y(is)18 b(needed)h(to)e (create)g(a)h(new)f(comm)o(unicator.)27 b(The)18 b(base)g(comm)o (unicator)f(for)g(all)i Fw(MPI)e Fv(com-)189 379 y(m)o(unicators)f(is)h (prede\014ned)h(outside)f(of)f Fw(MPI)p Fv(,)g(and)g(is)h Fi(MPI)p 1227 379 13 2 v 15 w(COMM)p 1372 379 V 14 w(W)o(ORLD)p Fv(.)f(This)h(mo)q(del)h(w)o(as)189 435 y(arriv)o(ed)c(at)g(after)g (considerable)i(debate,)e(and)h(w)o(as)e(c)o(hosen)i(to)f(increase)h (\\safet)o(y")e(of)h(programs)189 492 y(written)h(in)h Fw(MPI)p Fv(.)e(\()p Fl(End)i(of)g(r)n(ationale.)p Fv(\))75 665 y Fw(MPI)p 160 665 14 2 v 16 w(COMM)p 318 665 V 16 w(DUP\(comm,)d(new)o(comm\))117 742 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 814 y(OUT)108 b Fw(new)o(comm)397 b Fk(cop)o(y)14 b(of)f Fi(comm)f Fk(\(handle\))75 938 y Ft(int)23 b(MPI)p 245 938 15 2 v 17 w(Comm)p 358 938 V 17 w(dup\(MPI)p 543 938 V 16 w(Comm)g(comm,)h(MPI)p 894 938 V 16 w(Comm)g(*newcomm\))75 1024 y(MPI)p 150 1024 V 17 w(COMM)p 263 1024 V 16 w(DUP\(COMM,)f (NEWCOMM,)g(IERROR\))170 1081 y(INTEGER)g(COMM,)h(NEWCOMM,)e(IERROR)166 1167 y Fw(MPI)p 251 1167 14 2 v 16 w(COMM)p 409 1167 V 16 w(DUP)12 b Fv(Duplicates)i(the)e(existing)h(comm)o(unicator)f Fw(comm)e Fv(with)j(asso)q(ciated)f(k)o(ey)g(v)m(al-)75 1224 y(ues.)19 b(F)l(or)11 b(eac)o(h)h(k)o(ey)f(v)m(alue,)j(the)d(resp) q(ectiv)o(e)i(cop)o(y)f(callbac)o(k)g(function)h(determines)f(the)g (attribute)g(v)m(alue)75 1280 y(asso)q(ciated)f(with)h(this)f(k)o(ey)g (in)h(the)f(new)h(comm)o(unicator;)f(one)h(particular)f(action)h(that)e (a)h(cop)o(y)g(callbac)o(k)75 1337 y(ma)o(y)18 b(tak)o(e)g(is)h(to)f (delete)h(the)g(attribute)g(from)e(the)i(new)g(comm)o(unicator.)30 b(Returns)19 b(in)g Fw(new)o(comm)e Fv(a)75 1393 y(new)d(comm)o (unicator)f(with)g(the)h(same)f(group,)g(an)o(y)g(copied)i(cac)o(hed)f (information,)f(but)h(a)f(new)h(con)o(text)75 1450 y(\(see)h(section)h (5.7.1\).)189 1528 y Fl(A)n(dvic)n(e)d(to)h(users.)39 b Fv(This)14 b(op)q(eration)f(is)g(used)h(to)e(pro)o(vide)i(a)e (parallel)j(library)f(call)g(with)f(a)g(dupli-)189 1585 y(cate)f(comm)o(unication)i(space)f(that)f(has)h(the)f(same)h(prop)q (erties)g(as)g(the)g(original)h(comm)o(unicator.)189 1641 y(This)i(includes)j(an)o(y)d(attributes)g(\(see)g(b)q(elo)o(w\),)g (and)g(top)q(ologies)h(\(see)f(c)o(hapter)g(6\).)22 b(This)17 b(call)g(is)189 1698 y(v)m(alid)g(ev)o(en)g(if)f(there)g(are)g(p)q (ending)i(p)q(oin)o(t-to-p)q(oin)o(t)e(comm)o(unications)h(in)o(v)o (olving)g(the)f(comm)o(u-)189 1754 y(nicator)i Fw(comm)p Fv(.)26 b(A)18 b(t)o(ypical)h(call)g(migh)o(t)e(in)o(v)o(olv)o(e)i(a)f Fw(MPI)p 1206 1754 V 15 w(COMM)p 1363 1754 V 17 w(DUP)g Fv(at)f(the)h(b)q(eginning)i(of)189 1811 y(the)e(parallel)h(call,)g (and)f(an)g Fw(MPI)p 775 1811 V 16 w(COMM)p 933 1811 V 16 w(FREE)h Fv(of)e(that)g(duplicated)j(comm)o(unicator)e(at)f(the) 189 1867 y(end)e(of)g(the)h(call.)21 b(Other)15 b(mo)q(dels)h(of)f (comm)o(unicator)g(managemen)o(t)f(are)h(also)g(p)q(ossible.)189 1938 y(This)g(call)i(applies)g(to)d(b)q(oth)h(in)o(tra-)h(and)f(in)o (ter-comm)o(unicators.)20 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g (users.)p Fv(\))189 2017 y Fl(A)n(dvic)n(e)c(to)j(implementors.)38 b Fv(One)13 b(need)g(not)f(actually)i(cop)o(y)e(the)h(group)f (information,)h(but)f(only)189 2073 y(add)17 b(a)g(new)g(reference)h (and)g(incremen)o(t)g(the)f(reference)h(coun)o(t.)26 b(Cop)o(y)17 b(on)g(write)g(can)g(b)q(e)h(used)189 2130 y(for)c(the)h(cac)o(hed)h(information.\()p Fl(End)g(of)g(advic)n(e)g (to)h(implementors.)p Fv(\))75 2303 y Fw(MPI)p 160 2303 V 16 w(COMM)p 318 2303 V 16 w(CREA)l(TE\(comm,)d(group,)h(new)o(comm\)) 117 2380 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(\(handle\))117 2451 y(IN)155 b Fw(group)479 b Fk(Group,)14 b(whic)o(h)h(is)f(a)g(subset)i(of)e(the)h(group)g(of)e Fi(comm)g Fk(\(han-)905 2508 y(dle\))117 2579 y(OUT)108 b Fw(new)o(comm)397 b Fk(new)15 b(comm)o(uni)o(cator)d(\(handle\))75 2704 y Ft(int)23 b(MPI)p 245 2704 15 2 v 17 w(Comm)p 358 2704 V 17 w(create\(MPI)p 615 2704 V 16 w(Comm)g(comm,)g(MPI)p 965 2704 V 17 w(Group)g(group,)g(MPI)p 1364 2704 V 17 w(Comm)g(*newcomm\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 147 155 147 154 bop 75 -100 a Fm(5.4.)34 b(COMMUNICA)l(TOR)16 b(MANA)o(GEMENT)860 b Fv(147)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(COMM)p 263 45 V 16 w(CREATE\(COMM,)23 b(GROUP,)g(NEWCOMM,)f (IERROR\))170 102 y(INTEGER)h(COMM,)h(GROUP,)f(NEWCOMM,)f(IERROR)75 188 y Fv(This)14 b(function)g(creates)f(a)g(new)h(comm)o(unicator)f Fw(new)o(comm)f Fv(with)i(comm)o(unication)g(group)f(de\014ned)i(b)o(y) 75 244 y Fw(group)f Fv(and)h(a)e(new)i(con)o(text.)k(No)14 b(cac)o(hed)h(information)f(propagates)f(from)h Fw(comm)e Fv(to)i Fw(new)o(comm)p Fv(.)k(The)75 301 y(function)e(returns)f Fi(MPI)p 487 301 13 2 v 14 w(COMM)p 631 301 V 15 w(NULL)h Fv(to)e(pro)q(cesses)i(that)e(are)h(not)g(in)h Fw(group)p Fv(.)k(The)15 b(call)h(is)g(erroneous)75 357 y(if)k(not)f(all)h Fw(group)f Fv(argumen)o(ts)g(ha)o(v)o(e)f(the)i(same)f(v)m(alue,)i(or)e (if)h Fw(group)f Fv(is)h(not)f(a)f(subset)i(of)f(the)g(group)75 414 y(asso)q(ciated)d(with)f Fw(comm)p Fv(.)k(Note)c(that)g(the)g(call) i(is)f(to)e(b)q(e)i(executed)h(b)o(y)e(all)h(pro)q(cesses)g(in)g Fw(comm)p Fv(,)e(ev)o(en)75 470 y(if)i(they)f(do)g(not)g(b)q(elong)h (to)f(the)g(new)g(group.)20 b(This)c(call)g(applies)h(only)f(to)e(in)o (tra-comm)o(unicators.)189 562 y Fl(R)n(ationale.)58 b Fv(The)19 b(requiremen)o(t)g(that)f(the)g(en)o(tire)h(group)f(of)g Fw(comm)e Fv(participate)j(in)g(the)g(call)189 619 y(stems)14 b(from)h(the)g(follo)o(wing)h(considerations:)243 708 y Fx(\017)23 b Fv(It)e(allo)o(ws)g(the)g(implemen)o(tation)h(to)e(la)o (y)o(er)h Fw(MPI)p 1161 708 14 2 v 16 w(COMM)p 1319 708 V 16 w(CREA)l(TE)h Fv(on)f(top)f(of)h(regular)289 765 y(collectiv)o(e)c(comm)o(unications.)243 834 y Fx(\017)23 b Fv(It)12 b(pro)o(vides)h(additional)h(safet)o(y)l(,)e(in)h (particular)g(in)g(the)g(case)f(where)h(partially)g(o)o(v)o(erlapping) 289 890 y(groups)h(are)h(used)h(to)f(create)g(new)g(comm)o(unicators.) 243 959 y Fx(\017)23 b Fv(It)11 b(p)q(ermits)g(implemen)o(tations)i (sometimes)e(to)f(a)o(v)o(oid)h(comm)o(unication)h(related)f(to)g(con)o (text)289 1016 y(creation.)189 1105 y(\()p Fl(End)k(of)i(r)n(ationale.) p Fv(\))189 1197 y Fl(A)n(dvic)n(e)g(to)i(users.)53 b Fw(MPI)p 647 1197 V 16 w(COMM)p 805 1197 V 16 w(CREA)l(TE)19 b Fv(pro)o(vides)f(a)f(means)g(to)g(subset)h(a)f(group)g(of)g(pro-)189 1254 y(cesses)e(for)f(the)g(purp)q(ose)i(of)e(separate)g(MIMD)g (computation,)g(with)h(separate)f(comm)o(unication)189 1310 y(space.)33 b Fw(new)o(comm)p Fv(,)20 b(whic)o(h)g(emerges)g(from) f Fw(MPI)p 1072 1310 V 16 w(COMM)p 1230 1310 V 16 w(CREA)l(TE)i Fv(can)f(b)q(e)g(used)h(in)f(subse-)189 1367 y(quen)o(t)15 b(calls)i(to)e Fw(MPI)p 558 1367 V 16 w(COMM)p 716 1367 V 16 w(CREA)l(TE)i Fv(\(or)d(other)i(comm)o(unicator)f(constructors\))f (further)i(to)189 1423 y(sub)q(divide)21 b(a)d(computation)h(in)o(to)g (parallel)h(sub-computations.)31 b(A)18 b(more)h(general)g(service)g (is)189 1480 y(pro)o(vided)d(b)o(y)f Fw(MPI)p 524 1480 V 16 w(COMM)p 682 1480 V 16 w(SPLIT)p Fv(,)f(b)q(elo)o(w.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fv(\))189 1572 y Fl(A)n(dvic)n(e)32 b(to)i(implementors.)151 b Fv(Since)35 b(all)g(pro)q(cesses)f(calling)h Fw(MPI)p 1536 1572 V 16 w(COMM)p 1694 1572 V 17 w(DUP)e Fv(or)189 1628 y Fw(MPI)p 274 1628 V 15 w(COMM)p 431 1628 V 17 w(CREA)l(TE)21 b Fv(pro)o(vide)f(the)g(same)g Fw(group)g Fv(argumen)o(t,)f(it)i(is)f(theoretically)h(p)q(ossible)189 1684 y(to)13 b(agree)g(on)g(a)h(group-wide)g(unique)h(con)o(text)e (with)h(no)g(comm)o(unication.)20 b(Ho)o(w)o(ev)o(er,)13 b(lo)q(cal)h(exe-)189 1741 y(cution)f(of)f(these)h(functions)g (requires)h(use)f(of)f(a)g(larger)h(con)o(text)f(name)h(space)f(and)h (reduces)h(error)189 1797 y(c)o(hec)o(king.)20 b(Implemen)o(tations)c (ma)o(y)e(strik)o(e)h(v)m(arious)g(compromises)g(b)q(et)o(w)o(een)g (these)f(con\015icting)189 1854 y(goals,)g(suc)o(h)i(as)e(bulk)j(allo)q (cation)f(of)f(m)o(ultiple)i(con)o(texts)d(in)i(one)g(collectiv)o(e)h (op)q(eration.)189 1927 y(Imp)q(ortan)o(t:)28 b(If)20 b(new)g(comm)o(unicators)g(are)f(created)h(without)g(sync)o(hronizing)i (the)e(pro)q(cesses)189 1983 y(in)o(v)o(olv)o(ed)11 b(then)g(the)g (comm)o(unication)g(system)f(should)i(b)q(e)f(able)g(to)f(cop)q(e)h (with)g(messages)f(arriving)189 2040 y(in)16 b(a)g(con)o(text)f(that)g (has)g(not)h(y)o(et)f(b)q(een)i(allo)q(cated)f(at)f(the)h(receiving)i (pro)q(cess.)j(\()p Fl(End)16 b(of)h(advic)n(e)189 2096 y(to)f(implementors.)p Fv(\))75 2283 y Fw(MPI)p 160 2283 V 16 w(COMM)p 318 2283 V 16 w(SPLIT\(comm,)c(colo)o(r,)i(k)o(ey)l(,)h (new)o(comm\))117 2360 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator) 11 b(\(handle\))117 2433 y(IN)155 b Fw(colo)o(r)495 b Fk(con)o(trol)14 b(of)f(subset)i(assignmen)o(t)e(\(in)o(teger\))117 2506 y(IN)155 b Fw(k)o(ey)525 b Fk(con)o(trol)14 b(of)f(rank)h (assigmen)o(t)f(\(in)o(teger\))117 2579 y(OUT)108 b Fw(new)o(comm)397 b Fk(new)15 b(comm)o(uni)o(cator)d(\(handle\))75 2704 y Ft(int)23 b(MPI)p 245 2704 15 2 v 17 w(Comm)p 358 2704 V 17 w(split\(MPI)p 591 2704 V 16 w(Comm)g(comm,)g(int)h(color,)f(int)g (key,)h(MPI)p 1419 2704 V 16 w(Comm)g(*newcomm\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 148 156 148 155 bop 75 -100 a Fv(148)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(COMM)p 263 45 V 16 w(SPLIT\(COMM,)23 b(COLOR,)g(KEY,)g(NEWCOMM,)g(IERROR\))170 102 y(INTEGER)g(COMM,)h (COLOR,)f(KEY,)g(NEWCOMM,)g(IERROR)75 188 y Fv(This)c(function)f (partitions)g(the)h(group)e(asso)q(ciated)h(with)h Fw(comm)d Fv(in)o(to)i(disjoin)o(t)g(subgroups,)h(one)f(for)75 244 y(eac)o(h)g(v)m(alue)g(of)f Fw(colo)o(r)p Fv(.)26 b(Eac)o(h)17 b(subgroup)h(con)o(tains)f(all)i(pro)q(cesses)f(of)f(the)g (same)g(color.)27 b(Within)19 b(eac)o(h)75 301 y(subgroup,)24 b(the)f(pro)q(cesses)g(are)f(rank)o(ed)h(in)g(the)g(order)g(de\014ned)h (b)o(y)e(the)h(v)m(alue)h(of)e(the)h(argumen)o(t)75 357 y Fw(k)o(ey)p Fv(,)d(with)g(ties)g(brok)o(en)g(according)g(to)f(their)h (rank)f(in)i(the)f(old)g(group.)33 b(A)19 b(new)h(comm)o(unicator)f(is) 75 414 y(created)14 b(for)f(eac)o(h)h(subgroup)g(and)g(returned)g(in)g Fw(new)o(comm)p Fv(.)k(A)c(pro)q(cess)g(ma)o(y)f(supply)i(the)f(color)g (v)m(alue)75 470 y Fw(MPI)p 160 470 14 2 v 16 w(UNDEFINED)p Fv(,)19 b(in)i(whic)o(h)g(case)e Fw(new)o(comm)g Fv(returns)h Fi(MPI)p 1200 470 13 2 v 14 w(COMM)p 1344 470 V 15 w(NULL)p Fv(.)g(This)g(is)h(a)e(collectiv)o(e)75 527 y(call,)d(but)f(eac)o(h)h (pro)q(cess)f(is)h(p)q(ermitted)g(to)e(pro)o(vide)i(di\013eren)o(t)f(v) m(alues)i(for)d Fw(colo)o(r)g Fv(and)i Fw(k)o(ey)p Fv(.)166 583 y(A)47 b(call)i(to)e Fw(MPI)p 536 583 14 2 v 16 w(COMM)p 694 583 V 16 w(CREA)l(TE\(comm,)54 b(group,)i(new)o(comm\))45 b Fv(is)j(equiv)m(alen)o(t)i(to)75 640 y(a)15 b(call)i(to)d Fw(MPI)p 337 640 V 16 w(COMM)p 495 640 V 17 w(SPLIT\(comm,)e(colo)o(r,) i(k)o(ey)l(,)h(new)o(comm\))p Fv(,)e(where)j(all)g(mem)o(b)q(ers)g(of)f Fw(group)g Fv(pro-)75 696 y(vide)i Fw(colo)o(r)e Fv(=)30 b(0)15 b(and)h Fw(k)o(ey)h Fv(=)32 b(rank)15 b(in)i Fw(group)p Fv(,)f(and)g(all)h(pro)q(cesses)f(that)f(are)h(not)f(mem)o(b)q(ers)i (of)e Fw(group)75 753 y Fv(pro)o(vide)k Fw(colo)o(r)e Fv(=)37 b Fw(MPI)p 506 753 V 16 w(UNDEFINED)p Fv(.)18 b(The)g(function)h Fw(MPI)p 1166 753 V 16 w(COMM)p 1324 753 V 16 w(SPLIT)f Fv(allo)o(ws)h(more)e(general)75 809 y(partitioning)22 b(of)e(a)g(group)h(in)o(to)g(one)g(or)f(more)g (subgroups)h(with)g(optional)h(reordering.)36 b(This)22 b(call)75 865 y(applies)17 b(only)f(in)o(tra-comm)o(unicators.)-910 b Fx(>)166 922 y Fv(The)15 b(v)m(alue)i(of)e Fw(colo)o(r)e Fv(m)o(ust)i(b)q(e)h(nonnegativ)o(e.)-1119 b Fx(?)189 1017 y Fl(A)n(dvic)n(e)14 b(to)h(users.)39 b Fv(This)15 b(is)f(an)g(extremely)h(p)q(o)o(w)o(erful)f(mec)o(hanism)h(for)e (dividing)j(a)e(single)h(com-)189 1074 y(m)o(unicating)h(group)e(of)h (pro)q(cesses)g(in)o(to)g Fp(k)h Fv(subgroups,)f(with)g Fp(k)h Fv(c)o(hosen)g(implicitly)i(b)o(y)d(the)g(user)189 1130 y(\(b)o(y)f(the)h(n)o(um)o(b)q(er)g(of)g(colors)g(asserted)f(o)o (v)o(er)g(all)i(the)f(pro)q(cesses\).)20 b(Eac)o(h)14 b(resulting)i(comm)o(unica-)189 1187 y(tor)g(will)i(b)q(e)f(non-o)o(v)o (erlapping.)26 b(Suc)o(h)17 b(a)g(division)i(could)e(b)q(e)h(useful)g (for)e(de\014ning)i(a)f(hierarc)o(h)o(y)189 1243 y(of)d(computations,)h (suc)o(h)h(as)e(for)h(m)o(ultigrid,)h(or)f(linear)h(algebra.)189 1317 y(Multiple)k(calls)f(to)f Fw(MPI)p 623 1317 V 16 w(COMM)p 781 1317 V 16 w(SPLIT)g Fv(can)h(b)q(e)g(used)g(to)f(o)o(v)o (ercome)f(the)i(requiremen)o(t)g(that)189 1373 y(an)o(y)d(call)h(ha)o (v)o(e)f(no)g(o)o(v)o(erlap)g(of)g(the)g(resulting)i(comm)o(unicators)e (\(eac)o(h)g(pro)q(cess)g(is)h(of)f(only)h(one)189 1430 y(color)i(p)q(er)g(call\).)32 b(In)19 b(this)g(w)o(a)o(y)l(,)g(m)o (ultiple)i(o)o(v)o(erlapping)e(comm)o(unication)h(structures)e(can)h(b) q(e)189 1486 y(created.)h(Creativ)o(e)14 b(use)i(of)f(the)g Fw(colo)o(r)f Fv(and)h Fw(k)o(ey)h Fv(in)g(suc)o(h)f(splitting)i(op)q (erations)e(is)h(encouraged.)189 1560 y(Note)g(that,)g(for)h(a)f (\014xed)i(color,)f(the)g(k)o(eys)f(need)i(not)e(b)q(e)i(unique.)26 b(It)17 b(is)h Fw(MPI)p 1551 1560 V 16 w(COMM)p 1709 1560 V 16 w(SPLIT)p Fv('s)189 1616 y(resp)q(onsibilit)o(y)f(to)c(sort)h (pro)q(cesses)h(in)g(ascending)h(order)e(according)h(to)e(this)i(k)o (ey)l(,)g(and)f(to)g(break)189 1672 y(ties)19 b(in)h(a)e(consisten)o(t) i(w)o(a)o(y)l(.)30 b(If)19 b(all)h(the)f(k)o(eys)g(are)f(sp)q (eci\014ed)k(in)d(the)g(same)g(w)o(a)o(y)l(,)g(then)g(all)h(the)189 1729 y(pro)q(cesses)15 b(in)g(a)f(giv)o(en)h(color)g(will)h(ha)o(v)o(e) e(the)g(relativ)o(e)h(rank)g(order)f(as)g(they)g(did)i(in)f(their)g (paren)o(t)189 1785 y(group.)k(\(In)d(general,)f(they)g(will)i(ha)o(v)o (e)e(di\013eren)o(t)g(ranks.\))189 1859 y(Essen)o(tially)l(,)f(making)e (the)h(k)o(ey)f(v)m(alue)i(zero)f(for)e(all)j(pro)q(cesses)f(of)f(a)g (giv)o(en)h(color)g(means)f(that)g(one)189 1915 y(do)q(esn't)17 b(really)i(care)e(ab)q(out)h(the)f(rank-order)h(of)f(the)h(pro)q (cesses)g(in)g(the)g(new)g(comm)o(unicator.)189 1972 y(\()p Fl(End)d(of)i(advic)n(e)f(to)g(users.)p Fv(\))-160 2022 y Fx(>)189 2106 y Fl(R)n(ationale.)49 b Fw(colo)o(r)15 b Fv(is)i(restricted)g(to)f(b)q(e)i(nonnegativ)o(e,)f(so)f(as)h(not)f (to)g(con\014ct)h(with)g(the)g(v)m(alue)189 2163 y(assigned)e(to)g Fi(MPI)p 501 2163 13 2 v 14 w(UNDEFINED)p Fv(.)f(\()p Fl(End)i(of)g(r)n(ationale.)p Fv(\))-160 2213 y Fx(?)75 2322 y Fj(5.4.3)49 b(Communicato)o(r)16 b(Destructo)o(rs)75 2502 y Fw(MPI)p 160 2502 14 2 v 16 w(COMM)p 318 2502 V 16 w(FREE\(comm\))117 2579 y Fk(INOUT)62 b Fw(comm)470 b Fk(comm)o(unicator)11 b(to)j(b)q(e)h(destro)o(y)o(ed)g(\(handle\))75 2704 y Ft(int)23 b(MPI)p 245 2704 15 2 v 17 w(Comm)p 358 2704 V 17 w(free\(MPI)p 567 2704 V 16 w(Comm)g(*comm\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 149 157 149 156 bop 75 -100 a Fm(5.5.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b Fv(149)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(COMM)p 263 45 V 16 w(FREE\(COMM,)23 b(IERROR\))170 102 y(INTEGER)g(COMM,)h(IERROR)166 188 y Fv(This)13 b(collectiv)o(e)g (op)q(eration)g(marks)e(the)h(comm)o(unication)h(ob)s(ject)f(for)f (deallo)q(cation.)20 b(The)13 b(handle)75 244 y(is)e(set)g(to)f Fi(MPI)p 311 244 13 2 v 14 w(COMM)p 455 244 V 15 w(NULL)p Fv(.)h(An)o(y)g(p)q(ending)h(op)q(erations)f(that)f(use)h(this)g(comm)o (unicator)g(will)h(complete)75 301 y(normally;)18 b(the)f(ob)s(ject)g (is)g(actually)h(deallo)q(cated)h(only)e(if)h(there)f(are)g(no)g(other) g(activ)o(e)g(references)h(to)75 357 y(it.)28 b(This)18 b(call)h(applies)g(to)e(in)o(tra-)h(and)f(in)o(ter-comm)o(unicators.)28 b(The)18 b(delete)g(callbac)o(k)h(functions)g(for)75 414 y(all)d(cac)o(hed)g(attributes)f(\(see)g(section)h(5.7\))e(are)h (called)h(in)g(arbitrary)f(order.)189 520 y Fl(A)n(dvic)n(e)i(to)h (implementors.)52 b Fv(A)17 b(reference-coun)o(t)h(mec)o(hanism)f(ma)o (y)g(b)q(e)h(used:)24 b(the)17 b(reference)189 577 y(coun)o(t)d(is)h (incremen)o(ted)h(b)o(y)e(eac)o(h)h(call)g(to)f Fw(MPI)p 1000 577 14 2 v 16 w(COMM)p 1158 577 V 17 w(DUP)p Fv(,)g(and)g (decremen)o(ted)i(b)o(y)e(eac)o(h)h(call)189 633 y(to)j Fw(MPI)p 333 633 V 16 w(COMM)p 491 633 V 16 w(FREE)p Fv(.)h(The)h(ob)s(ject)e(is)i(ultimately)g(deallo)q(cated)g(when)g(the) f(coun)o(t)g(reac)o(hes)189 689 y(zero.)189 765 y(Though)10 b(collectiv)o(e,)j(it)d(is)h(an)o(ticipated)g(that)f(this)g(op)q (eration)h(will)h(normally)f(b)q(e)g(implemen)o(ted)h(to)189 821 y(b)q(e)h(lo)q(cal,)h(though)f(the)f(debugging)i(v)o(ersion)f(of)f (an)h Fw(MPI)f Fv(library)i(migh)o(t)e(c)o(ho)q(ose)h(to)f(sync)o (hronize.)189 878 y(\()p Fl(End)j(of)i(advic)n(e)f(to)g(implementors.)p Fv(\))75 1021 y Fn(5.5)59 b(Motivating)19 b(Examples)75 1124 y Fj(5.5.1)49 b(Current)15 b(Practice)g(#1)75 1210 y Fv(Example)h(#1a:)147 1303 y Ft(main\(int)22 b(argc,)i(char)f (**argv\))147 1360 y({)194 1416 y(int)h(me,)f(size;)194 1473 y(...)194 1529 y(MPI_Init)g(\()h(&argc,)f(&argv)g(\);)194 1586 y(MPI_Comm_rank)f(\(MPI_COMM_WORLD,)g(&me\);)194 1642 y(MPI_Comm_size)g(\(MPI_COMM_WORLD,)g(&size\);)194 1755 y(\(void\)printf)h(\("Process)f(\045d)i(size)f(\045d\\n",)g(me,)h (size\);)194 1812 y(...)194 1868 y(MPI_Finalize\(\);)147 1924 y(})75 2018 y Fv(Example)19 b(#1a)f(is)h(a)f(do-nothing)h(program) e(that)h(initializes)k(itself)d(legally)l(,)i(and)d(refers)g(to)g(the)h (the)75 2075 y(\\all")f(comm)o(unicator,)g(and)g(prin)o(ts)g(a)f (message.)27 b(It)18 b(terminates)g(itself)h(legally)g(to)q(o.)27 b(This)18 b(example)75 2131 y(do)q(es)e(not)e(imply)j(that)d Fw(MPI)h Fv(supp)q(orts)g Ft(printf)p Fv(-lik)o(e)h(comm)o(unication)g (itself.)75 2188 y(Example)g(#1b)f(\(supp)q(osing)h(that)e Ft(size)h Fv(is)h(ev)o(en\):)170 2281 y Ft(main\(int)23 b(argc,)g(char)h(**argv\))170 2338 y({)242 2394 y(int)f(me,)h(size;)242 2451 y(int)f(SOME_TAG)g(=)h(0;)242 2507 y(...)242 2564 y(MPI_Init\(&argc,)e(&argv\);)242 2677 y(MPI_Comm_rank\(MPI_COMM_WOR)o (LD,)f(&me\);)71 b(/*)23 b(local)h(*/)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 150 158 150 157 bop 75 -100 a Fv(150)370 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)242 45 y Ft(MPI_Comm_size\(MPI_COMM_WOR)o(LD,)21 b(&size\);)i(/*)g(local)h (*/)242 158 y(if\(\(me)f(\045)h(2\))f(==)h(0\))242 214 y({)314 271 y(/*)f(send)h(unless)f(highest-numbered)e(process)i(*/)314 327 y(if\(\(me)g(+)g(1\))h(<)g(size\))385 384 y(MPI_Send\(...,)e(me)i (+)g(1,)f(SOME_TAG,)g(MPI_COMM_WORLD\);)242 440 y(})242 497 y(else)314 553 y(MPI_Recv\(...,)f(me)h(-)h(1,)g(SOME_TAG,)e (MPI_COMM_WORLD\);)242 666 y(...)242 723 y(MPI_Finalize\(\);)170 779 y(})75 873 y Fv(Example)16 b(#1b)f(sc)o(hematically)i(illustrates)g (message)e(exc)o(hanges)h(b)q(et)o(w)o(een)g(\\ev)o(en")f(and)h(\\o)q (dd")f(pro-)75 929 y(cesses)h(in)g(the)f(\\all")g(comm)o(unicator.)75 1051 y Fj(5.5.2)49 b(Current)15 b(Practice)g(#2)147 1137 y Ft(main\(int)22 b(argc,)i(char)f(**argv\))147 1193 y({)194 1250 y(int)h(me,)f(count;)194 1306 y(void)h(*data;)194 1363 y(...)194 1476 y(MPI_Init\(&argc,)e(&argv\);)194 1532 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&me\);)194 1645 y(if\(me)i(==)h(0\))194 1701 y({)290 1758 y(/*)f(get)h(input,)f (create)g(buffer)g(``data'')g(*/)290 1814 y(...)194 1871 y(})194 1984 y(MPI_Bcast\(data,)f(count,)h(MPI_BYTE,)g(0,)g (MPI_COMM_WORLD\);)194 2097 y(...)194 2209 y(MPI_Finalize\(\);)147 2266 y(})75 2372 y Fv(This)16 b(example)g(illustrates)g(the)f(use)h(of) f(a)g(collectiv)o(e)h(comm)o(unication.)75 2494 y Fj(5.5.3)49 b(\(App)o(ro)o(ximate\))16 b(Current)f(Practice)g(#3)123 2580 y Ft(main\(int)22 b(argc,)i(char)f(**argv\))123 2636 y({)170 2693 y(int)h(me,)f(count,)g(count2;)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 151 159 151 158 bop 75 -100 a Fm(5.5.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b Fv(151)170 45 y Ft(void)24 b(*send_buf,)e(*recv_buf,) h(*send_buf2,)f(*recv_buf2;)170 102 y(MPI_Group)h(MPI_GROUP_WORLD,)f (grprem;)170 158 y(MPI_Comm)h(commslave;)170 214 y(static)g(int)h (ranks[])f(=)h({0};)170 271 y(...)170 327 y(MPI_Init\(&argc,)e (&argv\);)170 384 y(MPI_Comm_group\(MPI_COMM_WORLD)o(,)f (&MPI_GROUP_WORLD\);)170 440 y(MPI_Comm_rank\(MPI_COMM_WORLD,)f(&me\);) 47 b(/*)24 b(local)f(*/)170 553 y(MPI_Group_excl\(MPI_GROUP_WORL)o(D,)e (1,)j(ranks,)f(&grprem\);)46 b(/*)24 b(local)f(*/)170 610 y(MPI_Comm_create\(MPI_COMM_WORL)o(D,)e(grprem,)i(&commslave\);)170 723 y(if\(me)h(!=)f(0\))170 779 y({)218 835 y(/*)h(compute)f(on)g (slave)g(*/)218 892 y(...)218 948 y(MPI_Reduce\(send_buf,recv_bu)o (ff,count)o(,)e(MPI_INT,)i(MPI_SUM,)g(1,)g(commslave\);)218 1005 y(...)170 1061 y(})170 1118 y(/*)h(zero)f(falls)h(through)e (immediately)h(to)g(this)h(reduce,)f(others)g(do)g(later...)g(*/)170 1174 y(MPI_Reduce\(send_buf2,)e(recv_buff2,)i(count2,)433 1231 y(MPI_INT,)g(MPI_SUM,)f(0,)i(MPI_COMM_WORLD\);)170 1344 y(MPI_Comm_free\(&commslave\);)170 1400 y (MPI_Group_free\(&MPI_GROUP_WOR)o(LD\);)170 1456 y (MPI_Group_free\(&grprem\);)170 1513 y(MPI_Finalize\(\);)123 1569 y(})75 1672 y Fv(This)18 b(example)h(illustrates)g(ho)o(w)e(a)h (group)f(consisting)i(of)e(all)i(but)f(the)g(zeroth)f(pro)q(cess)h(of)g (the)g(\\all")75 1728 y(group)13 b(is)h(created,)f(and)h(then)g(ho)o(w) f(a)g(comm)o(unicator)g(is)h(formed)f(\()g Fw(commslave)p Fv(\))e(for)i(that)g(new)h(group.)75 1785 y(The)f(new)h(comm)o (unicator)e(is)i(used)g(in)g(a)e(collectiv)o(e)j(call,)g(and)e(all)h (pro)q(cesses)f(execute)h(a)f(collectiv)o(e)i(call)75 1841 y(in)j(the)33 b Fi(MPI)p 303 1841 13 2 v 15 w(COMM)p 448 1841 V 14 w(W)o(ORLD)18 b Fv(con)o(text.)25 b(This)18 b(example)h(illustrates)f(ho)o(w)f(the)g(t)o(w)o(o)f(comm)o(unicators) 75 1898 y(\(that)c(inheren)o(tly)k(p)q(ossess)d(distinct)i(con)o (texts\))d(protect)h(comm)o(unication.)20 b(That)13 b(is,)h(comm)o (unication)75 1954 y(in)30 b Fi(MPI)p 219 1954 V 14 w(COMM)p 363 1954 V 15 w(W)o(ORLD)15 b Fv(is)h(insulated)g(from)f(comm)o (unication)h(in)31 b Fw(commslave)p Fv(,)13 b(and)i(vice)i(v)o(ersa.) 166 2010 y(In)h(summary)l(,)g(\\group)f(safet)o(y")f(is)i(ac)o(hiev)o (ed)h(via)f(comm)o(unicators)f(b)q(ecause)i(distinct)g(con)o(texts)75 2067 y(within)d(comm)o(unicators)f(are)g(enforced)h(to)e(b)q(e)i (unique)h(on)e(an)o(y)g(pro)q(cess.)75 2188 y Fj(5.5.4)49 b(Example)17 b(#4)75 2274 y Fv(The)e(follo)o(wing)g(example)g(is)g (mean)o(t)f(to)f(illustrate)j(\\safet)o(y")d(b)q(et)o(w)o(een)h(p)q (oin)o(t-to-p)q(oin)o(t)i(and)e(collectiv)o(e)75 2330 y(comm)o(unication.)20 b Fw(MPI)13 b Fv(guaran)o(tees)f(that)h(a)g (single)h(comm)o(unicator)f(can)h(do)f(safe)g(p)q(oin)o(t-to-p)q(oin)o (t)h(and)75 2387 y(collectiv)o(e)j(comm)o(unication.)147 2478 y Ft(#define)22 b(TAG_ARBITRARY)h(12345)147 2534 y(#define)f(SOME_COUNT)166 b(50)147 2647 y(main\(int)22 b(argc,)i(char)f(**argv\))147 2704 y({)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 152 160 152 159 bop 75 -100 a Fv(152)370 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)194 45 y Ft(int)24 b(me;)194 102 y(MPI_Request)f(request[2];)194 158 y(MPI_Status)g(status[2];)194 214 y(MPI_Group)g(MPI_GROUP_WORLD,)f (subgroup;)194 271 y(int)i(ranks[])f(=)g({2,)h(4,)f(6,)h(8};)194 327 y(MPI_Comm)f(the_comm;)194 384 y(...)194 440 y(MPI_Init\(&argc,)f (&argv\);)194 497 y(MPI_Comm_group\(MPI_COMM_WORL)o(D,)f (&MPI_GROUP_WORLD\);)194 610 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)g (4,)j(ranks,)f(&subgroup\);)f(/*)i(local)f(*/)194 666 y(MPI_Group_rank\(subgroup,)e(&me\);)119 b(/*)23 b(local)g(*/)194 779 y(MPI_Comm_create\(MPI_COMM_WOR)o(LD,)e(subgroup,)i(&the_comm\);) 194 892 y(if\(me)g(!=)h(MPI_UNDEFINED\))194 948 y({)290 1005 y(MPI_Irecv\(buff1,)d(count,)i(MPI_DOUBLE,)g(MPI_ANY_SOURCE,)f (TAG_ARBITRARY,)719 1061 y(the_comm,)h(request\);)290 1118 y(MPI_Isend\(buff2,)e(count,)i(MPI_DOUBLE,)g(\(me+1\)\0454,)f (TAG_ARBITRARY,)719 1174 y(the_comm,)h(request+1\);)194 1231 y(})194 1344 y(for\(i)g(=)h(0;)g(i)g(<)f(SOME_COUNT,)g(i++\))242 1400 y(MPI_Reduce\(...,)f(the_comm\);)194 1456 y(MPI_Waitall\(2,)g (request,)h(status\);)194 1569 y(MPI_Comm_free\(t&he_comm\);)194 1626 y(MPI_Group_free\(&MPI_GROUP_WO)o(RLD\);)194 1682 y(MPI_Group_free\(&subgroup\);)194 1739 y(MPI_Finalize\(\);)147 1795 y(})75 1915 y Fj(5.5.5)49 b(Lib)o(ra)o(ry)16 b(Example)h(#1)75 2001 y Fv(The)e(main)h(program:)147 2083 y Ft(main\(int)22 b(argc,)i(char)f(**argv\))147 2139 y({)194 2195 y(int)h(done)f(=)h(0;) 194 2252 y(user_lib_t)f(*libh_a,)g(*libh_b;)194 2308 y(void)h(*dataset1,)e(*dataset2;)194 2365 y(...)194 2421 y(MPI_Init\(&argc,)g(&argv\);)194 2478 y(...)194 2534 y(init_user_lib\(MPI_COMM_WORLD)o(,)f(&libh_a\);)194 2591 y(init_user_lib\(MPI_COMM_WORLD)o(,)g(&libh_b\);)194 2647 y(...)194 2704 y(user_start_op\(libh_a,)g(dataset1\);)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 153 161 153 160 bop 75 -100 a Fm(5.5.)29 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1055 b Fv(153)194 45 y Ft(user_start_op\(libh_b,)21 b(dataset2\);)194 102 y(...)194 158 y(while\(!done\))194 214 y({)266 271 y(/*)i(work)h(*/)266 327 y(...)266 384 y(MPI_Reduce\(...,)e(MPI_COMM_WORLD\);)266 440 y(...)266 497 y(/*)h(see)h(if)g(done)f(*/)266 553 y(...)194 610 y(})194 666 y(user_end_op\(libh_a\);)194 723 y(user_end_op\(libh_b\);) 194 835 y(uninit_user_lib\(libh_a\);)194 892 y (uninit_user_lib\(libh_b\);)194 948 y(MPI_Finalize\(\);)147 1005 y(})75 1099 y Fv(The)15 b(user)h(library)g(initialization)i(co)q (de:)147 1192 y Ft(void)23 b(init_user_lib\(MPI_Comm)e(comm,)i (user_lib_t)f(**handle\))147 1249 y({)194 1305 y(user_lib_t)h(*save;) 194 1418 y(user_lib_initsave\(&save\);)e(/*)i(local)h(*/)194 1475 y(MPI_Comm_dup\(comm,)e(&\(save)h(->)g(comm\)\);)194 1588 y(/*)h(other)f(inits)g(*/)194 1644 y(...)194 1757 y(*handle)g(=)h(save;)147 1813 y(})75 1907 y Fv(User)15 b(start-up)g(co)q(de:)147 2001 y Ft(void)23 b (user_start_op\(user_lib_t)d(*handle,)j(void)h(*data\))147 2058 y({)194 2114 y(MPI_Irecv\()f(...,)g(handle->comm,)f(&\(handle)h (->)h(irecv_handle\))e(\);)194 2170 y(MPI_Isend\()h(...,)g (handle->comm,)f(&\(handle)h(->)h(isend_handle\))e(\);)147 2227 y(})75 2321 y Fv(User)15 b(comm)o(unication)h(clean-up)h(co)q(de:) 147 2415 y Ft(void)23 b(user_end_op\(user_lib_t)e(*handle\))147 2471 y({)194 2527 y(MPI_Status)i(*status;)194 2584 y(MPI_Wait\(handle)f (->)i(isend_handle,)e(status\);)194 2640 y(MPI_Wait\(handle)g(->)i (irecv_handle,)e(status\);)147 2697 y(})-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 154 162 154 161 bop 75 -100 a Fv(154)370 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fv(User)g(ob)s(ject)g(clean-up)i(co)q(de:)147 139 y Ft(void)23 b(uninit_user_lib\(user_lib_)o(t)e(*handle\))147 195 y({)194 252 y(MPI_Comm_free\(&\(handle)g(->)j(comm\)\);)194 308 y(free\(handle\);)147 365 y(})75 486 y Fj(5.5.6)49 b(Lib)o(ra)o(ry)16 b(Example)h(#2)75 572 y Fv(The)e(main)h(program:)147 666 y Ft(main\(int)22 b(argc,)i(char)f(**argv\))147 723 y({)194 779 y(int)h(ma,)f(mb;)194 835 y(MPI_Group)g(MPI_GROUP_WORLD,)f (group_a,)g(group_b;)194 892 y(MPI_Comm)h(comm_a,)g(comm_b;)194 1005 y(static)g(int)h(list_a[])f(=)g({0,)h(1};)75 1061 y(#if)47 b(defined\(EXAMPLE_2B\))22 b(|)h(defined\(EXAMPLE_2C\))194 1118 y(static)g(int)h(list_b[])f(=)g({0,)h(2)f(,3};)75 1174 y(#else/*)g(EXAMPLE_2A)f(*/)194 1231 y(static)h(int)h(list_b[])f (=)g({0,)h(2};)75 1287 y(#endif)194 1344 y(int)g(size_list_a)e(=)i (sizeof\(list_a\)/sizeof\(int)o(\);)194 1400 y(int)g(size_list_b)e(=)i (sizeof\(list_b\)/sizeof\(int)o(\);)194 1513 y(...)194 1569 y(MPI_Init\(&argc,)e(&argv\);)194 1626 y (MPI_Comm_group\(MPI_COMM_WORL)o(D,)f(&MPI_GROUP_WORLD\);)194 1739 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)g(size_list_a,)h(list_a,)h (&group_a\);)194 1795 y(MPI_Group_incl\(MPI_GROUP_WOR)o(LD,)e (size_list_b,)h(list_b,)h(&group_b\);)194 1908 y (MPI_Comm_create\(MPI_COMM_WOR)o(LD,)e(group_a,)i(&comm_a\);)194 1965 y(MPI_Comm_create\(MPI_COMM_WOR)o(LD,)e(group_b,)i(&comm_b\);)194 2077 y(if\(comm_a)g(!=)h(MPI_COMM_NULL\))266 2134 y (MPI_Comm_rank\(comm_a,)d(&ma\);)194 2190 y(if\(comm_a)i(!=)h (MPI_COMM_NULL\))266 2247 y(MPI_Comm_rank\(comm_b,)d(&mb\);)194 2360 y(if\(comm_a)i(!=)h(MPI_COMM_NULL\))266 2416 y (lib_call\(comm_a\);)194 2529 y(if\(comm_b)f(!=)h(MPI_COMM_NULL\))194 2586 y({)242 2642 y(lib_call\(comm_b\);)242 2698 y(lib_call\(comm_b\);) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 155 163 155 162 bop 75 -100 a Fm(5.5.)34 b(MOTIV)-5 b(A)l(TING)16 b(EXAMPLES)1050 b Fv(155)194 45 y Ft(})194 158 y(if\(comm_a)23 b(!=)h(MPI_COMM_NULL\))242 214 y(MPI_Comm_free\(&comm_a\);)194 271 y(if\(comm_b)f(!=)h(MPI_COMM_NULL\))242 327 y (MPI_Comm_free\(&comm_b\);)194 384 y(MPI_Group_free\(&group_a\);)194 440 y(MPI_Group_free\(&group_b\);)194 497 y (MPI_Group_free\(&MPI_GROUP_WO)o(RLD\);)194 553 y(MPI_Finalize\(\);)147 610 y(})75 703 y Fv(The)15 b(library:)147 797 y Ft(void)23 b(lib_call\(MPI_Comm)e(comm\))147 854 y({)194 910 y(int)j(me,)f(done)h (=)f(0;)194 967 y(MPI_Comm_rank\(comm,)f(&me\);)194 1023 y(if\(me)h(==)h(0\))266 1080 y(while\(!done\))266 1136 y({)337 1192 y(MPI_Recv\(...,)f(MPI_ANY_SOURCE,)e(MPI_ANY_TAG,)i (comm\);)337 1249 y(...)266 1305 y(})194 1362 y(else)194 1418 y({)242 1475 y(/*)h(work)f(*/)242 1531 y(MPI_Send\(...,)f(0,)i (ARBITRARY_TAG,)e(comm\);)242 1588 y(....)194 1644 y(})75 1701 y(#ifdef)h(EXAMPLE_2C)194 1757 y(/*)h(include)f(\(resp,)g (exclude\))g(for)g(safety)g(\(resp,)g(no)h(safety\):)f(*/)194 1813 y(MPI_Barrier\(comm\);)75 1870 y(#endif)147 1926 y(})75 2020 y Fv(The)13 b(ab)q(o)o(v)o(e)f(example)h(is)h(really)f (three)g(examples,)g(dep)q(ending)i(on)e(whether)f(or)g(not)h(one)f (includes)j(rank)75 2077 y(3)f(in)h Fw(list)p 222 2077 14 2 v 17 w(b)p Fv(,)g(and)f(whether)h(or)f(not)f(a)i(sync)o(hronize)g (is)g(included)i(in)e Fw(lib)p 1290 2077 V 17 w(call)p Fv(.)k(This)c(example)g(illustrates)75 2133 y(that,)i(despite)h(con)o (texts,)f(subsequen)o(t)h(calls)h(to)d Fw(lib)p 984 2133 V 17 w(call)h Fv(with)h(the)f(same)g(con)o(text)g(need)h(not)f(b)q(e)h (safe)75 2190 y(from)g(one)h(another)g(\(collo)q(quially)l(,)j(\\bac)o (k-masking"\).)30 b(Safet)o(y)18 b(is)i(realized)g(if)g(the)37 b Fw(MPI)p 1684 2190 V 16 w(Ba)o(rrier)17 b Fv(is)75 2246 y(added.)34 b(What)19 b(this)h(demonstrates)f(is)h(that)f (libraries)i(ha)o(v)o(e)f(to)f(b)q(e)h(written)g(carefully)l(,)i(ev)o (en)e(with)75 2302 y(con)o(texts.)j(When)17 b(rank)f(3)g(is)h (excluded,)i(then)d(the)h(sync)o(hronize)g(is)g(not)g(needed)g(to)f (get)g(safet)o(y)g(from)75 2359 y(bac)o(k)f(masking.)166 2415 y(Algorithms)i(lik)o(e)h(\\reduce")e(and)h(\\allreduce")h(ha)o(v)o (e)e(strong)g(enough)h(source)g(selectivit)o(y)h(prop-)75 2472 y(erties)c(so)g(that)g(they)g(are)f(inheren)o(tly)j(ok)m(a)o(y)e (\(no)f(bac)o(kmasking\),)h(pro)o(vided)h(that)e Fw(MPI)h Fv(pro)o(vides)g(basic)75 2528 y(guaran)o(tees.)21 b(So)16 b(are)g(m)o(ultiple)i(calls)f(to)e(a)h(t)o(ypical)h(tree-broadcast)e (algorithm)h(with)g(the)g(same)g(ro)q(ot)75 2585 y(or)h(di\013eren)o(t) h(ro)q(ots)f(\(see)h([28)o(]\).)27 b(Here)18 b(w)o(e)f(rely)h(on)g(t)o (w)o(o)f(guaran)o(tees)f(of)i Fw(MPI)p Fv(:)f(pairwise)h(ordering)g(of) 75 2641 y(messages)f(b)q(et)o(w)o(een)h(pro)q(cesses)g(in)g(the)g(same) f(con)o(text,)g(and)h(source)f(selectivit)o(y)i(|)f(deleting)h(either) 75 2698 y(feature)c(remo)o(v)o(es)f(the)i(guaran)o(tee)e(that)g(bac)o (kmasking)i(cannot)f(b)q(e)h(required.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 156 164 156 163 bop 75 -100 a Fv(156)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)166 45 y Fv(Algorithms)d(that)g(try)g(to)f(do)h(non-deterministic)j (broadcasts)c(or)h(other)g(calls)h(that)f(include)j(wild-)75 102 y(card)g(op)q(erations)g(will)i(not)e(generally)h(ha)o(v)o(e)f(the) g(go)q(o)q(d)g(prop)q(erties)h(of)f(the)g(deterministic)i(implemen-)75 158 y(tations)g(of)h(\\reduce,")g(\\allreduce,")i(and)e(\\broadcast.") 27 b(Suc)o(h)18 b(algorithms)g(w)o(ould)g(ha)o(v)o(e)g(to)f(utilize)75 214 y(the)e(monotonically)h(increasing)h(tags)d(\(within)i(a)f(comm)o (unicator)g(scop)q(e\))g(to)g(k)o(eep)g(things)h(straigh)o(t.)166 271 y(All)k(of)e(the)h(foregoing)f(is)h(a)g(supp)q(osition)h(of)e (\\collectiv)o(e)i(calls")f(implemen)o(ted)i(with)e(p)q(oin)o(t-to-)75 327 y(p)q(oin)o(t)g(op)q(erations.)30 b Fw(MPI)18 b Fv(implemen)o (tations)i(ma)o(y)e(or)g(ma)o(y)g(not)g(implemen)o(t)i(collectiv)o(e)h (calls)e(using)75 384 y(p)q(oin)o(t-to-p)q(oin)o(t)e(op)q(erations.)22 b(These)16 b(algorithms)g(are)f(used)i(to)e(illustrate)i(the)f(issues)h (of)e(correctness)75 440 y(and)g(safet)o(y)l(,)g(indep)q(enden)o(t)i (of)e(ho)o(w)g Fw(MPI)f Fv(implemen)o(ts)j(its)e(collectiv)o(e)i (calls.)k(See)16 b(also)f(section)h(5.8.)75 584 y Fn(5.6)59 b(Inter-Communication)75 685 y Fv(This)18 b(section)h(in)o(tro)q(duces) f(the)g(concept)g(of)f(in)o(ter-comm)o(unication)i(and)f(describ)q(es)i (the)d(p)q(ortions)h(of)75 741 y Fw(MPI)j Fv(that)g(supp)q(ort)h(it.)40 b(It)22 b(describ)q(es)h(supp)q(ort)f(for)f(writing)i(programs)d(that)h (con)o(tain)h(user-lev)o(el)75 798 y(serv)o(ers.)166 854 y(All)c(p)q(oin)o(t-to-p)q(oin)o(t)g(comm)o(unication)g(describ)q (ed)h(th)o(us)d(far)h(has)g(in)o(v)o(olv)o(ed)g(comm)o(unication)h(b)q (e-)75 911 y(t)o(w)o(een)c(pro)q(cesses)g(that)f(are)h(mem)o(b)q(ers)g (of)f(the)h(same)g(group.)19 b(This)c(t)o(yp)q(e)f(of)f(comm)o (unication)i(is)f(called)75 967 y(\\in)o(tra-comm)o(unication")j(and)g (the)h(comm)o(unicator)e(used)i(is)f(called)i(an)e(\\in)o(tra-comm)o (unicator,")f(as)75 1024 y(w)o(e)f(ha)o(v)o(e)g(noted)g(earlier)h(in)g (the)f(c)o(hapter.)166 1080 y(In)c(mo)q(dular)g(and)f(m)o(ulti-discipl) q(inary)j(applications,)g(di\013eren)o(t)d(pro)q(cess)h(groups)f (execute)h(distinct)75 1137 y(mo)q(dules)i(and)f(pro)q(cesses)h(within) g(di\013eren)o(t)f(mo)q(dules)h(comm)o(unicate)f(with)h(one)f(another)f (in)i(a)f(pip)q(eline)75 1193 y(or)j(a)f(more)h(general)h(mo)q(dule)g (graph.)j(In)d(these)f(applications,)i(the)e(most)f(natural)h(w)o(a)o (y)f(for)h(a)g(pro)q(cess)75 1250 y(to)g(sp)q(ecify)i(a)e(target)f(pro) q(cess)i(is)g(b)o(y)g(the)g(rank)f(of)g(the)h(target)e(pro)q(cess)i (within)h(the)e(target)g(group.)20 b(In)75 1306 y(applications)13 b(that)e(con)o(tain)h(in)o(ternal)h(user-lev)o(el)g(serv)o(ers,)f(eac)o (h)f(serv)o(er)h(ma)o(y)f(b)q(e)h(a)f(pro)q(cess)h(group)g(that)75 1362 y(pro)o(vides)17 b(services)f(to)g(one)g(or)g(more)f(clien)o(ts,)i (and)g(eac)o(h)f(clien)o(t)h(ma)o(y)f(b)q(e)g(a)g(pro)q(cess)h(group)e (that)h(uses)75 1419 y(the)h(services)h(of)f(one)g(or)g(more)g(serv)o (ers.)25 b(It)17 b(is)h(again)f(most)f(natural)i(to)e(sp)q(ecify)j(the) e(target)f(pro)q(cess)75 1475 y(b)o(y)d(rank)g(within)i(the)e(target)f (group)i(in)g(these)f(applications.)21 b(This)14 b(t)o(yp)q(e)g(of)e (comm)o(unication)j(is)e(called)75 1532 y(\\in)o(ter-comm)o(unication") 18 b(and)g(the)g(comm)o(unicator)f(used)h(is)g(called)h(an)f(\\in)o (ter-comm)o(unicator,")f(as)75 1588 y(in)o(tro)q(duced)f(earlier.)166 1645 y(An)d(in)o(ter-comm)o(unication)g(is)g(a)f(p)q(oin)o(t-to-p)q (oin)o(t)h(comm)o(unication)g(b)q(et)o(w)o(een)g(pro)q(cesses)f(in)h (di\013er-)75 1701 y(en)o(t)f(groups.)19 b(The)12 b(group)g(con)o (taining)h(a)f(pro)q(cess)h(that)e(initiates)j(an)e(in)o(ter-comm)o (unication)i(op)q(eration)75 1758 y(is)i(called)i(the)d(\\lo)q(cal)i (group,")e(that)g(is,)h(the)g(sender)g(in)h(a)e(send)i(and)f(the)g (receiv)o(er)g(in)h(a)e(receiv)o(e.)23 b(The)75 1814 y(group)16 b(con)o(taining)g(the)g(target)f(pro)q(cess)h(is)g(called)i (the)e(\\remote)f(group,")g(that)g(is,)h(the)g(receiv)o(er)h(in)f(a)75 1871 y(send)f(and)f(the)g(sender)h(in)g(a)f(receiv)o(e.)20 b(As)14 b(in)h(in)o(tra-comm)o(unication,)g(the)f(target)f(pro)q(cess)h (is)h(sp)q(eci\014ed)75 1927 y(using)k(a)g Fw(\(communicato)o(r,)e (rank\))i Fv(pair.)30 b(Unlik)o(e)21 b(in)o(tra-comm)o(unication,)e (the)g(rank)g(is)g(relativ)o(e)g(to)f(a)75 1983 y(second,)d(remote)g (group.)166 2040 y(All)e(in)o(ter-comm)o(unicator)f(constructors)g(are) f(blo)q(c)o(king)j(and)e(require)h(that)e(the)h(lo)q(cal)h(and)g (remote)75 2096 y(groups)i(b)q(e)h(disjoin)o(t)g(in)g(order)f(to)f(a)o (v)o(oid)h(deadlo)q(c)o(k.)166 2153 y(Here)f(is)h(a)f(summary)g(of)g (the)g(prop)q(erties)h(of)f(in)o(ter-comm)o(unication)i(and)e(in)o (ter-comm)o(unicators:)143 2247 y Fx(\017)23 b Fv(The)c(syn)o(tax)g(of) g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o(unication)g(is)g(the)f(same)g (for)g(b)q(oth)h(in)o(ter-)f(and)h(in)o(tra-)189 2303 y(comm)o(unication.)26 b(The)17 b(same)g(comm)o(unicator)g(can)g(b)q(e) g(used)h(b)q(oth)f(for)g(send)g(and)h(for)e(receiv)o(e)189 2360 y(op)q(erations.)143 2453 y Fx(\017)23 b Fv(A)14 b(target)g(pro)q(cess)h(is)g(addressed)h(b)o(y)f(its)g(rank)f(in)i(the) f(remote)f(group,)g(b)q(oth)h(for)f(sends)i(and)f(for)189 2510 y(receiv)o(es.)143 2604 y Fx(\017)23 b Fv(Comm)o(unications)12 b(using)i(an)e(in)o(ter-comm)o(unicator)h(are)f(guaran)o(teed)g(not)g (to)g(con\015ict)h(with)g(an)o(y)189 2660 y(comm)o(unications)i(that)g (use)h(a)e(di\013eren)o(t)i(comm)o(unicator.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 157 165 157 164 bop 75 -100 a Fm(5.6.)34 b(INTER-COMMUNICA)l(TION)1047 b Fv(157)143 45 y Fx(\017)23 b Fv(An)15 b(in)o(ter-comm)o(unicator)g (cannot)g(b)q(e)h(used)g(for)f(collectiv)o(e)i(comm)o(unication.)143 139 y Fx(\017)23 b Fv(A)15 b(comm)o(unicator)g(will)i(pro)o(vide)e (either)h(in)o(tra-)f(or)g(in)o(ter-comm)o(unication,)h(nev)o(er)f(b)q (oth.)75 233 y(The)i(routine)f Fw(MPI)p 412 233 14 2 v 16 w(COMM)p 570 233 V 17 w(TEST)p 701 233 V 16 w(INTER)h Fv(ma)o(y)e(b)q(e)i(used)g(to)f(determine)h(if)g(a)f(comm)o(unicator)g (is)h(an)75 289 y(in)o(ter-)12 b(or)f(in)o(tra-comm)o(unicator.)19 b(In)o(ter-comm)o(unicators)11 b(can)h(b)q(e)h(used)f(as)f(argumen)o (ts)g(to)g(some)h(of)f(the)75 346 y(other)i(comm)o(unicator)h(access)g (routines.)20 b(In)o(ter-comm)o(unicators)13 b(cannot)h(b)q(e)g(used)h (as)e(input)i(to)e(some)75 402 y(of)i(the)g(constructor)f(routines)i (for)f(in)o(tra-comm)o(unicators)f(\(for)h(instance,)g Fw(MPI)p 1476 402 V 16 w(COMM)p 1634 402 V 17 w(CREA)l(TE)p Fv(\).)189 508 y Fl(A)n(dvic)n(e)g(to)i(implementors.)41 b Fv(F)l(or)15 b(the)g(purp)q(ose)h(of)f(p)q(oin)o(t-to-p)q(oin)o(t)h (comm)o(unication,)g(comm)o(u-)189 565 y(nicators)f(can)g(b)q(e)h (represen)o(ted)g(in)g(eac)o(h)f(pro)q(cess)g(b)o(y)g(a)g(tuple)i (consisting)f(of:)189 659 y Fq(group)189 732 y(send)p 295 732 16 2 v 18 w(con)o(text)189 805 y(receiv)o(e)p 350 805 V 18 w(con)o(text)189 878 y(source)189 972 y Fv(F)l(or)9 b(in)o(ter-comm)o(unicators,)i Fq(group)g Fv(describ)q(es)h(the)e(remote)g(group,)h(and)f Fq(source)g Fv(is)h(the)f(rank)g(of)189 1028 y(the)k(pro)q(cess)h(in)g(the)g(lo)q (cal)g(group.)20 b(F)l(or)14 b(in)o(tra-comm)o(unicators,)f Fq(group)i Fv(is)g(the)f(comm)o(unicator)189 1085 y(group)20 b(\(remote=lo)q(cal\),)j Fq(source)d Fv(is)i(the)f(rank)g(of)f(the)i (pro)q(cess)f(in)h(this)f(group,)h(and)f Fq(send)189 1141 y(con)o(text)e Fv(and)g Fq(receiv)o(e)h(con)o(text)f Fv(are)g(iden)o(tical.)32 b(A)19 b(group)g(is)g(represen)o(ted)g(b)o(y) g(a)g(rank-to-)189 1197 y(absolute-address)c(translation)h(table.)189 1273 y(The)11 b(in)o(ter-comm)o(unicator)h(cannot)g(b)q(e)g(discussed)h (sensibly)g(without)f(considering)h(pro)q(cesses)f(in)189 1329 y(b)q(oth)f(the)h(lo)q(cal)g(and)g(remote)e(groups.)19 b(Imagine)12 b(a)f(pro)q(cess)g Fq(P)g Fv(in)i(group)e Fx(P)t Fv(,)g(whic)o(h)h(has)f(an)h(in)o(ter-)189 1385 y(comm)o(unicator)18 b Fq(C)527 1392 y Fh(P)556 1385 y Fv(,)i(and)f(a)f(pro)q(cess)h Fq(Q)g Fv(in)h(group)e Fx(Q)p Fv(,)i(whic)o(h)f(has)g(an)g(in)o(ter-comm)o(unicator)189 1442 y Fq(C)227 1449 y Fh(Q)257 1442 y Fv(.)h(Then)243 1536 y Fx(\017)j Fq(C)327 1543 y Fh(P)356 1536 y Fq(.group)14 b Fv(describ)q(es)j(the)f(group)e Fx(Q)i Fv(and)f Fq(C)1097 1543 y Fh(Q)1128 1536 y Fq(.group)f Fv(describ)q(es)j(the)e(group)g Fx(P)t Fv(.)243 1609 y Fx(\017)23 b Fq(C)327 1616 y Fh(P)356 1609 y Fq(.send)p 477 1609 V 18 w(con)o(text)17 b(=)h Fv(C)770 1616 y Fh(Q)800 1609 y Fq(.receiv)o(e)p 976 1609 V 18 w(con)o(text)d Fv(and)h(the)f(con)o(text)g(is)g(unique)i(in)f Fx(Q)p Fv(;)289 1665 y Fq(C)327 1672 y Fh(P)356 1665 y Fq(.receiv)o(e)p 532 1665 V 18 w(con)o(text)h(=)35 b(C)847 1672 y Fh(Q)877 1665 y Fq(.send)p 998 1665 V 18 w(con)o(text)16 b Fv(and)f(this)h(con)o(text)e(is)i(unique)h(in)f Fx(P)t Fv(.)243 1738 y Fx(\017)23 b Fq(C)327 1745 y Fh(P)356 1738 y Fq(.source)14 b Fv(is)i(rank)f(of)f Fq(P)h Fv(in)h Fx(P)j Fv(and)c Fq(C)1013 1745 y Fh(Q)1044 1738 y Fq(.source)f Fv(is)i(rank)f(of)g Fq(Q)g Fv(in)h Fx(Q)p Fv(.)189 1832 y(Assume)j(that)f Fq(P)g Fv(sends)i(a)e(message)g(to)h Fq(Q)g Fv(using)g(the)g(in)o(ter-comm)o(unicator.)31 b(Then)19 b Fq(P)g Fv(uses)189 1889 y(the)g Fq(group)f Fv(table)i(to)e(\014nd)h(the)g(absolute)g(address)g(of)g Fq(Q)p Fv(;)f Fq(source)h Fv(and)g Fq(send)p 1611 1889 V 18 w(con)o(text)g Fv(are)189 1945 y(app)q(ended)d(to)f(the)g (message.)189 2020 y(Assume)22 b(that)f Fq(Q)h Fv(p)q(osts)f(a)h (receiv)o(e)g(with)h(an)e(explicit)j(source)e(argumen)o(t)f(using)i (the)f(in)o(ter-)189 2077 y(comm)o(unicator.)c(Then)c Fq(Q)f Fv(matc)o(hes)g Fq(receiv)o(e)p 1006 2077 V 18 w(con)o(text)g Fv(to)f(the)i(message)e(con)o(text)h(and)g(source)189 2133 y(argumen)o(t)h(to)h(the)g(message)g(source.)189 2208 y(The)g(same)g(algorithm)g(is)h(appropriate)f(for)g(in)o(tra-comm) o(unicators)f(as)h(w)o(ell.)189 2283 y(In)g(order)g(to)f(supp)q(ort)h (in)o(ter-comm)o(unicator)g(accessors)g(and)g(constructors,)f(it)h(is)g (necessary)h(to)189 2340 y(supplemen)o(t)j(this)f(mo)q(del)h(with)f (additional)i(structures,)e(that)f(store)g(information)h(ab)q(out)g (the)189 2396 y(lo)q(cal)i(comm)o(unication)f(group,)h(and)f (additional)h(safe)f(con)o(texts.)30 b(\()p Fl(End)19 b(of)h(advic)n(e)g(to)g(imple-)189 2453 y(mentors.)p Fv(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 158 166 158 165 bop 75 -100 a Fv(158)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fj(5.6.1)49 b(Inter-communicato)o(r)15 b(Accesso)o(rs)75 225 y Fw(MPI)p 160 225 14 2 v 16 w(COMM)p 318 225 V 16 w(TEST)p 448 225 V 17 w(INTER\(comm,)e(\015ag\))117 303 y Fk(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(\(handle\))117 378 y(OUT)108 b Fw(\015ag)518 b Fk(\(logical\))75 502 y Ft(int)23 b(MPI)p 245 502 15 2 v 17 w(Comm)p 358 502 V 17 w(test)p 471 502 V 16 w(inter\(MPI)p 703 502 V 16 w(Comm)h(comm,)f(int)g(*flag\))75 589 y(MPI)p 150 589 V 17 w(COMM)p 263 589 V 16 w(TEST)p 375 589 V 17 w(INTER\(COMM,)f (FLAG,)i(IERROR\))170 645 y(INTEGER)f(COMM,)h(IERROR)170 702 y(LOGICAL)f(FLAG)75 788 y Fv(This)d(lo)q(cal)g(routine)g(allo)o(ws) f(the)g(calling)i(pro)q(cess)f(to)e(determine)i(if)g(a)f(comm)o (unicator)g(is)g(an)h(in)o(ter-)75 844 y(comm)o(unicator)g(or)h(an)f (in)o(tra-comm)o(unicator.)36 b(It)21 b(returns)g Fi(true)g Fv(if)g(it)g(is)h(an)e(in)o(ter-comm)o(unicator,)75 901 y(otherwise)15 b Fi(false)p Fv(.)166 957 y(When)j(an)f(in)o(ter-comm)o (unicator)h(is)g(used)g(as)f(an)g(input)i(argumen)o(t)d(to)h(the)h (comm)o(unicator)f(ac-)75 1014 y(cessors)d(describ)q(ed)i(ab)q(o)o(v)o (e)e(under)h(in)o(tra-comm)o(unication,)f(the)h(follo)o(wing)f(table)h (describ)q(es)h(b)q(eha)o(vior.)p 270 1108 1410 2 v 269 1165 2 57 v 652 1148 a Fw(MPI)p 737 1148 14 2 v 16 w(COMM)p 895 1148 V 17 w(*)e Fv(F)l(unction)j(Beha)o(vior)p 1679 1165 2 57 v 269 1221 V 655 1204 a(\(in)f(In)o(ter-Comm)o(unication)f (Mo)q(de\))p 1679 1221 V 270 1223 1410 2 v 270 1233 V 269 1289 2 57 v 295 1272 a Fw(MPI)p 380 1272 14 2 v 16 w(COMM)p 538 1272 V 16 w(SIZE)p 729 1289 2 57 v 111 w Fv(returns)g(the)g(size)h(of)f(the)g(lo)q(cal)i(group.)p 1679 1289 V 269 1346 V 295 1329 a Fw(MPI)p 380 1329 14 2 v 16 w(COMM)p 538 1329 V 16 w(GROUP)p 729 1346 2 57 v 52 w Fv(returns)e(the)g(lo)q(cal)h(group.)p 1679 1346 V 269 1402 V 295 1385 a Fw(MPI)p 380 1385 14 2 v 16 w(COMM)p 538 1385 V 16 w(RANK)p 729 1402 2 57 v 81 w Fv(returns)f(the)g(rank)g (in)h(the)f(lo)q(cal)i(group)p 1679 1402 V 270 1404 1410 2 v 75 1489 a(F)l(urthermore,)d(the)g(op)q(eration)g Fw(MPI)p 715 1489 14 2 v 16 w(COMM)p 873 1489 V 17 w(COMP)l(ARE)h Fv(is)g(v)m(alid)h(for)e(in)o(ter-comm)o(unicators.)19 b(Both)75 1546 y(comm)o(unicators)13 b(m)o(ust)h(b)q(e)g(either)g(in)o (tra-)g(or)f(in)o(ter-comm)o(unicators,)h(or)f(else)i Fi(MPI)p 1515 1546 13 2 v 14 w(UNEQUAL)e Fv(results.)75 1602 y(Both)22 b(corresp)q(onding)g(lo)q(cal)h(and)f(remote)f(groups)h (m)o(ust)f(compare)g(correctly)h(to)f(get)h(the)g(results)75 1659 y Fi(MPI)p 152 1659 V 14 w(CONGRUENT)c Fv(and)g Fi(MPI)p 603 1659 V 15 w(SIMILAR)p Fv(.)f(In)i(particular,)h(it)e(is)h (p)q(ossible)i(for)d Fi(MPI)p 1515 1659 V 14 w(SIMILAR)g Fv(to)g(result)75 1715 y(b)q(ecause)e(either)g(the)f(lo)q(cal)i(or)d (remote)h(groups)g(w)o(ere)g(similar)h(but)f(not)g(iden)o(tical.)166 1772 y(The)23 b(follo)o(wing)g(accessors)f(pro)o(vide)h(consisten)o(t)f (access)h(to)f(the)g(remote)g(group)g(of)g(an)h(in)o(ter-)75 1828 y(comm)o(unicator:)166 1885 y(The)15 b(follo)o(wing)h(are)f(all)h (lo)q(cal)h(op)q(erations.)75 2036 y Fw(MPI)p 160 2036 14 2 v 16 w(COMM)p 318 2036 V 16 w(REMOTE)p 521 2036 V 17 w(SIZE\(comm,)c(size\))117 2113 y Fk(IN)155 b Fw(comm)470 b Fk(in)o(ter-comm)o(unicator)11 b(\(handle\))117 2188 y(OUT)108 b Fw(size)519 b Fk(n)o(um)o(b)q(er)19 b(of)g(pro)q(cesses)j (in)d(the)h(remote)f(group)g(of)38 b Fi(comm)905 2244 y Fk(\(in)o(teger\))75 2369 y Ft(int)23 b(MPI)p 245 2369 15 2 v 17 w(Comm)p 358 2369 V 17 w(remote)p 519 2369 V 16 w(size\(MPI)p 727 2369 V 16 w(Comm)h(comm,)f(int)g(*size\))75 2455 y(MPI)p 150 2455 V 17 w(COMM)p 263 2455 V 16 w(REMOTE)p 423 2455 V 17 w(SIZE\(COMM,)f(SIZE,)h(IERROR\))170 2512 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 159 167 159 166 bop 75 -100 a Fm(5.6.)34 b(INTER-COMMUNICA)l(TION)1047 b Fv(159)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(COMM)p 318 45 V 16 w(REMOTE)p 521 45 V 17 w(GROUP\(comm,)14 b(group\))117 122 y Fk(IN)155 b Fw(comm)470 b Fk(in)o(ter-comm)o(unicator)11 b(\(handle\))117 197 y(OUT)108 b Fw(group)479 b Fk(remote)14 b(group)f(corresp)q(onding)i(to)f Fi(comm)e Fk(\(handle\))75 322 y Ft(int)23 b(MPI)p 245 322 15 2 v 17 w(Comm)p 358 322 V 17 w(remote)p 519 322 V 16 w(group\(MPI)p 751 322 V 16 w(Comm)g(comm,)h(MPI)p 1102 322 V 16 w(Group)g(*group\))75 408 y(MPI)p 150 408 V 17 w(COMM)p 263 408 V 16 w(REMOTE)p 423 408 V 17 w(GROUP\(COMM,)e(GROUP,)h(IERROR\))170 465 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)189 601 y Fl(R)n(ationale.)83 b Fv(Symmetric)23 b(access)g(to)f(b)q(oth)g(the)h(lo)q(cal)g(and)g (remote)f(groups)g(of)g(an)h(in)o(ter-)189 658 y(comm)o(unicator)17 b(is)h(imp)q(ortan)o(t,)g(so)f(this)h(function,)h(as)f(w)o(ell)g(as)g Fw(MPI)p 1407 658 14 2 v 16 w(COMM)p 1565 658 V 16 w(REMOTE)p 1768 658 V 17 w(SIZE)189 714 y Fv(ha)o(v)o(e)c(b)q(een)j(pro)o(vided.)j (\()p Fl(End)c(of)g(r)n(ationale.)p Fv(\))75 836 y Fj(5.6.2)49 b(Inter-communicato)o(r)15 b(Op)q(erations)75 922 y Fv(This)j(section)g (in)o(tro)q(duces)g(four)f(blo)q(c)o(king)h(in)o(ter-comm)o(unicator)g (op)q(erations.)26 b Fw(MPI)p 1573 922 V 16 w(INTERCOMM-)p 75 978 V 89 978 a(CREA)l(TE)13 b Fv(is)f(used)h(to)f(bind)h(t)o(w)o(o)e (in)o(tra-comm)o(unicators)h(in)o(to)g(an)g(inter-comm)o(unicator;)h (the)f(function)75 1034 y Fw(MPI)p 160 1034 V 16 w(INTERCOMM)p 450 1034 V 16 w(MERGE)h Fv(creates)f(an)g(in)o(tra-comm)o(unicator)g(b) o(y)g(merging)h(the)f(lo)q(cal)h(and)g(remote)75 1091 y(groups)18 b(of)g(an)g(in)o(ter-comm)o(unicator.)30 b(The)19 b(functions)p Fw(MPI)p 1140 1091 V 16 w(COMM)p 1298 1091 V 17 w(DUP)f Fv(and)p Fw(MPI)p 1581 1091 V 16 w(COMM)p 1739 1091 V 17 w(FREE)p Fv(,)75 1147 y(in)o(tro)q(duced)e (previously)l(,)h(duplicate)g(and)e(free)g(an)h(in)o(ter-comm)o (unicator,)e(resp)q(ectiv)o(ely)l(.)166 1204 y(Ov)o(erlap)23 b(of)f(lo)q(cal)h(and)g(remote)f(groups)g(that)g(are)g(b)q(ound)h(in)o (to)g(an)f(in)o(ter-comm)o(unicator)g(is)75 1260 y(prohibited.)h(If)15 b(there)h(is)g(o)o(v)o(erlap,)f(then)h(the)f(program)g(is)h(erroneous)f (and)h(is)g(lik)o(ely)h(to)e(deadlo)q(c)o(k.)22 b(\(If)75 1317 y(a)16 b(pro)q(cess)g(is)g(m)o(ultithreaded,)h(and)f Fw(MPI)f Fv(calls)i(blo)q(c)o(k)g(only)f(a)g(thread,)f(rather)g(than)h (a)g(pro)q(cess,)g(then)75 1373 y(\\dual)f(mem)o(b)q(ership")g(can)f(b) q(e)g(supp)q(orted.)20 b(It)14 b(is)h(then)f(the)g(user's)g(resp)q (onsibilit)o(y)j(to)c(mak)o(e)h(sure)g(that)75 1430 y(calls)i(on)f(b)q (ehalf)i(of)e(the)g(t)o(w)o(o)f(\\roles")h(of)f(a)h(pro)q(cess)h(are)f (executed)h(b)o(y)f(t)o(w)o(o)f(indep)q(enden)o(t)j(threads.\))166 1486 y(The)12 b(function)h Fw(MPI)p 516 1486 V 16 w(INTERCOMM)p 806 1486 V 17 w(CREA)l(TE)g Fv(can)g(b)q(e)f(used)h(to)f(create)g(an)g (in)o(ter-comm)o(unicator)75 1543 y(from)19 b(t)o(w)o(o)g(existing)i (in)o(tra-comm)o(unicators,)g(in)g(the)f(follo)o(wing)h(situation:)30 b(A)o(t)20 b(least)g(one)h(selected)75 1599 y(mem)o(b)q(er)f(from)g (eac)o(h)g(group)g(\(the)g(\\group)g(leader"\))g(has)g(the)h(abilit)o (y)g(to)f(comm)o(unicate)g(with)h(the)75 1655 y(selected)14 b(mem)o(b)q(er)f(from)g(the)g(other)f(group;)h(that)g(is,)g(a)g(\\p)q (eer")g(comm)o(unicator)f(exists)i(to)e(whic)o(h)i(b)q(oth)75 1712 y(leaders)e(b)q(elong,)g(and)g(eac)o(h)f(leader)h(kno)o(ws)e(the)i (rank)e(of)h(the)g(other)g(leader)h(in)g(this)g(p)q(eer)f(comm)o (unicator)75 1768 y(\(the)18 b(t)o(w)o(o)e(leaders)j(could)g(b)q(e)g (the)f(same)f(pro)q(cess\).)28 b(F)l(urthermore,)18 b(mem)o(b)q(ers)g (of)g(eac)o(h)g(group)f(kno)o(w)75 1825 y(the)e(rank)g(of)g(their)h (leader.)166 1881 y(Construction)10 b(of)h(an)f(in)o(ter-comm)o (unicator)h(from)f(t)o(w)o(o)f(in)o(tra-comm)o(unicators)h(requires)h (separate)75 1938 y(collectiv)o(e)16 b(op)q(erations)e(in)h(the)f(lo)q (cal)h(group)e(and)i(in)f(the)g(remote)g(group,)f(as)h(w)o(ell)h(as)f (a)f(p)q(oin)o(t-to-p)q(oin)o(t)75 1994 y(comm)o(unication)j(b)q(et)o (w)o(een)f(a)g(pro)q(cess)h(in)g(the)f(lo)q(cal)h(group)f(and)h(a)f (pro)q(cess)g(in)h(the)f(remote)g(group.)166 2051 y(In)g(standard)g Fw(MPI)f Fv(implemen)o(tations)i(\(with)f(static)g(pro)q(cess)g(allo)q (cation)h(at)f(initialization\),)i(the)75 2107 y Fw(MPI)p 160 2107 V 16 w(COMM)p 318 2107 V 16 w(W)o(ORLD)j Fv(comm)o(unicator)f (\(or)f(preferably)i(a)f(dedicated)i(duplicate)g(thereof)t(\))e(can)g (b)q(e)75 2164 y(this)e(p)q(eer)g(comm)o(unicator.)24 b(In)18 b(dynamic)f Fw(MPI)f Fv(implemen)o(tations,)i(where,)f(for)f (example,)h(a)g(pro)q(cess)75 2220 y(ma)o(y)g(spa)o(wn)h(new)g(c)o (hild)i(pro)q(cesses)f(during)g(an)f Fw(MPI)f Fv(execution,)j(the)e (paren)o(t)f(pro)q(cess)i(ma)o(y)e(b)q(e)i(the)75 2276 y(\\bridge")d(b)q(et)o(w)o(een)g(the)g(old)g(comm)o(unication)h(univ)o (erse)g(and)f(the)f(new)h(comm)o(unication)h(w)o(orld)f(that)75 2333 y(includes)h(the)f(paren)o(t)f(and)g(its)g(c)o(hildren.)166 2389 y(The)25 b(application)h(top)q(ology)e(functions)h(describ)q(ed)h (in)g(c)o(hapter)e(6)g(do)g(not)g(apply)i(to)e(in)o(ter-)75 2446 y(comm)o(unicators.)18 b(Users)11 b(that)g(require)h(this)g (capabilit)o(y)h(should)f(utilize)i Fw(MPI)p 1418 2446 V 16 w(INTERCOMM)p 1708 2446 V 16 w(MERGE)75 2502 y Fv(to)g(build)j(an) e(in)o(tra-comm)o(unicator,)f(then)i(apply)f(the)g(graph)g(or)g (cartesian)g(top)q(ology)f(capabilities)k(to)75 2559 y(that)i(in)o(tra-comm)o(unicator,)h(creating)f(an)h(appropriate)f(top) q(ology-orien)o(ted)h(in)o(tra-comm)o(unicator.)75 2615 y(Alternativ)o(ely)l(,)f(it)e(ma)o(y)f(b)q(e)h(reasonable)g(to)g (devise)h(one's)e(o)o(wn)g(application)j(top)q(ology)d(mec)o(hanisms)75 2672 y(for)e(this)g(case,)g(without)g(loss)h(of)e(generalit)o(y)l(.) 1085 b Fx(>)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 160 168 160 167 bop 75 -100 a Fv(160)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(INTERCOMM)p 450 45 V 16 w(CREA)l(TE\(lo)q(cal)p 741 45 V 17 w(comm,)46 b(lo)q(cal)p 1022 45 V 16 w(leader,)i(p)q(eer)p 1291 45 V 17 w(comm,)e(remote)p 1616 45 V 16 w(leader,)h(tag,)75 102 y(newintercomm\))117 179 y Fk(IN)155 b Fw(lo)q(cal)p 409 179 V 17 w(comm)365 b Fk(lo)q(cal)13 b(in)o(tra-comm)o(unicator)e(\(handle\))117 254 y(IN)155 b Fw(lo)q(cal)p 409 254 V 17 w(leader)370 b Fk(rank)14 b(of)f(lo)q(cal)g(group)h(leader)g(in)g Fi(lo)q(cal)p 1517 254 13 2 v 15 w(comm)e Fk(\(in)o(teger\))117 329 y(IN)155 b Fw(p)q(eer)p 401 329 14 2 v 18 w(comm)372 b Fk(\\p)q(eer")10 b(comm)o(unicator;)e(signi\014can)o(t)h(only)f(at)h (the)h Fi(lo)q(cal)p 1787 329 13 2 v 15 w(leader)905 385 y Fk(\(handle\))117 461 y(IN)155 b Fw(remote)p 453 461 14 2 v 16 w(leader)327 b Fk(rank)12 b(of)f(remote)g(group)h(leader) g(in)23 b Fi(p)q(eer)p 1548 461 13 2 v 16 w(comm)p Fk(;)10 b(signi\014can)o(t)905 517 y(only)j(at)h(the)h Fi(lo)q(cal)p 1201 517 V 15 w(leader)g Fk(\(in)o(teger\))117 592 y(IN)155 b Fw(tag)526 b Fk(\\safe")14 b(tag)f(\(in)o(teger\))117 667 y(OUT)108 b Fw(newintercomm)310 b Fk(new)15 b(in)o(ter-comm)o (unicator)c(\(handle\))-160 746 y Fx(?)75 792 y Ft(int)23 b(MPI)p 245 792 15 2 v 17 w(Intercomm)p 478 792 V 16 w(create\(MPI)p 734 792 V 16 w(Comm)g(local)p 989 792 V 17 w(comm,)g(int)g(local)p 1364 792 V 17 w(leader,)393 848 y(MPI)p 468 848 V 17 w(Comm)g(peer)p 700 848 V 17 w(comm,)g(int)h(remote)p 1100 848 V 16 w(leader,)f(int)g(tag,)393 905 y(MPI)p 468 905 V 17 w(Comm)g(*newintercomm\))75 991 y(MPI)p 150 991 V 17 w(INTERCOMM)p 383 991 V 16 w(CREATE\(LOCAL)p 687 991 V 15 w(COMM,)g(LOCAL)p 965 991 V 17 w(LEADER,)g(PEER)p 1269 991 V 16 w(COMM,)h(REMOTE)p 1573 991 V 16 w(LEADER,)f(TAG,)393 1048 y(NEWINTERCOMM,)f(IERROR\))170 1104 y(INTEGER)h(LOCAL)p 484 1104 V 17 w(COMM,)g(LOCAL)p 764 1104 V 17 w(LEADER,)f(PEER)p 1067 1104 V 17 w(COMM,)h(REMOTE)p 1371 1104 V 17 w(LEADER,)g(TAG,)170 1161 y(NEWINTERCOMM,)g(IERROR)75 1247 y Fv(This)d(call)h(creates)e(an)h (in)o(ter-comm)o(unicator.)33 b(It)20 b(is)g(collectiv)o(e)i(o)o(v)o (er)d(the)g(union)i(of)e(the)h(lo)q(cal)h(and)75 1303 y(remote)d(groups.)28 b(Pro)q(cesses)18 b(should)h(pro)o(vide)g(iden)o (tical)h Fw(lo)q(cal)p 1197 1303 14 2 v 16 w(comm)c Fv(and)i Fw(lo)q(cal)p 1525 1303 V 17 w(leader)g Fv(argumen)o(ts)75 1360 y(within)e(eac)o(h)g(group.)j(Wildcards)e(are)d(not)h(p)q (ermitted)h(for)f Fw(remote)p 1254 1360 V 15 w(leader,)g(lo)q(cal)p 1497 1360 V 16 w(leader)p Fv(,)g(and)h Fw(tag)p Fv(.)166 1416 y(This)g(call)g(uses)f(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication)g(with)f(comm)o(unicator)g Fw(p)q(eer)p 1542 1416 V 17 w(comm)p Fv(,)d(and)k(with)75 1473 y(tag)e Fw(tag)g Fv(b)q(et)o(w)o(een)h(the)g(leaders.)20 b(Th)o(us,)15 b(care)f(m)o(ust)g(b)q(e)h(tak)o(en)f(that)g(there)h(b)q(e)g(no)g(p)q (ending)h(comm)o(uni-)75 1529 y(cation)f(on)g Fw(p)q(eer)p 357 1529 V 18 w(comm)e Fv(that)h(could)j(in)o(terfere)e(with)h(this)f (comm)o(unication.)189 1636 y Fl(A)n(dvic)n(e)j(to)i(users.)60 b Fv(W)l(e)19 b(recommend)g(using)g(a)f(dedicated)i(p)q(eer)f(comm)o (unicator,)g(suc)o(h)g(as)f(a)189 1692 y(duplicate)e(of)e(MPI)p 528 1692 V 16 w(COMM)p 696 1692 V 16 w(W)o(ORLD,)h(to)e(a)o(v)o(oid)i (trouble)g(with)f(p)q(eer)h(comm)o(unicators.)20 b(\()p Fl(End)189 1748 y(of)c(advic)n(e)g(to)h(users.)p Fv(\))75 1949 y Fw(MPI)p 160 1949 V 16 w(INTERCOMM)p 450 1949 V 16 w(MERGE\(intercomm,)d(high,)i(newintracomm\))117 2026 y Fk(IN)155 b Fw(intercomm)384 b Fk(In)o(ter-Comm)o(unicator)11 b(\(handle\))117 2102 y(IN)155 b Fw(high)507 b Fk(\(logical\))117 2177 y(OUT)108 b Fw(newintracomm)308 b Fk(new)15 b(in)o(tra-comm)n (unicator)c(\(handle\))75 2301 y Ft(int)23 b(MPI)p 245 2301 15 2 v 17 w(Intercomm)p 478 2301 V 16 w(merge\(MPI)p 710 2301 V 16 w(Comm)g(intercomm,)g(int)g(high,)393 2358 y(MPI)p 468 2358 V 17 w(Comm)g(*newintracomm\))75 2444 y(MPI)p 150 2444 V 17 w(INTERCOMM)p 383 2444 V 16 w(MERGE\(INTERCOMM,)e (HIGH,)j(INTRACOMM,)e(IERROR\))170 2501 y(INTEGER)h(INTERCOMM,)g (INTRACOMM,)f(IERROR)170 2557 y(LOGICAL)h(HIGH)75 2643 y Fv(This)e(function)f(creates)g(an)g(in)o(tra-comm)o(unicator)g(from)f (the)h(union)h(of)f(the)g(t)o(w)o(o)e(groups)i(that)f(are)75 2700 y(asso)q(ciated)f(with)g Fw(intercomm)p Fv(.)26 b(All)19 b(pro)q(cesses)f(should)h(pro)o(vide)f(the)g(same)f Fw(high)h Fv(v)m(alue)h(within)g(eac)o(h)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 161 169 161 168 bop 75 -100 a Fm(5.6.)29 b(INTER-COMMUNICA)l(TION)1052 b Fv(161)383 4 y 18945146 4357381 0 0 24865505 5920358 startTexFig 383 4 a %%BeginDocument: context-fig-1.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def /col8 {.68 .85 .9 setrgbcolor} bind def /col9 {0 .39 0 setrgbcolor} bind def /col10 {.65 .17 .17 setrgbcolor} bind def /col11 {1 .51 0 setrgbcolor} bind def /col12 {.63 .13 .94 setrgbcolor} bind def /col13 {1 .75 .8 setrgbcolor} bind def /col14 {.7 .13 .13 setrgbcolor} bind def /col15 {1 .84 0 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -90.0 198.0 translate 0.900 -0.900 scale 0.500 setlinewidth % Polyline n 199 219 m 199 119 l 99 119 l 99 219 l clp gs col-1 s gr % Polyline n 359 219 m 359 119 l 259 119 l 259 219 l clp gs col-1 s gr % Polyline n 519 219 m 519 119 l 419 119 l 419 219 l clp gs col-1 s gr 1.000 setlinewidth n 220.000 173.000 m 204.000 169.000 l 220.000 165.000 l gs 2 setlinejoin col-1 s gr % Polyline n 204 169 m 254 169 l gs col-1 s gr n 238.000 165.000 m 254.000 169.000 l 238.000 173.000 l gs 2 setlinejoin col-1 s gr n 380.000 173.000 m 364.000 169.000 l 380.000 165.000 l gs 2 setlinejoin col-1 s gr % Polyline n 364 169 m 414 169 l gs col-1 s gr n 398.000 165.000 m 414.000 169.000 l 398.000 173.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 16.00 scalefont setfont 279 174 m gs 1 -1 scale (Group 1) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 439 174 m gs 1 -1 scale (Group 2) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 174 m gs 1 -1 scale (Group 0) col-1 show gr $F2psEnd %%EndDocument endTexFig 644 378 a Fv(Figure)15 b(5.1:)k(Three-group)c(pip)q(eline)q(.)75 509 y(of)h(the)g(t)o(w)o(o)e(groups.)22 b(If)16 b(pro)q(cesses)h(in)g (one)f(group)f(pro)o(vided)i(the)f(v)m(alue)h Fw(high)g(=)f(false)g Fv(and)h(pro)q(cesses)75 566 y(in)h(the)g(other)f(group)g(pro)o(vided)i (the)e(v)m(alue)i Fw(high)f(=)g(true)g Fv(then)g(the)f(union)i(orders)e (the)g(\\lo)o(w")g(group)75 622 y(b)q(efore)g(the)g(\\high")g(group.)23 b(If)17 b(all)h(pro)q(cesses)f(pro)o(vided)g(the)g(same)g Fw(high)g Fv(argumen)o(t)f(then)g(the)h(order)75 679 y(of)g(the)h(union)h(is)f(arbitrary)l(.)27 b(This)18 b(call)h(is)f(blo)q(c)o(king)h(and)f(collectiv)o(e)i(within)f(the)e (union)i(of)f(the)f(t)o(w)o(o)75 735 y(groups.)189 839 y Fl(A)n(dvic)n(e)27 b(to)h(implementors.)118 b Fv(The)29 b(implemen)o(tation)g(of)f Fw(MPI)p 1405 839 14 2 v 16 w(INTERCOMM)p 1695 839 V 17 w(MERGE)p Fv(,)189 895 y Fw(MPI)p 274 895 V 15 w(COMM)p 431 895 V 17 w(FREE)k Fv(and)g Fw(MPI)p 776 895 V 16 w(COMM)p 934 895 V 16 w(DUP)g Fv(are)f(similar)i(to)e(the)h(implemen)o(tation)h(of)189 952 y Fw(MPI)p 274 952 V 15 w(INTERCOMM)p 563 952 V 17 w(CREA)l(TE)p Fv(,)13 b(except)h(that)e(con)o(texts)f(priv)m(ate)j(to)e (the)h(input)g(inter-comm)o(un-)189 1008 y(icator)h(are)h(used)h(for)e (comm)o(unication)i(b)q(et)o(w)o(een)g(group)e(leaders)i(rather)f(than) g(con)o(texts)f(inside)189 1065 y(a)h(bridge)h(comm)o(unicator.)j(\()p Fl(End)d(of)g(advic)n(e)g(to)h(implementors.)p Fv(\))75 1186 y Fj(5.6.3)49 b(Inter-Communication)16 b(Examples)75 1272 y Fw(Example)f(1:)k(Three-Group)e(\\Pip)q(eline")75 1358 y Fv(Groups)f(0)g(and)h(1)f(comm)o(unicate.)24 b(Groups)16 b(1)h(and)f(2)g(comm)o(unicate.)24 b(Therefore,)17 b(group)f(0)g (requires)75 1414 y(one)f(in)o(ter-comm)o(unicator,)f(group)g(1)g (requires)i(t)o(w)o(o)d(in)o(ter-comm)o(unicators,)h(and)h(group)f(2)g (requires)h(1)75 1471 y(in)o(ter-comm)o(unicator.)147 1574 y Ft(main\(int)22 b(argc,)i(char)f(**argv\))147 1631 y({)194 1687 y(MPI_Comm)71 b(myComm;)166 b(/*)23 b(intra-communicator)f(of)h(local)h(sub-group)e(*/)194 1744 y(MPI_Comm)71 b(myFirstComm;)46 b(/*)23 b(inter-communicator)f(*/) 194 1800 y(MPI_Comm)71 b(mySecondComm;)22 b(/*)h(second)h (inter-communicator)d(\(group)i(1)h(only\))f(*/)194 1857 y(int)h(membershipKey;)194 1913 y(int)g(rank;)194 2026 y(MPI_Init\(&argc,)e(&argv\);)194 2083 y(MPI_Comm_rank\(MPI_COMM_WORLD) o(,)f(&rank\);)194 2195 y(/*)j(User)f(code)h(must)f(generate)g (membershipKey)f(in)h(the)h(range)f([0,)g(1,)h(2])g(*/)194 2252 y(membershipKey)e(=)i(rank)f(\045)h(3;)194 2365 y(/*)g(Build)f(intra-communicator)e(for)j(local)f(sub-group)g(*/)194 2421 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e(membershipKey,)h(rank,)h (&myComm\);)194 2534 y(/*)h(Build)f(inter-communicators.)45 b(Tags)23 b(are)h(hard-coded.)e(*/)194 2591 y(if)i(\(membershipKey)e (==)h(0\))194 2647 y({)501 b(/*)24 b(Group)f(0)h(communicates)e(with)h (group)h(1.)f(*/)242 2704 y(MPI_Intercomm_create\()e(myComm,)i(0,)h (MPI_COMM_WORLD,)d(1,)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 162 170 162 169 bop 75 -100 a Fv(162)370 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)383 4 y 18945146 5115185 0 0 28417720 7696465 startTexFig 383 4 a %%BeginDocument: context-fig-2.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def /col8 {.68 .85 .9 setrgbcolor} bind def /col9 {0 .39 0 setrgbcolor} bind def /col10 {.65 .17 .17 setrgbcolor} bind def /col11 {1 .51 0 setrgbcolor} bind def /col12 {.63 .13 .94 setrgbcolor} bind def /col13 {1 .75 .8 setrgbcolor} bind def /col14 {.7 .13 .13 setrgbcolor} bind def /col15 {1 .84 0 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -63.0 198.0 translate 0.900 -0.900 scale 0.500 setlinewidth % Polyline n 199 219 m 199 119 l 99 119 l 99 219 l clp gs col-1 s gr % Polyline n 359 219 m 359 119 l 259 119 l 259 219 l clp gs col-1 s gr % Polyline n 519 219 m 519 119 l 419 119 l 419 219 l clp gs col-1 s gr 1.000 setlinewidth n 220.000 173.000 m 204.000 169.000 l 220.000 165.000 l gs 2 setlinejoin col-1 s gr % Polyline n 204 169 m 254 169 l gs col-1 s gr n 238.000 165.000 m 254.000 169.000 l 238.000 173.000 l gs 2 setlinejoin col-1 s gr n 380.000 173.000 m 364.000 169.000 l 380.000 165.000 l gs 2 setlinejoin col-1 s gr % Polyline n 364 169 m 414 169 l gs col-1 s gr n 398.000 165.000 m 414.000 169.000 l 398.000 173.000 l gs 2 setlinejoin col-1 s gr n 78.000 165.000 m 94.000 169.000 l 78.000 173.000 l gs 2 setlinejoin col-1 s gr % Polyline n 94 169 m 69 169 l 69 89 l 549 89 l 549 169 l 524 169 l gs col-1 s gr n 540.000 173.000 m 524.000 169.000 l 540.000 165.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 16.00 scalefont setfont 279 174 m gs 1 -1 scale (Group 1) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 439 174 m gs 1 -1 scale (Group 2) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 174 m gs 1 -1 scale (Group 0) col-1 show gr $F2psEnd %%EndDocument endTexFig 682 426 a Fv(Figure)h(5.2:)i(Three-group)e(ring.)743 559 y Ft(1,)24 b(&myFirstComm\);)194 615 y(})194 672 y(else)g(if)f(\(membershipKey)f(==)i(1\))194 728 y({)334 b(/*)24 b(Group)f(1)h(communicates)e(with)h(groups)g(0)h(and)g(2.)f(*/) 242 785 y(MPI_Intercomm_create\()e(myComm,)i(0,)h(MPI_COMM_WORLD,)d(0,) 743 841 y(1,)j(&myFirstComm\);)242 898 y(MPI_Intercomm_create\()d (myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 954 y(12,)j(&mySecondComm\);) 194 1011 y(})194 1067 y(else)g(if)f(\(membershipKey)f(==)i(2\))194 1123 y({)501 b(/*)24 b(Group)f(2)h(communicates)e(with)h(group)h(1.)f (*/)242 1180 y(MPI_Intercomm_create\()e(myComm,)i(0,)h(MPI_COMM_WORLD,) d(1,)743 1236 y(12,)j(&myFirstComm\);)194 1293 y(})194 1406 y(/*)g(Do)f(work)h(...)f(*/)194 1519 y(switch\(membershipKey\))45 b(/*)24 b(free)f(communicators)f(appropriately)g(*/)194 1575 y({)194 1632 y(case)i(1:)266 1688 y(MPI_Comm_free\(&mySecondCo)o (mm\);)194 1744 y(case)g(0:)194 1801 y(case)g(2:)266 1857 y(MPI_Comm_free\(&myFirstCom)o(m\);)266 1914 y(break;)194 1970 y(})194 2083 y(MPI_Finalize\(\);)147 2140 y(})75 2260 y Fw(Example)15 b(2:)k(Three-Group)e(\\Ring")75 2346 y Fv(Groups)12 b(0)g(and)h(1)f(comm)o(unicate.)19 b(Groups)13 b(1)f(and)g(2)h(comm)o(unicate.)19 b(Groups)12 b(0)g(and)h(2)f(comm)o(unicate.)75 2402 y(Therefore,)j(eac)o(h)g (requires)h(t)o(w)o(o)e(in)o(ter-comm)o(unicators.)147 2508 y Ft(main\(int)22 b(argc,)i(char)f(**argv\))147 2565 y({)194 2621 y(MPI_Comm)71 b(myComm;)142 b(/*)24 b(intra-communicator)d(of)j(local)f(sub-group)f(*/)194 2678 y(MPI_Comm)71 b(myFirstComm;)22 b(/*)i(inter-communicators)d(*/) 1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 163 171 163 170 bop 75 -100 a Fm(5.6.)34 b(INTER-COMMUNICA)l(TION)1047 b Fv(163)194 45 y Ft(MPI_Comm)71 b(mySecondComm;)194 102 y(MPI_Status)23 b(status;)194 158 y(int)h(membershipKey;)194 214 y(int)g(rank;)194 327 y(MPI_Init\(&argc,)e(&argv\);)194 384 y(MPI_Comm_rank\(MPI_COMM_WORLD)o(,)f(&rank\);)194 440 y(...)194 553 y(/*)j(User)f(code)h(must)f(generate)g(membershipKey) f(in)h(the)h(range)f([0,)g(1,)h(2])g(*/)194 610 y(membershipKey)e(=)i (rank)f(\045)h(3;)194 723 y(/*)g(Build)f(intra-communicator)e(for)j (local)f(sub-group)g(*/)194 779 y(MPI_Comm_split\(MPI_COMM_WORL)o(D,)e (membershipKey,)h(rank,)h(&myComm\);)194 892 y(/*)h(Build)f (inter-communicators.)45 b(Tags)23 b(are)h(hard-coded.)e(*/)194 948 y(if)i(\(membershipKey)e(==)h(0\))194 1005 y({)310 b(/*)24 b(Group)f(0)h(communicates)e(with)h(groups)g(1)h(and)g(2.)f(*/) 242 1061 y(MPI_Intercomm_create\()e(myComm,)i(0,)h(MPI_COMM_WORLD,)d (1,)743 1118 y(1,)j(&myFirstComm\);)242 1174 y(MPI_Intercomm_create\()d (myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 1231 y(2,)j(&mySecondComm\);) 194 1287 y(})194 1344 y(else)g(if)f(\(membershipKey)f(==)i(1\))194 1400 y({)215 b(/*)24 b(Group)f(1)g(communicates)g(with)g(groups)g(0)h (and)f(2.)h(*/)242 1456 y(MPI_Intercomm_create\()d(myComm,)i(0,)h (MPI_COMM_WORLD,)d(0,)743 1513 y(1,)j(&myFirstComm\);)242 1569 y(MPI_Intercomm_create\()d(myComm,)i(0,)h(MPI_COMM_WORLD,)d(2,)743 1626 y(12,)j(&mySecondComm\);)194 1682 y(})194 1739 y(else)g(if)f (\(membershipKey)f(==)i(2\))194 1795 y({)191 b(/*)24 b(Group)f(2)h(communicates)e(with)h(groups)g(0)h(and)f(1.)h(*/)242 1852 y(MPI_Intercomm_create\()d(myComm,)i(0,)h(MPI_COMM_WORLD,)d(0,)743 1908 y(2,)j(&myFirstComm\);)242 1965 y(MPI_Intercomm_create\()d (myComm,)i(0,)h(MPI_COMM_WORLD,)d(1,)743 2021 y(12,)j(&mySecondComm\);) 194 2077 y(})194 2190 y(/*)g(Do)f(some)h(work)f(...)h(*/)194 2303 y(/*)g(Then)f(free)h(communicators)e(before)h(terminating...)f(*/) 194 2360 y(MPI_Comm_free\(&myFirstComm\);)194 2416 y (MPI_Comm_free\(&mySecondComm\))o(;)194 2473 y (MPI_Comm_free\(&myComm\);)194 2529 y(MPI_Finalize\(\);)147 2586 y(})-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 164 172 164 171 bop 75 -100 a Fv(164)370 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fw(Example)g(3:)k(Building)e(Name)d(Service)h(fo)o(r)f (Intercommunication)75 131 y Fv(The)i(follo)o(wing)g(pro)q(cedures)h (exemplify)g(the)f(pro)q(cess)g(b)o(y)f(whic)o(h)i(a)e(user)h(could)h (create)e(name)h(service)75 187 y(for)g(building)j(in)o(tercomm)o (unicators)e(via)g(a)f(rendezv)o(ous)h(in)o(v)o(olving)h(a)f(serv)o(er) f(comm)o(unicator,)g(and)h(a)75 244 y(tag)d(name)h(selected)i(b)o(y)e (b)q(oth)g(groups.)166 300 y(After)k(all)h Fw(MPI)f Fv(pro)q(cesses)h (execute)g Fw(MPI)p 910 300 14 2 v 15 w(INIT)p Fv(,)f(ev)o(ery)g(pro)q (cess)h(calls)g(the)f(example)h(function,)75 357 y Fw(Init)p 141 357 V 17 w(server\(\))p Fv(,)c(de\014ned)i(b)q(elo)o(w.)26 b(Then,)18 b(if)g(the)f Fw(new)p 975 357 V 17 w(w)o(o)o(rld)f Fv(returned)h(is)h(NULL,)g(the)f(pro)q(cess)g(getting)75 413 y(NULL)e(is)g(required)h(to)e(implemen)o(t)i(a)e(serv)o(er)g (function,)h(in)g(a)f(reactiv)o(e)h(lo)q(op,)g Fw(Do)p 1502 413 V 16 w(server\(\))p Fv(.)j(Ev)o(ery)o(one)75 470 y(else)g(just)f(do)q(es)g(their)h(prescrib)q(ed)g(computation,)f (using)h Fw(new)p 1168 470 V 17 w(w)o(o)o(rld)e Fv(as)h(the)g(new)h (e\013ectiv)o(e)f(\\global")75 526 y(comm)o(unicator.)k(One)16 b(designated)h(pro)q(cess)e(calls)i Fw(Undo)p 1076 526 V 17 w(Server\(\))e Fv(to)g(get)g(rid)h(of)g(the)f(serv)o(er)h(when)g (it)75 583 y(is)g(not)f(needed)h(an)o(y)f(longer.)166 639 y(F)l(eatures)g(of)g(this)g(approac)o(h)g(include:)143 745 y Fx(\017)23 b Fv(Supp)q(ort)15 b(for)g(m)o(ultiple)i(name)e(serv)o (ers)143 839 y Fx(\017)23 b Fv(Abilit)o(y)16 b(to)f(scop)q(e)h(the)f (name)g(serv)o(ers)g(to)f(sp)q(eci\014c)k(pro)q(cesses)143 933 y Fx(\017)23 b Fv(Abilit)o(y)16 b(to)f(mak)o(e)g(suc)o(h)g(serv)o (ers)g(come)g(and)g(go)g(as)g(desired.)75 1039 y Ft(#define)23 b(INIT_SERVER_TAG_1)e(666)75 1096 y(#define)i(UNDO_SERVER_TAG_1)93 b(777)75 1209 y(static)23 b(int)g(server_key_val;)75 1321 y(/*)h(for)f(attribute)g(management)f(for)i(server_comm,)46 b(copy)23 b(callback:)g(*/)75 1378 y(void)g(handle_copy_fn\(MPI_Comm)e (*oldcomm,)i(int)g(*keyval,)g(void)g(*extra_state,)75 1434 y(void)g(*attribute_val_in,)f(void)h(**attribute_val_out,)e(int)j (*flag\))75 1491 y({)147 1547 y(/*)f(copy)g(the)h(handle)f(*/)147 1604 y(*attribute_val_out)e(=)j(attribute_val_in;)147 1660 y(*flag)f(=)g(1;)h(/*)g(indicate)e(that)i(copy)f(to)h(happen)f(*/) 75 1717 y(})75 1830 y(int)g(Init_server\(peer_comm,)e(rank_of_server,)h (server_comm,)g(new_world\))75 1886 y(MPI_Comm)h(peer_comm;)75 1942 y(int)g(rank_of_server;)75 1999 y(MPI_Comm)g(*server_comm;)75 2055 y(MPI_Comm)g(*new_world;)94 b(/*)23 b(new)h(effective)e(world,)h (sans)h(server)f(*/)75 2112 y({)170 2168 y(MPI_Comm)g(temp_comm,)g (lone_comm;)170 2225 y(MPI_Group)g(peer_group,)f(temp_group;)170 2281 y(int)i(rank_in_peer_comm,)d(size,)i(color,)h(key)f(=)h(0;)170 2338 y(int)g(peer_leader,)e(peer_leader_rank_in_temp_co)o(mm;)170 2451 y(MPI_Comm_rank\(peer_comm,)f(&rank_in_peer_comm\);)170 2507 y(MPI_Comm_size\(peer_comm,)g(&size\);)170 2620 y(if)j(\(\(size)f(<)h(2\))f(||)h(\(0)g(>)f(rank_of_server\))f(||)i (\(rank_of_server)e(>=)h(size\)\))266 2676 y(return)g (\(MPI_ERR_OTHER\);)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 165 173 165 172 bop 75 -100 a Fm(5.6.)34 b(INTER-COMMUNICA)l(TION)1047 b Fv(165)170 102 y Ft(/*)24 b(create)f(two)h(communicators,)d(by)j (splitting)f(peer_comm)242 158 y(into)g(the)h(server)f(process,)g(and)g (everyone)g(else)g(*/)170 271 y(peer_leader)g(=)h(\(rank_of_server)d(+) j(1\))g(\045)f(size;)47 b(/*)24 b(arbitrary)f(choice)g(*/)170 384 y(if)h(\(\(color)f(=)h(\(rank_in_peer_comm)d(==)j (rank_of_server\)\)\))170 440 y({)266 497 y(MPI_Comm_split\(peer_comm,) c(color,)j(key,)h(&lone_comm\);)266 610 y(MPI_Intercomm_create\(lone)o (_comm,)d(0,)i(peer_comm,)g(peer_leader,)719 666 y(INIT_SERVER_TAG_1,)f (server_comm\);)266 779 y(MPI_Comm_free\(&lone_comm\))o(;)266 835 y(*new_world)g(=)i(MPI_COMM_NULL;)170 892 y(})170 948 y(else)170 1005 y({)266 1061 y(MPI_Comm_Split\(peer_comm,)c(color,) j(key,)h(&temp_comm\);)266 1174 y(MPI_Comm_group\(peer_comm,)c (&peer_group\);)266 1231 y(MPI_Comm_group\(temp_comm,)g(&temp_group\);) 266 1287 y(MPI_Group_translate_ranks)o(\(peer_gr)o(oup,)h(1,)i (&peer_leader,)123 1344 y(temp_group,)f(&peer_leader_rank_in_temp_c)o (omm\);)266 1456 y(MPI_Intercomm_create\(temp)o(_comm,)f (peer_leader_rank_in_tem)o(p_comm,)719 1513 y(peer_comm,)i (rank_of_server,)719 1569 y(INIT_SERVER_TAG_1,)f(server_comm\);)266 1682 y(/*)h(attach)h(new_world)e(communication)g(attribute)h(to)g (server_comm:)g(*/)266 1795 y(/*)g(CRITICAL)g(SECTION)g(FOR)h (MULTITHREADING)e(*/)266 1852 y(if\(server_keyval)g(==)h (MPI_KEYVAL_INVALID\))266 1908 y({)361 1965 y(/*)h(acquire)f(the)g (process-local)f(name)i(for)f(the)h(server)f(keyval)g(*/)361 2021 y(MPI_keyval_create\(handle_copy)o(_fn,)e(NULL,)1197 2077 y(&server_keyval,)g(NULL\);)266 2134 y(})266 2247 y(*new_world)h(=)i(temp_comm;)266 2360 y(/*)f(Cache)h(handle)f(of)g (intra-communicator)f(on)h(inter-communicator:)f(*/)266 2416 y(MPI_Attr_put\(server_comm,)e(server_keyval,)i(\(void)h (*\)\(*new_world\)\);)170 2473 y(})170 2586 y(return)g (\(MPI_SUCCESS\);)75 2642 y(})-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 166 174 166 173 bop 75 -100 a Fv(166)370 b Fm(CHAPTER)15 b(5.)30 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)166 45 y Fv(The)g(actual)h(serv)o(er)f(pro)q(cess)g(w)o(ould)h(commit)f(to) f(running)j(the)e(follo)o(wing)h(co)q(de:)75 139 y Ft(int)23 b(Do_server\(server_comm\))75 195 y(MPI_Comm)g(server_comm;)75 252 y({)170 308 y(void)h(init_queue\(\);)170 365 y(int)g(en_queue\(\),) e(de_queue\(\);)h(/*)g(keep)h(triplets)e(of)i(integers)910 421 y(for)g(later)f(matching)g(\(fns)g(not)g(shown\))g(*/)170 534 y(MPI_Comm)g(comm;)170 591 y(MPI_Status)g(status;)170 647 y(int)h(client_tag,)e(client_source;)170 703 y(int)i (client_rank_in_new_world,)c(pairs_rank_in_new_world;)170 760 y(int)k(buffer[10],)e(count)h(=)h(1;)170 873 y(void)g(*queue;)170 929 y(init_queue\(&queue\);)170 1099 y(for)g(\(;;\))170 1155 y({)266 1212 y(MPI_Recv\(buffer,)e(count,)h(MPI_INT,)f (MPI_ANY_SOURCE,)g(MPI_ANY_TAG,)481 1268 y(server_comm,)g(&status\);)h (/*)g(accept)g(from)g(any)h(client)f(*/)266 1381 y(/*)g(determine)g (client:)g(*/)266 1437 y(client_tag)f(=)i(status.MPI_TAG;)266 1494 y(client_source)e(=)i(status.MPI_SOURCE;)266 1550 y(client_rank_in_new_world)c(=)k(buffer[0];)266 1663 y(if)f(\(client_tag)g(==)g(UNDO_SERVER_TAG_1\))165 b(/*)24 b(client)f(that)1292 1720 y(terminates)g(server)g(*/)266 1776 y({)361 1833 y(while)h(\(de_queue\(queue,)d(MPI_ANY_TAG,)h (&pairs_rank_in_new_world,)743 1889 y(&pairs_rank_in_server\)\))457 1945 y(;)361 2058 y(MPI_Intercomm_free\(&server_co)o(mm\);)361 2115 y(break;)266 2171 y(})266 2284 y(if)h(\(de_queue\(queue,)f (client_tag,)h(&pairs_rank_in_new_worl)o(d,)648 2341 y(&pairs_rank_in_server\)\))266 2397 y({)361 2454 y(/*)h(matched)f (pair)g(with)g(same)h(tag,)f(tell)g(them)433 2510 y(about)g(each)g (other!)g(*/)361 2566 y(buffer[0])g(=)h(pairs_rank_in_new_world;)361 2623 y(MPI_Send\(buffer,)e(1,)i(MPI_INT,)e(client_src,)h(client_tag,) 1340 2679 y(server_comm\);)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 167 175 167 174 bop 75 -100 a Fm(5.6.)29 b(INTER-COMMUNICA)l(TION)1052 b Fv(167)361 102 y Ft(buffer[0])23 b(=)h(client_rank_in_new_world)o(;) 361 158 y(MPI_Send\(buffer,)e(1,)i(MPI_INT,)e(pairs_rank_in_server,)f (client_tag,)576 214 y(server_comm\);)266 271 y(})266 327 y(else)361 384 y(en_queue\(queue,)h(client_tag,)h(client_source,) 1030 440 y(client_rank_in_new_worl)o(d\);)170 553 y(})75 610 y(})166 703 y Fv(A)18 b(particular)h(pro)q(cess)g(w)o(ould)f(b)q(e) h(resp)q(onsible)h(for)e(ending)h(the)g(serv)o(er)f(when)g(it)h(is)f (no)h(longer)75 760 y(needed.)i(Its)15 b(call)i(to)d Fw(Undo)p 553 760 14 2 v 18 w(server)h Fv(w)o(ould)g(terminate)g(serv)o (er)g(function.)75 854 y Ft(int)23 b(Undo_server\(server_comm\))116 b(/*)24 b(example)f(client)g(that)g(ends)h(server)f(*/)75 910 y(MPI_Comm)g(*server_comm;)75 967 y({)170 1023 y(int)h(buffer)f(=)h (0;)170 1080 y(MPI_Send\(&buffer,)e(1,)i(MPI_INT,)e(0,)i (UNDO_SERVER_TAG_1,)d(*server_comm\);)170 1136 y (MPI_Intercomm_free\(server_com)o(m\);)75 1192 y(})166 1286 y Fv(The)15 b(follo)o(wing)g(is)g(a)f(blo)q(c)o(king)i (name-service)f(for)f(in)o(ter-comm)o(unication,)i(with)f(same)f(seman) o(tic)75 1343 y(restrictions)k(as)g Fw(MPI)p 457 1343 V 16 w(Intercomm)p 678 1343 V 14 w(create)p Fv(,)h(but)f(simpli\014ed)i (syn)o(tax.)27 b(It)18 b(uses)g(the)g(functionalit)o(y)h(just)75 1399 y(de\014ned)e(to)d(create)h(the)g(name)h(service.)75 1493 y Ft(int)23 b(Intercomm_name_create\(local_c)o(omm,)e (server_comm,)h(tag,)h(comm\))75 1549 y(MPI_Comm)g(local_comm,)f (server_comm;)75 1606 y(int)h(tag;)75 1662 y(MPI_Comm)g(*comm;)75 1719 y({)170 1775 y(int)h(error;)170 1832 y(int)g(found;)71 b(/*)23 b(attribute)g(acquisition)f(mgmt)i(for)f(new_world)g(*/)481 1888 y(/*)g(comm)h(in)f(server_comm)g(*/)170 1945 y(void)h(*val;)170 2058 y(MPI_Comm)f(new_world;)170 2170 y(int)h(buffer[10],)e(rank;)170 2227 y(int)i(local_leader)e(=)i(0;)170 2340 y (MPI_Attr_get\(server_comm,)d(server_keyval,)h(&val,)h(&found\);)170 2396 y(new_world)g(=)h(\(MPI_Comm\)val;)e(/*)h(retrieve)g(cached)g (handle)g(*/)170 2509 y(MPI_Comm_rank\(server_comm,)e(&rank\);)47 b(/*)23 b(rank)g(in)h(local)f(group)g(*/)170 2622 y(if)h(\(rank)f(==)h (local_leader\))170 2679 y({)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 168 176 168 175 bop 75 -100 a Fv(168)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)266 45 y Ft(buffer[0])23 b(=)g(rank;)266 102 y(MPI_Send\(&buffer,)e(1,)j (MPI_INT,)f(0,)g(tag,)h(server_comm\);)266 158 y(MPI_Recv\(&buffer,)d (1,)j(MPI_INT,)f(0,)g(tag,)h(server_comm\);)170 214 y(})170 327 y(error)g(=)f(MPI_Intercomm_create\(local_co)o(mm,)e(local_leader,) h(new_world,)862 384 y(buffer[0],)h(tag,)g(comm\);)170 497 y(return\(error\);)75 553 y(})75 696 y Fn(5.7)59 b(Caching)75 798 y Fw(MPI)18 b Fv(pro)o(vides)g(a)g(\\cac)o(hing")g (facilit)o(y)h(that)f(allo)o(ws)g(an)g(application)i(to)d(attac)o(h)g (arbitrary)h(pieces)h(of)75 854 y(information,)24 b(called)g Fq(attributes)p Fv(,)g(to)e(comm)o(unicators.)40 b(More)22 b(precisely)l(,)k(the)c(cac)o(hing)h(facilit)o(y)75 911 y(allo)o(ws)15 b(a)g(p)q(ortable)h(library)g(to)e(do)i(the)f(follo)o (wing:)143 1005 y Fx(\017)23 b Fv(pass)14 b(information)g(b)q(et)o(w)o (een)h(calls)g(b)o(y)f(asso)q(ciating)h(it)f(with)h(an)f Fw(MPI)f Fv(in)o(tra-)i(or)e(inter-comm)o(un-)189 1061 y(icator,)143 1155 y Fx(\017)23 b Fv(quic)o(kly)16 b(retriev)o(e)g (that)e(information,)h(and)143 1249 y Fx(\017)23 b Fv(b)q(e)18 b(guaran)o(teed)e(that)h(out-of-date)g(information)g(is)h(nev)o(er)g (retriev)o(ed,)f(ev)o(en)h(if)g(the)f(comm)o(uni-)189 1305 y(cator)d(is)i(freed)f(and)h(its)f(handle)h(subsequen)o(tly)h (reused)f(b)o(y)f Fw(MPI)p Fv(.)166 1399 y(The)i(cac)o(hing)h (capabilities,)h(in)f(some)e(form,)g(are)h(required)h(b)o(y)e(built-in) k Fw(MPI)c Fv(routines)h(suc)o(h)g(as)75 1455 y(collectiv)o(e)22 b(comm)o(unication)e(and)g(application)i(top)q(ology)l(.)33 b(De\014ning)21 b(an)f(in)o(terface)g(to)f(these)h(capa-)75 1512 y(bilities)h(as)c(part)h(of)f(the)h Fw(MPI)g Fv(standard)f(is)i(v) m(aluable)h(b)q(ecause)f(it)f(p)q(ermits)h(routines)f(lik)o(e)h (collectiv)o(e)75 1568 y(comm)o(unication)h(and)f(application)h(top)q (ologies)g(to)e(b)q(e)h(implemen)o(ted)i(as)d(p)q(ortable)i(co)q(de,)g (and)f(also)75 1625 y(b)q(ecause)h(it)g(mak)o(es)e Fw(MPI)h Fv(more)g(extensible)i(b)o(y)e(allo)o(wing)h(user-written)g(routines)f (to)g(use)g(standard)75 1681 y Fw(MPI)c Fv(calling)i(sequences.)189 1788 y Fl(A)n(dvic)n(e)d(to)i(users.)40 b Fv(The)15 b(comm)o(unicator)f Fi(MPI)p 1009 1788 13 2 v 15 w(COMM)p 1154 1788 V 14 w(SELF)h Fv(is)g(a)f(suitable)i(c)o(hoice)g(for)e(p)q(osting)189 1844 y(pro)q(cess-lo)q(cal)21 b(attributes,)e(via)h(this)g (attributing-cac)o(hing)g(mec)o(hanism.)33 b(\()p Fl(End)20 b(of)g(advic)n(e)g(to)189 1900 y(users.)p Fv(\))75 2022 y Fj(5.7.1)49 b(F)o(unctionalit)o(y)75 2108 y Fv(A)o(ttributes)15 b(are)g(attac)o(hed)g(to)g(comm)o(unicators.)20 b(A)o(ttributes)c(are)f (lo)q(cal)h(to)f(the)h(pro)q(cess)f(and)h(sp)q(eci\014c)75 2164 y(to)h(the)i(comm)o(unicator)e(to)h(whic)o(h)h(they)f(are)g(attac) o(hed.)28 b(A)o(ttributes)18 b(are)f(not)h(propagated)g(b)o(y)g Fw(MPI)75 2221 y Fv(from)k(one)h(comm)o(unicator)f(to)g(another)g (except)h(when)g(the)g(comm)o(unicator)f(is)i(duplicated)g(using)75 2277 y Fw(MPI)p 160 2277 14 2 v 16 w(COMM)p 318 2277 V 16 w(DUP)e Fv(\(and)f(ev)o(en)h(then)f(the)h(application)h(m)o(ust)e (giv)o(e)g(sp)q(eci\014c)i(p)q(ermission)g(through)75 2334 y(callbac)o(k)16 b(functions)g(for)f(the)g(attribute)g(to)g(b)q(e) h(copied\).)-1219 b Fx(>)189 2440 y Fl(A)n(dvic)n(e)15 b(to)i(users.)41 b Fv(A)o(ttributes)15 b(in)h(C)f(are)g(of)g(t)o(yp)q (e)g Fi(void)f(*)p Fv(.)20 b(T)o(ypically)l(,)d(suc)o(h)f(an)f (attribute)g(will)189 2496 y(b)q(e)h(a)g(p)q(oin)o(ter)h(to)e(a)h (structure)f(that)h(con)o(tains)g(further)g(information,)g(or)f(a)h (handle)h(to)f(an)g Fw(MPI)189 2553 y Fv(ob)s(ject.)j(In)14 b(F)l(ortran,)f(attributes)h(are)f(of)h(t)o(yp)q(e)g Fi(INTEGER)p Fv(.)f(Suc)o(h)h(attribute)g(can)g(b)q(e)h(a)e(handle)j (to)189 2609 y(an)f Fw(MPI)f Fv(ob)s(ject,)h(or)f(just)h(an)g(in)o (teger-v)m(alued)i(attribute.)j(\()p Fl(End)c(of)g(advic)n(e)g(to)h (users.)p Fv(\))-160 2671 y Fx(?)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 169 177 169 176 bop 75 -100 a Fm(5.7.)29 b(CA)o(CHING)1412 b Fv(169)189 45 y Fl(A)n(dvic)n(e)14 b(to)i(implementors.)39 b Fv(A)o(ttributes)15 b(are)f(scalar)g(v)m(alues,)h(equal)g(in)g(size)h (to,)d(or)h(larger)g(than)189 102 y(a)i(C-language)g(p)q(oin)o(ter.)24 b(A)o(ttributes)17 b(can)f(alw)o(a)o(ys)g(hold)h(an)g Fw(MPI)e Fv(handle.)25 b(\()p Fl(End)17 b(of)g(advic)n(e)h(to)189 158 y(implementors.)p Fv(\))166 242 y(The)12 b(cac)o(hing)g(in)o (terface)g(de\014ned)g(here)g(represen)o(ts)g(that)e(attributes)i(b)q (e)g(stored)f(b)o(y)g Fw(MPI)g Fv(opaquely)75 299 y(within)16 b(a)f(comm)o(unicator.)20 b(Accessor)15 b(functions)h(include)i(the)d (follo)o(wing:)143 377 y Fx(\017)23 b Fv(obtain)16 b(a)f(k)o(ey)h(v)m (alue)h(\(used)f(to)g(iden)o(tify)h(an)f(attribute\);)f(the)h(user)g (sp)q(eci\014es)i(\\callbac)o(k")e(func-)189 433 y(tions)h(b)o(y)g (whic)o(h)h Fw(MPI)f Fv(informs)g(the)h(application)h(when)f(the)f (comm)o(unicator)g(is)h(destro)o(y)o(ed)e(or)189 490 y(copied.)143 577 y Fx(\017)23 b Fv(store)14 b(and)i(retriev)o(e)f(the) g(v)m(alue)i(of)d(an)i(attribute;)189 661 y Fl(A)n(dvic)n(e)10 b(to)i(implementors.)37 b Fv(Cac)o(hing)10 b(and)h(callbac)o(k)g (functions)g(are)f(only)g(called)i(sync)o(hronously)l(,)189 718 y(in)k(resp)q(onse)f(to)g(explicit)j(application)e(requests.)k (This)c(a)o(v)o(oid)f(problems)h(that)f(result)g(from)g(re-)189 774 y(p)q(eated)h(crossings)h(b)q(et)o(w)o(een)f(user)h(and)f(system)g (space.)24 b(\(This)16 b(sync)o(hronous)g(calling)i(rule)f(is)g(a)189 831 y(general)e(prop)q(ert)o(y)g(of)g Fw(MPI)p Fv(.\))189 903 y(The)20 b(c)o(hoice)h(of)f(k)o(ey)g(v)m(alues)i(is)f(under)g(con)o (trol)f(of)f Fw(MPI)p Fv(.)h(This)h(allo)o(ws)f Fw(MPI)g Fv(to)f(optimize)j(its)189 959 y(implemen)o(tation)14 b(of)e(attribute)h(sets.)19 b(It)12 b(also)h(a)o(v)o(oids)g(con\015ict) h(b)q(et)o(w)o(een)f(indep)q(enden)o(t)i(mo)q(dules)189 1016 y(cac)o(hing)h(information)f(on)g(the)g(same)g(comm)o(unicators.) 189 1088 y(A)g(m)o(uc)o(h)g(smaller)h(in)o(terface,)f(consisting)h(of)e (just)h(a)g(callbac)o(k)h(facilit)o(y)l(,)g(w)o(ould)f(allo)o(w)g(the)g (en)o(tire)189 1144 y(cac)o(hing)f(facilit)o(y)h(to)e(b)q(e)h(implemen) o(ted)h(b)o(y)f(p)q(ortable)g(co)q(de.)20 b(Ho)o(w)o(ev)o(er,)13 b(with)h(the)f(minimal)j(call-)189 1200 y(bac)o(k)c(in)o(terface,)h (some)f(form)g(of)g(table)h(searc)o(hing)g(is)g(implied)i(b)o(y)d(the)h (need)g(to)f(handle)i(arbitrary)189 1257 y(comm)o(unicators.)23 b(In)17 b(con)o(trast,)e(the)i(more)f(complete)h(in)o(terface)g (de\014ned)g(here)g(p)q(ermits)g(rapid)189 1313 y(access)11 b(to)f(attributes)h(through)f(the)h(use)g(of)g(p)q(oin)o(ters)g(in)h (comm)o(unicators)f(\(to)e(\014nd)j(the)f(attribute)189 1370 y(table\))h(and)h(clev)o(erly)g(c)o(hosen)g(k)o(ey)f(v)m(alues)i (\(to)d(retriev)o(e)i(individual)j(attributes\).)i(In)13 b(ligh)o(t)g(of)f(the)189 1426 y(e\016ciency)j(\\hit")e(inheren)o(t)h (in)h(the)e(minimal)i(in)o(terface,)f(the)g(more)f(complete)h(in)o (terface)g(de\014ned)189 1483 y(here)h(is)h(seen)g(to)e(b)q(e)i(sup)q (erior.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Fv(\))75 1567 y Fw(MPI)f Fv(pro)o(vides)g(the)h(follo)o(wing)f (services)i(related)e(to)g(cac)o(hing.)20 b(They)c(are)f(all)h(pro)q (cess)f(lo)q(cal.)75 1718 y Fw(MPI)p 160 1718 14 2 v 16 w(KEYV)l(AL)p 346 1718 V 17 w(CREA)l(TE\(cop)o(y)p 636 1718 V 17 w(fn,)g(delete)p 828 1718 V 18 w(fn,)g(k)o(eyval,)g (extra)p 1150 1718 V 16 w(state\))117 1795 y Fk(IN)155 b Fw(cop)o(y)p 407 1795 V 17 w(fn)447 b Fk(Cop)o(y)14 b(callbac)o(k)f(function)g(for)h Fi(k)o(eyval)117 1867 y Fk(IN)155 b Fw(delete)p 431 1867 V 18 w(fn)422 b Fk(Delete)15 b(callbac)o(k)e(function)g(for)h Fi(k)o(eyval)117 1939 y Fk(OUT)108 b Fw(k)o(eyval)471 b Fk(k)o(ey)14 b(v)n(alue)f(for)h (future)g(access)i(\(in)o(teger\))117 2011 y(IN)155 b Fw(extra)p 416 2011 V 17 w(state)384 b Fk(Extra)14 b(state)h(for)f (callbac)o(k)f(functions)75 2135 y Ft(int)23 b(MPI)p 245 2135 15 2 v 17 w(Keyval)p 406 2135 V 17 w(create\(MPI)p 663 2135 V 15 w(Copy)p 774 2135 V 17 w(function)g(*copy)p 1126 2135 V 16 w(fn,)h(MPI)p 1310 2135 V 17 w(Delete)p 1471 2135 V 16 w(function)393 2192 y(*delete)p 564 2192 V 16 w(fn,)g(int)f(*keyval,)g(void*)g(extra)p 1249 2192 V 17 w(state\))75 2278 y(MPI)p 150 2278 V 17 w(KEYVAL)p 311 2278 V 16 w(CREATE\(COPY)p 591 2278 V 16 w(FN,)g(DELETE)p 846 2278 V 17 w(FN,)g(KEYVAL,)g(EXTRA)p 1269 2278 V 16 w(STATE,)h(IERROR\))170 2335 y(EXTERNAL)f(COPY)p 484 2335 V 17 w(FN,)g(DELETE)p 740 2335 V 17 w(FN)170 2391 y(INTEGER)g(KEYVAL,)g(EXTRA)p 675 2391 V 17 w(STATE,)g(IERROR)166 2478 y Fv(Generates)18 b(a)g(new)g(attribute)h(k)o(ey)l(.)29 b(Keys)19 b(are)f(lo)q(cally)i(unique)g(in)f(a)f(pro)q(cess,)h(and)f (opaque)h(to)75 2534 y(user,)g(though)g(they)f(are)h(explicitly)i (stored)d(in)i(in)o(tegers.)30 b(Once)19 b(allo)q(cated,)i(the)d(k)o (ey)h(v)m(alue)h(can)e(b)q(e)75 2591 y(used)e(to)e(asso)q(ciate)i (attributes)e(and)i(access)f(them)g(on)h(an)o(y)e(lo)q(cally)j (de\014ned)g(comm)o(unicator.)166 2647 y(The)10 b Fw(cop)o(y)p 343 2647 14 2 v 17 w(fn)g Fv(function)h(is)g(in)o(v)o(ok)o(ed)f(when)h (a)f(comm)o(unicator)f(is)i(duplicated)h(b)o(y)e Fw(MPI)p 1617 2647 V 16 w(COMM)p 1775 2647 V 17 w(DUP)p Fv(.)75 2704 y Fw(cop)o(y)p 164 2704 V 16 w(fn)16 b Fv(should)g(b)q(e)g(of)f(t) o(yp)q(e)g Fi(MPI)p 667 2704 13 2 v 14 w(Cop)o(y)p 768 2704 V 15 w(function)p Fv(,)h(whic)o(h)g(is)g(de\014ned)g(as)f(follo)o (ws:)452 b Fx(>)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 170 178 170 177 bop 75 -100 a Fv(170)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Ft(typedef)23 b(int)g(MPI_Copy_function\(MPI_Comm)e(oldcomm,)h (int)i(keyval,)791 102 y(void)f(*extra_state,)f(void)i (*attribute_val_in,)791 158 y(void)f(*attribute_val_out,)e(int)j (*flag\))-160 203 y Fx(?)166 250 y Fv(A)15 b(F)l(ortran)f(declaration)i (for)f(suc)o(h)g(a)g(function)h(is)g(as)f(follo)o(ws:)-160 256 y Fx(>)75 306 y Ft(SUBROUTINE)22 b(COPY)p 436 306 15 2 v 17 w(FUNCTION\(OLDCOMM,)g(KEYVAL,)h(EXTRA)p 1194 306 V 16 w(STATE,)g(ATTRIBUTE)p 1593 306 V 16 w(VAL)p 1681 306 V 17 w(IN,)393 363 y(ATTRIBUTE)p 612 363 V 16 w(VAL)p 700 363 V 17 w(OUT,)g(FLAG,)g(IERR\))170 419 y(INTEGER)g(OLDCOMM,)g(KEYVAL,)g(EXTRA)p 890 419 V 17 w(STATE,)g(ATTRIBUTE)p 1290 419 V 16 w(VAL)p 1378 419 V 16 w(IN,)170 475 y(ATTRIBUTE)p 389 475 V 16 w(VAL)p 477 475 V 17 w(OUT,)g(IERR)170 532 y(LOGICAL)g(FLAG)-160 562 y Fx(?)166 618 y Fv(The)13 b(cop)o(y)g(callbac)o(k)i(function)f(is) f(in)o(v)o(ok)o(ed)h(for)e(eac)o(h)h(k)o(ey)g(v)m(alue)i(in)f Fw(oldcomm)d Fv(in)j(arbitrary)f(order.)75 675 y(Eac)o(h)18 b(call)i(to)e(the)g(cop)o(y)g(callbac)o(k)i(is)f(made)f(with)h(a)f(k)o (ey)g(v)m(alue)i(and)e(its)h(corresp)q(onding)g(attribute.)75 731 y(If)f(it)g(returns)g Fi(\015ag)e(=)g(0)p Fv(,)i(then)g(the)g (attribute)f(is)h(deleted)h(in)g(the)f(duplicated)i(comm)o(unicator.)27 b(Oth-)75 788 y(erwise)19 b(\()p Fi(\015ag)d(=)g(1)p Fv(\),)i(the)g(new)h(attribute)f(v)m(alue)h(is)g(set)e(to)h(the)g(v)m (alue)h(returned)g(in)g Fw(attribute)p 1715 788 14 2 v 18 w(val)p 1787 788 V 16 w(out)p Fv(.)-2035 b Fx(>)75 844 y Fv(The)17 b(function)g(returns)g Fi(MPI)p 585 844 13 2 v 14 w(SUCCESS)e Fv(on)i(success)g(and)g(an)g(error)f(co)q(de)h (on)g(failure)g(\(in)g(whic)o(h)h(case)-2035 b Fx(?)75 901 y Fw(MPI)p 160 901 14 2 v 16 w(COMM)p 318 901 V 16 w(DUP)15 b Fv(will)i(fail\).)-777 b Fx(>)166 957 y Fw(cop)o(y)p 255 957 V 16 w(fn)21 b Fv(ma)o(y)e(b)q(e)i(sp)q(eci\014ed)i(as)d Fw(MPI)p 831 957 V 15 w(NULL)p 959 957 V 16 w(COPY)p 1096 957 V 18 w(FN)g Fv(or)g Fw(MPI)p 1335 957 V 15 w(DUP)p 1443 957 V 17 w(FN)g Fv(from)g(either)g(C)g(or)75 1014 y(F)o(OR)l(TRAN;)15 b Fw(MPI)p 415 1014 V 15 w(NULL)p 543 1014 V 17 w(COPY)p 681 1014 V 17 w(FN)f Fv(is)h(a)f(function)h (that)e(do)q(es)i(nothing)f(other)g(than)g(returning)h Fi(\015ag)75 1070 y(=)i(0)h Fv(and)g Fi(MPI)p 331 1070 13 2 v 15 w(SUCCESS)p Fv(.)e Fw(MPI)p 635 1070 14 2 v 16 w(DUP)p 744 1070 V 16 w(FN)j Fv(is)f(a)g(simple-minded)k(cop)o(y)c (function)h(that)e(sets)h Fi(\015ag)f(=)g(1)p Fv(,)75 1126 y(returns)e(the)g(v)m(alue)i(of)e Fw(attribute)p 646 1126 V 18 w(val)p 718 1126 V 16 w(in)g Fv(in)h Fw(attribute)p 999 1126 V 18 w(val)p 1071 1126 V 16 w(out)p Fv(,)g(and)f(returns)g Fi(MPI)p 1497 1126 13 2 v 14 w(SUCCESS)p Fv(.)189 1230 y Fl(A)n(dvic)n(e)10 b(to)i(users.)37 b Fv(Ev)o(en)10 b(though)g(b)q(oth)g(formal)g(argumen)o(ts)f Fw(attribute)p 1412 1230 14 2 v 18 w(val)p 1484 1230 V 16 w(in)i Fv(and)f Fw(attribute)p 1791 1230 V 18 w(val)p 1863 1230 V 16 w(out)189 1286 y Fv(are)17 b(of)h(t)o(yp)q(e)g Fi(void)f(*)p Fv(,)h(their)h(usage)f(di\013ers.)28 b(The)19 b(C)e(cop)o(y)h(function) h(is)g(passed)f(b)o(y)g Fw(MPI)g Fv(in)h Fw(at-)189 1343 y(tribute)p 317 1343 V 17 w(val)p 388 1343 V 16 w(in)j Fv(the)f Fl(value)g Fv(of)g(the)g(attribute,)h(and)f(in)h Fw(attribute)p 1336 1343 V 18 w(val)p 1408 1343 V 16 w(out)f Fv(the)h Fl(addr)n(ess)e Fv(of)h(the)189 1399 y(attribute,)15 b(so)g(as)g(to)f(allo)o(w)i(the)f(function)i(to)d (return)i(the)f(\(new\))g(attribute)g(v)m(alue.)22 b(The)16 b(use)f(of)189 1456 y(t)o(yp)q(e)g Fi(void)f(*)h Fv(for)f(b)q(oth)i(is) g(to)e(a)o(v)o(oid)h(messy)g(t)o(yp)q(e)g(casts.)-1315 b Fx(?)189 1530 y Fv(A)18 b(v)m(alid)i(cop)o(y)f(function)g(is)g(one)f (that)g(completely)i(duplicates)g(the)f(information)f(b)o(y)h(making) 189 1587 y(a)f(full)i(duplicate)g(cop)o(y)e(of)h(the)f(data)g (structures)g(implied)j(b)o(y)e(an)f(attribute;)i(another)e(migh)o(t) 189 1643 y(just)k(mak)o(e)f(another)h(reference)h(to)e(that)h(data)f (structure,)j(while)f(using)g(a)f(reference-coun)o(t)189 1700 y(mec)o(hanism.)k(Other)18 b(t)o(yp)q(es)f(of)g(attributes)g(migh) o(t)g(not)g(cop)o(y)g(at)g(all)h(\(they)f(migh)o(t)g(b)q(e)h(sp)q (eci\014c)189 1756 y(to)c Fw(oldcomm)g Fv(only\).)20 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g(users.)p Fv(\))-160 1814 y Fx(>)189 1906 y Fl(A)n(dvic)n(e)21 b(to)i(implementors.)81 b Fv(A)22 b(C)g(in)o(terface)g(should)h(b)q(e)g(assumed)f(for)g(cop)o (y)g(and)g(delete)189 1962 y(functions)17 b(asso)q(ciated)g(with)h(k)o (ey)e(v)m(alues)i(created)f(in)h(C;)e(a)h(F)l(ortran)e(calling)k(in)o (terface)e(should)189 2019 y(b)q(e)f(assumed)f(for)f(k)o(ey)h(v)m (alues)i(created)e(in)h(F)l(ortran.)j(\()p Fl(End)c(of)i(advic)n(e)f (to)g(implementors.)p Fv(\))-160 2077 y Fx(?)166 2122 y Fv(Analogous)g(to)g Fw(cop)o(y)p 534 2122 V 17 w(fn)h Fv(is)g(a)f(callbac)o(k)i(deletion)g(function,)f(de\014ned)h(as)e (follo)o(ws.)24 b(The)17 b Fw(delete)p 1823 2122 V 18 w(fn)75 2179 y Fv(function)h(is)g(in)o(v)o(ok)o(ed)g(when)g(a)g(comm)o (unicator)f(is)h(deleted)h(b)o(y)e Fw(MPI)p 1289 2179 V 16 w(COMM)p 1447 2179 V 16 w(FREE)h Fv(or)g(when)g(a)f(call)75 2235 y(is)j(made)f(explicitly)k(to)18 b Fw(MPI)p 596 2235 V 16 w(A)l(TTR)p 729 2235 V 17 w(DELETE)p Fv(.)h Fw(delete)p 1058 2235 V 18 w(fn)h Fv(should)g(b)q(e)g(of)f(t)o(yp)q(e)h Fi(MPI)p 1584 2235 13 2 v 14 w(Delete)p 1707 2235 V 16 w(function)p Fv(,)75 2291 y(whic)o(h)c(is)g(de\014ned)g(as)f(follo)o (ws:)-770 b Fx(>)75 2395 y Ft(typedef)23 b(int)g (MPI_Delete_function\(MPI_Comm)d(comm,)k(int)f(keyval,)75 2451 y(void)g(*attribute_val,)f(void)h(*extra_state\);)166 2554 y Fv(A)15 b(F)l(ortran)f(declaration)i(for)f(suc)o(h)g(a)g (function)h(is)g(as)f(follo)o(ws:)75 2611 y Ft(SUBROUTINE)22 b(DELETE)p 484 2611 15 2 v 17 w(FUNCTION\(COMM,)g(KEYVAL,)h(ATTRIBUTE)p 1266 2611 V 16 w(VAL,)g(EXTRA)p 1521 2611 V 16 w(STATE,)h(IERR\))170 2667 y(INTEGER)f(COMM,)h(KEYVAL,)e(ATTRIBUTE)p 914 2667 V 16 w(VAL,)i(EXTRA)p 1170 2667 V 16 w(STATE,)f(IERR)-160 2704 y Fx(?)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 171 179 171 178 bop 75 -100 a Fm(5.7.)34 b(CA)o(CHING)1407 b Fv(171)166 45 y(This)11 b(function)g(is)f(called)i(b)o(y)e Fw(MPI)p 745 45 14 2 v 16 w(COMM)p 903 45 V 16 w(FREE)p Fv(,)h Fw(MPI)p 1134 45 V 15 w(A)l(TTR)p 1266 45 V 17 w(DELETE)p Fv(,)f(and)g Fw(MPI)p 1641 45 V 16 w(A)l(TTR)p 1774 45 V 17 w(PUT)-160 47 y Fx(>)75 102 y Fv(to)18 b(do)g(whatev)o(er) g(is)h(needed)h(to)e(remo)o(v)o(e)f(an)i(attribute.)29 b(The)19 b(function)g(returns)g Fi(MPI)p 1621 102 13 2 v 14 w(SUCCESS)e Fv(on)-2035 b Fx(?)-160 154 y(>)75 158 y Fv(success)16 b(and)f(an)g(error)g(co)q(de)h(on)f(failure)h(\(in) g(whic)o(h)g(case)f Fw(MPI)p 1183 158 14 2 v 16 w(COMM)p 1341 158 V 16 w(FREE)h Fv(will)h(fail\).)166 214 y Fw(delete)p 279 214 V 18 w(fn)k Fv(ma)o(y)g(b)q(e)h(sp)q(eci\014ed)i(as)d Fw(MPI)p 862 214 V 16 w(NULL)p 991 214 V 16 w(DELETE)p 1177 214 V 16 w(FN)g Fv(from)g(either)h(C)g(or)e(F)o(OR)l(TRAN;)75 271 y Fw(MPI)p 160 271 V 16 w(NULL)p 289 271 V 16 w(DELETE)p 475 271 V 16 w(FN)12 b Fv(is)h(a)f(function)h(that)e(do)q(es)h (nothing,)h(other)f(than)g(returning)g Fi(MPI)p 1674 271 13 2 v 15 w(SUCCESS)p Fv(.)1937 327 y Fx(?)166 384 y Fv(The)e(sp)q(ecial)i(k)o(ey)e(v)m(alue)i Fi(MPI)p 664 384 V 14 w(KEYV)m(AL)p 835 384 V 14 w(INV)m(ALID)c Fv(is)j(nev)o(er)f(returned)h(b)o(y)f Fw(MPI)p 1493 384 14 2 v 16 w(KEYV)l(AL)p 1679 384 V 17 w(CREA)l(TE)p Fv(.)75 440 y(Therefore,)15 b(it)g(can)g(b)q(e)h(used)g(for)f(static)g (initialization)j(of)d(k)o(ey)g(v)m(alues.)75 591 y Fw(MPI)p 160 591 V 16 w(KEYV)l(AL)p 346 591 V 17 w(FREE\(k)o(eyval\))117 668 y Fk(INOUT)62 b Fw(k)o(eyval)471 b Fk(F)m(rees)15 b(the)g(in)o(teger)f(k)o(ey)g(v)n(alue)f(\(in)o(teger\))75 793 y Ft(int)23 b(MPI)p 245 793 15 2 v 17 w(Keyval)p 406 793 V 17 w(free\(int)f(*keyval\))75 879 y(MPI)p 150 879 V 17 w(KEYVAL)p 311 879 V 16 w(FREE\(KEYVAL,)g(IERROR\))170 936 y(INTEGER)h(KEYVAL,)g(IERROR)166 1022 y Fv(F)l(rees)37 b(an)g(extan)o(t)g(attribute)g(k)o(ey)l(.)87 b(This)37 b(function)i(sets)e(the)g(v)m(alue)i(of)e Fw(k)o(eyval)g Fv(to)75 1079 y Fi(MPI)p 152 1079 13 2 v 14 w(KEYV)m(AL)p 323 1079 V 14 w(INV)m(ALID)p Fv(.)15 b(Note)h(that)f(it)i(is)g(not)f (erroneous)g(to)f(free)i(an)f(attribute)g(k)o(ey)g(that)g(is)h(in)g (use,)75 1135 y(b)q(ecause)e(the)e(actual)h(free)f(do)q(es)h(not)f (transpire)h(un)o(til)h(after)e(all)h(references)g(\(in)g(other)f(comm) o(unicators)75 1192 y(on)20 b(the)g(pro)q(cess\))f(to)h(the)g(k)o(ey)f (ha)o(v)o(e)h(b)q(een)h(freed.)34 b(These)20 b(references)g(need)h(to)e (b)q(e)i(explictly)h(freed)75 1248 y(b)o(y)d(the)h(program,)f(either)h (via)f(calls)i(to)e Fw(MPI)p 887 1248 14 2 v 15 w(A)l(TTR)p 1019 1248 V 18 w(DELETE)g Fv(that)f(free)i(one)f(attribute)h(instance,) 75 1305 y(or)c(b)o(y)g(calls)h(to)e Fw(MPI)p 439 1305 V 16 w(COMM)p 597 1305 V 17 w(FREE)h Fv(that)g(free)g(all)h(attribute)f (instances)h(asso)q(ciated)f(with)g(the)h(freed)75 1361 y(comm)o(unicator.)1567 b Fx(>)1937 1413 y(?)75 1512 y Fw(MPI)p 160 1512 V 16 w(A)l(TTR)p 293 1512 V 17 w(PUT\(comm,)13 b(k)o(eyval,)h(attribute)p 868 1512 V 18 w(val\))117 1589 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)7 b(to)i(whic)o(h)g(attribute)h(will)e(b)q(e)h(attac)o(hed)h(\(han-)905 1646 y(dle\))117 1721 y(IN)155 b Fw(k)o(eyval)471 b Fk(k)o(ey)14 b(v)n(alue,)f(as)h(returned)h(b)o(y)905 1777 y Fi(MPI)p 982 1777 13 2 v 15 w(KEYV)m(AL)p 1154 1777 V 13 w(CREA)m(TE)d Fk(\(in)o(teger\))117 1852 y(IN)155 b Fw(attribute)p 484 1852 14 2 v 18 w(val)352 b Fk(attribute)15 b(v)n(alue)75 1977 y Ft(int)23 b(MPI)p 245 1977 15 2 v 17 w(Attr)p 358 1977 V 17 w(put\(MPI)p 543 1977 V 16 w(Comm)g(comm,)h(int)f (keyval,)g(void*)g(attribute)p 1467 1977 V 16 w(val\))75 2063 y(MPI)p 150 2063 V 17 w(ATTR)p 263 2063 V 16 w(PUT\(COMM,)g (KEYVAL,)g(ATTRIBUTE)p 925 2063 V 16 w(VAL,)g(IERROR\))170 2120 y(INTEGER)g(COMM,)h(KEYVAL,)e(ATTRIBUTE)p 914 2120 V 16 w(VAL,)i(IERROR)166 2206 y Fv(This)14 b(function)g(stores)f(the)g (stipulated)i(attribute)e(v)m(alue)i Fw(attribute)p 1331 2206 14 2 v 18 w(val)e Fv(for)g(subsequen)o(t)h(retriev)m(al)75 2263 y(b)o(y)j Fw(MPI)p 225 2263 V 16 w(A)l(TTR)p 358 2263 V 17 w(GET)p Fv(.)h(If)g(the)f(v)m(alue)i(is)f(already)g(presen)o (t,)g(then)f(the)h(outcome)f(is)h(as)f(if)h Fw(MPI)p 1729 2263 V 16 w(A)l(TTR-)p 75 2319 V 89 2319 a(DELETE)f Fv(w)o(as)f(\014rst)h(called)i(to)d(delete)j(the)e(previous)h(v)m(alue) g(\(and)f(the)h(callbac)o(k)g(function)g Fw(delete)p 1823 2319 V 18 w(fn)75 2376 y Fv(w)o(as)f(executed\),)i(and)f(a)f(new)i (v)m(alue)g(w)o(as)e(next)h(stored.)27 b(The)18 b(call)h(is)g (erroneous)f(if)g(there)g(is)g(no)g(k)o(ey)75 2432 y(with)h(v)m(alue)h Fw(k)o(eyval)p Fv(;)f(in)g(particular)h Fi(MPI)p 798 2432 13 2 v 14 w(KEYV)m(AL)p 969 2432 V 14 w(INV)m(ALID)d Fv(is)i(an)f(erroneous)h(k)o(ey)f(v)m(alue.)31 b(The)19 b(call)63 b Fx(>)75 2488 y Fv(will)17 b(fail)f(if)g(the)f Fw(delete)p 471 2488 14 2 v 18 w(fn)g Fv(function)h(returned)g(an)f (error)f(co)q(de)i(other)f(than)g Fi(MPI)p 1482 2488 13 2 v 14 w(SUCCESS)p Fv(.)253 b Fx(?)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 172 180 172 179 bop 75 -100 a Fv(172)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(A)l(TTR)p 293 45 V 17 w(GET\(comm,)e(k)o(eyval,)h(attribute)p 865 45 V 18 w(val,)h(\015ag\)) 117 122 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)10 b(to)i(whic)o(h)g(attribute)h(is)f(attac)o(hed)h(\(handle\))117 197 y(IN)155 b Fw(k)o(eyval)471 b Fk(k)o(ey)14 b(v)n(alue)f(\(in)o (teger\))117 273 y(OUT)108 b Fw(attribute)p 484 273 V 18 w(val)352 b Fk(attribute)15 b(v)n(alue,)d(unless)j Fi(\015ag)f Fk(=)g(false)117 348 y(OUT)108 b Fw(\015ag)518 b Fb(true)14 b Fk(if)h(an)f(attribute)i(v)n(alue)e(w)o(as)h(extracted;) i Fb(false)d Fk(if)g(no)905 404 y(attribute)h(is)e(asso)q(ciated)i (with)f(the)g(k)o(ey)-160 480 y Fx(>)75 529 y Ft(int)23 b(MPI)p 245 529 15 2 v 17 w(Attr)p 358 529 V 17 w(get\(MPI)p 543 529 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void)g(*attribute)p 1467 529 V 16 w(val,)g(int)h(*flag\))-160 569 y Fx(?)75 615 y Ft(MPI)p 150 615 V 17 w(ATTR)p 263 615 V 16 w(GET\(COMM,)f (KEYVAL,)g(ATTRIBUTE)p 925 615 V 16 w(VAL,)g(FLAG,)g(IERROR\))170 672 y(INTEGER)g(COMM,)h(KEYVAL,)e(ATTRIBUTE)p 914 672 V 16 w(VAL,)i(IERROR)170 728 y(LOGICAL)f(FLAG)166 814 y Fv(Retriev)o(es)c(attribute)g(v)m(alue)h(b)o(y)f(k)o(ey)l(.)30 b(The)19 b(call)h(is)f(erroneous)g(if)g(there)g(is)g(no)g(k)o(ey)g (with)g(v)m(alue)75 871 y Fw(k)o(eyval)p Fv(.)30 b(On)19 b(the)g(other)f(hand,)h(the)g(call)h(is)f(correct)f(if)h(the)g(k)o(ey)f (v)m(alue)i(exists,)g(but)e(no)h(attribute)f(is)75 927 y(attac)o(hed)e(on)h Ft(comm)f Fv(for)g(that)g(k)o(ey;)g(in)i(suc)o(h)f (case,)g(the)f(call)i(returns)f Ft(flag)23 b(=)h(false)p Fv(.)f(In)17 b(particular)75 984 y Fi(MPI)p 152 984 13 2 v 14 w(KEYV)m(AL)p 323 984 V 14 w(INV)m(ALID)d Fv(is)i(an)f (erroneous)g(k)o(ey)g(v)m(alue.)-1177 b Fx(>)189 1090 y Fl(A)n(dvic)n(e)19 b(to)i(users.)66 b Fv(The)20 b(call)h(to)e Fw(MPI)p 910 1090 14 2 v 15 w(A)o(ttr)p 1002 1090 V 17 w(put)i Fv(passes)f(in)g Fw(attribute)p 1464 1090 V 18 w(val)f Fv(the)h Fl(value)g Fv(of)f(the)189 1147 y(attribute;)12 b(the)g(call)g(to)f Fw(MPI)p 682 1147 V 16 w(A)o(ttr)p 775 1147 V 17 w(get)h Fv(passes)f(in)i Fw(attribute)p 1209 1147 V 18 w(val)e Fv(the)h Fl(addr)n(ess)f Fv(of)g(the)g(the)h(lo) q(cation)189 1203 y(where)19 b(the)g(attribute)g(v)m(alue)h(is)g(to)e (b)q(e)i(returned.)32 b(Th)o(us,)19 b(if)h(the)f(attribute)g(v)m(alue)h (itself)g(is)f(a)189 1259 y(p)q(oin)o(ter)f(of)g(t)o(yp)q(e)h Fi(void*)p Fv(,)f(the)g(the)h(actual)f Fw(attribute)p 1100 1259 V 18 w(val)g Fv(parameter)g(to)f Fw(MPI)p 1551 1259 V 16 w(A)o(ttr)p 1644 1259 V 17 w(put)i Fv(will)h(b)q(e)189 1316 y(of)e(t)o(yp)q(e)g Fi(void*)h Fv(and)f(the)h(actual)g Fw(attribute)p 939 1316 V 18 w(val)f Fv(parameter)f(to)h Fw(MPI)p 1390 1316 V 16 w(A)o(ttr)p 1483 1316 V 17 w(put)i Fv(will)g(b)q(e)f(of)f(t)o(yp)q(e)189 1372 y Fi(void**)p Fv(.)h(\()p Fl(End)d(of)g(advic)n(e)g(to)h(users.)p Fv(\))189 1479 y Fl(R)n(ationale.)62 b Fv(The)19 b(use)g(of)f(a)h(formal)f (parameter)h Fw(attribute)p 1263 1479 V 18 w(val)f Fv(or)g(t)o(yp)q(e)h Fi(void*)g Fv(\(rather)f(than)189 1535 y Fi(void**)p Fv(\))e(a)o(v)o(oids)h(the)h(messy)f(t)o(yp)q(e)h(casting)f(that)g(w)o (ould)h(b)q(e)g(needed)g(if)g(the)g(attribute)f(v)m(alue)h(is)189 1592 y(declared)e(with)g(a)e(t)o(yp)q(e)i(other)f(than)g Fi(void*)p Fv(.)k(\()p Fl(End)d(of)g(r)n(ationale.)p Fv(\))-160 1653 y Fx(?)75 1792 y Fw(MPI)p 160 1792 V 16 w(A)l(TTR)p 293 1792 V 17 w(DELETE\(comm,)c(k)o(eyval\))117 1870 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)10 b(to)i(whic)o(h)g(attribute)h(is)f(attac)o(hed)h(\(handle\))117 1945 y(IN)155 b Fw(k)o(eyval)471 b Fk(The)15 b(k)o(ey)e(v)n(alue)h(of)f (the)i(deleted)f(attribute)h(\(in)o(teger\))75 2069 y Ft(int)23 b(MPI)p 245 2069 15 2 v 17 w(Attr)p 358 2069 V 17 w(delete\(MPI)p 615 2069 V 16 w(Comm)g(comm,)g(int)h(keyval\))75 2156 y(MPI)p 150 2156 V 17 w(ATTR)p 263 2156 V 16 w(DELETE\(COMM,)f (KEYVAL,)g(IERROR\))170 2212 y(INTEGER)g(COMM,)h(KEYVAL,)e(IERROR)166 2298 y Fv(Delete)14 b(attribute)g(from)f(cac)o(he)h(b)o(y)f(k)o(ey)l(.) 20 b(This)14 b(function)g(in)o(v)o(ok)o(es)g(the)g(attribute)f(delete)i (function)75 2355 y Fw(delete)p 188 2355 14 2 v 18 w(fn)j Fv(sp)q(eci\014ed)i(when)e(the)g Fw(k)o(eyval)f Fv(w)o(as)g(created.)28 b(The)18 b(call)h(will)h(fail)e(if)g(the)g Fw(delete)p 1642 2355 V 18 w(fn)g Fv(function)-2034 b Fx(>)75 2411 y Fv(returns)15 b(an)g(error)g(co)q(de)g(other)g(than)g Fi(MPI)p 813 2411 13 2 v 15 w(SUCCESS)p Fv(.)-1176 b Fx(?)166 2468 y Fv(Whenev)o(er)16 b(a)g(comm)o(unicator)f(is)h (replicated)i(using)e(the)g(function)h Fw(MPI)p 1435 2468 14 2 v 16 w(COMM)p 1593 2468 V 16 w(DUP)p Fv(,)f(all)h(call-)75 2524 y(bac)o(k)h(cop)o(y)f(functions)i(for)e(attributes)g(that)g(are)h (curren)o(tly)g(set)f(are)h(in)o(v)o(ok)o(ed)g(\(in)g(arbitrary)f (order\).)75 2581 y(Whenev)o(er)23 b(a)f(comm)o(unicator)g(is)g (deleted)i(using)f(the)g(function)g Fw(MPI)p 1343 2581 V 15 w(COMM)p 1500 2581 V 17 w(FREE)g Fv(all)g(callbac)o(k)75 2637 y(delete)16 b(functions)g(for)f(attributes)g(that)f(are)h(curren)o (tly)h(set)f(are)g(in)o(v)o(ok)o(ed.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 173 181 173 180 bop 75 -100 a Fm(5.7.)29 b(CA)o(CHING)1412 b Fv(173)75 45 y Fj(5.7.2)49 b(A)o(ttributes)14 b(Example)189 131 y Fl(A)n(dvic)n(e)22 b(to)h(users.)83 b Fv(This)23 b(example)h(sho)o(ws)e(ho)o(w)g(to)f(write)i(a)f(collectiv)o(e)i(comm)o (unication)189 187 y(op)q(eration)16 b(that)f(uses)i(cac)o(hing)g(to)e (b)q(e)i(more)f(e\016cien)o(t)h(after)e(the)i(\014rst)e(call.)24 b(The)17 b(co)q(ding)g(st)o(yle)189 244 y(assumes)11 b(that)h Fw(MPI)f Fv(function)i(results)f(return)g(only)h(error)e (statuses.)18 b(\()p Fl(End)13 b(of)g(advic)n(e)g(to)h(users.)p Fv(\))147 332 y Ft(/*)23 b(key)h(for)f(this)g(module's)g(stuff:)g(*/) 147 389 y(static)g(int)g(gop_key)g(=)h(MPI_KEYVAL_INVALID;)147 502 y(typedef)e(struct)147 558 y({)218 615 y(int)i(ref_count;)237 b(/*)23 b(reference)g(count)g(*/)218 671 y(/*)h(other)f(stuff,)g (whatever)g(else)g(we)h(want)f(*/)147 728 y(})g(gop_stuff_type;)147 841 y(Efficient_Collective_Op)d(\(comm,)j(...\))147 897 y(MPI_Comm)f(comm;)147 953 y({)194 1010 y(gop_stuff_type)g(*gop_stuff;) 194 1066 y(MPI_Group)166 b(group;)194 1123 y(int)310 b(foundflag;)194 1236 y(MPI_Comm_group\(comm,)21 b(&group\);)194 1349 y(if)j(\(gop_key)f(==)g(MPI_KEYVAL_INVALID\))f(/*)h(get)h(a)f(key) h(on)f(first)h(call)f(ever)g(*/)194 1405 y({)242 1462 y(if)h(\()f(!)h(MPI_keyval_create\()d(gop_stuff_copier,)839 1518 y(gop_stuff_destructor,)839 1574 y(&gop_key,)h(\(void)h(*\)0\)\);) 242 1631 y(/*)h(get)f(the)h(key)f(while)g(assigning)g(its)g(copy)h(and) f(delete)g(callback)314 1687 y(behavior.)f(*/)242 1800 y(MPI_Abort)h(\(comm,)g(99\);)194 1857 y(})194 1970 y(MPI_Attr_get)g (\(comm,)g(gop_key,)f(&gop_stuff,)h(&foundflag\);)194 2026 y(if)h(\(foundflag\))194 2083 y({)g(/*)g(This)f(module)g(has)g (executed)g(in)h(this)f(group)g(before.)314 2139 y(We)g(will)h(use)f (the)g(cached)g(information)g(*/)194 2195 y(})194 2252 y(else)194 2308 y({)h(/*)g(This)f(is)h(a)f(group)g(that)h(we)f(have)h (not)f(yet)h(cached)f(anything)f(in.)314 2365 y(We)h(will)h(now)f(do)h (so.)242 2421 y(*/)242 2534 y(/*)g(First,)f(allocate)f(storage)h(for)h (the)f(stuff)g(we)h(want,)314 2591 y(and)f(initialize)g(the)g (reference)g(count)g(*/)242 2704 y(gop_stuff)g(=)g(\(gop_stuff_type)f (*\))i(malloc)f(\(sizeof\(gop_stuff_type\)\);)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 174 182 174 181 bop 75 -100 a Fv(174)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)242 45 y Ft(if)24 b(\(gop_stuff)e(==)i(NULL\))f({)h(/*)f(abort)g(on)h (out-of-memory)e(error)h(*/)h(})242 158 y(gop_stuff)f(->)g(ref_count)g (=)h(1;)242 271 y(/*)g(Second,)f(fill)g(in)g(*gop_stuff)g(with)g (whatever)g(we)h(want.)314 327 y(This)f(part)g(isn't)g(shown)h(here)f (*/)242 440 y(/*)h(Third,)f(store)g(gop_stuff)f(as)i(the)g(attribute)e (value)h(*/)242 497 y(MPI_Attr_put)f(\()i(comm,)f(gop_key,)g (gop_stuff\);)194 553 y(})194 610 y(/*)h(Then,)f(in)h(any)f(case,)g (use)h(contents)f(of)g(*gop_stuff)266 666 y(to)g(do)h(the)g(global)f (op)g(...)h(*/)147 723 y(})147 835 y(/*)f(The)h(following)e(routine)h (is)h(called)f(by)g(MPI)h(when)f(a)h(group)f(is)h(freed)f(*/)147 948 y(gop_stuff_destructor)e(\(comm,)i(keyval,)g(gop_stuff,)f(extra\)) 147 1005 y(MPI_Comm)g(comm;)147 1061 y(int)h(keyval;)147 1118 y(gop_stuff_type)f(*gop_stuff;)147 1174 y(void)h(*extra;)147 1231 y({)194 1287 y(if)h(\(keyval)f(!=)g(gop_key\))g({)h(/*)f(abort)h (--)f(programming)g(error)g(*/)g(})194 1400 y(/*)h(The)f(group's)g (being)g(freed)h(removes)f(one)g(reference)g(to)g(gop_stuff)g(*/)194 1456 y(gop_stuff)g(->)h(ref_count)e(-=)i(1;)194 1569 y(/*)g(If)f(no)h(references)f(remain,)f(then)i(free)f(the)h(storage)e (*/)194 1626 y(if)i(\(gop_stuff)e(->)i(ref_count)f(==)g(0\))h({)242 1682 y(free\(\(void)f(*\)gop_stuff\);)194 1739 y(})147 1795 y(})147 1908 y(/*)g(The)h(following)e(routine)h(is)h(called)f(by)g (MPI)h(when)f(a)h(group)f(is)h(copied)f(*/)147 1965 y(gop_stuff_copier) e(\(comm,)i(keyval,)g(extra,)g(gop_stuff_in,)f(gop_stuff_out,)g(flag\)) 147 2021 y(MPI_Comm)g(comm;)147 2077 y(int)h(keyval;)147 2134 y(gop_stuff_type)f(*gop_stuff_in,)f(*gop_stuff_out;)147 2190 y(void)i(*extra;)147 2247 y({)194 2303 y(if)h(\(keyval)f(!=)g (gop_key\))g({)h(/*)f(abort)h(--)f(programming)g(error)g(*/)g(})194 2416 y(/*)h(The)f(new)h(group)f(adds)g(one)h(reference)e(to)i(this)f (gop_stuff)g(*/)194 2473 y(gop_stuff)g(->)h(ref_count)e(+=)i(1;)194 2529 y(gop_stuff_out)e(=)i(gop_stuff_in;)147 2586 y(})1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 175 183 175 182 bop 75 -100 a Fm(5.8.)34 b(F)o(ORMALIZING)16 b(THE)f(LOOSEL)l(Y)i(SYNCHR)o(ONOUS)g(MODEL)394 b Fv(175)75 45 y Fn(5.8)59 b(F)n(o)n(rmalizing)21 b(the)e(Lo)r(osely)g(Synchronous) g(Mo)r(del)75 147 y Fv(In)i(this)h(section,)g(w)o(e)f(mak)o(e)f (further)h(statemen)o(ts)e(ab)q(out)i(the)g(lo)q(osely)h(sync)o (hronous)f(mo)q(del,)i(with)75 203 y(particular)16 b(atten)o(tion)f(to) f(in)o(tra-comm)o(unication.)75 325 y Fj(5.8.1)49 b(Basic)17 b(Statements)75 411 y Fv(When)e(a)g(caller)h(passes)e(a)h(comm)o (unicator)f(\(that)g(con)o(tains)h(a)g(con)o(text)f(and)h(group\))f(to) g(a)h(callee,)h(that)75 467 y(comm)o(unicator)g(m)o(ust)h(b)q(e)g(free) g(of)f(side)i(e\013ects)e(throughout)g(execution)i(of)f(the)g (subprogram:)22 b(there)75 523 y(should)16 b(b)q(e)f(no)g(activ)o(e)g (op)q(erations)g(on)g(that)f(comm)o(unicator)g(that)g(migh)o(t)h(in)o (v)o(olv)o(e)g(the)g(pro)q(cess.)20 b(This)75 580 y(pro)o(vides)i(one)g (mo)q(del)h(in)f(whic)o(h)h(libraries)g(can)f(b)q(e)h(written,)g(and)f (w)o(ork)f(\\safely)l(.")39 b(F)l(or)21 b(libraries)75 636 y(so)e(designated,)h(the)g(callee)h(has)e(p)q(ermission)h(to)f(do)g (whatev)o(er)g(comm)o(unication)h(it)f(lik)o(es)h(with)g(the)75 693 y(comm)o(unicator,)e(and)h(under)g(the)f(ab)q(o)o(v)o(e)g(guaran)o (tee)g(kno)o(ws)g(that)f(no)h(other)g(comm)o(unications)h(will)75 749 y(in)o(terfere.)36 b(Since)22 b(w)o(e)f(p)q(ermit)g(go)q(o)q(d)f (implemen)o(tations)i(to)e(create)g(new)h(comm)o(unicators)f(without)75 806 y(sync)o(hronization)14 b(\(suc)o(h)g(as)f(b)o(y)g(preallo)q(cated) i(con)o(texts)e(on)g(comm)o(unicators\),)g(this)g(do)q(es)h(not)f(imp)q (ose)75 862 y(a)i(signi\014can)o(t)h(o)o(v)o(erhead.)166 919 y(This)k(form)f(of)g(safet)o(y)f(is)i(analogous)f(to)g(other)g (common)g(computer-science)j(usages,)e(suc)o(h)f(as)75 975 y(passing)c(a)f(descriptor)h(of)f(an)h(arra)o(y)e(to)h(a)g(library) i(routine.)k(The)15 b(library)g(routine)g(has)g(ev)o(ery)f(righ)o(t)h (to)75 1032 y(exp)q(ect)h(suc)o(h)f(a)g(descriptor)h(to)f(b)q(e)g(v)m (alid)i(and)f(mo)q(di\014able.)75 1153 y Fj(5.8.2)49 b(Mo)q(dels)17 b(of)g(Execution)75 1239 y Fv(In)12 b(the)g(lo)q(osely)h (sync)o(hronous)f(mo)q(del,)h(transfer)e(of)h(con)o(trol)f(to)h(a)f Fq(parallel)16 b(pro)q(cedure)c Fv(is)g(e\013ected)g(b)o(y)75 1296 y(ha)o(ving)f(eac)o(h)g(executing)g(pro)q(cess)g(in)o(v)o(ok)o(e)g (the)f(pro)q(cedure.)19 b(The)11 b(in)o(v)o(o)q(cation)g(is)g(a)g (collectiv)o(e)h(op)q(eration:)75 1352 y(it)i(is)h(executed)g(b)o(y)f (all)h(pro)q(cesses)f(in)h(the)g(execution)g(group,)e(and)i(in)o(v)o(o) q(cations)f(are)g(similarly)i(ordered)75 1408 y(at)f(all)h(pro)q (cesses.)k(Ho)o(w)o(ev)o(er,)14 b(the)h(in)o(v)o(o)q(cation)h(need)g (not)f(b)q(e)h(sync)o(hronized.)166 1465 y(W)l(e)d(sa)o(y)g(that)g(a)g (parallel)i(pro)q(cedure)f(is)g Fl(active)f Fv(in)h(a)g(pro)q(cess)f (if)h(the)f(pro)q(cess)h(b)q(elongs)g(to)f(a)g(group)75 1521 y(that)j(ma)o(y)h(collectiv)o(ely)i(execute)f(the)f(pro)q(cedure,) h(and)f(some)f(mem)o(b)q(er)h(of)g(that)f(group)h(is)g(curren)o(tly)75 1578 y(executing)c(the)f(pro)q(cedure)h(co)q(de.)19 b(If)13 b(a)e(parallel)j(pro)q(cedure)f(is)f(activ)o(e)g(in)h(a)f(pro)q(cess,)g (then)g(this)h(pro)q(cess)75 1634 y(ma)o(y)e(b)q(e)i(receiving)g (messages)e(p)q(ertaining)i(to)f(this)g(pro)q(cedure,)h(ev)o(en)f(if)g (it)g(do)q(es)h(not)e(curren)o(tly)h(execute)75 1691 y(the)j(co)q(de)h(of)f(this)g(pro)q(cedure.)75 1811 y Fw(Static)h(communicato)o(r)e(allo)q(cation)75 1897 y Fv(This)22 b(co)o(v)o(ers)f(the)h(case)g(where,)h(at)e(an)o(y)g(p)q (oin)o(t)h(in)h(time,)g(at)e(most)g(one)h(in)o(v)o(o)q(cation)g(of)f(a) g(parallel)75 1953 y(pro)q(cedure)c(can)f(b)q(e)h(activ)o(e)f(at)f(an)o (y)h(pro)q(cess,)g(and)g(the)g(group)f(of)h(executing)h(pro)q(cesses)f (is)h(\014xed.)23 b(F)l(or)75 2010 y(example,)d(all)g(in)o(v)o(o)q (cations)f(of)g(parallel)h(pro)q(cedures)g(in)o(v)o(olv)o(e)f(all)h (pro)q(cesses,)g(pro)q(cesses)f(are)f(single-)75 2066 y(threaded,)d(and)h(there)f(are)g(no)g(recursiv)o(e)h(in)o(v)o(o)q (cations.)166 2122 y(In)k(suc)o(h)g(a)g(case,)g(a)g(comm)o(unicator)f (can)h(b)q(e)g(statically)h(allo)q(cated)f(to)g(eac)o(h)f(pro)q (cedure.)35 b(The)75 2179 y(static)18 b(allo)q(cation)h(can)f(b)q(e)h (done)g(in)g(a)e(pream)o(ble,)j(as)d(part)h(of)g(initialization)j(co)q (de.)29 b(If)18 b(the)g(parallel)75 2235 y(pro)q(cedures)g(can)f(b)q(e) g(organized)g(in)o(to)g(libraries,)h(so)f(that)f(only)h(one)g(pro)q (cedure)h(of)e(eac)o(h)h(library)h(can)75 2292 y(b)q(e)e(concurren)o (tly)g(activ)o(e)g(in)g(eac)o(h)f(pro)q(cessor,)g(then)h(it)g(is)g (su\016cien)o(t)g(to)f(allo)q(cate)h(one)f(comm)o(unicator)75 2348 y(p)q(er)h(library)l(.)75 2468 y Fw(Dynamic)e(communicato)o(r)g (allo)q(cation)75 2554 y Fv(Calls)h(of)g(parallel)h(pro)q(cedures)g (are)e(w)o(ell-nested)i(if)g(a)e(new)h(parallel)h(pro)q(cedure)g(is)f (alw)o(a)o(ys)f(in)o(v)o(ok)o(ed)h(in)75 2611 y(a)j(subset)h(of)e(a)h (group)g(executing)i(the)e(same)g(parallel)i(pro)q(cedure.)30 b(Th)o(us,)19 b(pro)q(cesses)g(that)e(execute)75 2667 y(the)e(same)g(parallel)i(pro)q(cedure)f(ha)o(v)o(e)f(the)g(same)g (execution)h(stac)o(k.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 176 184 176 183 bop 75 -100 a Fv(176)365 b Fm(CHAPTER)15 b(5.)35 b(GR)o(OUPS,)15 b(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)166 45 y Fv(In)j(suc)o(h)g(a)g(case,)f(a)h(new)g(comm)o(unicator)f(needs)h (to)f(b)q(e)i(dynamically)g(allo)q(cated)g(for)e(eac)o(h)h(new)75 102 y(in)o(v)o(o)q(cation)11 b(of)f(a)g(parallel)i(pro)q(cedure.)19 b(The)11 b(allo)q(cation)g(is)g(done)g(b)o(y)f(the)h(caller.)19 b(A)11 b(new)f(comm)o(unicator)75 158 y(can)k(b)q(e)h(generated)g(b)o (y)f(a)g(call)h(to)f Fw(MPI)p 746 158 14 2 v 15 w(COMM)p 903 158 V 17 w(DUP)p Fv(,)g(if)g(the)h(callee)g(execution)h(group)e(is) g(iden)o(tical)j(to)75 214 y(the)f(caller)h(execution)g(group,)e(or)g (b)o(y)h(a)f(call)i(to)e Fw(MPI)p 1005 214 V 16 w(COMM)p 1163 214 V 16 w(SPLIT)g Fv(if)i(the)f(caller)g(execution)h(group)75 271 y(is)f(split)h(in)o(to)f(sev)o(eral)f(subgroups)h(executing)h (distinct)g(parallel)g(routines.)22 b(The)15 b(new)h(comm)o(unicator)75 327 y(is)g(passed)f(as)g(an)g(argumen)o(t)g(to)f(the)h(in)o(v)o(ok)o (ed)h(routine.)166 384 y(The)h(need)h(for)e(generating)h(a)f(new)h (comm)o(unicator)f(at)h(eac)o(h)f(in)o(v)o(o)q(cation)i(can)f(b)q(e)g (alleviated)h(or)75 440 y(a)o(v)o(oided)e(altogether)e(in)j(some)e (cases:)20 b(If)15 b(the)h(execution)g(group)f(is)h(not)f(split,)h (then)f(one)h(can)f(allo)q(cate)75 497 y(a)g(stac)o(k)g(of)g(comm)o (unicators)h(in)g(a)f(pream)o(ble,)h(and)g(next)g(manage)f(the)h(stac)o (k)f(in)h(a)f(w)o(a)o(y)g(that)g(mimics)75 553 y(the)g(stac)o(k)g(of)f (recursiv)o(e)i(calls.)166 610 y(One)h(can)f(also)g(tak)o(e)f(adv)m(an) o(tage)g(of)h(the)g(w)o(ell-ordering)h(prop)q(ert)o(y)f(of)f(comm)o (unication)i(to)e(a)o(v)o(oid)75 666 y(confusing)g(caller)g(and)f (callee)h(comm)o(unication,)f(ev)o(en)h(if)f(b)q(oth)g(use)g(the)g (same)f(comm)o(unicator.)19 b(T)l(o)14 b(do)75 723 y(so,)g(one)i(needs) g(to)e(abide)j(b)o(y)e(the)g(follo)o(wing)h(t)o(w)o(o)e(rules:)143 816 y Fx(\017)23 b Fv(messages)13 b(sen)o(t)g(b)q(efore)h(a)f(pro)q (cedure)h(call)h(\(or)e(b)q(efore)h(a)f(return)g(from)g(the)h(pro)q (cedure\))g(are)f(also)189 873 y(receiv)o(ed)j(b)q(efore)f(the)h(matc)o (hing)f(call)h(\(or)f(return\))g(at)f(the)h(receiving)i(end;)143 967 y Fx(\017)23 b Fv(messages)14 b(are)h(alw)o(a)o(ys)g(selected)h(b)o (y)f(source)g(\(no)g(use)h(is)g(made)f(of)g Fi(MPI)p 1430 967 13 2 v 14 w(ANY)p 1529 967 V 15 w(SOURCE)p Fv(\).)75 1087 y Fw(The)h(General)f(case)75 1173 y Fv(In)22 b(the)g(general)g (case,)h(there)e(ma)o(y)g(b)q(e)h(m)o(ultiple)i(concurren)o(tly)e (activ)o(e)g(in)o(v)o(o)q(cations)g(of)f(the)g(same)75 1229 y(parallel)i(pro)q(cedure)g(within)g(the)f(same)f(group;)j(in)o(v) o(o)q(cations)e(ma)o(y)f(not)h(b)q(e)g(w)o(ell-nested.)41 b(A)22 b(new)75 1285 y(comm)o(unicator)11 b(needs)i(to)e(b)q(e)i (created)e(for)h(eac)o(h)f(in)o(v)o(o)q(cation.)20 b(It)11 b(is)i(the)f(user's)f(resp)q(onsibilit)o(y)j(to)d(mak)o(e)75 1342 y(sure)18 b(that,)f(should)h(t)o(w)o(o)e(distinct)j(parallel)g (pro)q(cedures)f(b)q(e)g(in)o(v)o(ok)o(ed)g(concurren)o(tly)g(on)f(o)o (v)o(erlapping)75 1398 y(sets)e(of)g(pro)q(cesses,)g(then)g(comm)o (unicator)g(creation)h(b)q(e)f(prop)q(erly)i(co)q(ordinated.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 177 185 177 184 bop 75 356 a Fo(Chapter)34 b(6)75 564 y Fr(Pro)s(cess)40 b(T)-10 b(op)s(ologi)q(es)75 805 y Fn(6.1)59 b(Intro)r(duction)75 906 y Fv(This)22 b(c)o(hapter)g(discusses)h(the)f Fw(MPI)f Fv(top)q(ology)g(mec)o(hanism.)40 b(A)22 b(top)q(ology)f(is)h(an)g (extra,)g(optional)75 963 y(attribute)14 b(that)g(one)g(can)h(giv)o(e)g (to)e(an)i(in)o(tra-comm)o(unicator;)f(top)q(ologies)g(cannot)h(b)q(e)g (added)g(to)f(in)o(ter-)75 1019 y(comm)o(unicators.)19 b(A)d(top)q(ology)e(can)i(pro)o(vide)f(a)g(con)o(v)o(enien)o(t)h (naming)f(mec)o(hanism)h(for)e(the)i(pro)q(cesses)75 1076 y(of)j(a)f(group)h(\(within)h(a)e(comm)o(unicator\),)h(and)g (additionally)l(,)j(ma)o(y)c(assist)h(the)g(run)o(time)h(system)e(in)75 1132 y(mapping)e(the)f(pro)q(cesses)h(on)o(to)e(hardw)o(are.)166 1188 y(As)19 b(stated)g(in)h(c)o(hapter)f(5,)h(a)f(pro)q(cess)h(group)f (in)h Fw(MPI)e Fv(is)i(a)f(collection)i(of)e Ft(n)g Fv(pro)q(cesses.)33 b(Eac)o(h)75 1245 y(pro)q(cess)17 b(in)h(the)e(group)h(is)g(assigned)g (a)g(rank)f(b)q(et)o(w)o(een)h Ft(0)g Fv(and)g Ft(n-1)p Fv(.)24 b(In)17 b(man)o(y)f(parallel)i(applications)75 1301 y(a)c(linear)h(ranking)g(of)f(pro)q(cesses)h(do)q(es)f(not)g (adequately)h(re\015ect)g(the)f(logical)h(comm)o(unication)g(pattern)75 1358 y(of)21 b(the)g(pro)q(cesses)g(\(whic)o(h)h(is)g(usually)g (determined)g(b)o(y)f(the)h(underlying)h(problem)f(geometry)e(and)75 1414 y(the)f(n)o(umerical)h(algorithm)g(used\).)31 b(Often)20 b(the)f(pro)q(cesses)g(are)g(arranged)g(in)h(top)q(ological)f(patterns) 75 1471 y(suc)o(h)c(as)g(t)o(w)o(o-)e(or)i(three-dimensional)i(grids.)j (More)14 b(generally)l(,)i(the)f(logical)h(pro)q(cess)g(arrangemen)o(t) d(is)75 1527 y(describ)q(ed)19 b(b)o(y)f(a)f(graph.)25 b(In)18 b(this)g(c)o(hapter)f(w)o(e)g(will)i(refer)f(to)e(this)i (logical)g(pro)q(cess)g(arrangemen)o(t)e(as)75 1584 y(the)f(\\virtual)h (top)q(ology)l(.")166 1640 y(A)11 b(clear)g(distinction)i(m)o(ust)e(b)q (e)g(made)g(b)q(et)o(w)o(een)g(the)h(virtual)f(pro)q(cess)g(top)q (ology)g(and)g(the)g(top)q(ology)75 1697 y(of)j(the)h(underlying,)i(ph) o(ysical)f(hardw)o(are.)j(The)c(virtual)g(top)q(ology)f(can)h(b)q(e)h (exploited)g(b)o(y)f(the)f(system)75 1753 y(in)j(the)f(assignmen)o(t)h (of)e(pro)q(cesses)i(to)f(ph)o(ysical)h(pro)q(cessors,)f(if)h(this)g (helps)g(to)f(impro)o(v)o(e)g(the)g(comm)o(u-)75 1810 y(nication)i(p)q(erformance)f(on)g(a)g(giv)o(en)h(mac)o(hine.)26 b(Ho)o(w)17 b(this)g(mapping)h(is)g(done,)f(ho)o(w)o(ev)o(er,)g(is)g (outside)75 1866 y(the)e(scop)q(e)g(of)f Fw(MPI)p Fv(.)g(The)g (description)j(of)d(the)g(virtual)i(top)q(ology)l(,)e(on)h(the)f(other) g(hand,)h(dep)q(ends)h(only)75 1922 y(on)i(the)g(application,)i(and)e (is)h(mac)o(hine-indep)q(endent.)30 b(The)19 b(functions)f(that)g(are)f (prop)q(osed)i(in)g(this)75 1979 y(c)o(hapter)c(deal)h(only)g(with)f (mac)o(hine-indep)q(en)q(den)o(t)j(mapping.)189 2067 y Fl(R)n(ationale.)55 b Fv(Though)18 b(ph)o(ysical)h(mapping)g(is)f (not)f(discussed,)j(the)e(existence)h(of)e(the)h(virtual)189 2123 y(top)q(ology)d(information)i(ma)o(y)e(b)q(e)i(used)f(as)g(advice) h(b)o(y)f(the)g(run)o(time)h(system.)22 b(There)16 b(are)g(w)o(ell-)189 2179 y(kno)o(wn)h(tec)o(hniques)h(for)f(mapping)h(grid/torus)f (structures)g(to)g(hardw)o(are)g(top)q(ologies)h(suc)o(h)f(as)189 2236 y(h)o(yp)q(ercub)q(es)k(or)f(grids.)35 b(F)l(or)20 b(more)g(complicated)h(graph)f(structures)g(go)q(o)q(d)h(heuristics)g (often)189 2292 y(yield)e(nearly)e(optimal)h(results)g([20)o(].)25 b(On)18 b(the)f(other)g(hand,)h(if)g(there)f(is)h(no)f(w)o(a)o(y)f(for) h(the)g(user)189 2349 y(to)12 b(sp)q(ecify)j(the)f(logical)g(pro)q (cess)g(arrangemen)o(t)e(as)h(a)g(\\virtual)h(top)q(ology)l(,")f(a)g (random)g(mapping)189 2405 y(is)k(most)f(lik)o(ely)j(to)d(result.)26 b(On)17 b(some)g(mac)o(hines,)g(this)h(will)g(lead)g(to)e(unnecessary)i (con)o(ten)o(tion)189 2462 y(in)g(the)f(in)o(terconnection)i(net)o(w)o (ork.)25 b(Some)17 b(details)h(ab)q(out)f(predicted)i(and)e(measured)h (p)q(erfor-)189 2518 y(mance)d(impro)o(v)o(emen)o(ts)f(that)g(result)i (from)e(go)q(o)q(d)h(pro)q(cess-to-pro)q(cessor)f(mapping)i(on)f(mo)q (dern)189 2575 y(w)o(ormhole-routing)g(arc)o(hitectures)g(can)h(b)q(e)f (found)h(in)g([10)o(,)f(9].)189 2647 y(Besides)22 b(p)q(ossible)g(p)q (erformance)f(b)q(ene\014ts,)h(the)f(virtual)g(top)q(ology)g(can)g (function)g(as)f(a)h(con-)189 2704 y(v)o(enien)o(t,)16 b(pro)q(cess-naming)h(structure,)f(with)g(tremendous)g(b)q(ene\014ts)h (for)f(program)f(readabilit)o(y)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 178 186 178 185 bop 75 -100 a Fv(178)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)189 45 y Fv(and)g(notational)g(p)q(o)o(w)o (er)g(in)h(message-passing)f(programming.)20 b(\()p Fl(End)15 b(of)h(r)n(ationale.)p Fv(\))75 188 y Fn(6.2)59 b(Virtual)20 b(T)-5 b(op)r(ologies)75 290 y Fv(The)22 b(comm)o(unication)h(pattern)e (of)g(a)h(set)g(of)f(pro)q(cesses)h(can)g(b)q(e)h(represen)o(ted)f(b)o (y)g(a)f(graph.)40 b(The)75 346 y(no)q(des)14 b(stand)f(for)f(the)i (pro)q(cesses,)f(and)g(the)h(edges)f(connect)h(pro)q(cesses)f(that)g (comm)o(unicate)g(with)h(eac)o(h)75 403 y(other.)37 b Fw(MPI)21 b Fv(pro)o(vides)h(message-passing)f(b)q(et)o(w)o(een)h(an)o (y)e(pair)i(of)f(pro)q(cesses)g(in)h(a)f(group.)38 b(There)75 459 y(is)23 b(no)g(requiremen)o(t)g(for)f(op)q(ening)i(a)e(c)o(hannel)i (explicitly)l(.)46 b(Therefore,)24 b(a)e(\\missing)h(link")h(in)g(the) 75 516 y(user-de\014ned)19 b(pro)q(cess)e(graph)g(do)q(es)g(not)g(prev) o(en)o(t)g(the)g(corresp)q(onding)h(pro)q(cesses)g(from)e(exc)o (hanging)75 572 y(messages.)26 b(It)17 b(means)g(rather)g(that)g(this)h (connection)g(is)g(neglected)h(in)f(the)f(virtual)h(top)q(ology)l(.)26 b(This)75 629 y(strategy)21 b(implies)k(that)c(the)i(top)q(ology)f(giv) o(es)h(no)f(con)o(v)o(enien)o(t)h(w)o(a)o(y)f(of)g(naming)g(this)h (path)o(w)o(a)o(y)e(of)75 685 y(comm)o(unication.)28 b(Another)17 b(p)q(ossible)j(consequence)f(is)f(that)f(an)g(automatic)g (mapping)i(to)q(ol)e(\(if)h(one)75 741 y(exists)12 b(for)g(the)g(run)o (time)h(en)o(vironmen)o(t\))f(will)i(not)e(tak)o(e)f(accoun)o(t)h(of)g (this)g(edge)h(when)f(mapping.)20 b(Edges)75 798 y(in)13 b(the)g(comm)o(unication)g(graph)g(are)f(not)g(w)o(eigh)o(ted,)h(so)f (that)g(pro)q(cesses)h(are)f(either)h(simply)h(connected)75 854 y(or)h(not)f(connected)j(at)d(all.)189 961 y Fl(R)n(ationale.)55 b Fv(Exp)q(erience)20 b(with)e(similar)h(tec)o(hniques)g(in)g(P)l(ARMA) o(CS)f([5,)f(8])g(sho)o(w)g(that)h(this)189 1017 y(information)i(is)h (usually)h(su\016cien)o(t)f(for)e(a)h(go)q(o)q(d)g(mapping.)36 b(Additionally)l(,)24 b(a)c(more)g(precise)189 1074 y(sp)q (eci\014cation)e(is)f(more)f(di\016cult)i(for)e(the)g(user)h(to)f(set)g (up,)h(and)g(it)f(w)o(ould)h(mak)o(e)f(the)h(in)o(terface)189 1130 y(functions)f(substan)o(tially)g(more)f(complicated.)21 b(\()p Fl(End)15 b(of)i(r)n(ationale.)p Fv(\))166 1236 y(Sp)q(ecifying)22 b(the)e(virtual)g(top)q(ology)g(in)g(terms)g(of)f(a) g(graph)h(is)g(su\016cien)o(t)h(for)e(all)i(applications.)75 1293 y(Ho)o(w)o(ev)o(er,)12 b(in)i(man)o(y)e(applications)j(the)e (graph)g(structure)g(is)g(regular,)g(and)g(the)g(detailed)i(set-up)e (of)g(the)75 1349 y(graph)d(w)o(ould)i(b)q(e)f(incon)o(v)o(enien)o(t)h (for)e(the)h(user)g(and)g(migh)o(t)g(b)q(e)g(less)g(e\016cien)o(t)h(at) e(run)h(time.)19 b(A)10 b(large)h(frac-)75 1406 y(tion)j(of)g(all)h (parallel)h(applications)g(use)e(pro)q(cess)h(top)q(ologies)g(lik)o(e)g (rings,)f(t)o(w)o(o-)f(or)h(higher-dimensional)75 1462 y(grids,)19 b(or)f(tori.)28 b(These)19 b(structures)f(are)g(completely) h(de\014ned)h(b)o(y)e(the)h(n)o(um)o(b)q(er)f(of)g(dimensions)i(and)75 1519 y(the)15 b(n)o(um)o(b)q(ers)h(of)f(pro)q(cesses)g(in)h(eac)o(h)g (co)q(ordinate)f(direction.)22 b(Also,)15 b(the)g(mapping)h(of)f(grids) h(and)f(tori)75 1575 y(is)h(generally)h(an)f(easier)g(problem)g(then)g (that)f(of)g(general)h(graphs.)21 b(Th)o(us,)16 b(it)f(is)i(desirable)g (to)e(address)75 1631 y(these)g(cases)h(explicitly)l(.)166 1688 y(Pro)q(cess)i(co)q(ordinates)g(in)h(a)f(cartesian)g(structure)g (b)q(egin)h(their)f(n)o(um)o(b)q(ering)h(at)f(0.)28 b(Ro)o(w-ma)s(jor) 75 1744 y(n)o(um)o(b)q(ering)18 b(is)g(alw)o(a)o(ys)e(used)i(for)e(the) h(pro)q(cesses)h(in)f(a)g(cartesian)g(structure.)26 b(This)17 b(means)g(that,)g(for)75 1801 y(example,)i(the)f(relation)g(b)q(et)o(w) o(een)g(group)f(rank)h(and)g(co)q(ordinates)g(for)f(four)g(pro)q (cesses)h(in)h(a)e(\(2)11 b Fx(\002)i Fv(2\))75 1857 y(grid)j(is)f(as)g(follo)o(ws.)206 1949 y(co)q(ord)g(\(0,0\):)48 b(rank)15 b(0)206 2005 y(co)q(ord)g(\(0,1\):)48 b(rank)15 b(1)206 2062 y(co)q(ord)g(\(1,0\):)48 b(rank)15 b(2)206 2118 y(co)q(ord)g(\(1,1\):)48 b(rank)15 b(3)75 2249 y Fn(6.3)59 b(Emb)r(edding)18 b(in)i(MPI)75 2351 y Fv(The)13 b(supp)q(ort)f(for)g(virtual)h(top)q(ologies)g(as)f(de\014ned)h(in)h (this)e(c)o(hapter)h(is)f(consisten)o(t)h(with)g(other)f(parts)f(of)75 2407 y Fw(MPI)p Fv(,)k(and,)h(whenev)o(er)g(p)q(ossible,)h(mak)o(es)f (use)g(of)f(functions)i(that)e(are)h(de\014ned)h(elsewhere.)23 b(T)l(op)q(ology)75 2464 y(information)e(is)g(asso)q(ciated)h(with)f (comm)o(unicators.)36 b(It)21 b(is)h(added)f(to)g(comm)o(unicators)f (using)i(the)75 2520 y(cac)o(hing)16 b(mec)o(hanism)g(describ)q(ed)h (in)f(Chapter)f(5.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 179 187 179 186 bop 75 -100 a Fm(6.4.)34 b(O)o(VER)-5 b(VIEW)16 b(OF)f(THE)g(FUNCTIONS)888 b Fv(179)75 45 y Fn(6.4)59 b(Overview)19 b(of)h(the)f(F)n(unctions)75 147 y Fv(The)h(functions)h Fw(MPI)p 459 147 14 2 v 15 w(GRAPH)p 624 147 V 18 w(CREA)l(TE)g Fv(and)f Fw(MPI)p 1007 147 V 16 w(CART)p 1142 147 V 17 w(CREA)l(TE)h Fv(are)e(used)i(to)e(create)g(general)75 203 y(\(graph\))e(virtual)h(top)q(ologies)g(and)g(cartesian)g(top)q (ologies,)g(resp)q(ectiv)o(ely)l(.)29 b(These)18 b(top)q(ology)g (creation)75 259 y(functions)13 b(are)g(collectiv)o(e.)21 b(As)12 b(with)h(other)g(collectiv)o(e)h(calls,)g(the)f(program)f(m)o (ust)g(b)q(e)h(written)g(to)f(w)o(ork)75 316 y(correctly)l(,)j(whether) h(the)f(call)h(sync)o(hronizes)g(or)f(not.)166 372 y(The)21 b(top)q(ology)f(creation)h(functions)g(tak)o(e)f(as)g(input)i(an)e (existing)i(comm)o(unicator)e Fw(comm)p 1794 372 V 14 w(old)p Fv(,)75 429 y(whic)o(h)14 b(de\014nes)h(the)e(set)g(of)g(pro)q (cesses)h(on)g(whic)o(h)g(the)f(top)q(ology)g(is)h(to)f(b)q(e)h(mapp)q (ed.)20 b(A)14 b(new)f(comm)o(uni-)75 485 y(cator)e Fw(comm)p 308 485 V 15 w(top)q(ol)h Fv(is)h(created)f(that)f(carries)h(the)g(top) q(ological)h(structure)f(as)f(cac)o(hed)i(information)f(\(see)75 542 y(Chapter)f(5\).)18 b(In)12 b(analogy)f(to)g(function)i Fw(MPI)p 849 542 V 15 w(COMM)p 1006 542 V 17 w(CREA)l(TE)p Fv(,)f(no)f(cac)o(hed)h(information)g(propagates)75 598 y(from)i Fw(comm)p 302 598 V 15 w(old)h Fv(to)g Fw(comm)p 562 598 V 14 w(top)q(ol)p Fv(.)166 655 y Fw(MPI)p 251 655 V 16 w(CART)p 386 655 V 17 w(CREA)l(TE)j Fv(can)g(b)q(e)g(used)g (to)f(describ)q(e)i(cartesian)f(structures)f(of)g(arbitrary)g(dimen-)75 711 y(sion.)i(F)l(or)12 b(eac)o(h)h(co)q(ordinate)g(direction)g(one)g (sp)q(eci\014es)h(whether)f(the)f(pro)q(cess)h(structure)f(is)h(p)q (erio)q(dic)h(or)75 768 y(not.)19 b(Note)c(that)f(an)h Fp(n)p Fv(-dimensional)j(h)o(yp)q(ercub)q(e)e(is)g(an)e Fp(n)p Fv(-dimensional)k(torus)c(with)i(2)e(pro)q(cesses)h(p)q(er)75 824 y(co)q(ordinate)h(direction.)22 b(Th)o(us,)15 b(sp)q(ecial)i(supp)q (ort)e(for)g(h)o(yp)q(ercub)q(e)i(structures)e(is)h(not)f(necessary)l (.)21 b(The)75 880 y(lo)q(cal)13 b(auxiliary)h(function)f Fw(MPI)p 626 880 V 15 w(DIMS)p 752 880 V 16 w(CREA)l(TE)h Fv(can)e(b)q(e)h(used)f(to)g(compute)g(a)g(balanced)h(distribution)75 937 y(of)i(pro)q(cesses)g(among)g(a)g(giv)o(en)g(n)o(um)o(b)q(er)h(of)f (dimensions.)189 1043 y Fl(R)n(ationale.)26 b Fv(Similar)18 b(functions)g(are)f(con)o(tained)h(in)g(EXPRESS)g([22)o(])f(and)h(P)l (ARMA)o(CS.)f(\()p Fl(End)189 1100 y(of)f(r)n(ationale.)p Fv(\))166 1206 y(The)h(function)g Fw(MPI)p 525 1206 V 16 w(TOPO)p 667 1206 V 17 w(TEST)f Fv(can)h(b)q(e)g(used)g(to)f (inquire)i(ab)q(out)e(the)h(top)q(ology)f(asso)q(ciated)75 1262 y(with)g(a)f(comm)o(unicator.)20 b(The)c(top)q(ological)g (information)f(can)h(b)q(e)g(extracted)f(from)g(the)g(comm)o(unica-)75 1319 y(tor)h(using)h(the)g(functions)h Fw(MPI)p 633 1319 V 15 w(GRAPHDIMS)p 909 1319 V 17 w(GET)f Fv(and)g Fw(MPI)p 1203 1319 V 16 w(GRAPH)p 1369 1319 V 18 w(GET)p Fv(,)f(for)g(general)i (graphs,)75 1375 y(and)e Fw(MPI)p 249 1375 V 15 w(CARTDIM)p 469 1375 V 17 w(GET)f Fv(and)h Fw(MPI)p 760 1375 V 16 w(CART)p 895 1375 V 17 w(GET)p Fv(,)f(for)g(cartesian)g(top)q(ologies.) 21 b(Sev)o(eral)16 b(additional)75 1432 y(functions)f(are)f(pro)o (vided)h(to)f(manipulate)i(cartesian)e(top)q(ologies:)20 b(the)15 b(functions)g Fw(MPI)p 1603 1432 V 16 w(CART)p 1738 1432 V 17 w(RANK)75 1488 y Fv(and)f Fw(MPI)p 247 1488 V 16 w(CART)p 382 1488 V 17 w(COORDS)h Fv(translate)e(cartesian)h (co)q(ordinates)f(in)o(to)h(a)f(group)h(rank,)f(and)h(vice-v)o(ersa;)75 1545 y(the)20 b(function)h Fw(MPI)p 426 1545 V 16 w(CART)p 561 1545 V 17 w(SUB)g Fv(can)f(b)q(e)h(used)g(to)f(extract)f(a)h (cartesian)g(subspace)h(\(analogous)f(to)75 1601 y Fw(MPI)p 160 1601 V 16 w(COMM)p 318 1601 V 16 w(SPLIT)p Fv(\).)e(The)i(function) g Fw(MPI)p 868 1601 V 16 w(CART)p 1003 1601 V 17 w(SHIFT)f Fv(pro)o(vides)g(the)h(information)f(needed)i(to)75 1658 y(comm)o(unicate)46 b(with)h(neigh)o(b)q(ors)f(in)h(a)f(cartesian)g (dimension.)114 b(The)47 b(t)o(w)o(o)d(functions)75 1714 y Fw(MPI)p 160 1714 V 16 w(GRAPH)p 326 1714 V 17 w(NEIGHBORS)p 594 1714 V 19 w(COUNT)19 b Fv(and)h Fw(MPI)p 963 1714 V 15 w(GRAPH)p 1128 1714 V 18 w(NEIGHBORS)h Fv(can)e(b)q(e)h(used)f(to) g(extract)75 1770 y(the)h(neigh)o(b)q(ors)g(of)f(a)g(no)q(de)h(in)h(a)e (graph.)33 b(The)19 b(function)i Fw(MPI)p 1204 1770 V 16 w(CART)p 1339 1770 V 17 w(SUB)f Fv(is)g(collectiv)o(e)h(o)o(v)o(er)e (the)75 1827 y(input)d(comm)o(unicator's)f(group;)f(all)i(other)f (functions)h(are)f(lo)q(cal.)166 1883 y(Tw)o(o)c(additional)i (functions,)g Fw(MPI)p 767 1883 V 16 w(GRAPH)p 933 1883 V 18 w(MAP)f Fv(and)g Fw(MPI)p 1229 1883 V 16 w(CART)p 1364 1883 V 17 w(MAP)g Fv(are)g(presen)o(ted)g(in)h(the)75 1940 y(last)f(section.)19 b(In)12 b(general)g(these)g(functions)h(are)e (not)g(called)i(b)o(y)f(the)g(user)f(directly)l(.)21 b(Ho)o(w)o(ev)o(er,)11 b(together)75 1996 y(with)16 b(the)h(comm)o (unicator)e(manipulation)j(functions)f(presen)o(ted)f(in)h(Chapter)f (5,)f(they)i(are)e(su\016cien)o(t)75 2053 y(to)g(implemen)o(t)h(all)g (other)f(top)q(ology)g(functions.)20 b(Section)d(6.5.7)c(outlines)k (suc)o(h)e(an)g(implemen)o(tation.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 180 188 180 187 bop 75 -100 a Fv(180)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fn(6.5)59 b(T)-5 b(op)r(ology)19 b(Constructo)n(rs)75 148 y Fj(6.5.1)49 b(Ca)o(rtesian)16 b(Constructo)o(r)75 329 y Fw(MPI)p 160 329 14 2 v 16 w(CART)p 295 329 V 17 w(CREA)l(TE\(comm)p 616 329 V 15 w(old,)f(ndims,)g(dims,)g(p)q(erio)q(ds,)h(reo)o(rder,)e (comm)p 1412 329 V 14 w(ca)o(rt\))117 406 y Fk(IN)171 b Fw(comm)p 454 406 V 14 w(old)383 b Fk(input)14 b(comm)o(unicator)d (\(handle\))117 481 y(IN)171 b Fw(ndims)460 b Fk(n)o(um)o(b)q(er)13 b(of)h(dimensions)e(of)i(cartesian)g(grid)g(\(in)o(teger\))117 556 y(IN)171 b Fw(dims)483 b Fk(in)o(teger)19 b(arra)o(y)f(of)f(size)i Fi(ndims)f Fk(sp)q(ecifying)g(the)g(n)o(um)o(b)q(er)g(of)905 612 y(pro)q(cesses)f(in)c(eac)o(h)i(dimension)117 688 y(IN)171 b Fw(p)q(erio)q(ds)436 b Fk(logical)12 b(arra)o(y)h(of)g(size) h Fi(ndims)f Fk(sp)q(ecifying)g(whether)h(the)g(grid)905 744 y(is)g(p)q(erio)q(dic)g(\()p Fi(true)p Fk(\))h(or)f(not)g(\()p Fi(false)p Fk(\))g(in)g(eac)o(h)g(dimension)117 819 y(IN)171 b Fw(reo)o(rder)438 b Fk(ranking)10 b(ma)o(y)g(b)q(e)h(reordered)i(\()p Fi(true)p Fk(\))f(or)f(not)g(\()p Fi(false)p Fk(\))g(\(logical\))117 894 y(OUT)124 b Fw(comm)p 454 894 V 14 w(ca)o(rt)367 b Fk(comm)o(unicator)11 b(with)j(new)g(cartesian)h(top)q(ology)d (\(handle\))75 1019 y Ft(int)23 b(MPI)p 245 1019 15 2 v 17 w(Cart)p 358 1019 V 17 w(create\(MPI)p 615 1019 V 16 w(Comm)g(comm)p 846 1019 V 17 w(old,)g(int)g(ndims,)g(int)h (*dims,)f(int)g(*periods,)393 1075 y(int)h(reorder,)e(MPI)p 778 1075 V 17 w(Comm)h(*comm)p 1034 1075 V 17 w(cart\))75 1162 y(MPI)p 150 1162 V 17 w(CART)p 263 1162 V 16 w(CREATE\(COMM)p 543 1162 V 16 w(OLD,)g(NDIMS,)g(DIMS,)h(PERIODS,)e(REORDER,)h(COMM)p 1514 1162 V 17 w(CART,)g(IERROR\))170 1218 y(INTEGER)g(COMM)p 460 1218 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g(COMM)p 1074 1218 V 17 w(CART,)g(IERROR)170 1275 y(LOGICAL)g(PERIODS\(*\),)g (REORDER)166 1361 y Fw(MPI)p 251 1361 14 2 v 16 w(CART)p 386 1361 V 17 w(CREA)l(TE)18 b Fv(returns)f(a)g(handle)i(to)e(a)g(new)g (comm)o(unicator)g(to)g(whic)o(h)h(the)g(cartesian)75 1418 y(top)q(ology)13 b(information)g(is)g(attac)o(hed.)19 b(If)13 b Fw(reo)o(rder)f(=)h(false)h Fv(then)f(the)g(rank)g(of)f(eac)o (h)h(pro)q(cess)h(in)g(the)f(new)75 1474 y(group)g(is)g(iden)o(tical)i (to)d(its)h(rank)g(in)h(the)f(old)h(group.)k(Otherwise,)c(the)f (function)h(ma)o(y)e(reorder)h(the)g(pro-)75 1530 y(cesses)h(\(p)q (ossibly)i(so)d(as)h(to)g(c)o(ho)q(ose)g(a)f(go)q(o)q(d)h(em)o(b)q (edding)i(of)e(the)g(virtual)h(top)q(ology)e(on)o(to)g(the)i(ph)o (ysical)75 1587 y(mac)o(hine\).)k(If)12 b(the)g(total)f(size)i(of)e (the)h(cartesian)g(grid)g(is)g(smaller)h(than)f(the)f(size)i(of)f(the)f (group)h(of)f Fw(comm)p Fv(,)75 1643 y(then)k(some)g(pro)q(cesses)h (are)f(returned)g Fi(MPI)p 826 1643 13 2 v 15 w(COMM)p 971 1643 V 14 w(NULL)p Fv(,)h(in)g(analogy)f(to)f Fw(MPI)p 1475 1643 14 2 v 16 w(COMM)p 1633 1643 V 17 w(SPLIT)p Fv(.)g(The)75 1700 y(call)i(is)g(erroneous)f(if)h(it)f(sp)q(eci\014es)i (a)e(grid)h(that)e(is)i(larger)f(than)g(the)g(group)g(size.)75 1822 y Fj(6.5.2)49 b(Ca)o(rtesian)16 b(Convenience)g(F)o(unction:)21 b(MPI)p 996 1822 15 2 v 18 w(DIMS)p 1131 1822 V 18 w(CREA)l(TE)75 1907 y Fv(F)l(or)15 b(cartesian)h(top)q(ologies,)g(the)g(function)g Fw(MPI)p 919 1907 14 2 v 16 w(DIMS)p 1046 1907 V 16 w(CREA)l(TE)h Fv(helps)g(the)f(user)g(select)g(a)g(balanced)75 1964 y(distribution)k(of)f(pro)q(cesses)g(p)q(er)g(co)q(ordinate)g (direction,)i(dep)q(ending)g(on)d(the)h(n)o(um)o(b)q(er)g(of)g(pro)q (cesses)75 2020 y(in)i(the)f(group)g(to)f(b)q(e)i(balanced)h(and)e (optional)h(constrain)o(ts)e(that)h(can)g(b)q(e)h(sp)q(eci\014ed)h(b)o (y)e(the)g(user.)75 2077 y(One)f(use)g(is)h(to)e(partition)h(all)g(the) g(pro)q(cesses)g(\(the)f(size)i(of)e Fi(MPI)p 1227 2077 13 2 v 15 w(COMM)p 1372 2077 V 14 w(W)o(ORLD)p Fv('s)g(group\))g(in)o (to)h(an)75 2133 y Fp(n)p Fv(-dimensional)f(top)q(ology)l(.)75 2284 y Fw(MPI)p 160 2284 14 2 v 16 w(DIMS)p 287 2284 V 16 w(CREA)l(TE\(nno)q(des,)f(ndims,)e(dims\))117 2361 y Fk(IN)171 b Fw(nno)q(des)441 b Fk(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g(a)f(grid)h(\(in)o(teger\))117 2436 y(IN)171 b Fw(ndims)460 b Fk(n)o(um)o(b)q(er)13 b(of)h(cartesian)g(dimensions)f(\(in)o(teger\))117 2512 y(INOUT)78 b Fw(dims)483 b Fk(in)o(teger)18 b(arra)o(y)f(of)f(size)i Fb(ndims)e Fk(sp)q(ecifying)h(the)h(n)o(um)o(b)q(er)f(of)905 2568 y(no)q(des)e(in)e(eac)o(h)i(dimension)75 2692 y Ft(int)23 b(MPI)p 245 2692 15 2 v 17 w(Dims)p 358 2692 V 17 w(create\(int)f(nnodes,)h(int)h(ndims,)f(int)g(*dims\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 181 189 181 188 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fv(181)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(DIMS)p 263 45 V 16 w(CREATE\(NNODES,)22 b(NDIMS,)h(DIMS,)h(IERROR\))170 102 y(INTEGER)f(NNODES,)g(NDIMS,)g(DIMS\(*\),)g(IERROR)166 188 y Fv(The)13 b(en)o(tries)h(in)f(the)g(arra)o(y)f Fw(dims)h Fv(are)g(set)f(to)h(describ)q(e)i(a)d(cartesian)h(grid)h (with)f Fw(ndims)g Fv(dimensions)75 244 y(and)g(a)g(total)g(of)f Fw(nno)q(des)k Fv(no)q(des.)j(The)14 b(dimensions)h(are)d(set)h(to)g(b) q(e)h(as)e(close)i(to)f(eac)o(h)g(other)g(as)g(p)q(ossible,)75 301 y(using)i(an)g(appropriate)g(divisibili)q(t)o(y)i(algorithm.)i(The) c(caller)h(ma)o(y)e(further)g(constrain)h(the)g(op)q(eration)75 357 y(of)e(this)h(routine)g(b)o(y)g(sp)q(ecifying)h(elemen)o(ts)f(of)f (arra)o(y)g Ft(dims)p Fv(.)19 b(If)13 b Ft(dims[i])g Fv(is)h(set)f(to)g(a)g(p)q(ositiv)o(e)i(n)o(um)o(b)q(er,)75 414 y(the)i(routine)g(will)h(not)e(mo)q(dify)i(the)e(n)o(um)o(b)q(er)h (of)f(no)q(des)i(in)f(dimension)h Ft(i)p Fv(;)f(only)g(those)g(en)o (tries)g(where)75 470 y Ft(dims[i])23 b(=)h(0)15 b Fv(are)g(mo)q (di\014ed)h(b)o(y)f(the)h(call.)166 527 y(Negativ)o(e)e(input)i(v)m (alues)f(of)f Ft(dims[i])f Fv(are)h(erroneous.)20 b(An)15 b(error)e(will)j(o)q(ccur)f(if)g Ft(nnodes)e Fv(is)i(not)f(a)75 583 y(m)o(ultiple)j(of)366 543 y Fe(Y)305 637 y Ff(i;dims)p Fg([)p Ff(i)p Fg(])p Fh(6)p Fg(=0)489 583 y Fp(dims)p Fv([)p Fp(i)p Fv(].)166 685 y(F)l(or)g Ft(dims[i])g Fv(set)g(b)o(y)g (the)h(call,)h Ft(dims[i])e Fv(will)i(b)q(e)f(ordered)g(in)h (non-increasing)g(order.)27 b(Arra)o(y)75 741 y Ft(dims)20 b Fv(is)i(suitable)g(for)f(use)g(as)g(input)h(to)e(routine)i Fw(MPI)p 1076 741 14 2 v 16 w(CART)p 1211 741 V 17 w(CREA)l(TE)p Fv(.)g Fw(MPI)p 1514 741 V 15 w(DIMS)p 1640 741 V 16 w(CREA)l(TE)g Fv(is)75 798 y(lo)q(cal.)75 1055 y Fq(Example)c(6.1)p 380 873 1247 2 v 379 930 2 57 v 405 913 a Ft(dims)p 634 930 V 159 w Fv(function)e(call)p 1303 930 V 424 w Ft(dims)p 1626 930 V 379 986 V 405 969 a Fv(b)q(efore)g(call)p 634 986 V 1303 986 V 720 w(on)f(return)p 1626 986 V 380 988 1247 2 v 379 1044 2 57 v 405 1027 a(\(0,0\))p 634 1044 V 160 w Fw(MPI)p 745 1027 14 2 v 16 w(DIMS)p 872 1027 V 15 w(CREA)l(TE\(6,)h(2,)e(dims\))p 1303 1044 2 57 v 50 w Fv(\(3,2\))p 1626 1044 V 379 1101 V 405 1084 a(\(0,0\))p 634 1101 V 160 w Fw(MPI)p 745 1084 14 2 v 16 w(DIMS)p 872 1084 V 15 w(CREA)l(TE\(7,)i(2,)e(dims\))p 1303 1101 2 57 v 50 w Fv(\(7,1\))p 1626 1101 V 379 1157 V 405 1140 a(\(0,3,0\))p 634 1157 V 124 w Fw(MPI)p 745 1140 14 2 v 16 w(DIMS)p 872 1140 V 15 w(CREA)l(TE\(6,)i(3,)e(dims\))p 1303 1157 2 57 v 50 w Fv(\(2,3,1\))p 1626 1157 V 379 1214 V 405 1197 a(\(0,3,0\))p 634 1214 V 124 w Fw(MPI)p 745 1197 14 2 v 16 w(DIMS)p 872 1197 V 15 w(CREA)l(TE\(7,)i(3,)e (dims\))p 1303 1214 2 57 v 50 w Fv(erroneous)h(call)p 1626 1214 V 380 1215 1247 2 v 75 1399 a Fj(6.5.3)49 b(General)16 b(\(Graph\))f(Constructo)o(r)75 1579 y Fw(MPI)p 160 1579 14 2 v 16 w(GRAPH)p 326 1579 V 17 w(CREA)l(TE\(comm)p 647 1579 V 16 w(old,)g(nno)q(des,)h(index,)g(edges,)g(reo)o(rder,)e (comm)p 1443 1579 V 14 w(graph\))358 b Fx(>)117 1656 y Fk(IN)171 b Fw(comm)p 454 1656 V 14 w(old)383 b Fk(input)14 b(comm)o(unicator)d(\(handle\))498 b Fx(?)117 1731 y Fk(IN)171 b Fw(nno)q(des)441 b Fk(n)o(um)o(b)q(er)13 b(of)h(no)q(des)g(in)g(graph)g(\(in)o(teger\))117 1806 y(IN)171 b Fw(index)473 b Fk(arra)o(y)14 b(of)f(in)o(tegers)i (describing)f(no)q(de)h(degrees)g(\(see)g(b)q(elo)o(w\))117 1882 y(IN)171 b Fw(edges)468 b Fk(arra)o(y)14 b(of)f(in)o(tegers)i (describing)f(graph)g(edges)h(\(see)g(b)q(elo)o(w\))117 1957 y(IN)171 b Fw(reo)o(rder)438 b Fk(ranking)10 b(ma)o(y)g(b)q(e)h (reordered)i(\()p Fi(true)p Fk(\))f(or)f(not)g(\()p Fi(false)p Fk(\))g(\(logical\))117 2032 y(OUT)124 b Fw(comm)p 454 2032 V 14 w(graph)333 b Fk(comm)o(unicator)11 b(with)j(graph)g(top)q (ology)e(added)i(\(handle\))75 2156 y Ft(int)23 b(MPI)p 245 2156 15 2 v 17 w(Graph)p 382 2156 V 17 w(create\(MPI)p 639 2156 V 16 w(Comm)g(comm)p 870 2156 V 16 w(old,)h(int)f(nnodes,)g (int)h(*index,)f(int)g(*edges,)393 2213 y(int)h(reorder,)e(MPI)p 778 2213 V 17 w(Comm)h(*comm)p 1034 2213 V 17 w(graph\))75 2299 y(MPI)p 150 2299 V 17 w(GRAPH)p 287 2299 V 16 w(CREATE\(COMM)p 567 2299 V 16 w(OLD,)g(NNODES,)g(INDEX,)g(EDGES,)g(REORDER,)g(COMM)p 1538 2299 V 17 w(GRAPH,)393 2356 y(IERROR\))170 2412 y(INTEGER)g(COMM)p 460 2412 V 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g (EDGES\(*\),)f(COMM)p 1360 2412 V 17 w(GRAPH,)h(IERROR)170 2469 y(LOGICAL)g(REORDER)166 2555 y Fw(MPI)p 251 2555 14 2 v 16 w(GRAPH)p 417 2555 V 17 w(CREA)l(TE)f Fv(returns)e(a)g (handle)h(to)f(a)g(new)g(comm)o(unicator)g(to)g(whic)o(h)h(the)f(graph) 75 2611 y(top)q(ology)e(information)h(is)g(attac)o(hed.)30 b(If)19 b Fw(reo)o(rder)f(=)h(false)g Fv(then)g(the)g(rank)f(of)h(eac)o (h)f(pro)q(cess)h(in)h(the)75 2668 y(new)13 b(group)g(is)h(iden)o (tical)h(to)d(its)i(rank)f(in)h(the)f(old)g(group.)19 b(Otherwise,)14 b(the)g(function)g(ma)o(y)e(reorder)h(the)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 182 190 182 189 bop 75 -100 a Fv(182)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fv(pro)q(cesses.)25 b(If)17 b(the)g(size,)h Fw(nno)q(des)p Fv(,)h(of)e(the)g(graph)f(is)i (smaller)f(than)g(the)g(size)h(of)e(the)h(group)g(of)f Fw(comm)p Fv(,)75 102 y(then)f(some)f(pro)q(cesses)h(are)f(returned)g Fi(MPI)p 822 102 13 2 v 15 w(COMM)p 967 102 V 14 w(NULL)p Fv(,)h(in)h(analogy)e(to)f Fw(MPI)p 1468 102 14 2 v 16 w(CART)p 1603 102 V 17 w(CREA)l(TE)j Fv(and)75 158 y Fw(MPI)p 160 158 V 16 w(COMM)p 318 158 V 16 w(SPLIT)p Fv(.)c(The)h(call)h(is)g(erroneous)e(if)i(it)f(sp)q(eci\014es)h(a)f (graph)f(that)h(is)g(larger)g(than)f(the)h(group)75 214 y(size)j(of)f(the)g(input)h(comm)o(unicator.)166 271 y(The)d(three)h(parameters)e Fw(nno)q(des,)j(index)f Fv(and)g Fw(edges)g Fv(de\014ne)g(the)f(graph)g(structure.)19 b Fw(nno)q(des)c Fv(is)f(the)75 327 y(n)o(um)o(b)q(er)f(of)g(no)q(des)g (of)f(the)h(graph.)19 b(The)13 b(no)q(des)h(are)e(n)o(um)o(b)q(ered)i (from)e Ft(0)h Fv(to)f Ft(nnodes-1)p Fv(.)18 b(The)13 b Ft(i)p Fv(th)f(en)o(try)75 384 y(of)k(arra)o(y)e Fw(index)j Fv(stores)f(the)g(total)f(n)o(um)o(b)q(er)h(of)g(neigh)o(b)q(ors)h(of)e (the)h(\014rst)g Ft(i)g Fv(graph)g(no)q(des.)22 b(The)17 b(lists)f(of)75 440 y(neigh)o(b)q(ors)d(of)g(no)q(des)g Ft(0,)24 b(1,)f Fp(:)8 b(:)g(:)e Ft(,)24 b(nnodes-1)11 b Fv(are)h(stored)h(in)g(consecutiv)o(e)h(lo)q(cations)f(in)h(arra)o(y) d Fw(edges)p Fv(.)75 497 y(The)16 b(arra)o(y)e Fw(edges)i Fv(is)g(a)f(\015attened)h(represen)o(tation)f(of)g(the)h(edge)f(lists.) 21 b(The)16 b(total)f(n)o(um)o(b)q(er)h(of)f(en)o(tries)75 553 y(in)h Fw(index)g Fv(is)g Fw(nno)q(des)h Fv(and)e(the)g(total)g(n)o (um)o(b)q(er)g(of)g(en)o(tries)h(in)g Fw(edges)g Fv(is)f(equal)h(to)f (the)g(n)o(um)o(b)q(er)h(of)e(graph)75 610 y(edges.)166 666 y(The)19 b(de\014nitions)h(of)e(the)h(argumen)o(ts)f Ft(nnodes)p Fv(,)g Ft(index)p Fv(,)h(and)g Ft(edges)f Fv(are)g(illustrated)i(with)f(the)75 723 y(follo)o(wing)d(simple)h (example.)75 827 y Fq(Example)h(6.2)k Fv(Assume)g(there)g(are)g(four)f (pro)q(cesses)i(0,)g(1,)f(2,)h(3)f(with)g(the)g(follo)o(wing)h (adjacency)75 884 y(matrix:)p 181 929 433 2 v 180 985 2 57 v 206 968 a(pro)q(cess)p 373 985 V 50 w(neigh)o(b)q(ors)p 613 985 V 181 987 433 2 v 180 1043 2 57 v 266 1026 a(0)p 373 1043 V 110 w(1,)15 b(3)p 613 1043 V 180 1100 V 266 1083 a(1)p 373 1100 V 110 w(0)p 613 1100 V 180 1156 V 266 1139 a(2)p 373 1156 V 110 w(3)p 613 1156 V 180 1213 V 266 1196 a(3)p 373 1213 V 110 w(0,)g(2)p 613 1213 V 181 1214 433 2 v 166 1289 a(Then,)g(the)h(input)g(argumen)o(ts)e(are:) 206 1381 y(nno)q(des)i(=)50 b(4)206 1437 y(index)16 b(=)81 b(2,)15 b(3,)f(4,)h(6)206 1494 y(edges)g(=)82 b(1,)15 b(3,)f(0,)h(3,)f(0,)h(2)166 1586 y(Th)o(us,)h(in)h(C,)e Ft(index[0])g Fv(is)i(the)f(degree)g(of)g(no)q(de)h(zero,)f(and)g Ft(index[i])23 b(-)h(index[i-1])14 b Fv(is)j(the)75 1642 y(degree)22 b(of)g(no)q(de)g Ft(i,)i(i=1,)f Fp(:)8 b(:)g(:)e Ft(,)23 b(nnodes-1)p Fv(;)h(the)e(list)h(of)e(neigh)o(b)q(ors)i(of)e (no)q(de)i(zero)e(is)i(stored)e(in)75 1699 y Ft(edges[j])p Fv(,)14 b(for)h Ft(0)e Fx(\024)h Ft(j)f Fx(\024)h Ft(index)p Fv([)p Ft(0)p Fv(])9 b Fx(\000)h Ft(1)16 b Fv(and)g(the)f(list)i(of)e (neigh)o(b)q(ors)h(of)f(no)q(de)i Ft(i)p Fv(,)e Ft(i)e Fp(>)h Ft(0)p Fv(,)h(is)h(stored)f(in)75 1755 y Ft(edges[j])p Fv(,)f Ft(index)p Fv([)p Ft(i)9 b Fx(\000)h Ft(1)p Fv(])i Fx(\024)h Ft(j)g Fx(\024)g Ft(index)p Fv([)p Ft(i)p Fv(])8 b Fx(\000)j Ft(1)p Fv(.)166 1811 y(In)17 b(F)l(ortran,)f Ft(index\(1\))f Fv(is)j(the)e(degree)i(of)e(no)q(de)h(zero,)g(and)g Ft(index\(i+1\))22 b(-)i(index\(i\))16 b Fv(is)h(the)75 1868 y(degree)22 b(of)g(no)q(de)g Ft(i,)i(i=1,)f Fp(:)8 b(:)g(:)e Ft(,)23 b(nnodes-1)p Fv(;)h(the)e(list)h(of)e(neigh)o(b)q (ors)i(of)e(no)q(de)i(zero)e(is)i(stored)e(in)75 1924 y Ft(edges\(j\))p Fv(,)d(for)g Ft(1)f Fx(\024)i Ft(j)f Fx(\024)g Ft(index)p Fv(\()p Ft(1)p Fv(\))f(and)i(the)f(list)h(of)f (neigh)o(b)q(ors)h(of)f(no)q(de)i Ft(i)p Fv(,)e Ft(i)g Fp(>)h Ft(0)p Fv(,)f(is)h(stored)f(in)75 1981 y Ft(edges\(j\))p Fv(,)c Ft(index)p Fv(\()p Ft(i)p Fv(\))8 b(+)j Ft(1)h Fx(\024)h Ft(j)g Fx(\024)g Ft(index)p Fv(\()p Ft(i)c Fv(+)h Ft(1)p Fv(\).)189 2086 y Fl(A)n(dvic)n(e)17 b(to)i (implementors.)56 b Fv(The)19 b(follo)o(wing)f(top)q(ology)g (information)g(is)g(lik)o(ely)i(to)d(b)q(e)h(stored)189 2142 y(with)d(a)g(comm)o(unicator:)243 2236 y Fx(\017)23 b Fv(T)o(yp)q(e)15 b(of)g(top)q(ology)g(\(cartesian/graph\),)243 2308 y Fx(\017)23 b Fv(F)l(or)14 b(a)h(cartesian)g(top)q(ology:)316 2381 y(1.)22 b Ft(ndims)14 b Fv(\(n)o(um)o(b)q(er)h(of)g(dimensions\),) 316 2445 y(2.)22 b Ft(dims)14 b Fv(\(n)o(um)o(b)q(ers)h(of)g(pro)q (cesses)h(p)q(er)f(co)q(ordinate)h(direction\),)316 2510 y(3.)22 b Ft(periods)14 b Fv(\(p)q(erio)q(dicit)o(y)j(information\),) 316 2574 y(4.)22 b Ft(own_position)12 b Fv(\(o)o(wn)h(p)q(osition)i(in) f(grid,)h(could)f(also)g(b)q(e)h(computed)f(from)f(rank)h(and)374 2631 y(dims\))243 2704 y Fx(\017)23 b Fv(F)l(or)14 b(a)h(graph)g(top)q (ology:)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 183 191 183 190 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fv(183)316 45 y(1.)22 b Ft(index)p Fv(,)316 110 y(2.)g Ft(edges)p Fv(,)289 183 y(whic)o(h)16 b(are)f(the)g(v)o(ectors)f (de\014ning)j(the)e(graph)g(structure.)189 277 y(F)l(or)k(a)g(graph)h (structure)f(the)h(n)o(um)o(b)q(er)g(of)f(no)q(des)i(is)f(equal)h(to)e (the)h(n)o(um)o(b)q(er)g(of)f(pro)q(cesses)h(in)189 333 y(the)h(group.)36 b(Therefore,)22 b(the)f(n)o(um)o(b)q(er)g(of)g(no)q (des)g(do)q(es)g(not)g(ha)o(v)o(e)f(to)g(b)q(e)i(stored)e(explicitly)l (.)189 390 y(An)c(additional)i(zero)f(en)o(try)f(at)f(the)i(start)e(of) h(arra)o(y)f Fw(index)j Fv(simpli\014es)g(access)f(to)f(the)g(top)q (ology)189 446 y(information.)k(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g (implementors.)p Fv(\))75 568 y Fj(6.5.4)49 b(T)l(op)q(ology)19 b(inquiry)e(functions)75 654 y Fv(If)11 b(a)f(top)q(ology)g(has)h(b)q (een)g(de\014ned)h(with)f(one)g(of)f(the)g(ab)q(o)o(v)o(e)g(functions,) i(then)f(the)g(top)q(ology)f(information)75 710 y(can)15 b(b)q(e)h(lo)q(ok)o(ed)g(up)g(using)f(inquiry)i(functions.)k(They)15 b(all)h(are)f(lo)q(cal)i(calls.)75 861 y Fw(MPI)p 160 861 14 2 v 16 w(TOPO)p 302 861 V 17 w(TEST\(comm,)c(status\))117 938 y Fk(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(\(handle\))117 1013 y(OUT)124 b Fw(status)460 b Fk(top)q(ology)13 b(t)o(yp)q(e)h(of)g(comm)o(uni)o(cator)e Fb(comm)h Fk(\(c)o(hoice\))75 1138 y Ft(int)23 b(MPI)p 245 1138 15 2 v 17 w(Topo)p 358 1138 V 17 w(test\(MPI)p 567 1138 V 16 w(Comm)g(comm,)g(int)h (*status\))75 1224 y(MPI)p 150 1224 V 17 w(TOPO)p 263 1224 V 16 w(TEST\(COMM,)f(STATUS,)g(IERROR\))170 1281 y(INTEGER)g(COMM,)h(STATUS,)e(IERROR)166 1367 y Fv(The)h(function)g Fw(MPI)p 537 1367 14 2 v 16 w(TOPO)p 679 1367 V 17 w(TEST)g Fv(returns)f(the)h(t)o(yp)q(e)g(of)f(top)q(ology)g(that)g(is)h (assigned)g(to)f(a)75 1424 y(comm)o(unicator.)166 1480 y(The)15 b(output)g(v)m(alue)i Ft(status)d Fv(is)i(one)f(of)g(the)g (follo)o(wing:)117 1584 y Fi(MPI)p 194 1584 13 2 v 14 w(GRAPH)602 b Fv(graph)15 b(top)q(ology)117 1638 y Fi(MPI)p 194 1638 V 14 w(CART)631 b Fv(cartesian)15 b(top)q(ology)117 1693 y Fi(MPI)p 194 1693 V 14 w(UNDEFINED)508 b Fv(no)15 b(top)q(ology)75 1870 y Fw(MPI)p 160 1870 14 2 v 16 w(GRAPHDIMS)p 437 1870 V 17 w(GET\(comm,)e(nno)q(des,)k(nedges\))117 1947 y Fk(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)7 b(for)i(group)h(with)f(graph)h(structure)i(\(handle\))117 2022 y(OUT)124 b Fw(nno)q(des)441 b Fk(n)o(um)o(b)q(er)15 b(of)f(no)q(des)i(in)f(graph)g(\(in)o(teger\))h(\(same)e(as)i(n)o(um)o (b)q(er)905 2079 y(of)d(pro)q(cesses)k(in)c(the)i(group\))117 2154 y(OUT)124 b Fw(nedges)445 b Fk(n)o(um)o(b)q(er)13 b(of)h(edges)h(in)e(graph)h(\(in)o(teger\))75 2278 y Ft(int)23 b(MPI)p 245 2278 15 2 v 17 w(Graphdims)p 478 2278 V 16 w(get\(MPI)p 662 2278 V 16 w(Comm)h(comm,)f(int)g(*nnodes,)g (int)h(*nedges\))75 2365 y(MPI)p 150 2365 V 17 w(GRAPHDIMS)p 383 2365 V 16 w(GET\(COMM,)e(NNODES,)h(NEDGES,)g(IERROR\))170 2421 y(INTEGER)g(COMM,)h(NNODES,)e(NEDGES,)h(IERROR)166 2508 y Fv(F)l(unctions)c Fw(MPI)p 462 2508 14 2 v 16 w(GRAPHDIMS)p 739 2508 V 17 w(GET)g Fv(and)g Fw(MPI)p 1037 2508 V 16 w(GRAPH)p 1203 2508 V 18 w(GET)g Fv(retriev)o(e)g(the)f (graph-top)q(ology)75 2564 y(information)d(that)g(w)o(as)f(asso)q (ciated)i(with)f(a)g(comm)o(unicator)g(b)o(y)g Fw(MPI)p 1310 2564 V 16 w(GRAPH)p 1476 2564 V 17 w(CREA)l(TE)p Fv(.)166 2621 y(The)21 b(information)g(pro)o(vided)h(b)o(y)f Fw(MPI)p 864 2621 V 16 w(GRAPHDIMS)p 1141 2621 V 17 w(GET)g Fv(can)g(b)q(e)g(used)h(to)e(dimension)j(the)75 2677 y(v)o(ectors)14 b Ft(index)h Fv(and)g Ft(edges)g Fv(correctly)g(for)g (the)g(follo)o(wing)h(call)g(to)f Fw(MPI)p 1339 2677 V 16 w(GRAPH)p 1505 2677 V 17 w(GET)p Fv(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 184 192 184 191 bop 75 -100 a Fv(184)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(GRAPH)p 326 45 V 17 w(GET\(comm,)e(maxindex,)i(maxedges,)g (index,)h(edges\))117 122 y Fk(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(with)j(graph)g(structure)i(\(handle\))117 197 y(IN)171 b Fw(maxindex)393 b Fk(length)14 b(of)f(v)o(ector)i Fi(index)g Fk(in)f(the)g(calling)f(program)905 254 y(\(in)o(teger\))117 329 y(IN)171 b Fw(maxedges)388 b Fk(length)14 b(of)f(v)o(ector)i Fi(edges)g Fk(in)e(the)i(calling)d(program)905 385 y(\(in)o(teger\))117 461 y(OUT)124 b Fw(index)473 b Fk(arra)o(y)17 b(of)g(in)o(tegers)i(con) o(taining)d(the)i(graph)g(structure)h(\(for)905 517 y(details)14 b(see)h(the)g(de\014nition)e(of)g Fi(MPI)p 1484 517 13 2 v 15 w(GRAPH)p 1638 517 V 14 w(CREA)m(TE)p Fk(\))117 592 y(OUT)124 b Fw(edges)468 b Fk(arra)o(y)14 b(of)f(in)o(tegers)i(con) o(taining)e(the)h(graph)g(structure)75 717 y Ft(int)23 b(MPI)p 245 717 15 2 v 17 w(Graph)p 382 717 V 17 w(get\(MPI)p 567 717 V 16 w(Comm)g(comm,)g(int)h(maxindex,)f(int)g(maxedges,)g(int)g (*index,)393 773 y(int)h(*edges\))75 860 y(MPI)p 150 860 V 17 w(GRAPH)p 287 860 V 16 w(GET\(COMM,)f(MAXINDEX,)g(MAXEDGES,)f (INDEX,)h(EDGES,)g(IERROR\))170 916 y(INTEGER)g(COMM,)h(MAXINDEX,)e (MAXEDGES,)h(INDEX\(*\),)f(EDGES\(*\),)h(IERROR)75 1097 y Fw(MPI)p 160 1097 14 2 v 16 w(CARTDIM)p 381 1097 V 16 w(GET\(comm,)13 b(ndims\))117 1174 y Fk(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j(\(handle\))117 1249 y(OUT)124 b Fw(ndims)460 b Fk(n)o(um)o(b)q(er)12 b(of)f(dimensions)g(of)h(the)h(cartesian)g(structure)h(\(in)o(te-)905 1306 y(ger\))75 1430 y Ft(int)23 b(MPI)p 245 1430 15 2 v 17 w(Cartdim)p 430 1430 V 16 w(get\(MPI)p 614 1430 V 17 w(Comm)g(comm,)g(int)h(*ndims\))75 1517 y(MPI)p 150 1517 V 17 w(CARTDIM)p 335 1517 V 16 w(GET\(COMM,)f(NDIMS,)g (IERROR\))170 1573 y(INTEGER)g(COMM,)h(NDIMS,)f(IERROR)166 1660 y Fv(The)14 b(functions)h Fw(MPI)p 538 1660 14 2 v 16 w(CARTDIM)p 759 1660 V 16 w(GET)f Fv(and)g Fw(MPI)p 1046 1660 V 16 w(CART)p 1181 1660 V 17 w(GET)g Fv(return)g(the)g (cartesian)g(top)q(ology)75 1716 y(information)h(that)g(w)o(as)f(asso)q (ciated)i(with)f(a)g(comm)o(unicator)g(b)o(y)g Fw(MPI)p 1310 1716 V 16 w(CART)p 1445 1716 V 17 w(CREA)l(TE)p Fv(.)75 1867 y Fw(MPI)p 160 1867 V 16 w(CART)p 295 1867 V 17 w(GET\(comm,)e(maxdims,)g(dims,)i(p)q(erio)q(ds,)h(co)q(o)o(rds\)) 117 1944 y Fk(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j(\(handle\))117 2019 y(IN)171 b Fw(maxdims)403 b Fk(length)16 b(of)f(v)o(ectors)i Fb(dims,)k(periods)p Fk(,)15 b(and)g Fb(coords)g Fk(in)g(the)905 2076 y(calling)e(program)f(\(in)o(teger\))117 2151 y(OUT)124 b Fw(dims)483 b Fk(n)o(um)o(b)q(er)13 b(of)g(pro)q(cesses)i(for)e(eac)o (h)h(cartesian)g(dimension)e(\(ar-)905 2207 y(ra)o(y)i(of)f(in)o (teger\))117 2283 y(OUT)124 b Fw(p)q(erio)q(ds)436 b Fk(p)q(erio)q(dicit)o(y)21 b(\()p Fi(true)p Fk(/)p Fi(false)p Fk(\))g(for)g(eac)o(h)g(cartesian)h(dimension)905 2339 y(\(arra)o(y)14 b(of)f(logical\))117 2414 y(OUT)124 b Fw(co)q(o)o(rds)449 b Fk(co)q(ordinates)21 b(of)f(calling)e(pro)q(cess) k(in)e(cartesian)h(structure)905 2471 y(\(arra)o(y)14 b(of)f(in)o(teger\))75 2595 y Ft(int)23 b(MPI)p 245 2595 15 2 v 17 w(Cart)p 358 2595 V 17 w(get\(MPI)p 543 2595 V 16 w(Comm)g(comm,)h(int)f(maxdims,)g(int)g(*dims,)g(int)h(*periods,) 393 2651 y(int)g(*coords\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 185 193 185 192 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fv(185)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(CART)p 263 45 V 16 w(GET\(COMM,)23 b(MAXDIMS,)g(DIMS,)g(PERIODS,)g(COORDS,)g (IERROR\))170 102 y(INTEGER)g(COMM,)h(MAXDIMS,)e(DIMS\(*\),)h (COORDS\(*\),)g(IERROR)170 158 y(LOGICAL)g(PERIODS\(*\))75 339 y Fw(MPI)p 160 339 14 2 v 16 w(CART)p 295 339 V 17 w(RANK\(comm,)13 b(co)q(o)o(rds,)i(rank\))117 416 y Fk(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j (\(handle\))117 491 y(IN)171 b Fw(co)q(o)o(rds)449 b Fk(in)o(teger)17 b(arra)o(y)f(\(of)g(size)h Fb(ndims)p Fk(\))e(sp)q(ecifying)h(the)h(cartesian)905 548 y(co)q(ordinates)e(of)e (a)h(pro)q(cess)117 623 y(OUT)124 b Fw(rank)488 b Fk(rank)14 b(of)f(sp)q(eci\014ed)j(pro)q(cess)f(\(in)o(teger\))75 747 y Ft(int)23 b(MPI)p 245 747 15 2 v 17 w(Cart)p 358 747 V 17 w(rank\(MPI)p 567 747 V 16 w(Comm)g(comm,)g(int)h(*coords,)f (int)g(*rank\))75 834 y(MPI)p 150 834 V 17 w(CART)p 263 834 V 16 w(RANK\(COMM,)g(COORDS,)g(RANK,)g(IERROR\))170 890 y(INTEGER)g(COMM,)h(COORDS\(*\),)e(RANK,)h(IERROR)166 977 y Fv(F)l(or)12 b(a)h(pro)q(cess)g(group)g(with)g(cartesian)g (structure,)g(the)g(function)g Fw(MPI)p 1397 977 14 2 v 16 w(CART)p 1532 977 V 17 w(RANK)g Fv(translates)75 1033 y(the)22 b(logical)h(pro)q(cess)f(co)q(ordinates)g(to)g(pro)q (cess)g(ranks)f(as)h(they)g(are)f(used)i(b)o(y)e(the)h(p)q(oin)o (t-to-p)q(oin)o(t)75 1090 y(routines.)166 1146 y(F)l(or)f(dimension)i Ft(i)e Fv(with)h Ft(periods\(i\))h(=)h(true)p Fv(,)e(if)g(the)f(co)q (ordinate,)j Ft(coords\(i\))p Fv(,)d(is)h(out)f(of)75 1203 y(range,)14 b(that)g(is,)h Ft(coords\(i\))22 b Fp(<)j Ft(0)14 b Fv(or)g Ft(coords\(i\))23 b Fx(\025)h Ft(dims\(i\))p Fv(,)13 b(it)i(is)g(shifted)h(bac)o(k)e(to)g(the)h(in)o(terv)m(al)75 1259 y Ft(0)24 b Fx(\024)g Ft(coords\(i\))f Fp(<)h Ft(dims\(i\))e Fv(automatically)l(.)46 b(Out-of-range)23 b(co)q(ordinates)h(are)f (erroneous)h(for)75 1315 y(non-p)q(erio)q(dic)18 b(dimensions.)75 1466 y Fw(MPI)p 160 1466 V 16 w(CART)p 295 1466 V 17 w(COORDS\(comm,)c(rank,)h(maxdims,)e(co)q(o)o(rds\))117 1544 y Fk(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j(\(handle\))117 1619 y(IN)171 b Fw(rank)488 b Fk(rank)14 b(of)f(a)h(pro)q(cess)i(within)d (group)g(of)h Fi(comm)e Fk(\(in)o(teger\))117 1694 y(IN)171 b Fw(maxdims)403 b Fk(length)11 b(of)f(v)o(ector)i Fb(coord)d Fk(in)h(the)i(calling)d(program)g(\(in)o(teger\))117 1769 y(OUT)124 b Fw(co)q(o)o(rds)449 b Fk(in)o(teger)16 b(arra)o(y)e(\(of)h(size)h Fb(ndims)p Fk(\))e(con)o(taining)g(the)h (cartesian)905 1825 y(co)q(ordinates)g(of)e(sp)q(eci\014ed)j(pro)q (cess)f(\(in)o(teger\))75 1950 y Ft(int)23 b(MPI)p 245 1950 15 2 v 17 w(Cart)p 358 1950 V 17 w(coords\(MPI)p 615 1950 V 16 w(Comm)g(comm,)g(int)h(rank,)f(int)g(maxdims,)g(int)g (*coords\))75 2036 y(MPI)p 150 2036 V 17 w(CART)p 263 2036 V 16 w(COORDS\(COMM,)g(RANK,)g(MAXDIMS,)g(COORDS,)f(IERROR\))170 2093 y(INTEGER)h(COMM,)h(RANK,)f(MAXDIMS,)f(COORDS\(*\),)h(IERROR)166 2179 y Fv(The)28 b(in)o(v)o(erse)g(mapping,)k(rank-to-co)q(ordinates)c (translation)f(is)i(pro)o(vided)g(b)o(y)e Fw(MPI)p 1727 2179 14 2 v 16 w(CART-)p 75 2236 V 89 2236 a(COORDS)p Fv(.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 186 194 186 193 bop 75 -100 a Fv(186)887 b Fm(CHAPTER)15 b(6.)30 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(GRAPH)p 326 45 V 17 w(NEIGHBORS)p 594 45 V 19 w(COUNT\(comm,)e(rank,)i(nneighb)q(o)o(rs\))117 122 y Fk(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(with)j(graph)g(top)q(ology)e(\(handle\))117 196 y(IN)171 b Fw(rank)488 b Fk(rank)14 b(of)f(pro)q(cess)j(in)d(group)h(of)f Fi(comm)f Fk(\(in)o(teger\))117 270 y(OUT)124 b Fw(nneighb)q(o)o(rs)369 b Fk(n)o(um)o(b)q(er)13 b(of)h(neigh)o(b)q(ors)g(of)f(sp)q(eci\014ed)j (pro)q(cess)f(\(in)o(teger\))75 394 y Ft(int)23 b(MPI)p 245 394 15 2 v 17 w(Graph)p 382 394 V 17 w(neighbors)p 615 394 V 16 w(count\(MPI)p 847 394 V 16 w(Comm)g(comm,)g(int)h(rank,)f (int)g(*nneighbors\))75 481 y(MPI)p 150 481 V 17 w(GRAPH)p 287 481 V 16 w(NEIGHBORS)p 519 481 V 16 w(COUNT\(COMM,)g(RANK,)g (NNEIGHBORS,)f(IERROR\))170 537 y(INTEGER)h(COMM,)h(RANK,)f (NNEIGHBORS,)f(IERROR)166 624 y Fw(MPI)p 251 624 14 2 v 16 w(GRAPH)p 417 624 V 17 w(NEIGHBORS)p 685 624 V 18 w(COUNT)h Fv(and)e Fw(MPI)p 1058 624 V 16 w(GRAPH)p 1224 624 V 18 w(NEIGHBORS)i Fv(pro)o(vide)f(adjacency)75 680 y(information)15 b(for)g(a)g(general,)g(graph)g(top)q(ology)l(.)75 831 y Fw(MPI)p 160 831 V 16 w(GRAPH)p 326 831 V 17 w(NEIGHBORS\(comm,)g (rank,)g(maxneighb)q(o)o(rs,)f(neighb)q(o)o(rs\))117 908 y Fk(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(with)j(graph)g(top)q(ology)e(\(handle\))117 982 y(IN)171 b Fw(rank)488 b Fk(rank)14 b(of)f(pro)q(cess)j(in)d(group)h(of)f Fi(comm)f Fk(\(in)o(teger\))117 1056 y(IN)171 b Fw(maxneighb)q(o)o(rs) 312 b Fk(size)15 b(of)e(arra)o(y)h Fi(neighb)q(o)o(rs)h Fk(\(in)o(teger\))117 1130 y(OUT)124 b Fw(neighb)q(o)o(rs)392 b Fk(ranks)13 b(of)g(pro)q(cesses)i(that)e(are)h(neigh)o(b)q(ors)f(to)g (sp)q(eci\014ed)h(pro-)905 1187 y(cess)i(\(arra)o(y)d(of)h(in)o (teger\))75 1311 y Ft(int)23 b(MPI)p 245 1311 15 2 v 17 w(Graph)p 382 1311 V 17 w(neighbors\(MPI)p 711 1311 V 15 w(Comm)g(comm,)h(int)f(rank,)g(int)h(maxneighbors,)393 1367 y(int)g(*neighbors\))75 1454 y(MPI)p 150 1454 V 17 w(GRAPH)p 287 1454 V 16 w(NEIGHBORS\(COMM,)e(RANK,)h(MAXNEIGHBORS,)f (NEIGHBORS,)h(IERROR\))170 1510 y(INTEGER)g(COMM,)h(RANK,)f (MAXNEIGHBORS,)f(NEIGHBORS\(*\),)g(IERROR)75 1638 y Fq(Example)c(6.3)k Fv(Supp)q(ose)14 b(that)d Ft(comm)h Fv(is)g(a)g(comm)o(unicator)g(with) g(a)g(sh)o(u\017e-exc)o(hange)h(top)q(ology)l(.)19 b(The)75 1694 y(group)h(has)f(2)317 1678 y Ff(n)360 1694 y Fv(mem)o(b)q(ers.)34 b(Eac)o(h)19 b(pro)q(cess)h(is)h(lab)q(eled)h(b)o(y)d Fp(a)1166 1701 y Fg(1)1186 1694 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)1312 1701 y Ff(n)1354 1694 y Fv(with)21 b Fp(a)1487 1701 y Ff(i)1521 1694 y Fx(2)f(f)p Fv(0)p Fp(;)8 b Fv(1)p Fx(g)p Fv(,)19 b(and)h(has)75 1751 y(three)d(neigh)o(b)q(ors:)23 b(exc)o(hange\()p Fp(a)639 1758 y Fg(1)659 1751 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)785 1758 y Ff(n)808 1751 y Fv(\))14 b(=)i Fp(a)915 1758 y Fg(1)935 1751 y Fp(;)8 b(:)g(:)g(:)t(;)g(a)1060 1758 y Ff(n)p Fh(\000)p Fg(1)1128 1751 y Fp(;)g Fv(\026)-23 b Fp(a)1173 1758 y Ff(n)1213 1751 y Fv(\()q(\026)f Fp(a)14 b Fv(=)i(1)11 b Fx(\000)g Fp(a)p Fv(\),)17 b(sh)o(u\017e\()p Fp(a)1639 1758 y Fg(1)1658 1751 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)1784 1758 y Ff(n)1807 1751 y Fv(\))15 b(=)75 1807 y Fp(a)99 1814 y Fg(2)119 1807 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)245 1814 y Ff(n)267 1807 y Fp(;)g(a)312 1814 y Fg(1)331 1807 y Fv(,)24 b(and)e(unsh)o(u\017e\()p Fp(a)680 1814 y Fg(1)700 1807 y Fp(;)8 b(:)g(:)g(:)d(;)j(a)826 1814 y Ff(n)849 1807 y Fv(\))24 b(=)g Fp(a)974 1814 y Ff(n)998 1807 y Fp(;)8 b(a)1043 1814 y Fg(1)1062 1807 y Fp(;)g(:)g(:)g(:)t(;)g(a)1187 1814 y Ff(n)p Fh(\000)p Fg(1)1255 1807 y Fv(.)41 b(The)22 b(graph)g(adjacency)h(list)f(is)75 1864 y(illustrated)17 b(b)q(elo)o(w)e(for)g Fp(n)e Fv(=)g(3.)p 75 1928 1119 2 v 74 1984 2 57 v 133 1968 a Fq(no)q(de)p 300 1984 V 104 w(exc)o(hange)122 b(sh)o(u\017e)h(unsh)o(u\017e)p 1193 1984 V 74 2041 V 300 2041 V 326 2024 a Fv(neigh)o(b)q(ors\(1\))49 b(neigh)o(b)q(ors\(2\))h(neigh)o(b)q(ors\(3\))p 1193 2041 V 75 2043 1119 2 v 74 2099 2 57 v 100 2082 a(0)f(\(000\))p 300 2099 V 161 w(1)275 b(0)g(0)p 1193 2099 V 74 2156 V 100 2139 a(1)49 b(\(001\))p 300 2156 V 161 w(0)275 b(2)g(4)p 1193 2156 V 74 2212 V 100 2195 a(2)49 b(\(010\))p 300 2212 V 161 w(3)275 b(4)g(1)p 1193 2212 V 74 2268 V 100 2251 a(3)49 b(\(011\))p 300 2268 V 161 w(2)275 b(6)g(5)p 1193 2268 V 74 2325 V 100 2308 a(4)49 b(\(100\))p 300 2325 V 161 w(5)275 b(1)g(2)p 1193 2325 V 74 2381 V 100 2364 a(5)49 b(\(101\))p 300 2381 V 161 w(4)275 b(3)g(6)p 1193 2381 V 74 2438 V 100 2421 a(6)49 b(\(110\))p 300 2438 V 161 w(7)275 b(5)g(3)p 1193 2438 V 74 2494 V 100 2477 a(7)49 b(\(111\))p 300 2494 V 161 w(6)275 b(7)g(7)p 1193 2494 V 75 2496 1119 2 v 166 2591 a(Supp)q(ose)15 b(that)e(the)g(comm)o(unicator)h Ft(comm)f Fv(has)g(this)h(top)q(ology) f(asso)q(ciated)h(with)g(it.)20 b(The)14 b(follo)o(w-)75 2647 y(ing)g(co)q(de)h(fragmen)o(t)e(cycles)h(through)g(the)g(three)g (t)o(yp)q(es)g(of)f(neigh)o(b)q(ors)i(and)f(p)q(erforms)f(an)h (appropriate)75 2704 y(p)q(erm)o(utation)h(for)g(eac)o(h.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 187 195 187 194 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fv(187)75 45 y Ft(C)48 b(assume:)22 b(each)i(process)f(has)g(stored)g (a)h(real)f(number)g(A.)75 102 y(C)48 b(extract)22 b(neighborhood)h (information)218 158 y(CALL)g(MPI_COMM_RANK\(comm,)f(myrank,)h(ierr\)) 218 214 y(CALL)g(MPI_GRAPH_NEIGHBORS\(comm,)e(myrank,)i(3,)g (neighbors,)g(ierr\))75 271 y(C)48 b(perform)22 b(exchange)h (permutation)218 327 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e(1,)j (MPI_REAL,)e(neighbors\(1\),)h(0,)194 384 y(+)119 b(neighbors\(1\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 440 y(C)48 b(perform)22 b(shuffle)h(permutation)218 497 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e(1,) j(MPI_REAL,)e(neighbors\(2\),)h(0,)194 553 y(+)119 b(neighbors\(3\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 610 y(C)48 b(perform)22 b(unshuffle)h(permutation)218 666 y(CALL)g(MPI_SENDRECV_REPLACE\(A,)e (1,)j(MPI_REAL,)e(neighbors\(3\),)h(0,)194 723 y(+)119 b(neighbors\(2\),)23 b(0,)g(comm,)g(status,)g(ierr\))75 844 y Fj(6.5.5)49 b(Ca)o(rtesian)16 b(Shift)g(Co)q(o)o(rdinates)75 930 y Fv(If)f(the)g(pro)q(cess)f(top)q(ology)h(is)g(a)f(cartesian)h (structure,)f(a)g Fw(MPI)p 1145 930 14 2 v 16 w(SENDRECV)i Fv(op)q(eration)f(is)g(lik)o(ely)h(to)e(b)q(e)75 987 y(used)19 b(along)g(a)f(co)q(ordinate)h(direction)h(to)e(p)q(erform)g (a)g(shift)h(of)f(data.)29 b(As)19 b(input,)h Fw(MPI)p 1629 987 V 16 w(SENDRECV)75 1043 y Fv(tak)o(es)14 b(the)h(rank)g(of)f (a)h(source)f(pro)q(cess)i(for)e(the)h(receiv)o(e,)g(and)g(the)g(rank)g (of)f(a)h(destination)g(pro)q(cess)h(for)75 1099 y(the)11 b(send.)18 b(If)11 b(the)g(function)g Fw(MPI)p 641 1099 V 16 w(CART)p 776 1099 V 17 w(SHIFT)f Fv(is)h(called)h(for)e(a)h (cartesian)f(pro)q(cess)h(group,)g(it)f(pro)o(vides)75 1156 y(the)j(calling)h(pro)q(cess)e(with)h(the)g(ab)q(o)o(v)o(e)f(iden) o(ti\014ers,)i(whic)o(h)f(then)g(can)g(b)q(e)g(passed)f(to)g Fw(MPI)p 1616 1156 V 16 w(SENDRECV)p Fv(.)75 1212 y(The)17 b(user)h(sp)q(eci\014es)h(the)e(co)q(ordinate)h(direction)g(and)f(the)h (size)g(of)f(the)g(step)g(\(p)q(ositiv)o(e)h(or)f(negativ)o(e\).)75 1269 y(The)e(function)h(is)g(lo)q(cal.)75 1420 y Fw(MPI)p 160 1420 V 16 w(CART)p 295 1420 V 17 w(SHIFT\(comm,)c(direction,)k (disp,)g(rank)p 977 1420 V 16 w(source,)g(rank)p 1224 1420 V 16 w(dest\))117 1497 y Fk(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j(\(handle\))117 1572 y(IN)171 b Fw(direction)408 b Fk(co)q(ordinate)14 b(dimension)f(of)g(shift)h(\(in)o(teger\))117 1647 y(IN)171 b Fw(disp)497 b Fk(displacemen)o(t)18 b(\()p Fa(>)h Fk(0:)27 b(up)o(w)o(ards)18 b(shift,)h Fa(<)g Fk(0:)27 b(do)o(wn)o(w)o(ards)905 1704 y(shift\))14 b(\(in)o(teger\))117 1779 y(OUT)124 b Fw(rank)p 420 1779 V 16 w(source)353 b Fk(rank)14 b(of)f(source)j (pro)q(cess)f(\(in)o(teger\))117 1854 y(OUT)124 b Fw(rank)p 420 1854 V 16 w(dest)396 b Fk(rank)14 b(of)f(destination)h(pro)q(cess)i (\(in)o(teger\))75 1978 y Ft(int)23 b(MPI)p 245 1978 15 2 v 17 w(Cart)p 358 1978 V 17 w(shift\(MPI)p 591 1978 V 16 w(Comm)g(comm,)g(int)h(direction,)e(int)i(disp,)f(int)g(*rank)p 1681 1978 V 17 w(source,)393 2035 y(int)h(*rank)p 612 2035 V 16 w(dest\))75 2121 y(MPI)p 150 2121 V 17 w(CART)p 263 2121 V 16 w(SHIFT\(COMM,)f(DIRECTION,)f(DISP,)h(RANK)p 1067 2121 V 17 w(SOURCE,)g(RANK)p 1371 2121 V 17 w(DEST,)g(IERROR\))170 2178 y(INTEGER)g(COMM,)h(DIRECTION,)e(DISP,)h(RANK)p 1009 2178 V 17 w(SOURCE,)g(RANK)p 1313 2178 V 16 w(DEST,)h(IERROR)1937 2214 y Fx(>)166 2264 y Fv(The)16 b Fw(direction)g Fv(argumen)o(t)e (indicates)j(the)f(dimension)h(of)e(the)g(shift,)h(i.e.,)f(the)g(co)q (ordinate)h(whic)o(h)75 2321 y(v)m(alue)21 b(is)f(mo)q(di\014ed)g(b)o (y)g(the)f(shift.)33 b(The)20 b(co)q(ordinates)g(are)f(n)o(um)o(b)q (ered)h(from)e(0)i(to)e Ft(ndims-1)p Fv(,)h(when)75 2377 y Ft(ndims)14 b Fv(is)i(the)f(n)o(um)o(b)q(er)h(of)f(dimensions.)1157 b Fx(?)166 2434 y Fv(Dep)q(ending)18 b(on)e(the)g(p)q(erio)q(dicit)o(y) i(of)e(the)g(cartesian)h(group)f(in)h(the)f(sp)q(eci\014ed)i(co)q (ordinate)f(direc-)75 2490 y(tion,)c Fw(MPI)p 264 2490 14 2 v 16 w(CART)p 399 2490 V 17 w(SHIFT)g Fv(pro)o(vides)h(the)f(iden) o(ti\014ers)i(for)e(a)g(circular)h(or)f(an)g(end-o\013)g(shift.)20 b(In)14 b(the)f(case)75 2547 y(of)j(an)g(end-o\013)g(shift,)h(the)f(v)m (alue)i Fi(MPI)p 735 2547 13 2 v 14 w(PROC)p 861 2547 V 13 w(NULL)f Fv(ma)o(y)f(b)q(e)h(returned)f(in)h Fw(rank)p 1479 2547 14 2 v 17 w(source)g Fv(or)e Fw(rank)p 1771 2547 V 17 w(dest)p Fv(,)75 2603 y(indicating)i(that)e(the)g(source)g (or)g(the)g(destination)h(for)f(the)g(shift)h(is)f(out)g(of)g(range.) -32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 188 196 188 195 bop 75 -100 a Fv(188)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fq(Example)j(6.4)k Fv(The)14 b(comm)o(unicator,)e Fw(comm)p Fv(,)g(has)g(a)h(t)o(w)o (o-dimensional,)h(p)q(erio)q(dic,)h(cartesian)f(top)q(ol-)75 102 y(ogy)f(asso)q(ciated)h(with)g(it.)19 b(A)14 b(t)o(w)o (o-dimensional)g(arra)o(y)f(of)g Ft(REAL)p Fv(s)g(is)h(stored)f(one)h (elemen)o(t)g(p)q(er)g(pro)q(cess,)75 158 y(in)h(v)m(ariable)h Ft(A)p Fv(.)e(One)h(wishes)g(to)e(sk)o(ew)h(this)h(arra)o(y)l(,)e(b)o (y)i(shifting)g(column)g Ft(i)f Fv(\(v)o(ertically)l(,)h(i.e.,)g(along) f(the)75 214 y(column\))i(b)o(y)f Ft(i)g Fv(steps.)75 305 y Ft(....)75 362 y(C)24 b(find)f(process)g(rank)218 418 y(CALL)g(MPI_COMM_RANK\(comm,)f(rank,)h(ierr\)\))75 475 y(C)h(find)f(cartesian)g(coordinates)218 531 y(CALL)g (MPI_CART_COORDS\(comm,)e(rank,)j(maxdims,)e(coords,)h(ierr\))75 588 y(C)h(compute)f(shift)g(source)g(and)g(destination)218 644 y(CALL)g(MPI_CART_SHIFT\(comm,)f(0,)h(coords\(2\),)g(source,)g (dest,)g(ierr\))75 701 y(C)h(skew)f(array)218 757 y(CALL)g (MPI_SENDRECV_REPLACE\(A,)e(1,)j(MPI_REAL,)e(dest,)i(0,)f(source,)g(0,) h(comm,)194 814 y(+)621 b(status,)23 b(ierr\))-160 866 y Fx(>)189 962 y Fl(A)n(dvic)n(e)12 b(to)i(users.)37 b Fv(In)13 b(F)l(ortran,)e(the)h(dimension)i(indicated)g(b)o(y)e Fi(DIRECTION)d(=)i(i)h Fv(has)g Fi(DIMS\(i+1\))189 1018 y Fv(no)q(des,)j(where)f Fi(DIMS)h Fv(is)g(the)g(arra)o(y)e(that)h(w)o (as)g(used)h(to)f(create)g(the)h(grid.)20 b(In)15 b(C,)g(the)f (dimension)189 1075 y(indicated)g(b)o(y)f Fi(direction)g(=)f(i)h Fv(is)g(the)g(dimension)i(sp)q(eci\014ed)g(b)o(y)e Fi(dims[i])p Fv(.)18 b(\()p Fl(End)13 b(of)h(advic)n(e)g(to)h(users.)p Fv(\))-160 1132 y Fx(?)75 1242 y Fj(6.5.6)49 b(P)o(a)o(rtitioning)17 b(of)g(Ca)o(rtesian)e(structures)75 1422 y Fw(MPI)p 160 1422 14 2 v 16 w(CART)p 295 1422 V 17 w(SUB\(comm,)e(remain)p 688 1422 V 16 w(dims,)h(new)o(comm\))117 1499 y Fk(IN)171 b Fw(comm)454 b Fk(comm)o(unicator)11 b(with)j(cartesian)g(structure)j (\(handle\))117 1574 y(IN)171 b Fw(remain)p 466 1574 V 15 w(dims)339 b Fk(the)24 b Fb(i)p Fk(th)f(en)o(try)h(of)e Fi(remain)p 1363 1574 13 2 v 15 w(dims)g Fk(sp)q(eci\014es)j(whether)f (the)905 1630 y Fb(i)p Fk(th)11 b(dimension)f(is)h(k)o(ept)h(in)f(the)h (subgrid)f(\()p Fb(true)p Fk(\))g(or)g(is)g(drop-)905 1687 y(p)q(ed)k(\()p Fb(false)p Fk(\))e(\(logical)f(v)o(ector\))117 1761 y(OUT)124 b Fw(new)o(comm)381 b Fk(comm)o(unicator)20 b(con)o(taining)h(the)i(subgrid)f(that)g(includes)905 1818 y(the)15 b(calling)d(pro)q(cess)k(\(handle\))75 1942 y Ft(int)23 b(MPI)p 245 1942 15 2 v 17 w(Cart)p 358 1942 V 17 w(sub\(MPI)p 543 1942 V 16 w(Comm)g(comm,)h(int)f (*remain)p 1085 1942 V 16 w(dims,)g(MPI)p 1316 1942 V 17 w(Comm)h(*newcomm\))75 2029 y(MPI)p 150 2029 V 17 w(CART)p 263 2029 V 16 w(SUB\(COMM,)f(REMAIN)p 662 2029 V 16 w(DIMS,)h(NEWCOMM,)e(IERROR\))170 2085 y(INTEGER)h(COMM,)h (NEWCOMM,)e(IERROR)170 2142 y(LOGICAL)h(REMAIN)p 508 2142 V 17 w(DIMS\(*\))166 2228 y Fv(If)28 b(a)f(cartesian)g(top)q (ology)g(has)h(b)q(een)g(created)g(with)g Fw(MPI)p 1263 2228 14 2 v 15 w(CART)p 1397 2228 V 17 w(CREA)l(TE)p Fv(,)h(the)e(function)75 2284 y Fw(MPI)p 160 2284 V 16 w(CART)p 295 2284 V 17 w(SUB)c Fv(can)f(b)q(e)h(used)f(to)g(partition)g (the)g(comm)o(unicator)g(group)g(in)o(to)g(subgroups)g(that)75 2341 y(form)17 b(lo)o(w)o(er-dimensional)j(cartesian)f(subgrids,)g(and) f(to)g(build)i(for)d(eac)o(h)h(subgroup)h(a)e(comm)o(unica-)75 2397 y(tor)k(with)h(the)g(asso)q(ciated)f(subgrid)i(cartesian)f(top)q (ology)l(.)39 b(\(This)22 b(function)g(is)h(closely)f(related)h(to)75 2454 y Fw(MPI)p 160 2454 V 16 w(COMM)p 318 2454 V 16 w(SPLIT)p Fv(.\))75 2556 y Fq(Example)18 b(6.5)k Fv(Assume)12 b(that)g Fw(MPI)p 726 2556 V 15 w(CART)p 860 2556 V 17 w(CREA)l(TE)p Ft(\(...,)25 b(comm\))11 b Fv(has)h(de\014ned)h(a)e(\(2)s Fx(\002)s Fv(3)s Fx(\002)s Fv(4\))i(grid.)75 2613 y(Let)i Ft(remain)p 303 2613 15 2 v 17 w(dims)23 b(=)h(\(true,)f(false,)g (true\))p Fv(.)c(Then)d(a)f(call)h(to,)194 2704 y Ft (MPI_CART_SUB\(comm,)22 b(remain_dims,)g(comm_new\),)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 189 197 189 196 bop 75 -100 a Fm(6.5.)34 b(TOPOLOGY)16 b(CONSTR)o(UCTORS)958 b Fv(189)75 45 y(will)26 b(create)d(three)i(comm)o(unicators)e(eac)o(h) h(with)h(eigh)o(t)f(pro)q(cesses)g(in)h(a)f(2)15 b Fx(\002)i Fv(4)23 b(cartesian)i(top)q(ol-)75 102 y(ogy)l(.)36 b(If)21 b Ft(remain)p 388 102 15 2 v 16 w(dims)j(=)f(\(false,)g(false,)g (true\))d Fv(then)h(the)g(call)h(to)e Fw(MPI)p 1494 102 14 2 v 16 w(CART)p 1629 102 V 17 w(SUB\(comm,)75 158 y(remain)p 207 158 V 15 w(dims,)k(comm)p 464 158 V 14 w(new\))f Fv(will)h(create)e(six)g(non-o)o(v)o(erlapping)h(comm)o (unicators,)h(eac)o(h)e(with)g(four)75 214 y(pro)q(cesses,)15 b(in)h(a)f(one-dimensional)j(cartesian)d(top)q(ology)l(.)75 336 y Fj(6.5.7)49 b(Lo)o(w-level)18 b(top)q(ology)g(functions)75 422 y Fv(The)d(t)o(w)o(o)e(additional)k(functions)e(in)o(tro)q(duced)h (in)g(this)f(section)h(can)e(b)q(e)i(used)f(to)g(implemen)o(t)h(all)f (other)75 478 y(top)q(ology)g(functions.)21 b(In)15 b(general)h(they)f (will)i(not)e(b)q(e)h(called)h(b)o(y)e(the)g(user)h(directly)l(,)g (unless)g(he)g(or)f(she)75 535 y(is)h(creating)f(additional)i(virtual)f (top)q(ology)e(capabilit)o(y)j(other)e(than)g(that)f(pro)o(vided)i(b)o (y)g Fw(MPI)p Fv(.)75 686 y Fw(MPI)p 160 686 V 16 w(CART)p 295 686 V 17 w(MAP\(comm,)d(ndims,)i(dims,)f(p)q(erio)q(ds,)i (newrank\))117 763 y Fk(IN)171 b Fw(comm)454 b Fk(input)14 b(comm)o(unicator)d(\(handle\))117 838 y(IN)171 b Fw(ndims)460 b Fk(n)o(um)o(b)q(er)13 b(of)h(dimensions)e(of)i(cartesian)g(structure) i(\(in)o(teger\))117 913 y(IN)171 b Fw(dims)483 b Fk(in)o(teger)18 b(arra)o(y)f(of)f(size)i Fb(ndims)e Fk(sp)q(ecifying)h(the)h(n)o(um)o (b)q(er)f(of)905 970 y(pro)q(cesses)g(in)c(eac)o(h)i(co)q(ordinate)f (direction)117 1045 y(IN)171 b Fw(p)q(erio)q(ds)436 b Fk(logical)16 b(arra)o(y)h(of)g(size)h Fb(ndims)f Fk(sp)q(ecifying)g (the)h(p)q(erio)q(dicit)o(y)905 1101 y(sp)q(eci\014cation)d(in)e(eac)o (h)i(co)q(ordinate)f(direction)117 1177 y(OUT)124 b Fw(newrank)414 b Fk(reordered)11 b(rank)f(of)e(the)i(calling)e(pro)q(cess;)k Fi(MPI)p 1643 1177 13 2 v 15 w(UNDEFINED)905 1233 y Fk(if)h(calling)g (pro)q(cess)i(do)q(es)g(not)f(b)q(elong)f(to)h(grid)g(\(in)o(teger\))75 1357 y Ft(int)23 b(MPI)p 245 1357 15 2 v 17 w(Cart)p 358 1357 V 17 w(map\(MPI)p 543 1357 V 16 w(Comm)g(comm,)h(int)f(ndims,) g(int)h(*dims,)f(int)g(*periods,)393 1414 y(int)h(*newrank\))75 1500 y(MPI)p 150 1500 V 17 w(CART)p 263 1500 V 16 w(MAP\(COMM,)f (NDIMS,)g(DIMS,)g(PERIODS,)g(NEWRANK,)g(IERROR\))170 1557 y(INTEGER)g(COMM,)h(NDIMS,)f(DIMS\(*\),)f(NEWRANK,)h(IERROR)170 1613 y(LOGICAL)g(PERIODS\(*\))166 1700 y Fw(MPI)p 251 1700 14 2 v 16 w(CART)p 386 1700 V 17 w(MAP)12 b Fv(computes)h(an)f (\\optimal")g(placemen)o(t)i(for)d(the)i(calling)h(pro)q(cess)f(on)f (the)h(ph)o(ys-)75 1756 y(ical)i(mac)o(hine.)20 b(A)13 b(p)q(ossible)i(implemen)o(tation)g(of)e(this)h(function)g(is)g(to)e (alw)o(a)o(ys)h(return)g(the)h(rank)f(of)g(the)75 1813 y(calling)k(pro)q(cess,)e(that)g(is,)g(not)g(to)f(p)q(erform)h(an)o(y)g (reordering.)189 1919 y Fl(A)n(dvic)n(e)22 b(to)h(implementors.)84 b Fv(The)22 b(function)i Fw(MPI)p 1138 1919 V 15 w(CART)p 1272 1919 V 17 w(CREA)l(TE\(comm,)f(ndims,)h(dims,)189 1975 y(p)q(erio)q(ds,)f(reo)o(rder,)d(comm)p 646 1975 V 15 w(ca)o(rt\))p Fv(,)h(with)g Ft(reorder)i(=)h(true)c Fv(can)h(b)q(e)g(implemen)o(ted)i(b)o(y)d(calling)189 2032 y Fw(MPI)p 274 2032 V 15 w(CART)p 408 2032 V 17 w(MAP\(comm,)72 b(ndims,)h(dims,)g(p)q(erio)q(ds,)h(newrank\))p Fv(,)g(then)62 b(calling)189 2088 y Fw(MPI)p 274 2088 V 15 w(COMM)p 431 2088 V 17 w(SPLIT\(comm,)26 b(colo)o(r,)i(k)o(ey)l(,) h(comm)p 1095 2088 V 15 w(ca)o(rt\))p Fv(,)f(with)f Ft(color)d(=)f(0)k Fv(if)g Ft(newrank)c Fx(6)p Fv(=)189 2145 y Ft(MPI)p 264 2145 15 2 v 16 w(UNDEFINED)p Fv(,)14 b Ft(color)23 b(=)h(MPI)p 786 2145 V 17 w(UNDEFINED)14 b Fv(otherwise,)h(and)g Ft(key)23 b(=)h(newrank)p Fv(.)189 2220 y(The)c(function)h Fw(MPI)p 555 2220 14 2 v 15 w(CART)p 689 2220 V 17 w(SUB\(comm,)f (remain)p 1089 2220 V 15 w(dims,)h(comm)p 1343 2220 V 14 w(new\))g Fv(can)f(b)q(e)g(implemen)o(ted)189 2276 y(b)o(y)c(a)g(call)i(to)e Fw(MPI)p 519 2276 V 16 w(COMM)p 677 2276 V 16 w(SPLIT\(comm,)e(colo)o(r,)i(k)o(ey)l(,)h(comm)p 1304 2276 V 14 w(new\))p Fv(,)g(using)g(a)g(single)h(n)o(um)o(b)q(er) 189 2333 y(enco)q(ding)11 b(of)f(the)g(lost)g(dimensions)i(as)e Ft(color)g Fv(and)g(a)g(single)h(n)o(um)o(b)q(er)g(enco)q(ding)h(of)d (the)i(preserv)o(ed)189 2389 y(dimensions)16 b(as)f Ft(key)p Fv(.)189 2464 y(All)f(other)g(cartesian)f(top)q(ology)g(functions)i (can)e(b)q(e)i(implemen)o(ted)g(lo)q(cally)l(,)g(using)g(the)e(top)q (ology)189 2521 y(information)h(that)g(is)i(cac)o(hed)f(with)g(the)g (comm)o(unicator.)k(\()p Fl(End)c(of)h(advic)n(e)g(to)g(implementors.)p Fv(\))166 2627 y(The)f(corresp)q(onding)i(new)e(function)h(for)f (general)g(graph)g(structures)g(is)h(as)f(follo)o(ws.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 190 198 190 197 bop 75 -100 a Fv(190)882 b Fm(CHAPTER)15 b(6.)35 b(PR)o(OCESS)15 b(TOPOLOGIES)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(GRAPH)p 326 45 V 17 w(MAP\(comm,)e(nno)q(des,)k(index,)f (edges,)g(newrank\))117 122 y Fk(IN)171 b Fw(comm)454 b Fk(input)14 b(comm)o(unicator)d(\(handle\))117 197 y(IN)171 b Fw(nno)q(des)441 b Fk(n)o(um)o(b)q(er)13 b(of)h(graph)f(no)q (des)i(\(in)o(teger\))117 273 y(IN)171 b Fw(index)473 b Fk(in)o(teger)30 b(arra)o(y)g(sp)q(ecifying)f(the)i(graph)e (structure,)36 b(see)905 329 y Fi(MPI)p 982 329 13 2 v 15 w(GRAPH)p 1136 329 V 14 w(CREA)m(TE)117 404 y Fk(IN)171 b Fw(edges)468 b Fk(in)o(teger)15 b(arra)o(y)e(sp)q(ecifying)h(the)g (graph)g(structure)117 479 y(OUT)124 b Fw(newrank)414 b Fk(reordered)11 b(rank)f(of)e(the)i(calling)e(pro)q(cess;)k Fi(MPI)p 1643 479 V 15 w(UNDEFINED)905 536 y Fk(if)j(the)i(calling)d (pro)q(cess)k(do)q(es)e(not)g(b)q(elong)g(to)f(graph)h(\(in)o(te-)905 592 y(ger\))75 717 y Ft(int)23 b(MPI)p 245 717 15 2 v 17 w(Graph)p 382 717 V 17 w(map\(MPI)p 567 717 V 16 w(Comm)g(comm,)g (int)h(nnodes,)f(int)g(*index,)g(int)h(*edges,)393 773 y(int)g(*newrank\))75 860 y(MPI)p 150 860 V 17 w(GRAPH)p 287 860 V 16 w(MAP\(COMM,)f(NNODES,)g(INDEX,)g(EDGES,)g(NEWRANK,)g (IERROR\))170 916 y(INTEGER)g(COMM,)h(NNODES,)e(INDEX\(*\),)h (EDGES\(*\),)g(NEWRANK,)f(IERROR)189 1052 y Fl(A)n(dvic)n(e)17 b(to)h(implementors.)53 b Fv(The)18 b(function)g Fw(MPI)p 1087 1052 14 2 v 16 w(GRAPH)p 1253 1052 V 17 w(CREA)l(TE\(comm,)f(nno)q (des,)i(index,)189 1109 y(edges,)j(reo)o(rder,)e(comm)p 613 1109 V 15 w(graph\))p Fv(,)h(with)g Ft(reorder)i(=)h(true)c Fv(can)g(b)q(e)h(implemen)o(ted)i(b)o(y)d(calling)189 1165 y Fw(MPI)p 274 1165 V 15 w(GRAPH)p 439 1165 V 18 w(MAP\(comm,)66 b(nno)q(des,)k(index,)e(edges,)h(newrank\))p Fv(,)f(then)58 b(calling)189 1222 y Fw(MPI)p 274 1222 V 15 w(COMM)p 431 1222 V 17 w(SPLIT\(comm,)21 b(colo)o(r,)h(k)o(ey)l(,) i(comm)p 1079 1222 V 15 w(graph\))p Fv(,)f(with)g Ft(color)g(=)h(0)e Fv(if)h Ft(newrank)g Fx(6)p Fv(=)189 1278 y Ft(MPI)p 264 1278 15 2 v 16 w(UNDEFINED)p Fv(,)14 b Ft(color)23 b(=)h(MPI)p 786 1278 V 17 w(UNDEFINED)14 b Fv(otherwise,)h(and)g Ft(key)23 b(=)h(newrank)p Fv(.)189 1353 y(All)c(other)f(graph)g(top)q (ology)g(functions)h(can)f(b)q(e)h(implemen)o(ted)h(lo)q(cally)l(,)h (using)e(the)f(top)q(ology)189 1410 y(information)14 b(that)g(is)i(cac)o(hed)f(with)g(the)g(comm)o(unicator.)k(\()p Fl(End)c(of)h(advic)n(e)g(to)g(implementors.)p Fv(\))75 1553 y Fn(6.6)59 b(An)20 b(Application)f(Example)75 1654 y Fq(Example)f(6.6)k Fv(The)11 b(example)h(in)f(\014gure)g(6.1)f(sho)o (ws)g(ho)o(w)g(the)h(grid)g(de\014nition)h(and)f(inquiry)h(functions)75 1711 y(can)20 b(b)q(e)g(used)h(in)f(an)g(application)h(program.)33 b(A)19 b(partial)h(di\013eren)o(tial)h(equation,)g(for)e(instance)i (the)75 1767 y(P)o(oisson)e(equation,)g(is)g(to)g(b)q(e)g(solv)o(ed)g (on)g(a)f(rectangular)h(domain.)31 b(First,)19 b(the)g(pro)q(cesses)g (organize)75 1824 y(themselv)o(es)e(in)h(a)e(t)o(w)o(o-dimensional)h (structure.)24 b(Eac)o(h)16 b(pro)q(cess)h(then)g(inquires)h(ab)q(out)f (the)f(ranks)g(of)75 1880 y(its)e(neigh)o(b)q(ors)h(in)g(the)f(four)g (directions)h(\(up,)f(do)o(wn,)g(righ)o(t,)g(left\).)19 b(The)c(n)o(umerical)g(problem)g(is)g(solv)o(ed)75 1937 y(b)o(y)g(an)g(iterativ)o(e)h(metho)q(d,)f(the)g(details)h(of)f(whic)o (h)h(are)f(hidden)i(in)f(the)f(subroutine)h Ft(relax)p Fv(.)166 1993 y(In)e(eac)o(h)f(relaxation)g(step)g(eac)o(h)g(pro)q (cess)h(computes)f(new)g(v)m(alues)h(for)f(the)g(solution)h(grid)f (function)75 2050 y(at)h(all)i(p)q(oin)o(ts)f(o)o(wned)g(b)o(y)g(the)g (pro)q(cess.)20 b(Then)15 b(the)g(v)m(alues)h(at)e(in)o(ter-pro)q(cess) i(b)q(oundaries)g(ha)o(v)o(e)e(to)g(b)q(e)75 2106 y(exc)o(hanged)e (with)h(neigh)o(b)q(oring)g(pro)q(cesses.)19 b(F)l(or)11 b(example,)i(the)f(exc)o(hange)g(subroutine)h(migh)o(t)f(con)o(tain)75 2162 y(a)i(call)i(lik)o(e)g Fw(MPI)p 363 2162 14 2 v 16 w(SEND\(...,neigh)p 666 2162 V 15 w(rank\(1\),...\))i Fv(to)c(send)h(up)q(dated)g(v)m(alues)h(to)e(the)h(left-hand)g(neigh)o (b)q(or)75 2219 y Ft(\(i-1,j\))p Fv(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 191 199 191 198 bop 75 -100 a Fm(6.6.)34 b(AN)15 b(APPLICA)l(TION)i(EXAMPLE)974 b Fv(191)194 163 y Ft(integer)23 b(ndims,)g(num)p 627 163 15 2 v 17 w(neigh)194 219 y(logical)g(reorder)194 276 y(parameter)g(\(ndims=2,)g(num)p 747 276 V 16 w(neigh=4,)g (reorder=.true.\))194 332 y(integer)g(comm,)g(comm)p 627 332 V 17 w(cart,)g(dims\(ndims\),)f(neigh)p 1217 332 V 17 w(def\(ndims\),)g(ierr)194 388 y(integer)h(neigh)p 508 388 V 17 w(rank\(num)p 717 388 V 16 w(neigh\),)g(own)p 996 388 V 16 w(position\(ndims\),)f(i,)i(j)194 445 y(logical)f (periods\(ndims\))194 501 y(real)p Fx(\003)p Ft(8)g(u\(0:101,0:101\),)f (f\(0:101,0:101\))194 558 y(data)i(dims)f(/)h(ndims)f Fx(\003)g Ft(0)h(/)194 614 y(comm)g(=)f(MPI)p 436 614 V 17 w(COMM)p 549 614 V 17 w(WORLD)75 671 y(C)107 b(Set)23 b(process)g(grid)g(size)h(and)f(periodicity)194 727 y(call)h(MPI)p 389 727 V 16 w(DIMS)p 501 727 V 17 w(CREATE\(comm,)e(ndims,)h (dims,ierr\))194 784 y(periods\(1\))g(=)h(.TRUE.)194 840 y(periods\(2\))f(=)h(.TRUE.)75 897 y(C)107 b(Create)23 b(a)g(grid)h(structure)e(in)i(WORLD)f(group)g(and)h(inquire)f(about)g (own)g(position)194 953 y(call)h(MPI)p 389 953 V 16 w(CART)p 501 953 V 17 w(CREATE)f(\(comm,)g(ndims,)g(dims,)g(periods,)g(reorder,) g(comm)p 1688 953 V 17 w(cart,ierr\))194 1009 y(call)h(MPI)p 389 1009 V 16 w(CART)p 501 1009 V 17 w(GET)f(\(comm)p 733 1009 V 17 w(cart,)g(ndims,)g(dims,)g(periods,)g(own)p 1490 1009 V 17 w(position,ierr\))75 1066 y(C)107 b(Look)23 b(up)h(the)f(ranks)g(for)h(the)f(neighbors.)46 b(Own)24 b(process)f(coordinates)f(are)i(\(i,j\).)75 1122 y(C)107 b(Neighbors)22 b(are)i(\(i-1,j\),)f(\(i+1,j\),)f(\(i,j-1\),)h (\(i,j+1\))194 1179 y(i)h(=)g(own)p 365 1179 V 17 w(position\(1\))194 1235 y(j)g(=)g(own)p 365 1235 V 17 w(position\(2\))194 1292 y(neigh)p 317 1292 V 17 w(def\(1\))f(=)h(i-1)194 1348 y(neigh)p 317 1348 V 17 w(def\(2\))f(=)h(j)194 1405 y(call)g(MPI)p 389 1405 V 16 w(CART)p 501 1405 V 17 w(RANK)f(\(comm)p 757 1405 V 17 w(cart,)g(neigh)p 1037 1405 V 17 w(def,)g(neigh)p 1293 1405 V 16 w(rank\(1\),ierr\))194 1461 y(neigh)p 317 1461 V 17 w(def\(1\))g(=)h(i+1)194 1518 y(neigh)p 317 1518 V 17 w(def\(2\))f(=)h(j)194 1574 y(call)g(MPI)p 389 1574 V 16 w(CART)p 501 1574 V 17 w(RANK)f(\(comm)p 757 1574 V 17 w(cart,)g(neigh)p 1037 1574 V 17 w(def,)g(neigh)p 1293 1574 V 16 w(rank\(2\),ierr\))194 1630 y(neigh)p 317 1630 V 17 w(def\(1\))g(=)h(i)194 1687 y(neigh)p 317 1687 V 17 w(def\(2\))f(=)h(j-1)194 1743 y(call)g(MPI)p 389 1743 V 16 w(CART)p 501 1743 V 17 w(RANK)f(\(comm)p 757 1743 V 17 w(cart,)g(neigh)p 1037 1743 V 17 w(def,)g(neigh)p 1293 1743 V 16 w(rank\(3\),ierr\))194 1800 y(neigh)p 317 1800 V 17 w(def\(1\))g(=)h(i)194 1856 y(neigh)p 317 1856 V 17 w(def\(2\))f(=)h(j+1)194 1913 y(call)g(MPI)p 389 1913 V 16 w(CART)p 501 1913 V 17 w(RANK)f(\(comm)p 757 1913 V 17 w(cart,)g(neigh)p 1037 1913 V 17 w(def,)g(neigh)p 1293 1913 V 16 w(rank\(4\),ierr\))75 1969 y(C)107 b(Initialize)22 b(the)i(grid)f(functions)g(and)g(start)g(the)h(iteration)194 2026 y(call)g(init)f(\(u,)g(f\))194 2082 y(do)h(10)f(it=1,100)242 2139 y(call)g(relax)h(\(u,)f(f\))75 2195 y(C)107 b(Exchange)22 b(data)i(with)f(neighbor)g(processes)242 2251 y(call)g(exchange)g(\(u,) h(comm)p 771 2251 V 16 w(cart,)f(neigh)p 1050 2251 V 17 w(rank,)g(num)p 1282 2251 V 17 w(neigh\))75 2308 y(10)71 b(continue)194 2364 y(call)24 b(output)f(\(u\))194 2421 y(end)154 2625 y Fv(Figure)16 b(6.1:)j(Set-up)d(of)e(pro)q(cess)i (structure)f(for)f(t)o(w)o(o-dimensional)i(parallel)h(P)o(oisson)e (solv)o(er.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 192 200 192 199 bop 75 356 a Fo(Chapter)34 b(7)75 564 y Fr(MPI)40 b(En)m(vironmen)m(tal)h(Managemen)m(t)75 786 y Fv(This)16 b(c)o(hapter)g(discusses)h(routines)g(for)e(getting)h(and,)f(where)h (appropriate,)g(setting)g(v)m(arious)g(param-)75 843 y(eters)g(that)f(relate)h(to)f(the)h Fw(MPI)f Fv(implemen)o(tation)i (and)f(the)g(execution)g(en)o(vironmen)o(t)h(\(suc)o(h)e(as)h(error)75 899 y(handling\).)21 b(The)13 b(pro)q(cedures)h(for)e(en)o(tering)i (and)f(lea)o(ving)h(the)f Fw(MPI)g Fv(execution)h(en)o(vironmen)o(t)f (are)g(also)75 956 y(describ)q(ed)k(here.)75 1099 y Fn(7.1)59 b(Implementation)17 b(info)n(rmation)75 1202 y Fj(7.1.1)49 b(Environmental)17 b(Inquiries)75 1288 y Fv(A)i(set)g(of)f(attributes)h (that)f(describ)q(e)i(the)f(execution)h(en)o(vironmen)o(t)f(are)g (attac)o(hed)f(to)g(the)h(comm)o(u-)75 1344 y(nicator)f Fi(MPI)p 309 1344 13 2 v 14 w(COMM)p 453 1344 V 15 w(W)o(ORLD)g Fv(when)h Fw(MPI)e Fv(is)i(initialized.)32 b(The)19 b(v)m(alue)g(of)f (these)g(attributes)g(can)g(b)q(e)75 1401 y(inquired)h(b)o(y)d(using)h (the)g(function)h Fw(MPI)p 784 1401 14 2 v 15 w(A)l(TTR)p 916 1401 V 17 w(GET)f Fv(describ)q(ed)i(in)e(Chapter)g(5.)23 b(It)17 b(is)g(erroneous)g(to)75 1457 y(delete)f(these)g(attributes,)e (free)i(their)f(k)o(eys,)g(or)g(c)o(hange)g(their)h(v)m(alues.)-1461 b Fx(>)-160 1510 y(?)166 1514 y Fv(The)15 b(list)h(of)f(prede\014ned)i (attribute)e(k)o(eys)g(include)75 1608 y Fi(MPI)p 152 1608 13 2 v 14 w(T)m(A)o(G)p 246 1608 V 14 w(UB)22 b Fv(Upp)q(er)16 b(b)q(ound)g(for)f(tag)f(v)m(alue.)75 1701 y Fi(MPI)p 152 1701 V 14 w(HOST)23 b Fv(Host)15 b(pro)q(cess)g(rank,)g(if)h(suc)o(h)f(exists,)g Fi(MPI)p 1041 1701 V 15 w(PROC)p 1168 1701 V 13 w(NULL)p Fv(,)h(otherwise.)75 1795 y Fi(MPI)p 152 1795 V 14 w(IO)22 b Fv(rank)14 b(of)f(a)g(no)q(de)h (that)f(has)h(regular)g(I/O)g(facilities)i(\(p)q(ossibly)f(m)o (yrank\).)j(No)q(des)c(in)h(the)e(same)189 1852 y(comm)o(unicator)h(ma) o(y)h(return)g(di\013eren)o(t)h(v)m(alues)g(for)e(this)i(parameter.) -1572 b Fx(>)75 1946 y Fi(MPI)p 152 1946 V 14 w(WTIME)p 306 1946 V 15 w(IS)p 356 1946 V 15 w(GLOBAL)22 b Fv(Bo)q(olean)16 b(v)m(ariable)h(that)d(indicates)j(whether)e(clo)q(c)o(ks)h(are)f(sync) o(hronized.)-1994 b Fx(?)166 2039 y Fv(V)l(endors)12 b(ma)o(y)g(add)g(implemen)o(tation)h(sp)q(eci\014c)h(parameters)d (\(suc)o(h)i(as)e(no)q(de)i(n)o(um)o(b)q(er,)g(real)f(mem-)75 2096 y(ory)j(size,)g(virtual)h(memory)f(size,)h(etc.\))-925 b Fx(>)166 2152 y Fv(These)15 b(prede\014ned)h(attributes)f(do)f(not)h (c)o(hange)f(v)m(alue)i(b)q(et)o(w)o(een)f Fw(MPI)f Fv(initializati)q (on)j(\()p Fw(MPI)p 1774 2152 14 2 v 16 w(INIT)75 2209 y Fv(and)e Fw(MPI)g Fv(completion)h(\()p Fw(MPI)p 595 2209 V 16 w(FINALIZE)p Fv(\),)d(and)j(cannot)f(b)q(e)h(up)q(dated)g(or) e(deleted)j(b)o(y)e(users.)189 2315 y Fl(A)n(dvic)n(e)g(to)j(users.)43 b Fv(Note)16 b(that)f(in)h(the)g(C)g(binding,)h(the)f(v)m(alue)h (returned)f(b)o(y)g(these)g(attributes)189 2371 y(is)f(a)g Fl(p)n(ointer)g Fv(to)g(an)g Ft(int)g Fv(con)o(taining)h(the)f (requested)h(v)m(alue.)21 b(\()p Fl(End)15 b(of)i(advic)n(e)f(to)g (users.)p Fv(\))-160 2433 y Fx(?)166 2478 y Fv(The)f(required)i (parameter)d(v)m(alues)i(are)f(discussed)i(in)f(more)f(detail)h(b)q (elo)o(w:)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 193 201 193 200 bop 75 -100 a Fm(7.1.)34 b(IMPLEMENT)l(A)l(TION)16 b(INF)o(ORMA)l(TION)841 b Fv(193)75 45 y Fw(T)l(ag)15 b(values)75 131 y Fv(T)l(ag)h(v)m(alues)h(range)f(from)f Ft(0)h Fv(to)f(the)i(v)m(alue)g(returned)f(for)g Fi(MPI)p 1162 131 13 2 v 14 w(T)m(A)o(G)p 1256 131 V 14 w(UB)f Fv(inclusiv)o(e.)26 b(These)16 b(v)m(alues)h(are)75 187 y(guaran)o(teed)c(to)g(b)q(e)i(unc)o(hanging)f(during)h(the)f (execution)h(of)e(an)h Fw(MPI)f Fv(program.)18 b(In)c(addition,)h(the)f (tag)75 244 y(upp)q(er)20 b(b)q(ound)g(v)m(alue)g(m)o(ust)f(b)q(e)g Fl(at)h(le)n(ast)j Fv(32767.)30 b(An)19 b Fw(MPI)f Fv(implemen)o (tation)i(is)g(free)f(to)f(mak)o(e)h(the)75 300 y(v)m(alue)c(of)f Fi(MPI)p 319 300 V 14 w(T)m(A)o(G)p 413 300 V 14 w(UB)f Fv(larger)h(than)g(this;)h(for)e(example,)i(the)f(v)m(alue)h(2)1303 284 y Fg(30)1348 300 y Fx(\000)8 b Fv(1)14 b(is)h(also)f(a)f(legal)i(v) m(alue)h(for)75 357 y Fi(MPI)p 152 357 V 14 w(T)m(A)o(G)p 246 357 V 14 w(UB)p Fv(.)1609 b Fx(>)166 413 y Fv(The)15 b(attribute)f Fi(MPI)p 527 413 V 15 w(T)m(A)o(G)p 622 413 V 14 w(UB)g Fv(has)g(the)h(same)f(v)m(alue)i(on)f(all)g(pro)q (cesses)h(of)e Fi(MPI)p 1543 413 V 14 w(COMM)p 1687 413 V 15 w(W)o(ORLD)p Fv(.)76 b Fx(?)75 533 y Fw(Host)16 b(rank)75 619 y Fv(The)e(v)m(alue)h(returned)g(for)e Fi(MPI)p 612 619 V 14 w(HOST)i Fv(gets)e(the)h(rank)g(of)f(the)i Ft(HOST)e Fv(pro)q(cess)h(in)h(the)f(group)g(asso)q(ciated)75 676 y(with)22 b(comm)o(unicator)f Fi(MPI)p 565 676 V 14 w(COMM)p 709 676 V 15 w(W)o(ORLD)p Fv(,)g(if)h(there)g(is)g(suc)o (h.)39 b Fi(MPI)p 1343 676 V 15 w(PROC)p 1470 676 V 13 w(NULL)22 b Fv(is)h(returned)e(if)75 732 y(there)16 b(is)h(no)f(host.) 21 b Fw(MPI)16 b Fv(do)q(es)g(not)g(sp)q(ecify)h(what)f(it)g(means)g (for)f(a)h(pro)q(cess)g(to)f(b)q(e)i(a)f Ft(HOST)p Fv(,)f(nor)h(do)q (es)75 788 y(it)f(requires)h(that)f(a)g Ft(HOST)f Fv(exists.)1276 b Fx(>)166 845 y Fv(The)15 b(attribute)g Fi(MPI)p 528 845 V 15 w(HOST)g Fv(has)g(the)g(same)g(v)m(alue)i(on)e(all)h(pro)q (cesses)g(of)e Fi(MPI)p 1509 845 V 15 w(COMM)p 1654 845 V 14 w(W)o(ORLD)p Fv(.)110 b Fx(?)75 965 y Fw(IO)15 b(rank)75 1051 y Fv(The)e(v)m(alue)h(returned)f(for)f Fi(MPI)p 607 1051 V 14 w(IO)f Fv(is)j(the)e(rank)h(of)f(a)g(pro)q(cessor)g(that) g(can)h(pro)o(vide)g(language-standard)75 1107 y(I/O)i(facilities.)21 b(F)l(or)14 b(F)l(ortran,)e(this)j(means)f(that)f(all)i(of)f(the)g(F)l (ortran)f(I/O)h(op)q(erations)h(are)e(supp)q(orted)75 1164 y(\(e.g.,)i Ft(OPEN)p Fv(,)h Ft(REWIND)p Fv(,)f Ft(WRITE)p Fv(\).)g(F)l(or)h(C,)g(this)h(means)g(that)f(all)h(of)f(the) h(ANSI-C)h(I/O)f(op)q(erations)f(are)75 1220 y(supp)q(orted)g(\(e.g.,)d Ft(fopen)p Fv(,)h Ft(fprintf)p Fv(,)g Ft(lseek)p Fv(\).)1042 b Fx(>)166 1277 y Fv(If)16 b(ev)o(ery)g(pro)q(cess)h(can)f(pro)o(vide)h (language-standard)f(I/O,)g(then)h(the)f(v)m(alue)i Fi(MPI)p 1603 1277 V 14 w(ANY)p 1702 1277 V 15 w(SOURCE)75 1333 y Fv(will)24 b(b)q(e)e(returned.)40 b(Otherwise,)24 b(if)f(the)f (calling)h(pro)q(cess)g(can)f(pro)o(vide)g(language-standard)g(I/O,)75 1389 y(then)17 b(its)h(rank)f(will)h(b)q(e)g(returned.)26 b(Otherwise,)18 b(if)g(some)f(pro)q(cess)g(can)g(pro)o(vide)h (language-standard)75 1446 y(I/O)k(then)g(the)g(rank)g(of)f(one)h(suc)o (h)g(pro)q(cess)g(will)i(b)q(e)e(returned.)41 b(The)22 b(same)f(v)m(alue)i(need)g(not)e(b)q(e)75 1502 y(returned)15 b(b)o(y)g(all)h(pro)q(cesses.)k(If)c(no)f(pro)q(cess)g(can)g(pro)o (vide)g(language-standard)g(I/O,)g(then)h(the)f(v)m(alue)75 1559 y Fi(MPI)p 152 1559 V 14 w(PROC)p 278 1559 V 14 w(NULL)h Fv(will)h(b)q(e)e(returned.)189 1665 y Fl(A)n(dvic)n(e)d(to)i (users.)38 b Fv(Note)11 b(that)h(input)h(is)f(not)g(collectiv)o(e,)i (and)e(this)g(attribute)g(do)q(es)h Fl(not)e Fv(indicate)189 1722 y(whic)o(h)16 b(pro)q(cess)f(can)g(or)g(do)q(es)h(pro)o(vide)f (input.)21 b(\()p Fl(End)16 b(of)g(advic)n(e)g(to)h(users.)p Fv(\))75 1842 y Fw(Clo)q(ck)e(synchronization)75 1927 y Fv(The)37 b(v)m(alue)h(returned)e(for)g Fi(MPI)p 702 1927 V 15 w(WTIME)p 857 1927 V 15 w(IS)p 907 1927 V 14 w(GLOBAL)g Fv(is)h(1)g(if)g(clo)q(c)o(ks)g(at)f(all)h(pro)q(cesses)g (in)75 1984 y Fi(MPI)p 152 1984 V 14 w(COMM)p 296 1984 V 15 w(W)o(ORLD)15 b Fv(are)g(sync)o(hronized,)h(0)f(otherwise.)20 b(A)15 b(collection)i(of)e(clo)q(c)o(ks)g(is)h(considered)h(syn-)75 2040 y(c)o(hronized)j(if)g(explicit)h(e\013ort)e(has)g(b)q(een)h(tak)o (en)f(to)f(sync)o(hronize)j(them.)32 b(The)19 b(exp)q(ectation)h(is)g (that)75 2097 y(the)d(v)m(ariation)h(in)g(time,)g(as)f(measured)g(b)o (y)g(calls)i(to)d Fw(MPI)p 1095 2097 14 2 v 16 w(WTIME)p Fv(,)g(will)j(b)q(e)f(less)g(then)g(one)f(half)h(the)75 2153 y(round-trip)g(time)g(for)f(an)h Fw(MPI)f Fv(message)g(of)h (length)g(zero.)27 b(If)18 b(time)g(is)g(measured)g(at)f(a)g(pro)q (cess)h(just)75 2210 y(b)q(efore)d(a)g(send)h(and)g(at)e(another)h(pro) q(cess)g(just)g(after)g(a)f(matc)o(hing)i(receiv)o(e,)f(the)h(second)f (time)h(should)75 2266 y(b)q(e)g(alw)o(a)o(ys)e(higher)i(than)f(the)h (\014rst)e(one.)166 2323 y(The)22 b(attribute)h Fi(MPI)p 543 2323 13 2 v 14 w(WTIME)p 697 2323 V 15 w(IS)p 747 2323 V 14 w(GLOBAL)f Fv(need)h(not)f(b)q(e)h(presen)o(t)f(when)h(the)f (clo)q(c)o(ks)h(are)f(not)75 2379 y(sync)o(hronized)i(\(ho)o(w)o(ev)o (er,)e(the)h(attribute)f(k)o(ey)g Fi(MPI)p 1017 2379 V 14 w(WTIME)p 1171 2379 V 15 w(IS)p 1221 2379 V 15 w(GLOBAL)g Fv(is)g(alw)o(a)o(ys)g(v)m(alid\).)42 b(This)75 2436 y(attribute)15 b(ma)o(y)g(b)q(e)g(asso)q(ciated)h(with)f(comm)o (unicators)g(other)g(then)g Fi(MPI)p 1361 2436 V 15 w(COMM)p 1506 2436 V 14 w(W)o(ORLD)p Fv(.)166 2492 y(The)10 b(attribute)g Fi(MPI)p 518 2492 V 15 w(WTIME)p 673 2492 V 15 w(IS)p 723 2492 V 14 w(GLOBAL)g Fv(has)g(the)g(same)g(v)m(alue)h(on)f(all)h (pro)q(cesses)g(of)f Fi(MPI)p 1706 2492 V 14 w(COMM)p 1850 2492 V 15 w(W)o(ORLD)p Fv(.)1937 2548 y Fx(?)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 194 202 194 201 bop 75 -100 a Fv(194)532 b Fm(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(GET)p 264 45 V 17 w(PROCESSOR)p 540 45 V 18 w(NAME\()g(name,)e(resultlen)j(\))117 122 y Fk(OUT)108 b Fw(name)485 b Fk(A)17 b(unique)g(sp)q(eci\014er)i(for)e (the)g(actual)g(\(as)g(opp)q(osed)h(to)e(vir-)905 179 y(tual\))e(no)q(de.)117 254 y(OUT)108 b Fw(resultlen)430 b Fk(Length)12 b(\(in)g(prin)o(table)f(c)o(haracters\))i(of)f(the)g (result)g(returned)905 310 y(in)i Fb(name)75 435 y Ft(int)23 b(MPI)p 245 435 15 2 v 17 w(Get)p 334 435 V 17 w(processor)p 567 435 V 16 w(name\(char)g(*name,)g(int)g(*resultlen\))75 521 y(MPI)p 150 521 V 17 w(GET)p 239 521 V 17 w(PROCESSOR)p 472 521 V 15 w(NAME\()h(NAME,)f(RESULTLEN,)f(IERROR\))170 578 y(CHARACTER*\(*\))h(NAME)170 634 y(INTEGER)g(RESULTLEN,IERROR)166 721 y Fv(This)17 b(routine)f(returns)g(the)g(name)h(of)e(the)i(pro)q (cessor)e(on)i(whic)o(h)g(it)f(w)o(as)f(called)j(at)d(the)i(momen)o(t) 75 777 y(of)j(the)g(call.)35 b(The)20 b(name)g(is)h(a)e(c)o(haracter)h (string)g(for)f(maxim)o(um)h(\015exibilit)o(y)l(.)37 b(F)l(rom)19 b(this)h(v)m(alue)i(it)75 834 y(m)o(ust)14 b(b)q(e)h(p)q(ossible)h(to)e(iden)o(tify)i(a)e(sp)q(eci\014c)j(piece)f (of)e(hardw)o(are;)g(p)q(ossible)i(v)m(alues)g(include)g(\\pro)q (cessor)75 890 y(9)k(in)h(rac)o(k)e(4)h(of)f(mpp.cs.org")g(and)h (\\231")f(\(where)h(231)f(is)i(the)f(actual)g(pro)q(cessor)g(n)o(um)o (b)q(er)g(in)h(the)75 946 y(running)16 b(homogeneous)f(system\).)k(The) c(argumen)o(t)f Fw(name)g Fv(m)o(ust)h(represen)o(t)f(storage)g(that)g (is)i(at)e(least)75 1003 y Fi(MPI)p 152 1003 13 2 v 14 w(MAX)p 258 1003 V 15 w(PROCESSOR)p 514 1003 V 13 w(NAME)g Fv(c)o(haracters)f(long.)20 b Fw(MPI)p 1073 1003 14 2 v 16 w(GET)p 1177 1003 V 17 w(PROCESSOR)p 1453 1003 V 18 w(NAME)15 b Fv(ma)o(y)e(write)h(up)75 1059 y(to)h(this)g(man)o(y)g (c)o(haracters)f(in)o(to)i Fw(name)p Fv(.)166 1116 y(The)11 b(n)o(um)o(b)q(er)h(of)e(c)o(haracters)h(actually)h(written)f(is)g (returned)h(in)g(the)f(output)g(argumen)o(t,)g Fw(resultlen)p Fv(.)189 1222 y Fl(R)n(ationale.)72 b Fv(This)21 b(function)h(allo)o (ws)f Fw(MPI)f Fv(implemen)o(tations)i(that)d(do)i(pro)q(cess)g (migration)189 1279 y(to)d(return)i(the)f(curren)o(t)g(pro)q(cessor.)32 b(Note)19 b(that)g(nothing)h(in)g Fw(MPI)f Fl(r)n(e)n(quir)n(es)f Fv(or)h(de\014nes)h(pro-)189 1335 y(cess)g(migration;)i(this)e (de\014nition)i(of)d Fw(MPI)p 953 1335 V 16 w(GET)p 1057 1335 V 17 w(PROCESSOR)p 1333 1335 V 18 w(NAME)h Fv(simply)i(allo)o(ws)e (suc)o(h)189 1391 y(an)15 b(implemen)o(tation.)21 b(\()p Fl(End)16 b(of)g(r)n(ationale.)p Fv(\))189 1498 y Fl(A)n(dvic)n(e)g(to) h(users.)47 b Fv(The)16 b(user)g(m)o(ust)g(pro)o(vide)h(at)e(least)i Fi(MPI)p 1262 1498 13 2 v 14 w(MAX)p 1368 1498 V 15 w(PROCESSOR)p 1624 1498 V 12 w(NAME)g Fv(space)189 1554 y(to)g(write)i(the)f(pro)q (cessor)g(name)g(|)h(pro)q(cessor)f(names)g(can)g(b)q(e)h(this)g(long.) 29 b(The)19 b(user)f(should)189 1611 y(examine)g(the)f(ouput)h(argumen) o(t,)e Fw(resultlen)p Fv(,)j(to)e(determine)h(the)g(actual)f(length)h (of)f(the)g(name.)189 1667 y(\()p Fl(End)e(of)i(advic)n(e)f(to)g (users.)p Fv(\))-160 1728 y Fx(>)166 1773 y Fv(The)h(constan)o(t)f Fi(MPI)p 523 1773 V 14 w(BSEND)p 672 1773 V 15 w(O)o(VERHEAD)g Fv(pro)o(vides)h(an)g(upp)q(er)h(b)q(ound)g(on)f(the)g(\014xed)g(o)o(v) o(erhead)75 1830 y(p)q(er)f(message)e(bu\013ered)i(b)o(y)f(a)g(call)h (to)f Fw(MPI)p 830 1830 14 2 v 16 w(BSEND)h Fv(\(see)f(Section)h (3.6.1\).)-1542 b Fx(?)75 1973 y Fn(7.2)59 b(Erro)n(r)21 b(handling)75 2074 y Fv(An)14 b Fw(MPI)f Fv(implemen)o(tation)i(cannot) f(or)f(ma)o(y)h(c)o(ho)q(ose)f(not)h(to)f(handle)i(some)f(errors)f (that)g(o)q(ccur)h(during)75 2131 y Fw(MPI)e Fv(calls.)21 b(These)13 b(can)g(include)i(errors)e(that)f(generate)h(exceptions)h (or)e(traps,)h(suc)o(h)g(as)g(\015oating)f(p)q(oin)o(t)75 2187 y(errors)17 b(or)g(access)g(violations.)27 b(The)18 b(set)f(of)g(errors)g(that)f(are)i(handled)g(b)o(y)g Fw(MPI)f Fv(is)h(implemen)o(tation-)75 2244 y(dep)q(enden)o(t.)j(Eac)o (h)15 b(suc)o(h)h(error)e(generates)h(an)g Fq(MPI)i(exception)p Fv(.)-1415 b Fx(>)166 2300 y Fv(The)13 b(ab)q(o)o(v)o(e)f(text)h(tak)o (es)f(precedence)i(o)o(v)o(er)e(an)o(y)h(text)f(on)h(error)f(handling)j (within)f(this)f(do)q(cumen)o(t.)75 2357 y(Sp)q(eci\014cally)l(,)k (text)c(that)f(states)h(that)g(errors)f Fl(wil)r(l)h Fv(b)q(e)h(handled)h(should)g(b)q(e)f(read)f(as)g Fl(may)h Fv(b)q(e)g(handled.)-2014 b Fx(?)166 2413 y Fv(A)16 b(user)h(can)f (asso)q(ciate)h(an)f(error)g(handler)h(with)g(a)f(comm)o(unicator.)23 b(The)16 b(sp)q(eci\014ed)j(error)d(han-)75 2470 y(dling)21 b(routine)f(will)h(b)q(e)f(used)h(for)e(an)o(y)g Fw(MPI)g Fv(exception)h(that)f(o)q(ccurs)h(during)h(a)e(call)h(to)f Fw(MPI)g Fv(for)g(a)75 2526 y(comm)o(unication)c(with)g(this)g(comm)o (unicator.)k Fw(MPI)14 b Fv(calls)i(that)d(are)h(not)h(related)f(to)g (an)o(y)g(comm)o(unica-)75 2583 y(tor)d(are)h(considered)i(to)d(b)q(e)i (attac)o(hed)f(to)f(the)h(comm)o(unicator)g Fi(MPI)p 1230 2583 13 2 v 15 w(COMM)p 1375 2583 V 14 w(W)o(ORLD)p Fv(.)g(The)g(attac)o(hmen)o(t)75 2639 y(of)i(error)g(handlers)i(to)e (comm)o(unicators)g(is)i(purely)f(lo)q(cal:)21 b(di\013eren)o(t)15 b(pro)q(cesses)g(ma)o(y)f(attac)o(h)g(di\013eren)o(t)75 2695 y(error)g(handlers)j(to)d(the)i(same)e(comm)o(unicator.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 195 203 195 202 bop 75 -100 a Fm(7.2.)34 b(ERR)o(OR)17 b(HANDLING)1194 b Fv(195)166 45 y(A)17 b(newly)i(created)e(comm)o(unicator)h(inherits)g (the)g(error)f(handler)h(that)f(is)h(asso)q(ciated)g(with)g(the)75 102 y(\\paren)o(t")h(comm)o(unicator.)32 b(In)20 b(particular,)h(the)f (user)g(can)f(sp)q(ecify)i(a)e(\\global")h(error)f(handler)i(for)75 158 y(all)d(comm)o(unicators)e(b)o(y)h(asso)q(ciating)g(this)g(handler) h(with)f(the)g(comm)o(unicator)f Fi(MPI)p 1570 158 13 2 v 14 w(COMM)p 1714 158 V 15 w(W)o(ORLD)75 214 y Fv(immediately)h (after)d(initialization.)166 271 y(Sev)o(eral)i(prede\014ned)g(error)f (handlers)h(are)f(a)o(v)m(ailable)i(in)f Fw(MPI)p Fv(:)75 365 y Fi(MPI)p 152 365 V 14 w(ERRORS)p 326 365 V 14 w(ARE)p 420 365 V 14 w(F)m(A)m(T)m(AL)21 b Fv(The)15 b(handler,)f(when)h (called,)g(causes)f(the)g(program)f(to)g(ab)q(ort)h(on)g(all)g(exe-)189 421 y(cuting)f(pro)q(cesses.)20 b(This)13 b(has)g(the)g(same)g (e\013ect)g(as)f(if)i Fw(MPI)p 1203 421 14 2 v 16 w(ABORT)g Fv(w)o(as)e(called)j(b)o(y)e(the)g(pro)q(cess)189 478 y(that)h(in)o(v)o(ok)o(ed)i(the)f(handler.)75 571 y Fi(MPI)p 152 571 13 2 v 14 w(ERRORS)p 326 571 V 14 w(RETURN)22 b Fv(The)15 b(handler)g(has)g(no)f(e\013ect)g(other)g(than)g(returning) h(the)g(error)e(co)q(de)i(to)f(the)62 b Fx(>)189 628 y Fv(user.)1654 b Fx(?)166 722 y Fv(Implemen)o(tations)19 b(ma)o(y)f(pro)o(vide)h(additional)h(prede\014ned)g(error)d(handlers)j (and)e(programmers)75 778 y(can)d(co)q(de)h(their)g(o)o(wn)f(error)f (handlers.)166 835 y(The)19 b(error)f(handler)i Fi(MPI)p 623 835 V 15 w(ERRORS)p 798 835 V 13 w(ARE)p 891 835 V 15 w(F)m(A)m(T)m(AL)d Fv(is)i(asso)q(ciated)g(b)o(y)g(default)h(with) f Fi(MPI)p 1719 835 V 14 w(COMM-)p 75 891 V 87 891 a(W)o(ORLD)e Fv(after)f(initialization.)27 b(Th)o(us,)17 b(if)g(the)g(user)f(c)o(ho) q(oses)h(not)f(to)g(con)o(trol)h(error)f(handling,)i(ev)o(ery)75 948 y(error)13 b(that)f Fw(MPI)h Fv(handles)h(is)g(treated)f(as)g (fatal.)19 b(Since)14 b(\(almost\))e(all)j Fw(MPI)d Fv(calls)j(return)e (an)g(error)g(co)q(de,)75 1004 y(a)g(user)h(ma)o(y)g(c)o(ho)q(ose)f(to) g(handle)j(errors)d(in)h(its)g(main)g(co)q(de,)h(b)o(y)e(testing)h(the) g(return)g(co)q(de)g(of)g Fw(MPI)f Fv(calls)75 1060 y(and)k(executing)h (a)e(suitable)i(reco)o(v)o(ery)e(co)q(de)i(when)f(the)g(call)g(w)o(as)f (not)h(successful.)26 b(In)17 b(this)g(case,)g(the)75 1117 y(error)g(handler)h Fi(MPI)p 432 1117 V 15 w(ERRORS)p 607 1117 V 13 w(RETURN)f Fv(will)i(b)q(e)f(used.)28 b(Usually)18 b(it)g(is)g(more)f(con)o(v)o(enien)o(t)h(and)f(more)75 1173 y(e\016cien)o(t)i(not)f(to)f(test)h(for)g(errors)f(after)h(eac)o (h)g Fw(MPI)g Fv(call,)h(and)g(ha)o(v)o(e)f(suc)o(h)g(error)g(handled)i (b)o(y)e(a)g(non)75 1230 y(trivial)e Fw(MPI)f Fv(error)f(handler.)166 1286 y(After)h(an)g(error)f(is)i(detected,)g(the)f(state)f(of)h Fw(MPI)g Fv(is)h(unde\014ned.)21 b(That)15 b(is,)g(using)h(a)f (user-de\014ned)75 1343 y(error)h(handler,)i(or)e Fi(MPI)p 501 1343 V 14 w(ERRORS)p 675 1343 V 14 w(RETURN)p Fv(,)g(do)q(es)h Fl(not)k Fv(necessarily)d(allo)o(w)f(the)f(user)h(to)f(con)o(tin)o(ue)i (to)75 1399 y(use)c Fw(MPI)g Fv(after)f(an)h(error)f(is)h(detected.)20 b(The)15 b(purp)q(ose)f(of)g(these)g(error)f(handlers)i(is)f(to)g(allo) o(w)g(a)g(user)g(to)75 1456 y(issue)f(user-de\014ned)i(error)d (messages)g(and)g(to)g(tak)o(e)g(actions)h(unrelated)g(to)f Fw(MPI)g Fv(\(suc)o(h)h(as)f(\015ushing)i(I/O)75 1512 y(bu\013ers\))j(b)q(efore)g(a)g(program)f(exits.)27 b(An)17 b Fw(MPI)g Fv(implemen)o(tation)h(is)g(free)f(to)g(allo)o(w)g Fw(MPI)g Fv(to)f(con)o(tin)o(ue)75 1569 y(after)e(an)i(error)e(but)h (is)h(not)f(required)h(to)f(do)g(so.)189 1675 y Fl(A)n(dvic)n(e)i(to)i (implementors.)56 b Fv(A)17 b(go)q(o)q(d)h(qualit)o(y)g(implemen)o (tation)h(will,)h(to)d(the)h(greatest)f(p)q(os-)189 1731 y(sible,)25 b(exten)o(t,)e(circumscrib)q(e)i(the)d(impact)g(of)g(an)g (error,)h(so)f(that)g(normal)g(pro)q(cessing)h(can)189 1788 y(con)o(tin)o(ue)14 b(after)f(an)h(error)g(handler)h(w)o(as)e(in)o (v)o(ok)o(ed.)19 b(The)c(implemen)o(tation)g(do)q(cumen)o(tation)f (will)189 1844 y(pro)o(vide)k(information)g(on)g(the)g(p)q(ossible)i (e\013ect)e(of)f(eac)o(h)h(class)h(of)e(errors.)28 b(\()p Fl(End)18 b(of)h(advic)n(e)g(to)189 1901 y(implementors.)p Fv(\))166 2007 y(An)f Fw(MPI)f Fv(error)g(handler)i(is)f(an)g(opaque)g (ob)s(ject,)f(whic)o(h)i(is)f(accessed)h(b)o(y)e(a)h(handle.)29 b Fw(MPI)17 b Fv(calls)75 2063 y(are)d(pro)o(vided)h(to)f(create)h(new) f(error)g(handlers,)h(to)f(asso)q(ciate)h(error)e(handlers)j(with)f (comm)o(unicators,)75 2120 y(and)g(to)g(test)g(whic)o(h)h(error)e (handler)i(is)g(asso)q(ciated)g(with)f(a)g(comm)o(unicator.)75 2271 y Fw(MPI)p 160 2271 14 2 v 16 w(ERRHANDLER)p 469 2271 V 18 w(CREA)l(TE\()h(function,)g(errhandler)f(\))117 2348 y Fk(IN)155 b Fw(function)434 b Fk(user)15 b(de\014ned)g(error)g (handling)e(pro)q(cedure)117 2423 y(OUT)108 b Fw(errhandler)397 b Fk(MPI)14 b(error)h(handler)f(\(handle\))75 2548 y Ft(int)23 b(MPI)p 245 2548 15 2 v 17 w(Errhandler)p 502 2548 V 16 w(create\(MPI)p 758 2548 V 16 w(Handler)p 942 2548 V 16 w(function)g(*function,)393 2604 y(MPI)p 468 2604 V 17 w(Errhandler)f(*errhandler\))75 2690 y(MPI)p 150 2690 V 17 w(ERRHANDLER)p 407 2690 V 16 w(CREATE\(FUNCTION,)f (HANDLER,)i(IERROR\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 196 204 196 203 bop 75 -100 a Fv(196)532 b Fm(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)170 45 y Ft(EXTERNAL)23 b(FUNCTION)170 102 y(INTEGER)g(ERRHANDLER,)g(IERROR) 166 188 y Fv(Register)d(the)g(user)g(routine)h Fw(function)g Fv(for)f(use)g(as)f(an)h Fw(MPI)g Fv(exception)h(handler.)35 b(Returns)20 b(in)75 244 y Fw(errhandler)15 b Fv(a)g(handle)i(to)d(the) h(registered)h(exception)g(handler.)-1330 b Fx(>)166 301 y Fv(In)14 b(the)g(C)f(language,)g(the)h(user)f(routine)h(should)h (b)q(e)f(a)f(C)g(function)i(of)e(t)o(yp)q(e)g Fi(MPI)p 1559 301 13 2 v 14 w(Handler)p 1706 301 V 17 w(function)p Fv(,)-2034 b Fx(?)75 357 y Fv(whic)o(h)16 b(is)g(de\014ned)g(as:)75 447 y Ft(typedef)23 b(void)g(\(MPI_Handler_function\)\(MPI_C)o(omm)e (*,)i(int)h(*,)f(...\);)75 537 y Fv(The)d(\014rst)g(argumen)o(t)f(is)i (the)f(comm)o(unicator)g(in)h(use.)35 b(The)20 b(second)h(is)f(the)h (error)e(co)q(de)i(to)e(b)q(e)i(re-)-2035 b Fx(>)75 593 y Fv(turned)24 b(b)o(y)f(the)h Fw(MPI)f Fv(routine)h(that)f(raised)h (the)g(error.)44 b(If)24 b(the)f(routine)h(w)o(ould)g(ha)o(v)o(e)g (returned)75 650 y Fi(MPI)p 152 650 V 14 w(ERR)p 245 650 V 15 w(IN)p 301 650 V 15 w(ST)m(A)m(TUS)p Fv(,)14 b(it)i(is)g(the)g(error)g(co)q(de)g(returned)g(in)h(the)f(status)f(for) g(the)h(request)g(that)f(caused)75 706 y(the)e(error)f(handler)i(to)f (b)q(e)g(in)o(v)o(ok)o(ed.)20 b(The)13 b(remaining)h(argumen)o(ts)e (are)h(\\)p Ft(stdargs)p Fv(")e(argumen)o(ts)h(whose)-2035 b Fx(?)75 763 y Fv(n)o(um)o(b)q(er)14 b(and)f(meaning)h(is)g(implemen)o (tation-dep)q(enden)o(t.)22 b(An)13 b(implemen)o(tation)i(should)f (clearly)g(do)q(c-)75 819 y(umen)o(t)g(these)g(argumen)o(ts.)k (Addresses)d(are)e(used)h(so)g(that)f(the)h(handler)h(ma)o(y)e(b)q(e)h (written)g(in)g(F)l(ortran.)189 909 y Fl(R)n(ationale.)72 b Fv(The)21 b(v)m(ariable)h(argumen)o(t)e(list)h(is)g(pro)o(vided)h(b)q (ecause)f(it)g(pro)o(vides)g(an)g(ANSI-)189 966 y(standard)13 b(ho)q(ok)h(for)g(pro)o(viding)h(additional)g(information)f(to)g(the)g (error)f(handler;)i(without)g(this)189 1022 y(ho)q(ok,)f(ANSI)i(C)f (prohibits)i(additional)f(argumen)o(ts.)j(\()p Fl(End)d(of)g(r)n (ationale.)p Fv(\))75 1206 y Fw(MPI)p 160 1206 14 2 v 16 w(ERRHANDLER)p 469 1206 V 18 w(SET\()f(comm,)e(errhandler)i(\))117 1283 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(to)j(set)h(the)f(error)h(handler)f(for)g(\(handle\))117 1356 y(IN)155 b Fw(errhandler)397 b Fk(new)15 b(MPI)f(error)g(handler)h (for)e(comm)o(unicator)e(\(handle\))75 1481 y Ft(int)23 b(MPI)p 245 1481 15 2 v 17 w(Errhandler)p 502 1481 V 16 w(set\(MPI)p 686 1481 V 16 w(Comm)h(comm,)f(MPI)p 1037 1481 V 17 w(Errhandler)f(errhandler\))75 1567 y(MPI)p 150 1567 V 17 w(ERRHANDLER)p 407 1567 V 16 w(SET\(COMM,)g(ERRHANDLER,)h (IERROR\))170 1624 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)166 1710 y Fv(Asso)q(ciates)16 b(the)g(new)g(error)f(handler)h Fw(erro)o(rhandler)f Fv(with)h(comm)o(unicator)g Fw(comm)d Fv(at)i(the)h(calling)75 1767 y(pro)q(cess.)k(Note)15 b(that)f(an)i(error)e(handler)i(is)g(alw)o(a)o(ys)f(asso)q(ciated)g (with)g(the)h(comm)o(unicator.)75 1917 y Fw(MPI)p 160 1917 14 2 v 16 w(ERRHANDLER)p 469 1917 V 18 w(GET\()f(comm,)e (errhandler)i(\))117 1995 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(to)j(get)g(the)h(error)g(handler)f(from)e (\(handle\))117 2067 y(OUT)108 b Fw(errhandler)397 b Fk(MPI)16 b(error)g(handler)g(curren)o(tly)g(asso)q(ciated)g(with)f (comm)o(u-)905 2124 y(nicator)f(\(handle\))75 2248 y Ft(int)23 b(MPI)p 245 2248 15 2 v 17 w(Errhandler)p 502 2248 V 16 w(get\(MPI)p 686 2248 V 16 w(Comm)h(comm,)f(MPI)p 1037 2248 V 17 w(Errhandler)f(*errhandler\))75 2335 y(MPI)p 150 2335 V 17 w(ERRHANDLER)p 407 2335 V 16 w(GET\(COMM,)g(ERRHANDLER,)h (IERROR\))170 2391 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)166 2478 y Fv(Returns)16 b(in)g Fw(errhandler)g Fv(\(a)f(handle)i(to\))e (the)g(error)g(handler)i(that)e(is)h(curren)o(tly)g(asso)q(ciated)g (with)75 2534 y(comm)o(unicator)f Fw(comm)p Fv(.)166 2591 y(Example:)22 b(A)16 b(library)h(function)g(ma)o(y)f(register)g (at)f(its)i(en)o(try)f(p)q(oin)o(t)g(the)g(curren)o(t)g(error)g (handler)75 2647 y(for)h(a)h(comm)o(unicator,)f(set)h(its)g(o)o(wn)f (priv)m(ate)i(error)e(handler)i(for)e(this)h(comm)o(unicator,)g(and)g (restore)75 2704 y(b)q(efore)d(exiting)i(the)e(previous)h(error)e (handler.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 197 205 197 204 bop 75 -100 a Fm(7.3.)34 b(ERR)o(OR)17 b(CODES)e(AND)g(CLASSES) 952 b Fv(197)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(ERRHANDLER)p 469 45 V 18 w(FREE\()15 b(errhandler)g(\))117 122 y Fk(IN)155 b Fw(errhandler)397 b Fk(MPI)14 b(error)h(handler)f(\(handle\))75 247 y Ft(int)23 b(MPI)p 245 247 15 2 v 17 w(Errhandler)p 502 247 V 16 w(free\(MPI)p 710 247 V 16 w(Errhandler)g(*errhandler\))75 333 y(MPI)p 150 333 V 17 w(ERRHANDLER)p 407 333 V 16 w(FREE\(ERRHANDLER,)e(IERROR\))170 390 y(INTEGER)i(ERRHANDLER,)g (IERROR)166 476 y Fv(Marks)14 b(the)h(error)g(handler)h(asso)q(ciated)g (with)f Fw(errhandler)g Fv(for)g(deallo)q(cation)i(and)e(sets)g Fw(errhandler)75 533 y Fv(to)h Fi(MPI)p 209 533 13 2 v 14 w(ERRHANDLER)p 492 533 V 15 w(NULL)p Fv(.)h(The)g(error)f(handler) i(will)g(b)q(e)g(deallo)q(cated)g(after)e(all)h(comm)o(unicators)75 589 y(asso)q(ciated)e(with)h(it)f(ha)o(v)o(e)g(b)q(een)i(deallo)q (cated.)75 740 y Fw(MPI)p 160 740 14 2 v 16 w(ERROR)p 323 740 V 18 w(STRING\()e(erro)o(rco)q(de,)f(string,)h(resultlen)i(\)) 117 817 y Fk(IN)155 b Fw(erro)o(rco)q(de)410 b Fk(Error)15 b(co)q(de)g(returned)g(b)o(y)f(an)f(MPI)i(routine)117 892 y(OUT)108 b Fw(string)481 b Fk(T)m(ext)14 b(that)g(corresp)q(onds)i (to)e(the)g Fi(erro)o(rco)q(de)117 967 y Fk(OUT)108 b Fw(resultlen)430 b Fk(Length)12 b(\(in)g(prin)o(table)f(c)o (haracters\))i(of)f(the)g(result)g(returned)905 1024 y(in)i Fi(string)75 1148 y Ft(int)23 b(MPI)p 245 1148 15 2 v 17 w(Error)p 382 1148 V 17 w(string\(int)f(errorcode,)h(char)g (*string,)g(int)g(*resultlen\))75 1235 y(MPI)p 150 1235 V 17 w(ERROR)p 287 1235 V 16 w(STRING\(ERRORCODE,)f(STRING,)h (RESULTLEN,)f(IERROR\))170 1291 y(INTEGER)h(ERRORCODE,)g(RESULTLEN,)f (IERROR)170 1348 y(CHARACTER*\(*\))h(STRING)166 1434 y Fv(Returns)18 b(the)g(error)e(string)i(asso)q(ciated)g(with)f(an)h (error)f(co)q(de)h(or)f(class.)27 b(The)18 b(argumen)o(t)f Fw(string)62 b Fx(>)1937 1487 y(?)75 1491 y Fv(m)o(ust)15 b(represen)o(t)g(storage)f(that)g(is)i(at)f(least)g Fi(MPI)p 924 1491 13 2 v 14 w(MAX)p 1030 1491 V 15 w(ERROR)p 1182 1491 V 13 w(STRING)g Fv(c)o(haracters)g(long.)166 1547 y(The)c(n)o(um)o(b)q(er)h(of)e(c)o(haracters)h(actually)h(written)f(is) g(returned)h(in)g(the)f(output)g(argumen)o(t,)g Fw(resultlen)p Fv(.)189 1653 y Fl(R)n(ationale.)38 b Fv(The)14 b(form)e(of)h(this)g (function)h(w)o(as)e(c)o(hosen)i(to)e(mak)o(e)h(the)g(F)l(ortran)f(and) h(C)g(bindings)189 1710 y(similar.)30 b(A)18 b(v)o(ersion)g(that)g (returns)g(a)g(p)q(oin)o(ter)g(to)g(a)g(string)g(has)g(t)o(w)o(o)f (di\016culties.)31 b(First,)18 b(the)189 1766 y(return)10 b(string)h(m)o(ust)g(b)q(e)g(statically)h(allo)q(cated)g(and)f (di\013eren)o(t)g(for)f(eac)o(h)h(error)f(message)g(\(allo)o(wing)189 1823 y(the)j(p)q(oin)o(ters)g(returned)g(b)o(y)g(successiv)o(e)h(calls) g(to)e Fi(MPI)p 1117 1823 V 14 w(ERROR)p 1268 1823 V 14 w(STRING)h Fv(to)f(p)q(oin)o(t)h(to)f(the)h(correct)189 1879 y(message\).)25 b(Second,)19 b(in)f(F)l(ortran,)e(a)i(function)g (declared)g(as)f(returning)h Fi(CHARA)o(CTER*\(*\))e Fv(can)189 1936 y(not)e(b)q(e)i(referenced)g(in,)g(for)f(example,)g(a)g Fi(PRINT)g Fv(statemen)o(t.)k(\()p Fl(End)c(of)i(r)n(ationale.)p Fv(\))75 2079 y Fn(7.3)59 b(Erro)n(r)21 b(co)r(des)e(and)g(classes)75 2180 y Fv(The)13 b(error)f(co)q(des)h(returned)f(b)o(y)h Fw(MPI)f Fv(are)g(left)h(en)o(tirely)g(to)f(the)h(implemen)o(tation)h (\(with)e(the)h(exception)75 2237 y(of)h Fi(MPI)p 203 2237 V 15 w(SUCCESS)p Fv(\).)e(This)j(is)h(done)f(to)f(allo)o(w)h(an)f (implemen)o(tation)i(to)f(pro)o(vide)g(as)f(m)o(uc)o(h)h(information)75 2293 y(as)g(p)q(ossible)i(in)f(the)f(error)g(co)q(de)g(\(for)g(use)g (with)h Fw(MPI)p 1004 2293 14 2 v 15 w(ERROR)p 1166 2293 V 19 w(STRING)p Fv(\).)166 2350 y(T)l(o)d(mak)o(e)f(it)h(p)q(ossible)i (for)e(an)f(application)j(to)d(in)o(terpret)i(an)f(error)f(co)q(de,)i (the)f(routine)g Fw(MPI)p 1761 2350 V 16 w(ERR-)75 2406 y(OR)p 140 2406 V 17 w(CLASS)18 b Fv(con)o(v)o(erts)f(an)o(y)g(error)f (co)q(de)i(in)o(to)g(one)f(of)g(a)g(small)h(set)g(of)f(standard)g (error)f(co)q(des,)i(called)64 b Fx(>)75 2463 y Fl(err)n(or)17 b(classes)p Fv(.)h(V)l(alid)f(error)d(classes)i(include)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 198 206 198 205 bop 75 -100 a Fv(198)532 b Fm(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)381 43 y Fi(MPI)p 458 43 13 2 v 14 w(SUCCESS)190 b Fv(No)15 b(error)381 100 y Fi(MPI)p 458 100 V 14 w(ERR)p 551 100 V 14 w(BUFFER)117 b Fv(In)o(v)m(alid)18 b(bu\013er)d(p)q(oin)o(ter)381 156 y Fi(MPI)p 458 156 V 14 w(ERR)p 551 156 V 14 w(COUNT)130 b Fv(In)o(v)m(alid)18 b(coun)o(t)d(argumen)o(t)381 212 y Fi(MPI)p 458 212 V 14 w(ERR)p 551 212 V 14 w(TYPE)166 b Fv(In)o(v)m(alid)18 b(datat)o(yp)q(e)c(argumen)o(t)381 269 y Fi(MPI)p 458 269 V 14 w(ERR)p 551 269 V 14 w(T)m(A)o(G)194 b Fv(In)o(v)m(alid)18 b(tag)c(argumen)o(t)381 325 y Fi(MPI)p 458 325 V 14 w(ERR)p 551 325 V 14 w(COMM)144 b Fv(In)o(v)m(alid)18 b(comm)o(unicator)381 382 y Fi(MPI)p 458 382 V 14 w(ERR)p 551 382 V 14 w(RANK)161 b Fv(In)o(v)m(alid)18 b(rank)381 438 y Fi(MPI)p 458 438 V 14 w(ERR)p 551 438 V 14 w(REQUEST)86 b Fv(In)o(v)m(alid)18 b(request)d(\(handle\))381 495 y Fi(MPI)p 458 495 V 14 w(ERR)p 551 495 V 14 w(ROOT)157 b Fv(In)o(v)m(alid)18 b(ro)q(ot)381 551 y Fi(MPI)p 458 551 V 14 w(ERR)p 551 551 V 14 w(GROUP)132 b Fv(In)o(v)m(alid)18 b(group)381 608 y Fi(MPI)p 458 608 V 14 w(ERR)p 551 608 V 14 w(OP)216 b Fv(In)o(v)m(alid)18 b(op)q(eration)381 664 y Fi(MPI)p 458 664 V 14 w(ERR)p 551 664 V 14 w(TOPOLOGY)48 b Fv(In)o(v)m(alid)18 b(top)q(ology)381 721 y Fi(MPI)p 458 721 V 14 w(ERR)p 551 721 V 14 w(DIMS)173 b Fv(In)o(v)m(alid)18 b(dimension)e(argumen)o(t)381 777 y Fi(MPI)p 458 777 V 14 w(ERR)p 551 777 V 14 w(ARG)191 b Fv(In)o(v)m(alid)18 b(argumen)o(t)c(of)h(some)g(other)f(kind)381 833 y Fi(MPI)p 458 833 V 14 w(ERR)p 551 833 V 14 w(UNKNO)o(WN)60 b Fv(Unkno)o(wn)16 b(error)381 890 y Fi(MPI)p 458 890 V 14 w(ERR)p 551 890 V 14 w(TRUNCA)m(TE)56 b Fv(Message)15 b(truncated)g(on)g(receiv)o(e)381 946 y Fi(MPI)p 458 946 V 14 w(ERR)p 551 946 V 14 w(OTHER)134 b Fv(Kno)o(wn)16 b(error)e(not)h(in)h(this)g(list)381 1003 y Fi(MPI)p 458 1003 V 14 w(ERR)p 551 1003 V 14 w(INTERN)124 b Fv(In)o(ternal)16 b(MPI)f(\(implemen)o(tation\))h(error)381 1059 y Fi(MPI)p 458 1059 V 14 w(ERR)p 551 1059 V 14 w(IN)p 606 1059 V 15 w(ST)m(A)m(TUS)65 b Fv(Error)15 b(co)q(de)g(is)h(in)g (status)381 1116 y Fi(MPI)p 458 1116 V 14 w(ERR)p 551 1116 V 14 w(PENDING)94 b Fv(P)o(ending)16 b(request)381 1172 y Fi(MPI)p 458 1172 V 14 w(ERR)p 551 1172 V 14 w(LASTCODE)60 b Fv(Last)15 b(error)g(co)q(de)166 1263 y(The)g(error)e(classes)j(are)e (a)g(subset)h(of)f(the)g(error)g(co)q(des:)20 b(an)15 b Fw(MPI)f Fv(function)h(ma)o(y)f(return)g(an)h(error)75 1319 y(class)21 b(n)o(um)o(b)q(er;)i(and)e(the)g(function)h Fw(MPI)p 820 1319 14 2 v 15 w(ERROR)p 982 1319 V 18 w(STRING)g Fv(can)e(b)q(e)i(used)f(to)f(compute)h(the)g(error)75 1376 y(string)15 b(asso)q(ciated)h(with)f(an)g(error)g(class.)166 1432 y(The)g(error)g(co)q(des)h(satisfy)l(,)389 1524 y(0)d(=)g Fw(MPI)p 557 1524 13 2 v 14 w(SUCCESS)h Fp(<)f Fw(MPI)p 906 1524 V 14 w(ERR)p 1005 1524 V 16 w Fp(:::)e Fx(\024)i Fw(MPI)p 1201 1524 V 15 w(ERR)p 1301 1524 V 16 w(LASTCODE)p Fp(:)-160 1616 y Fx(?)189 1711 y Fl(R)n(ationale.)42 b Fv(The)16 b(di\013erence)h(b)q(et)o(w)o(een)e Fi(MPI)p 983 1711 V 15 w(ERR)p 1077 1711 V 14 w(UNKNO)o(WN)h Fv(and)g Fi(MPI)p 1485 1711 V 14 w(ERR)p 1578 1711 V 14 w(OTHER)g Fv(is)g(that)189 1767 y Fw(MPI)p 274 1767 14 2 v 15 w(ERROR)p 436 1767 V 18 w(STRING)g Fv(can)g(return)f(useful)h(information)f(ab)q (out)g Fi(MPI)p 1433 1767 13 2 v 15 w(ERR)p 1527 1767 V 14 w(OTHER)p Fv(.)189 1841 y(Note)h(that)g Fi(MPI)p 477 1841 V 14 w(SUCCESS)f Fv(=)g(0)i(is)g(necessary)f(to)g(b)q(e)i (consisten)o(t)e(with)h(C)g(practice;)g(the)g(sepa-)189 1897 y(ration)g(of)g(error)f(classes)i(and)g(error)e(co)q(des)i(allo)o (ws)g(us)f(to)g(de\014ne)h(the)g(error)e(classes)i(this)g(w)o(a)o(y)l (.)189 1954 y(Ha)o(ving)d(a)g(kno)o(wn)g Fi(LASTCODE)f Fv(is)i(often)f(a)g(nice)h(sanit)o(y)f(c)o(hec)o(k)h(as)f(w)o(ell.)20 b(\()p Fl(End)c(of)g(r)n(ationale.)p Fv(\))75 2143 y Fw(MPI)p 160 2143 14 2 v 16 w(ERROR)p 323 2143 V 18 w(CLASS\()f(erro)o (rco)q(de,)f(erro)o(rclass)g(\))117 2220 y Fk(IN)155 b Fw(erro)o(rco)q(de)410 b Fk(Error)15 b(co)q(de)g(returned)g(b)o(y)f (an)f(MPI)i(routine)117 2293 y(OUT)108 b Fw(erro)o(rclass)410 b Fk(Error)15 b(class)f(asso)q(ciated)h(with)e Fi(erro)o(rco)q(de)75 2418 y Ft(int)23 b(MPI)p 245 2418 15 2 v 17 w(Error)p 382 2418 V 17 w(class\(int)f(errorcode,)h(int)g(*errorclass\))75 2504 y(MPI)p 150 2504 V 17 w(ERROR)p 287 2504 V 16 w(CLASS\(ERRORCODE,) f(ERRORCLASS,)g(IERROR\))170 2561 y(INTEGER)h(ERRORCODE,)g(ERRORCLASS,) f(IERROR)-160 2597 y Fx(>)166 2647 y Fv(The)f(function)h Fw(MPI)p 534 2647 14 2 v 16 w(ERROR)p 697 2647 V 18 w(CLASS)f Fv(maps)g(eac)o(h)g(standard)f(error)g(co)q(de)i(\(error)e(class\))h (on)o(to)75 2704 y(itself.)-342 b Fx(?)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 199 207 199 206 bop 75 -100 a Fm(7.4.)34 b(TIMERS)16 b(AND)f(SYNCHR)o(ONIZA)l (TION)859 b Fv(199)75 45 y Fn(7.4)59 b(Timers)19 b(and)h (synchronization)75 147 y Fw(MPI)15 b Fv(de\014nes)h(a)f(timer.)21 b(A)15 b(timer)h(is)f(sp)q(eci\014ed)j(ev)o(en)e(though)f(it)g(is)h (not)f(\\message-passing,")g(b)q(ecause)75 203 y(timing)21 b(parallel)g(programs)e(is)i(imp)q(ortan)o(t)e(in)i(\\p)q(erformance)f (debugging")h(and)f(b)q(ecause)h(existing)75 259 y(timers)16 b(\(b)q(oth)f(in)i(POSIX)f(1003.1-1988)d(and)j(1003.4D)e(14.1)g(and)i (in)h(F)l(ortran)d(90\))h(are)g(either)i(incon-)75 316 y(v)o(enien)o(t)f(or)f(do)g(not)f(pro)o(vide)i(adequate)f(access)h(to)e (high-resolution)j(timers.)75 467 y Fw(MPI)p 160 467 14 2 v 16 w(WTIME\(\))75 571 y Ft(double)23 b(MPI)p 317 571 15 2 v 17 w(Wtime\(void\))75 657 y(DOUBLE)g(PRECISION)g(MPI)p 556 657 V 16 w(WTIME\(\))166 743 y Fw(MPI)p 251 743 14 2 v 16 w(WTIME)10 b Fv(returns)g(a)h(\015oating-p)q(oin)o(t)g(n)o(um)o (b)q(er)g(of)f(seconds,)i(represen)o(ting)f(elapsed)h(w)o(all-clo)q(c)o (k)75 800 y(time)k(since)g(some)f(time)g(in)h(the)g(past.)166 856 y(The)21 b(\\time)g(in)h(the)f(past")g(is)g(guaran)o(teed)g(not)f (to)h(c)o(hange)g(during)h(the)f(life)h(of)f(the)g(pro)q(cess.)75 913 y(The)d(user)g(is)g(resp)q(onsible)h(for)e(con)o(v)o(erting)h (large)f(n)o(um)o(b)q(ers)h(of)f(seconds)h(to)f(other)g(units)i(if)f (they)f(are)75 969 y(preferred.)166 1026 y(This)j(function)h(is)f(p)q (ortable)g(\(it)g(returns)f(seconds,)i(not)e(\\tic)o(ks"\),)h(it)g (allo)o(ws)g(high-resolution,)75 1082 y(and)15 b(carries)h(no)f (unnecessary)h(baggage.)j(One)d(w)o(ould)f(use)h(it)f(lik)o(e)i(this:) 600 b Fx(>)75 1188 y Ft({)147 1245 y(double)23 b(starttime,)f(endtime;) 147 1301 y(starttime)g(=)i(MPI_Wtime\(\);)170 1358 y(....)48 b(stuff)23 b(to)g(be)h(timed)47 b(...)147 1414 y(endtime)70 b(=)24 b(MPI_Wtime\(\);)147 1471 y(printf\("That)e(took)h(\045f)h (seconds\\n",endtime-start)o(time\);)75 1527 y(})1937 1581 y Fx(?)166 1633 y Fv(The)12 b(times)f(returned)h(are)f(lo)q(cal)h (to)f(the)h(no)q(de)g(that)e(called)j(them.)19 b(There)12 b(is)f(no)h(requiremen)o(t)g(that)75 1690 y(di\013eren)o(t)e(no)q(des)h (return)f(\\the)g(same)g(time.")18 b(\(But)10 b(see)h(also)f(the)g (discussion)i(of)d Fi(MPI)p 1515 1690 13 2 v 15 w(WTIME)p 1670 1690 V 15 w(IS)p 1720 1690 V 14 w(GLOBAL)p Fv(\).)75 1841 y Fw(MPI)p 160 1841 14 2 v 16 w(WTICK\(\))75 1945 y Ft(double)23 b(MPI)p 317 1945 15 2 v 17 w(Wtick\(void\))75 2031 y(DOUBLE)g(PRECISION)g(MPI)p 556 2031 V 16 w(WTICK\(\))166 2117 y Fw(MPI)p 251 2117 14 2 v 16 w(WTICK)d Fv(returns)h(the)g (resolution)h(of)f Fw(MPI)p 1038 2117 V 15 w(WTIME)g Fv(in)h(seconds.)38 b(That)20 b(is,)j(it)e(returns,)75 2174 y(as)g(a)f(double)j(precision)f(v)m(alue,)h(the)f(n)o(um)o(b)q(er) f(of)g(seconds)g(b)q(et)o(w)o(een)g(successiv)o(e)h(clo)q(c)o(k)g(tic)o (ks.)38 b(F)l(or)75 2230 y(example,)22 b(if)g(the)e(clo)q(c)o(k)i(is)f (implemen)o(ted)h(b)o(y)f(the)g(hardw)o(are)f(as)g(a)g(coun)o(ter)h (that)f(is)h(incremen)o(ted)75 2287 y(ev)o(ery)15 b(millisecond,)j(the) d(v)m(alue)i(returned)e(b)o(y)g Fw(MPI)p 971 2287 V 16 w(WTICK)f Fv(should)i(b)q(e)g(10)1398 2270 y Fh(\000)p Fg(3)1445 2287 y Fv(.)75 2430 y Fn(7.5)59 b(Sta)n(rtup)75 2532 y Fv(One)11 b(goal)f(of)g Fw(MPI)f Fv(is)i(to)f(ac)o(hiev)o(e)g Fl(sour)n(c)n(e)i(c)n(o)n(de)f(p)n(ortability)p Fv(.)18 b(By)11 b(this)f(w)o(e)g(mean)g(that)g(a)g(program)f(written)75 2588 y(using)16 b Fw(MPI)e Fv(and)i(complying)g(with)g(the)f(relev)m (an)o(t)h(language)f(standards)g(is)h(p)q(ortable)f(as)g(written,)g (and)75 2644 y(m)o(ust)h(not)g(require)i(an)o(y)e(source)h(co)q(de)g(c) o(hanges)g(when)g(mo)o(v)o(ed)f(from)g(one)h(system)f(to)g(another.)24 b(This)75 2701 y(explicitly)d(do)q(es)e Fl(not)j Fv(sa)o(y)c(an)o (ything)g(ab)q(out)h(ho)o(w)e(an)i Fw(MPI)e Fv(program)h(is)h(started)e (or)h(launc)o(hed)i(from)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 200 208 200 207 bop 75 -100 a Fv(200)532 b Fm(CHAPTER)16 b(7.)34 b(MPI)16 b(ENVIR)o(ONMENT)l(AL)g(MANA)o(GEMENT)75 45 y Fv(the)g(command)g(line,)h(nor)f(what)f(the)h(user)g(m)o(ust)f(do)h (to)f(set)h(up)g(the)g(en)o(vironmen)o(t)g(in)h(whic)o(h)g(an)f Fw(MPI)75 102 y Fv(program)h(will)j(run.)28 b(Ho)o(w)o(ev)o(er,)18 b(an)g(implemen)o(tation)h(ma)o(y)f(require)g(some)g(setup)h(to)e(b)q (e)i(p)q(erformed)75 158 y(b)q(efore)14 b(other)g Fw(MPI)f Fv(routines)h(ma)o(y)g(b)q(e)g(called.)21 b(T)l(o)14 b(pro)o(vide)g(for)f(this,)i Fw(MPI)e Fv(includes)j(an)e (initialization)75 214 y(routine)i Fw(MPI)p 317 214 14 2 v 15 w(INIT)p Fv(.)75 365 y Fw(MPI)p 160 365 V 16 w(INIT\(\))75 469 y Ft(int)23 b(MPI)p 245 469 15 2 v 17 w(Init\(int)g(*argc,)g(char)g (***argv\))75 556 y(MPI)p 150 556 V 17 w(INIT\(IERROR\))170 612 y(INTEGER)g(IERROR)166 698 y Fv(This)17 b(routine)h(m)o(ust)e(b)q (e)i(called)g(b)q(efore)f(an)o(y)g(other)f Fw(MPI)h Fv(routine.)25 b(It)17 b(m)o(ust)f(b)q(e)i(called)h(at)d(most)75 755 y(once;)f(subsequen)o(t)h(calls)g(are)f(erroneous)g(\(see)g Fw(MPI)p 982 755 14 2 v 16 w(INITIALIZED)p Fv(\).)166 811 y(All)i Fw(MPI)e Fv(programs)f(m)o(ust)h(con)o(tain)h(a)f(call)h (to)f Fw(MPI)p 1074 811 V 16 w(INIT)p Fv(;)f(this)i(routine)g(m)o(ust)f (b)q(e)i(called)g(b)q(efore)75 868 y(an)o(y)h(other)g Fw(MPI)f Fv(routine)i(\(apart)e(from)g Ft(MPI)p 872 868 15 2 v 17 w(INITIALIZED)p Fv(\))f(is)j(called.)30 b(The)18 b(v)o(ersion)h(for)e(ANSI)i(C)75 924 y(accepts)c(the)h Fi(a)o(rgc)f Fv(and)g Fi(a)o(rgv)g Fv(that)g(are)g(pro)o(vided)h(b)o(y) f(the)g(argumen)o(ts)f(to)h Ft(main)p Fv(:)-1625 b Fx(>)75 1016 y Ft(int)23 b(main\(argc,)g(argv\))75 1073 y(int)g(argc;)75 1129 y(char)g(**argv;)75 1186 y({)170 1242 y(MPI_Init\(&argc,)f (&argv\);)170 1355 y(/*)i(parse)f(arguments)g(*/)170 1412 y(/*)h(main)f(program)95 b(*/)170 1525 y(MPI_Finalize\(\);)118 b(/*)23 b(see)h(below)f(*/)75 1581 y(})-160 1620 y Fx(?)75 1673 y Fv(The)15 b(F)l(ortran)f(v)o(ersion)i(tak)o(es)e(only)i Fi(IERROR)p Fv(.)-1026 b Fx(>)166 1730 y Fv(An)17 b(MPI)g(implemen)o (tation)i(is)e(free)g(to)f(require)i(that)f(the)g(argumen)o(ts)f(in)i (the)f(C)g(binding)i(m)o(ust)75 1786 y(b)q(e)d(the)f(argumen)o(ts)f(to) h Ft(main)p Fv(.)189 1890 y Fl(R)n(ationale.)40 b Fv(The)15 b(command)g(line)i(arguemen)o(ts)d(are)h(pro)o(vided)h Fl(to)f Fw(MPI)p 1459 1890 14 2 v 16 w(Init)g Fv(to)g(allo)o(w)g(an)g (MPI)189 1946 y(implemen)o(tation)g(to)e(use)i(them)f(in)g(initializi)q (ng)j(the)d(MPI)g(en)o(vironmen)o(t.)20 b(They)14 b(are)g(passed)g(b)o (y)189 2003 y(reference)f(to)g(allo)o(w)g(an)g(MPI)g(implemen)o(tation) i(to)d Fl(pr)n(ovide)h Fv(them)g(in)h(en)o(vironmen)o(ts)g(where)f(the) 189 2059 y(command-line)k(argumen)o(ts)d(are)h(not)g(pro)o(vided)h(to)e Ft(main)p Fv(.)19 b(\()p Fl(End)d(of)g(r)n(ationale.)p Fv(\))-160 2118 y Fx(?)75 2258 y Fw(MPI)p 160 2258 V 16 w(FINALIZE\(\))75 2361 y Ft(int)23 b(MPI)p 245 2361 15 2 v 17 w(Finalize\(void\))75 2448 y(MPI)p 150 2448 V 17 w(FINALIZE\(IERROR\))170 2504 y(INTEGER)g(IERROR)166 2591 y Fv(This)14 b(routines)f(cleans)h(up)g(all)g Fw(MPI)f Fv(state.)18 b(Once)c(this)g(routine)f(is)h(called,)h(no)e Fw(MPI)f Fv(routine)i(\(ev)o(en)75 2647 y Fw(MPI)p 160 2647 14 2 v 16 w(INIT)p Fv(\))e(ma)o(y)h(b)q(e)h(called.)21 b(The)14 b(user)g(m)o(ust)f(ensure)h(that)f(all)h(p)q(ending)i(comm)o (unications)e(in)o(v)o(olving)75 2704 y(a)h(pro)q(cess)g(completes)h(b) q(efore)g(the)f(pro)q(cess)g(calls)i Fw(MPI)p 1041 2704 V 15 w(FINALIZE)p Fv(.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 201 209 201 208 bop 75 -100 a Fm(7.5.)34 b(ST)l(AR)l(TUP)1411 b Fv(201)75 45 y Fw(MPI)p 160 45 14 2 v 16 w(INITIALIZED\()13 b(\015ag)i(\))117 122 y Fk(OUT)108 b Fw(\015ag)518 b Fk(F)p Fv(lag)15 b(is)f(true)h(if)f Fw(MPI)p 1270 122 V 16 w(INIT)g Fv(has)g(b)q(een)h(called)h(and)f(false)905 179 y(otherwise.)75 303 y Ft(int)23 b(MPI)p 245 303 15 2 v 17 w(Initialized\(int)f(*flag\))75 390 y(MPI)p 150 390 V 17 w(INITIALIZED\(FLAG,)f(IERROR\))170 446 y(LOGICAL)i(FLAG)170 503 y(INTEGER)g(IERROR)166 589 y Fv(This)c(routine)f(ma)o(y)g(b)q(e)h (used)g(to)e(determine)i(whether)g Fw(MPI)p 1242 589 14 2 v 16 w(INIT)e Fv(has)h(b)q(een)h(called.)31 b(It)18 b(is)h(the)75 645 y Fl(only)c Fv(routine)g(that)g(ma)o(y)f(b)q(e)i (called)h(b)q(efore)e Fw(MPI)p 938 645 V 16 w(INIT)g Fv(is)g(called.)75 796 y Fw(MPI)p 160 796 V 16 w(ABORT\()h(comm,)d (erro)o(rco)q(de)h(\))117 874 y Fk(IN)155 b Fw(comm)470 b Fk(comm)o(unicator)11 b(of)j(tasks)g(to)g(ab)q(ort)117 949 y(IN)155 b Fw(erro)o(rco)q(de)410 b Fk(error)15 b(co)q(de)g(to)e (return)j(to)d(in)o(v)o(oking)f(en)o(vironmen)o(t)75 1073 y Ft(int)23 b(MPI)p 245 1073 15 2 v 17 w(Abort\(MPI)p 478 1073 V 16 w(Comm)g(comm,)h(int)f(errorcode\))75 1160 y(MPI)p 150 1160 V 17 w(ABORT\(COMM,)f(ERRORCODE,)h(IERROR\))170 1216 y(INTEGER)g(COMM,)h(ERRORCODE,)e(IERROR)166 1303 y Fv(This)d(routine)f(mak)o(es)g(a)g(\\b)q(est)g(attempt")f(to)h(ab)q (ort)g(all)h(tasks)e(in)i(the)g(group)e(of)h Fw(comm)p Fv(.)27 b(This)75 1359 y(function)21 b(do)q(es)g(not)f(require)h(that)f (the)g(in)o(v)o(oking)h(en)o(vironmen)o(t)f(tak)o(e)g(an)o(y)g(action)h (with)f(the)h(error)75 1416 y(co)q(de.)g(Ho)o(w)o(ev)o(er,)14 b(a)h(Unix)i(or)e(POSIX)h(en)o(vironmen)o(t)g(should)g(handle)h(this)f (as)f(a)g Ft(return)23 b(errorcode)75 1472 y Fv(from)14 b(the)i(main)f(program)f(or)h(an)g Ft(abort\(errorcode\))p Fv(.)166 1528 y Fw(MPI)f Fv(implemen)o(tations)i(are)e(required)i(to)d (de\014ne)j(the)f(b)q(eha)o(vior)g(of)f Fw(MPI)p 1441 1528 14 2 v 16 w(ABORT)i Fv(at)e(least)g(for)g(a)75 1585 y Ft(comm)i Fv(of)g Fi(MPI)p 317 1585 13 2 v 14 w(COMM)p 461 1585 V 15 w(W)o(ORLD)p Fv(.)f Fw(MPI)h Fv(implemen)o(tations)i(ma)o (y)d(ignore)i(the)g Ft(comm)e Fv(argumen)o(t)h(and)g(act)75 1641 y(as)f(if)g(the)h Ft(comm)e Fv(w)o(as)h Fi(MPI)p 526 1641 V 14 w(COMM)p 670 1641 V 15 w(W)o(ORLD)p Fv(.)189 1748 y Fl(R)n(ationale.)54 b Fv(The)18 b(comm)o(unicator)f(argumen)o(t) g(is)h(pro)o(vided)h(to)e(allo)o(w)h(for)f(future)g(extensions)189 1804 y(of)i(MPI)g(to)f(en)o(vironmen)o(ts)i(with,)g(for)f(example,)i (dynamic)f(pro)q(cess)g(managemen)o(t.)31 b(In)20 b(par-)189 1861 y(ticular,)f(it)g(allo)o(ws)f(but)h(do)q(es)g(not)f(require)h(an)f (MPI)h(implemen)o(tation)g(to)f(ab)q(ort)g(a)g(subset)g(of)189 1917 y Fi(MPI)p 266 1917 V 14 w(COMM)p 410 1917 V 15 w(W)o(ORLD)p Fv(.)c(\()p Fl(End)i(of)g(r)n(ationale.)p Fv(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 202 210 202 209 bop 75 356 a Fo(Chapter)34 b(8)75 564 y Fr(Pro\014ling)42 b(In)m(terface)75 805 y Fn(8.1)59 b(Requirements)75 906 y Fv(T)l(o)15 b(meet)g(the)g Fw(MPI)g Fv(pro\014ling)h(in)o(terface,)f (an)h(implemen)o(tation)g(of)f(the)g Fw(MPI)g Fv(functions)h Fl(must)131 997 y Fv(1.)22 b(pro)o(vide)14 b(a)g(mec)o(hanism)g (through)g(whic)o(h)h(all)g(of)e(the)h Fw(MPI)f Fv(de\014ned)j (functions)f(ma)o(y)e(b)q(e)h(accessed)189 1054 y(with)e(a)h(name)f (shift.)19 b(Th)o(us)13 b(all)g(of)f(the)h Fw(MPI)f Fv(functions)h (\(whic)o(h)g(normally)g(start)e(with)i(the)g(pre\014x)189 1110 y(\\)p Ft(MPI)p 287 1110 15 2 v 16 w Fv("\))i(should)h(also)f(b)q (e)h(accessible)h(with)e(the)h(pre\014x)f(\\)p Ft(PMPI)p 1289 1110 V 17 w Fv(".)131 1203 y(2.)22 b(ensure)d(that)e(those)i Fw(MPI)e Fv(functions)j(whic)o(h)f(are)f(not)g(replaced)i(ma)o(y)e (still)h(b)q(e)h(link)o(ed)g(in)o(to)e(an)189 1260 y(executable)e (image)f(without)h(causing)f(name)h(clashes.)131 1352 y(3.)22 b(do)q(cumen)o(t)c(the)h(implemen)o(tation)g(of)f(di\013eren)o (t)h(language)f(bindings)i(of)e(the)g Fw(MPI)g Fv(in)o(terface)g(if)189 1409 y(they)f(are)h(la)o(y)o(ered)g(on)f(top)g(of)h(eac)o(h)f(other,)h (so)f(that)g(the)h(pro\014ler)g(dev)o(elop)q(er)h(kno)o(ws)e(whether) 189 1465 y(she)d(m)o(ust)f(implemen)o(t)i(the)e(pro\014le)i(in)o (terface)f(for)f(eac)o(h)h(binding,)h(or)e(can)h(economise)h(b)o(y)e (imple-)189 1522 y(men)o(ting)i(it)h(only)f(for)g(the)g(lo)o(w)o(est)g (lev)o(el)h(routines.)131 1615 y(4.)22 b(where)15 b(the)g(implemen)o (tation)i(of)d(di\013eren)o(t)i(language)f(bindings)i(is)f(is)g(done)f (through)g(a)g(la)o(y)o(ered)189 1671 y(approac)o(h)h(\(e.g.)24 b(the)17 b(F)l(ortran)f(binding)j(is)e(a)f(set)h(of)g(\\wrapp)q(er")f (functions)i(whic)o(h)f(call)h(the)f(C)189 1727 y(implemen)o(tation\),) g(ensure)g(that)f(these)g(wrapp)q(er)h(functions)g(are)f(separable)h (from)f(the)g(rest)g(of)189 1784 y(the)f(library)l(.)189 1859 y(This)k(is)h(necessary)f(to)f(allo)o(w)h(a)g(separate)f (pro\014ling)j(library)e(to)g(b)q(e)g(correctly)h(implemen)o(ted,)189 1915 y(since)g(\(at)f(least)g(with)h(Unix)g(link)o(er)h(seman)o(tics\)) e(the)h(pro\014ling)g(library)g(m)o(ust)f(con)o(tain)h(these)189 1971 y(wrapp)q(er)14 b(functions)h(if)g(it)f(is)h(to)f(p)q(erform)g(as) f(exp)q(ected.)21 b(This)15 b(requiremen)o(t)g(allo)o(ws)f(the)h(p)q (erson)189 2028 y(who)c(builds)i(the)e(pro\014ling)h(library)g(to)f (extract)f(these)i(functions)g(from)e(the)h(original)i Fw(MPI)d Fv(library)189 2084 y(and)k(add)g(them)g(in)o(to)g(the)f (pro\014ling)j(library)e(without)g(bringing)h(along)f(an)o(y)g(other)f (unnecessary)189 2141 y(co)q(de.)131 2234 y(5.)22 b(pro)o(vide)15 b(a)g(no-op)h(routine)f Fw(MPI)p 756 2234 14 2 v 16 w(PCONTROL)h Fv(in)g(the)g Fw(MPI)e Fv(library)l(.)75 2376 y Fn(8.2)59 b(Discussion)75 2478 y Fv(The)14 b(ob)s(jectiv)o(e)f(of)g(the)h Fw(MPI)f Fv(pro\014ling)h(in)o(terface)g(is)g(to)f(ensure)h(that)f(it)h (is)g(relativ)o(ely)g(easy)g(for)f(authors)75 2534 y(of)19 b(pro\014ling)h(\(and)f(other)g(similar\))h(to)q(ols)g(to)e(in)o (terface)i(their)f(co)q(des)h(to)f Fw(MPI)g Fv(implemen)o(tations)h(on) 75 2591 y(di\013eren)o(t)15 b(mac)o(hines.)166 2647 y(Since)20 b Fw(MPI)f Fv(is)g(a)g(mac)o(hine)h(indep)q(enden)o(t)h(standard)e (with)g(man)o(y)g(di\013eren)o(t)g(implemen)o(tations,)75 2704 y(it)e(is)g(unreasonable)g(to)f(exp)q(ect)i(that)e(the)g(authors)g (of)g(pro\014ling)i(to)q(ols)f(for)e Fw(MPI)h Fv(will)j(ha)o(v)o(e)d (access)h(to)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 203 211 203 210 bop 75 -100 a Fm(8.3.)34 b(LOGIC)16 b(OF)f(THE)g(DESIGN)1092 b Fv(203)75 45 y(the)14 b(source)g(co)q(de)g(whic)o(h)h(implemen)o(ts)g Fw(MPI)e Fv(on)h(an)o(y)f(particular)i(mac)o(hine.)20 b(It)14 b(is)g(therefore)g(necessary)75 102 y(to)21 b(pro)o(vide)i(a)f (mec)o(hanism)g(b)o(y)g(whic)o(h)h(the)f(implemen)o(tors)h(of)f(suc)o (h)g(to)q(ols)g(can)g(collect)i(whatev)o(er)75 158 y(p)q(erformance)15 b(information)h(they)f(wish)h Fl(without)g Fv(access)f(to)g(the)g (underlying)i(implemen)o(tation.)166 214 y(W)l(e)e(b)q(eliev)o(e)h (that)e(ha)o(ving)h(suc)o(h)g(an)f(in)o(terface)h(is)g(imp)q(ortan)o(t) f(if)i Fw(MPI)d Fv(is)j(to)e(b)q(e)h(attractiv)o(e)e(to)h(end)75 271 y(users,)h(since)i(the)e(a)o(v)m(ailabilit)o(y)j(of)d(man)o(y)g (di\013eren)o(t)h(to)q(ols)f(will)i(b)q(e)g(a)e(signi\014can)o(t)h (factor)f(in)h(attracting)75 327 y(users)f(to)g(the)g Fw(MPI)g Fv(standard.)166 384 y(The)e(pro\014ling)h(in)o(terface)e(is)h (just)g(that,)f(an)g(in)o(terface.)20 b(It)12 b(sa)o(ys)g Fl(nothing)g Fv(ab)q(out)g(the)h(w)o(a)o(y)f(in)h(whic)o(h)75 440 y(it)h(is)h(used.)20 b(There)14 b(is)h(therefore)f(no)g(attempt)f (to)g(la)o(y)h(do)o(wn)g(what)g(information)g(is)h(collected)g(through) 75 497 y(the)g(in)o(terface,)g(or)g(ho)o(w)g(the)g(collected)i (information)e(is)h(sa)o(v)o(ed,)e(\014ltered,)i(or)f(displa)o(y)o(ed.) 166 553 y(While)g(the)g(initial)h(imp)q(etus)f(for)f(the)g(dev)o (elopmen)o(t)h(of)e(this)i(in)o(terface)f(arose)g(from)f(the)h(desire)i (to)75 610 y(p)q(ermit)g(the)f(implemen)o(tation)i(of)d(pro\014ling)j (to)q(ols,)d(it)i(is)f(clear)h(that)f(an)g(in)o(terface)g(lik)o(e)h (that)f(sp)q(eci\014ed)75 666 y(ma)o(y)j(also)g(pro)o(v)o(e)g(useful)h (for)f(other)g(purp)q(oses,)i(suc)o(h)e(as)h(\\in)o(ternet)o(w)o (orking")e(m)o(ultiple)j Fw(MPI)e Fv(imple-)75 723 y(men)o(tations.)j (Since)c(all)f(that)f(is)h(de\014ned)h(is)f(an)g(in)o(terface,)f(there) h(is)g(no)f(ob)s(jection)h(to)f(its)h(b)q(eing)g(used)75 779 y(wherev)o(er)f(it)h(is)f(useful.)166 835 y(As)j(the)g(issues)h(b)q (eing)g(addressed)f(here)h(are)f(in)o(timately)h(tied)f(up)h(with)f (the)g(w)o(a)o(y)f(in)i(whic)o(h)g(ex-)75 892 y(ecutable)j(images)e (are)g(built,)j(whic)o(h)e(ma)o(y)f(di\013er)h(greatly)f(on)g (di\013eren)o(t)h(mac)o(hines,)h(the)e(examples)75 948 y(giv)o(en)d(b)q(elo)o(w)f(should)i(b)q(e)e(treated)g(solely)h(as)f (one)g(w)o(a)o(y)f(of)h(implemen)o(ting)i(the)e(ob)s(jectiv)o(e)h(of)e (the)i Fw(MPI)75 1005 y Fv(pro\014ling)g(in)o(terface.)k(The)16 b(actual)g(requiremen)o(ts)g(made)g(of)f(an)h(implemen)o(tation)h(are)e (those)g(detailed)75 1061 y(in)i(the)f(Requiremen)o(ts)i(section)e(ab)q (o)o(v)o(e,)g(the)g(whole)h(of)f(the)g(rest)g(of)f(this)i(c)o(hapter)f (is)g(only)h(presen)o(t)f(as)75 1118 y(justi\014cation)g(and)f (discussion)i(of)e(the)g(logic)h(for)f(those)g(requiremen)o(ts.)166 1174 y(The)i(examples)g(b)q(elo)o(w)h(sho)o(w)e(one)h(w)o(a)o(y)f(in)h (whic)o(h)h(an)f(implemen)o(tation)h(could)f(b)q(e)h(constructed)75 1231 y(to)h(meet)h(the)g(requiremen)o(ts)g(on)g(a)g(Unix)h(system)e (\(there)h(are)f(doubtless)i(others)e(whic)o(h)i(w)o(ould)f(b)q(e)75 1287 y(equally)d(v)m(alid\).)75 1428 y Fn(8.3)59 b(Logic)20 b(of)g(the)f(design)75 1529 y Fv(Pro)o(vided)d(that)g(an)g Fw(MPI)f Fv(implemen)o(tation)i(meets)f(the)g(requiremen)o(ts)h(ab)q(o) o(v)o(e,)e(it)h(is)h(p)q(ossible)h(for)d(the)75 1586 y(implemen)o(tor)20 b(of)f(the)h(pro\014ling)g(system)f(to)g(in)o (tercept)h(all)g(of)f(the)h Fw(MPI)f Fv(calls)h(whic)o(h)g(are)g(made)f (b)o(y)75 1642 y(the)g(user)h(program.)30 b(She)20 b(can)f(then)h (collect)g(whatev)o(er)f(information)g(she)h(requires)g(b)q(efore)f (calling)75 1698 y(the)d(underlying)h Fw(MPI)e Fv(implemen)o(tation)h (\(through)f(its)h(name)f(shifted)h(en)o(try)f(p)q(oin)o(ts\))h(to)f (ac)o(hiev)o(e)h(the)75 1755 y(desired)g(e\013ects.)75 1874 y Fj(8.3.1)49 b(Miscellaneous)17 b(control)g(of)f(p)o(ro\014ling) 75 1960 y Fv(There)e(is)g(a)g(clear)g(requiremen)o(t)h(for)e(the)h (user)g(co)q(de)g(to)f(b)q(e)i(able)f(to)f(con)o(trol)h(the)g (pro\014ler)g(dynamically)75 2016 y(at)h(run)g(time.)20 b(This)c(is)g(normally)g(used)f(for)g(\(at)f(least\))h(the)g(purp)q (oses)h(of)143 2096 y Fx(\017)23 b Fv(Enabling)16 b(and)g(disabling)h (pro\014ling)f(dep)q(ending)i(on)d(the)g(state)f(of)h(the)g (calculation.)143 2184 y Fx(\017)23 b Fv(Flushing)16 b(trace)f(bu\013ers)g(at)g(non-critical)i(p)q(oin)o(ts)e(in)h(the)g (calculation)143 2272 y Fx(\017)23 b Fv(Adding)16 b(user)f(ev)o(en)o (ts)g(to)g(a)g(trace)f(\014le.)166 2351 y(These)h(requiremen)o(ts)h (are)f(met)g(b)o(y)g(use)h(of)e(the)i Fw(MPI)p 1092 2351 14 2 v 15 w(PCONTROL)p Fv(.)75 2502 y Fw(MPI)p 160 2502 V 16 w(PCONTROL\(level,)f Fp(:)8 b(:)g(:)e Fw(\))117 2579 y Fk(IN)155 b Fw(level)504 b Fk(Pro\014ling)13 b(lev)o(el)75 2704 y Ft(int)23 b(MPI)p 245 2704 15 2 v 17 w(Pcontrol\(const)f(int)i (level,)f Fp(:)8 b(:)g(:)d Ft(\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 204 212 204 211 bop 75 -100 a Fv(204)878 b Fm(CHAPTER)15 b(8.)30 b(PR)o(OFILING)16 b(INTERF)-5 b(A)o(CE)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(PCONTROL\(level\))170 102 y(INTEGER)23 b(LEVEL,)g Fp(:)8 b(:)g(:)166 188 y Fw(MPI)16 b Fv(libraries)j (themselv)o(es)f(mak)o(e)e(no)h(use)h(of)e(this)i(routine,)g(and)f (simply)h(return)f(immediately)75 244 y(to)d(the)g(user)h(co)q(de.)20 b(Ho)o(w)o(ev)o(er)13 b(the)i(presence)g(of)f(calls)i(to)d(this)i (routine)g(allo)o(ws)g(a)f(pro\014ling)h(pac)o(k)m(age)g(to)75 301 y(b)q(e)h(explicitly)i(called)f(b)o(y)e(the)g(user.)166 357 y(Since)k Fw(MPI)f Fv(has)g(no)g(con)o(trol)g(of)f(the)i(implemen)o (tation)g(of)f(the)g(pro\014ling)h(co)q(de,)g(w)o(e)f(are)g(unable)75 414 y(to)13 b(sp)q(ecify)i(precisely)h(the)e(seman)o(tics)g(whic)o(h)h (will)g(b)q(e)g(pro)o(vided)f(b)o(y)g(calls)h(to)e Fw(MPI)p 1505 414 14 2 v 16 w(PCONTROL)p Fv(.)h(This)75 470 y(v)m(agueness)i (extends)f(to)g(the)g(n)o(um)o(b)q(er)h(of)f(argumen)o(ts)f(to)h(the)g (function,)g(and)h(their)g(datat)o(yp)q(es.)166 527 y(Ho)o(w)o(ev)o(er) 11 b(to)g(pro)o(vide)h(some)g(lev)o(el)h(of)e(p)q(ortabilit)o(y)i(of)e (user)h(co)q(des)h(to)e(di\013eren)o(t)h(pro\014ling)h(libraries,)75 583 y(w)o(e)i(request)g(the)g(follo)o(wing)h(meanings)g(for)f(certain)g (v)m(alues)i(of)d(lev)o(el.)143 677 y Fx(\017)23 b Ft(level==0)14 b Fv(Pro\014ling)i(is)g(disabled.)143 771 y Fx(\017)23 b Ft(level==1)14 b Fv(Pro\014ling)i(is)g(enabled)g(at)f(a)g(normal)g (default)h(lev)o(el)g(of)f(detail.)143 865 y Fx(\017)23 b Ft(level==2)14 b Fv(Pro\014le)h(bu\013ers)g(are)g(\015ushed.)21 b(\(This)16 b(ma)o(y)e(b)q(e)i(a)f(no-op)g(in)h(some)f(pro\014lers\).) 143 958 y Fx(\017)23 b Fv(All)13 b(other)f(v)m(alues)h(of)f Ft(level)f Fv(ha)o(v)o(e)h(pro\014le)h(library)g(de\014ned)g(e\013ects) f(and)h(additional)g(argumen)o(ts.)166 1052 y(W)l(e)18 b(also)f(request)h(that)f(the)h(default)g(state)f(after)g Fw(MPI)p 1147 1052 V 16 w(INIT)g Fv(has)g(b)q(een)i(called)g(is)g(for)e (pro\014ling)75 1109 y(to)e(b)q(e)h(enabled)h(at)e(the)h(normal)f (default)h(lev)o(el.)22 b(\(i.e.)f(as)15 b(if)h Fw(MPI)p 1204 1109 V 16 w(PCONTROL)h Fv(had)e(just)h(b)q(een)g(called)75 1165 y(with)h(the)f(argumen)o(t)g(1\).)23 b(This)17 b(allo)o(ws)f (users)h(to)e(link)j(with)f(a)f(pro\014ling)i(library)f(and)g(obtain)f (pro\014le)75 1222 y(output)f(without)g(ha)o(ving)h(to)e(mo)q(dify)i (their)g(source)f(co)q(de)h(at)e(all.)166 1278 y(The)h(pro)o(vision)g (of)f Fw(MPI)p 589 1278 V 16 w(PCONTROL)i Fv(as)e(a)g(no-op)h(in)g(the) g(standard)f Fw(MPI)g Fv(library)h(allo)o(ws)g(them)75 1335 y(to)g(mo)q(dify)h(their)g(source)f(co)q(de)h(to)f(obtain)h(more)f (detailed)i(pro\014ling)f(information,)g(but)f(still)i(b)q(e)f(able)75 1391 y(to)f(link)h(exactly)g(the)f(same)g(co)q(de)h(against)f(the)g (standard)g Fw(MPI)f Fv(library)l(.)75 1534 y Fn(8.4)59 b(Examples)75 1637 y Fj(8.4.1)49 b(Pro\014ler)16 b(implementation)75 1723 y Fv(Supp)q(ose)25 b(that)e(the)h(pro\014ler)h(wishes)g(to)e (accum)o(ulate)i(the)f(total)f(amoun)o(t)g(of)h(data)f(sen)o(t)h(b)o(y) g(the)75 1780 y(MPI)p 167 1780 V 16 w(SEND)18 b(function,)g(along)f (with)h(the)f(total)g(elapsed)h(time)f(sp)q(en)o(t)h(in)g(the)f (function.)27 b(This)18 b(could)75 1836 y(trivially)f(b)q(e)f(ac)o (hiev)o(ed)g(th)o(us)-737 b Fx(>)75 1942 y Ft(static)23 b(int)g(totalBytes;)75 1999 y(static)g(double)g(totalTime;)75 2112 y(int)g(MPI_SEND\(void)g(*)g(buffer,)g(const)g(int)h(count,)f (MPI_Datatype)f(datatype,)385 2168 y(int)i(dest,)f(int)g(tag,)h (MPI_comm)e(comm\))75 2225 y({)147 2281 y(double)h(tstart)g(=)g (MPI_Wtime\(\);)166 b(/*)23 b(Pass)h(on)f(all)h(the)f(arguments)g(*/) 147 2338 y(int)g(extent;)147 2394 y(int)g(result)95 b(=)23 b(PMPI_Send\(buffer,count,dataty)o(pe,dest)o(,tag,com)o(m\);)147 2507 y(MPI_Type_size\(datatype,)d(&extent\);)47 b(/*)23 b(Compute)g(size)h(*/)147 2563 y(totalBytes)e(+=)i(count*extent;)147 2676 y(totalTime)46 b(+=)24 b(MPI_Wtime\(\))e(-)i(tstart;)214 b(/*)23 b(and)h(time)238 b(*/)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 205 213 205 212 bop 75 -100 a Fm(8.4.)34 b(EXAMPLES)1370 b Fv(205)147 102 y Ft(return)23 b(result;)75 158 y(})1937 212 y Fx(?)75 330 y Fj(8.4.2)49 b(MPI)17 b(lib)o(ra)o(ry)f(implementation)75 415 y Fv(On)i(a)g(Unix)g(system,)g(in)h(whic)o(h)f(the)g Fw(MPI)f Fv(library)i(is)f(implemen)o(ted)h(in)g(C,)e(then)h(there)g (are)f(v)m(arious)75 472 y(p)q(ossible)22 b(options,)f(of)e(whic)o(h)i (t)o(w)o(o)e(of)g(the)h(most)g(ob)o(vious)g(are)g(presen)o(ted)g(here.) 35 b(Whic)o(h)20 b(is)h(b)q(etter)75 528 y(dep)q(ends)c(on)e(whether)g (the)h(link)o(er)g(and)f(compiler)i(supp)q(ort)e(w)o(eak)g(sym)o(b)q (ols.)75 648 y Fw(Systems)h(with)g(w)o(eak)f(symb)q(ols)75 734 y Fv(If)k(the)f(compiler)i(and)f(link)o(er)h(supp)q(ort)f(w)o(eak)f (external)h(sym)o(b)q(ols)f(\(e.g.)30 b(Solaris)19 b(2.x,)f(other)g (system)75 791 y(V.4)13 b(mac)o(hines\),)i(then)f(only)g(a)g(single)h (library)g(is)f(required)h(through)f(the)g(use)g(of)g Ft(#pragma)23 b(weak)13 b Fv(th)o(us)75 897 y Ft(#pragma)23 b(weak)g(MPI_Example)g(=)g(PMPI_Example)75 1010 y(int)g (PMPI_Example\(/*)f(appropriate)h(args)g(*/\))75 1066 y({)170 1123 y(/*)h(Useful)f(content)g(*/)75 1179 y(})166 1285 y Fv(The)d(e\013ect)g(of)g(this)g Ft(#pragma)f Fv(is)i(to)e (de\014ne)i(the)f(external)h(sym)o(b)q(ol)f Ft(MPI)p 1472 1285 15 2 v 17 w(Example)f Fv(as)h(a)g(w)o(eak)75 1342 y(de\014nition.)h(This)15 b(means)f(that)g(the)g(link)o(er)h(will) h(not)e(complain)i(if)e(there)h(is)f(another)g(de\014nition)i(of)e(the) 75 1398 y(sym)o(b)q(ol)h(\(for)e(instance)i(in)g(the)f(pro\014ling)i (library\),)e(ho)o(w)o(ev)o(er)f(if)i(no)f(other)g(de\014nition)i (exists,)e(then)h(the)75 1455 y(link)o(er)h(will)h(use)f(the)f(w)o(eak) g(de\014nition.)75 1575 y Fw(Systems)h(without)h(w)o(eak)e(symb)q(ols) 75 1661 y Fv(In)k(the)g(absence)g(of)f(w)o(eak)g(sym)o(b)q(ols)h(then)g (one)f(p)q(ossible)j(solution)e(w)o(ould)g(b)q(e)g(to)f(use)h(the)f(C)h (macro)75 1717 y(pre-pro)q(cessor)c(th)o(us)75 1823 y Ft(#ifdef)23 b(PROFILELIB)75 1880 y(#)95 b(ifdef)23 b(__STDC__)75 1936 y(#)191 b(define)23 b(FUNCTION\(name\))f(P##name)75 1993 y(#)95 b(else)75 2049 y(#)191 b(define)23 b(FUNCTION\(name\))f (P/**/name)75 2106 y(#)95 b(endif)75 2162 y(#else)75 2219 y(#)g(define)23 b(FUNCTION\(name\))f(name)75 2275 y(#endif)166 2381 y Fv(Eac)o(h)15 b(of)g(the)g(user)g(visible)j (functions)e(in)g(the)f(library)h(w)o(ould)g(then)f(b)q(e)h(declared)g (th)o(us)75 2488 y Ft(int)23 b(FUNCTION\(MPI_Example\)\(/*)e (appropriate)h(args)i(*/\))75 2544 y({)170 2600 y(/*)g(Useful)f (content)g(*/)75 2657 y(})-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 206 214 206 213 bop 75 -100 a Fv(206)873 b Fm(CHAPTER)15 b(8.)35 b(PR)o(OFILING)16 b(INTERF)-5 b(A)o(CE)166 45 y Fv(The)20 b(same)g(source)g(\014le)h(can)g(then)f(b)q(e)h(compiled)h(to)d(pro)q (duce)i(b)q(oth)f(v)o(ersions)g(of)g(the)g(library)l(,)75 102 y(dep)q(ending)d(on)f(the)f(state)f(of)h(the)g Ft(PROFILELIB)f Fv(macro)h(sym)o(b)q(ol.)166 158 y(It)e(is)h(required)g(that)f(the)g (standard)g Fw(MPI)f Fv(library)i(b)q(e)g(built)h(in)f(suc)o(h)f(a)g(w) o(a)o(y)f(that)g(the)i(inclusion)h(of)75 214 y Fw(MPI)g Fv(functions)h(can)f(b)q(e)h(ac)o(hiev)o(ed)h(one)e(at)g(a)g(time.)20 b(This)c(is)g(a)f(somewhat)g(unpleasan)o(t)h(requiremen)o(t,)75 271 y(since)22 b(it)f(ma)o(y)f(mean)h(that)f(eac)o(h)g(external)i (function)f(has)g(to)f(b)q(e)h(compiled)i(from)d(a)g(separate)g (\014le.)75 327 y(Ho)o(w)o(ev)o(er)14 b(this)i(is)g(necessary)g(so)f (that)g(the)g(author)g(of)g(the)h(pro\014ling)g(library)g(need)h(only)f (de\014ne)g(those)75 384 y Fw(MPI)f Fv(functions)g(whic)o(h)h(she)g (wishes)g(to)e(in)o(tercept,)i(references)g(to)e(an)o(y)h(others)g(b)q (eing)h(ful\014lled)i(b)o(y)d(the)75 440 y(normal)g Fw(MPI)g Fv(library)l(.)21 b(Therefore)15 b(the)g(link)i(step)e(can)g(lo)q(ok)h (something)f(lik)o(e)h(this)75 547 y Ft(\045)24 b(cc)f(...)h(-lmyprof)f (-lpmpi)g(-lmpi)166 653 y Fv(Here)f Ft(libmyprof.a)d Fv(con)o(tains)j(the)f(pro\014ler)h(functions)h(whic)o(h)f(in)o (tercept)g(some)f(of)g(the)h Fw(MPI)75 709 y Fv(functions.)k Ft(libpmpi.a)16 b Fv(con)o(tains)h(the)g(\\name)g(shifted")g Fw(MPI)g Fv(functions,)h(and)f Ft(libmpi.a)f Fv(con)o(tains)75 766 y(the)f(normal)g(de\014nitions)i(of)e(the)g Fw(MPI)g Fv(functions.)75 887 y Fj(8.4.3)49 b(Complications)75 973 y Fw(Multiple)16 b(counting)75 1059 y Fv(Since)g(parts)e(of)g(the)g Fw(MPI)g Fv(library)i(ma)o(y)d(themselv)o(es)i(b)q(e)h(implemen)o(ted)g (using)f(more)f(basic)h Fw(MPI)f Fv(func-)75 1116 y(tions)h(\(e.g.)k(a) c(p)q(ortable)h(implemen)o(tation)g(of)f(the)g(collectiv)o(e)i(op)q (erations)f(implemen)o(ted)h(using)e(p)q(oin)o(t)75 1172 y(to)e(p)q(oin)o(t)h(comm)o(unications\),)g(there)g(is)h(p)q(oten)o (tial)f(for)f(pro\014ling)i(functions)g(to)e(b)q(e)h(called)i(from)d (within)75 1228 y(an)19 b Fw(MPI)g Fv(function)h(whic)o(h)g(w)o(as)e (called)j(from)e(a)g(pro\014ling)h(function.)33 b(This)20 b(could)g(lead)g(to)f(\\double)75 1285 y(coun)o(ting")d(of)g(the)g (time)h(sp)q(en)o(t)f(in)h(the)f(inner)i(routine.)23 b(Since)17 b(this)g(e\013ect)f(could)h(actually)g(b)q(e)g(useful)75 1341 y(under)e(some)e(circumstances)i(\(e.g.)k(it)14 b(migh)o(t)f(allo)o(w)i(one)f(to)f(answ)o(er)g(the)h(question)h(\\Ho)o (w)e(m)o(uc)o(h)h(time)75 1398 y(is)h(sp)q(en)o(t)g(in)h(the)e(p)q(oin) o(t)h(to)f(p)q(oin)o(t)i(routines)f(when)g(they're)f(called)i(from)e (collectiv)o(e)j(functions)e(?"\),)f(w)o(e)75 1454 y(ha)o(v)o(e)h (decided)j(not)d(to)g(enforce)h(an)o(y)g(restrictions)g(on)f(the)h (author)f(of)h(the)f Fw(MPI)h Fv(library)g(whic)o(h)h(w)o(ould)75 1511 y(o)o(v)o(ercome)11 b(this.)19 b(Therefore)12 b(the)g(author)g(of) f(the)h(pro\014ling)i(library)e(should)h(b)q(e)g(a)o(w)o(are)e(of)g (this)i(problem,)75 1567 y(and)f(guard)g(against)g(it)h(herself.)19 b(In)13 b(a)f(single)i(threaded)e(w)o(orld)g(this)h(is)g(easily)g(ac)o (hiev)o(ed)g(through)f(use)g(of)75 1624 y(a)j(static)f(v)m(ariable)j (in)f(the)f(pro\014ling)h(co)q(de)f(whic)o(h)h(remem)o(b)q(ers)f(if)h (y)o(ou)e(are)h(already)g(inside)i(a)e(pro\014ling)75 1680 y(routine.)25 b(It)17 b(b)q(ecomes)g(more)f(complex)i(in)f(a)g(m)o (ulti-threaded)h(en)o(vironmen)o(t)f(\(as)f(do)q(es)h(the)f(meaning)75 1737 y(of)f(the)g(times)h(recorded)f(!\))75 1857 y Fw(Link)o(er)g(o)q (ddities)75 1942 y Fv(The)g(Unix)g(link)o(er)g(traditionally)h(op)q (erates)e(in)i(one)e(pass)g(:)20 b(the)14 b(e\013ect)g(of)g(this)h(is)g (that)f(functions)h(from)75 1999 y(libraries)f(are)e(only)h(included)j (in)d(the)g(image)f(if)h(they)g(are)f(needed)i(at)e(the)h(time)g(the)f (library)h(is)h(scanned.)75 2055 y(When)j(com)o(bined)g(with)g(w)o(eak) f(sym)o(b)q(ols,)g(or)g(m)o(ultiple)i(de\014nitions)g(of)e(the)h(same)f (function,)h(this)g(can)75 2112 y(cause)e(o)q(dd)h(\(and)f(unexp)q (ected\))i(e\013ects.)166 2168 y(Consider,)24 b(for)e(instance,)j(an)d (implemen)o(tation)i(of)e Fw(MPI)g Fv(in)h(whic)o(h)g(the)g(F)l(ortran) e(binding)j(is)75 2225 y(ac)o(hiev)o(ed)19 b(b)o(y)e(using)i(wrapp)q (er)f(functions)g(on)g(top)g(of)f(the)h(C)f(implemen)o(tation.)29 b(The)18 b(author)g(of)f(the)75 2281 y(pro\014le)e(library)h(then)e (assumes)h(that)f(it)g(is)h(reasonable)g(only)g(to)f(pro)o(vide)h (pro\014le)g(functions)h(for)e(the)g(C)75 2338 y(binding,)20 b(since)e(F)l(ortran)e(will)j(ev)o(en)o(tually)g(call)f(these,)g(and)g (the)f(cost)g(of)g(the)h(wrapp)q(ers)f(is)h(assumed)75 2394 y(to)f(b)q(e)g(small.)27 b(Ho)o(w)o(ev)o(er,)16 b(if)i(the)f(wrapp)q(er)g(functions)h(are)f(not)g(in)h(the)f (pro\014ling)i(library)l(,)f(then)g(none)75 2451 y(of)c(the)g (pro\014led)i(en)o(try)e(p)q(oin)o(ts)g(will)i(b)q(e)f(unde\014ned)h (when)f(the)g(pro\014ling)g(library)g(is)g(called.)21 b(Therefore)75 2507 y(none)d(of)f(the)g(pro\014ling)i(co)q(de)f(will)h (b)q(e)f(included)i(in)e(the)g(image.)26 b(When)18 b(the)f(standard)g Fw(MPI)g Fv(library)75 2563 y(is)f(scanned,)f(the)g(F)l(ortran)f(wrapp) q(ers)h(will)i(b)q(e)e(resolv)o(ed,)h(and)f(will)h(also)f(pull)i(in)f (the)f(base)g(v)o(ersions)g(of)75 2620 y(the)g Fw(MPI)g Fv(functions.)21 b(The)15 b(o)o(v)o(erall)h(e\013ect)f(is)h(that)e(the) h(co)q(de)h(will)h(link)g(successfully)l(,)g(but)e(will)i(not)e(b)q(e) 75 2676 y(pro\014led.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 207 215 207 214 bop 75 -100 a Fm(8.5.)34 b(MUL)l(TIPLE)16 b(LEVELS)g(OF)f (INTER)o(CEPTION)739 b Fv(207)166 45 y(T)l(o)17 b(o)o(v)o(ercome)f (this)h(w)o(e)g(m)o(ust)f(ensure)h(that)g(the)g(F)l(ortran)e(wrapp)q (er)i(functions)h(are)f(included)i(in)75 102 y(the)d(pro\014ling)i(v)o (ersion)e(of)g(the)g(library)l(.)23 b(W)l(e)17 b(ensure)f(that)g(this)g (is)h(p)q(ossible)h(b)o(y)e(requiring)h(that)f(these)75 158 y(b)q(e)g(separable)f(from)g(the)g(rest)f(of)h(the)g(base)g Fw(MPI)g Fv(library)l(.)21 b(This)15 b(allo)o(ws)g(them)g(to)g(b)q(e)g Ft(ar)p Fv(ed)h(out)e(of)h(the)75 214 y(base)g(library)h(and)g(in)o(to) f(the)g(pro\014ling)i(one.)75 358 y Fn(8.5)59 b(Multiple)20 b(levels)f(of)h(interception)75 459 y Fv(The)15 b(sc)o(heme)h(giv)o(en) g(here)f(do)q(es)h(not)f(directly)h(supp)q(ort)f(the)h(nesting)g(of)e (pro\014ling)j(functions,)e(since)i(it)75 516 y(pro)o(vides)f(only)g(a) g(single)g(alternativ)o(e)g(name)g(for)f(eac)o(h)h Fw(MPI)f Fv(function.)22 b(Consideration)16 b(w)o(as)e(giv)o(en)j(to)75 572 y(an)e(implemen)o(tation)i(whic)o(h)f(w)o(ould)g(allo)o(w)g(m)o (ultiple)h(lev)o(els)g(of)e(call)h(in)o(terception,)h(ho)o(w)o(ev)o(er) d(w)o(e)h(w)o(ere)75 629 y(unable)j(to)e(construct)g(an)g(implemen)o (tation)i(of)e(this)h(whic)o(h)h(did)f(not)g(ha)o(v)o(e)f(the)g(follo)o (wing)i(disadv)m(an-)75 685 y(tages)143 779 y Fx(\017)23 b Fv(assuming)15 b(a)g(particular)h(implemen)o(tation)g(language.)143 873 y Fx(\017)23 b Fv(imp)q(osing)16 b(a)f(run)g(time)h(cost)f(ev)o(en) g(when)h(no)f(pro\014ling)h(w)o(as)f(taking)g(place.)75 966 y(Since)i(one)f(of)g(the)g(ob)s(jectiv)o(es)g(of)f Fw(MPI)h Fv(is)g(to)f(p)q(ermit)i(e\016cien)o(t,)f(lo)o(w)g(latency)h (implemen)o(tations,)g(and)75 1023 y(it)j(is)f(not)g(the)h(business)g (of)f(a)g(standard)g(to)g(require)h(a)f(particular)h(implemen)o(tation) g(language,)g(w)o(e)75 1079 y(decided)d(to)e(accept)g(the)g(sc)o(heme)h (outlined)h(ab)q(o)o(v)o(e.)166 1136 y(Note,)f(ho)o(w)o(ev)o(er,)f (that)h(it)h(is)f(p)q(ossible)j(to)c(use)i(the)g(sc)o(heme)f(ab)q(o)o (v)o(e)g(to)g(implemen)o(t)i(a)e(m)o(ulti-lev)o(el)75 1192 y(system,)k(since)g(the)g(function)g(called)i(b)o(y)d(the)h(user)f (ma)o(y)g(call)i(man)o(y)e(di\013eren)o(t)g(pro\014ling)i(functions)75 1249 y(b)q(efore)15 b(calling)i(the)f(underlying)h Fw(MPI)d Fv(function.)166 1305 y(Unfortunately)g(suc)o(h)h(an)f(implemen)o (tation)i(ma)o(y)d(require)j(more)d(co)q(op)q(eration)i(b)q(et)o(w)o (een)g(the)f(dif-)75 1362 y(feren)o(t)g(pro\014ling)h(libraries)h(than) e(is)g(required)h(for)f(the)g(single)h(lev)o(el)h(implemen)o(tation)f (detailed)h(ab)q(o)o(v)o(e.)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 208 216 208 215 bop 75 377 a Fr(Biblio)q(graph)m(y)98 600 y Fv([1])21 b(V.)14 b(Bala)h(and)f(S.)h(Kipnis.)20 b(Pro)q(cess)14 b(groups:)19 b(a)14 b(mec)o(hanism)h(for)f(the)g(co)q(ordination)i(of)e (and)g(com-)168 656 y(m)o(unication)i(among)e(pro)q(cesses)i(in)f(the)g (Ven)o(us)h(collectiv)o(e)g(comm)o(unication)g(library)l(.)k(T)l(ec)o (hnical)168 713 y(rep)q(ort,)15 b(IBM)g(T.)g(J.)g(W)l(atson)g(Researc)o (h)g(Cen)o(ter,)g(Octob)q(er)g(1992.)k(Preprin)o(t.)98 806 y([2])i(V.)h(Bala,)h(S.)e(Kipnis,)k(L.)d(Rudolph,)j(and)c(Marc)g (Snir.)40 b(Designing)23 b(e\016cien)o(t,)h(scalable,)g(and)168 863 y(p)q(ortable)17 b(collectiv)o(e)i(comm)o(unication)e(libraries.)26 b(T)l(ec)o(hnical)18 b(rep)q(ort,)e(IBM)h(T.)f(J.)g(W)l(atson)g(Re-)168 919 y(searc)o(h)f(Cen)o(ter,)g(Octob)q(er)h(1992.)i(Preprin)o(t.)98 1013 y([3])j(Purushotham)16 b(V.)f(Bangalore,)g(Nathan)g(E.)g(Doss,)f (and)i(An)o(thon)o(y)f(Skjellum.)22 b(MPI++:)f(Issues)168 1070 y(and)16 b(Features.)j(In)d Fl(OON-SKI)f('94)p Fv(,)g(page)g(in)h (press,)f(1994.)98 1163 y([4])21 b(A.)14 b(Beguelin,)j(J.)d(Dongarra,)e (A.)i(Geist,)h(R.)f(Manc)o(hek,)g(and)g(V.)g(Sunderam.)19 b(Visualization)e(and)168 1220 y(debugging)g(in)f(a)f(heterogeneous)g (en)o(vironmen)o(t.)20 b Fl(IEEE)15 b(Computer)p Fv(,)h(26\(6\):88{95,) 11 b(June)17 b(1993.)98 1314 y([5])k(Luc)14 b(Bomans)f(and)h(Rolf)f (Hemp)q(el.)18 b(The)c(Argonne/GMD)e(macros)g(in)i(F)o(OR)l(TRAN)g(for) e(p)q(ortable)168 1370 y(parallel)20 b(programming)e(and)h(their)f (implemen)o(tation)i(on)e(the)h(In)o(tel)g(iPSC/2.)29 b Fl(Par)n(al)r(lel)19 b(Com-)168 1427 y(puting)p Fv(,)c(15:119{132,)d (1990.)98 1520 y([6])21 b(R.)15 b(Butler)g(and)f(E.)g(Lusk.)k(User's)c (guide)h(to)f(the)g(p4)g(programming)g(system.)k(T)l(ec)o(hnical)e(Rep) q(ort)168 1577 y(TM-ANL{92/17,)e(Argonne)h(National)h(Lab)q(oratory)l (,)e(1992.)98 1671 y([7])21 b(Ralph)j(Butler)f(and)g(Ewing)g(Lusk.)41 b(Monitors,)24 b(messages,)f(and)g(clusters:)35 b(the)22 b(p4)g(parallel)168 1727 y(programming)15 b(system.)21 b Fl(Journal)c(of)g(Par)n(al)r(lel)f(Computing)p Fv(,)f(1994.)20 b(to)15 b(app)q(ear)h(\(Also)g(Argonne)168 1784 y(National)11 b(Lab)q(oratory)e(Mathematics)h(and)g(Computer)g(Science)i(Division)g (preprin)o(t)f(P362-0493\).)98 1877 y([8])21 b(Robin)d(Calkin,)e(Rolf)h (Hemp)q(el,)g(Hans-Christian)g(Hopp)q(e,)g(and)f(P)o(eter)f(Wypior.)22 b(P)o(ortable)16 b(pro-)168 1934 y(gramming)11 b(with)h(the)f(parmacs)g (message{passing)g(library)l(.)k Fl(Par)n(al)r(lel)d(Computing,)h(Sp)n (e)n(cial)f(issue)168 1990 y(on)k(message{p)n(assing)f(interfac)n(es)p Fv(,)f(to)g(app)q(ear.)98 2084 y([9])21 b(S.)c(Chittor)g(and)h(R.)f(J.) g(En)o(b)q(o)q(dy)l(.)27 b(P)o(erformance)16 b(ev)m(aluation)j(of)d (mesh{connected)i(w)o(ormhole{)168 2141 y(routed)f(net)o(w)o(orks)e (for)i(in)o(terpro)q(cessor)f(comm)o(unication)i(in)f(m)o (ulticomputers.)25 b(In)17 b Fl(Pr)n(o)n(c)n(e)n(e)n(dings)168 2197 y(of)g(the)f(1990)i(Sup)n(er)n(c)n(omputing)e(Confer)n(enc)n(e)p Fv(,)c(pages)j(647{656,)e(1990.)75 2291 y([10])21 b(S.)f(Chittor)g(and) g(R.)g(J.)f(En)o(b)q(o)q(dy)l(.)35 b(Predicting)21 b(the)f(e\013ect)g (of)f(mapping)i(on)e(the)h(comm)o(unica-)168 2347 y(tion)g(p)q (erformance)g(of)f(large)g(m)o(ulticomputers.)34 b(In)20 b Fl(Pr)n(o)n(c)n(e)n(e)n(dings)d(of)k(the)f(1991)h(International)168 2404 y(Confer)n(enc)n(e)15 b(on)h(Par)n(al)r(lel)f(Pr)n(o)n(c)n (essing,)f(vol.)i(II)f(\(Softwar)n(e\))p Fv(,)f(pages)h(I)q(I{1)h({)f (I)q(I{4,)g(1991.)75 2498 y([11])21 b(J.)e(Dongarra,)f(A.)g(Geist,)h (R.)g(Manc)o(hek,)g(and)g(V.)f(Sunderam.)31 b(In)o(tegrated)18 b(PVM)g(framew)o(ork)168 2554 y(supp)q(orts)c(heterogeneous)g(net)o(w)o (ork)f(computing.)18 b Fl(Computers)e(in)e(Physics)p Fv(,)f(7\(2\):166{75,)e(April)168 2610 y(1993.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 209 217 209 216 bop 75 -100 a Fm(BIBLIOGRAPHY)1371 b Fv(209)75 45 y([12])21 b(J.)c(J.)g(Dongarra,)f(R.)h(Hemp)q(el,)i(A.)e(J.)g(G.)f (Hey)l(,)i(and)f(D.)f(W.)h(W)l(alk)o(er.)25 b(A)17 b(prop)q(osal)g(for) g(a)g(user-)168 102 y(lev)o(el,)c(message)d(passing)h(in)o(terface)g (in)h(a)e(distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec)o(hnical)g (Rep)q(ort)168 158 y(TM-12231,)g(Oak)j(Ridge)g(National)g(Lab)q (oratory)l(,)e(F)l(ebruary)h(1993.)75 249 y([13])21 b(Nathan)e(Doss,)f (William)i(Gropp,)e(Ewing)h(Lusk,)h(and)e(An)o(thon)o(y)g(Skjellum.)31 b(A)19 b(mo)q(del)g(imple-)168 305 y(men)o(tation)c(of)g(MPI.)20 b(T)l(ec)o(hnical)d(rep)q(ort,)d(Argonne)i(National)f(Lab)q(oratory)l (,)f(1993.)75 396 y([14])21 b(Edin)o(burgh)g(P)o(arallel)f(Computing)g (Cen)o(tre,)g(Univ)o(ersit)o(y)g(of)f(Edin)o(burgh.)34 b Fl(CHIMP)19 b(Conc)n(epts)p Fv(,)168 452 y(June)e(1991.)75 543 y([15])k(Edin)o(burgh)c(P)o(arallel)g(Computing)f(Cen)o(tre,)f (Univ)o(ersit)o(y)h(of)g(Edin)o(burgh.)22 b Fl(CHIMP)15 b(V)m(ersion)h(1.0)168 599 y(Interfac)n(e)p Fv(,)e(Ma)o(y)g(1992.)75 690 y([16])21 b(D.)d(F)l(eitelson.)28 b(Comm)o(unicators:)d(Ob)s (ject-based)18 b(m)o(ultipart)o(y)g(in)o(teractions)g(for)g(parallel)h (pro-)168 747 y(gramming.)j(T)l(ec)o(hnical)c(Rep)q(ort)f(91-12,)e (Dept.)g(Computer)h(Science,)i(The)e(Hebrew)g(Univ)o(ersit)o(y)168 803 y(of)f(Jerusalem,)h(No)o(v)o(em)o(b)q(er)f(1991.)75 894 y([17])21 b(Hub)q(ertus)i(F)l(rank)o(e,)f(P)o(eter)f(Ho)q(c)o(hsc)o (hild,)k(Pratap)c(P)o(attnaik,)h(and)g(Marc)e(Snir.)40 b(An)22 b(e\016cien)o(t)168 950 y(implemen)o(tation)i(of)f(MPI.)42 b(In)23 b Fl(1994)h(International)e(Confer)n(enc)n(e)f(on)i(Par)n(al)r (lel)g(Pr)n(o)n(c)n(essing)p Fv(,)168 1007 y(1994.)75 1097 y([18])e(G.)14 b(A.)g(Geist,)g(M.)g(T.)f(Heath,)h(B.)g(W.)g(P)o (eyton,)f(and)i(P)l(.)f(H.)g(W)l(orley)l(.)19 b(A)14 b(user's)g(guide)h(to)f(PICL:)g(a)168 1154 y(p)q(ortable)f(instrumen)o (ted)f(comm)o(unication)h(library)l(.)i(T)l(ec)o(hnical)f(Rep)q(ort)e (TM-11616,)f(Oak)h(Ridge)168 1210 y(National)k(Lab)q(oratory)l(,)e (Octob)q(er)i(1990.)75 1301 y([19])21 b(William)j(D.)c(Gropp)h(and)h (Barry)e(Smith.)39 b(Chameleon)22 b(parallel)h(programming)e(to)q(ols)g (users)168 1357 y(man)o(ual.)f(T)l(ec)o(hnical)d(Rep)q(ort)f (ANL-93/23,)e(Argonne)h(National)h(Lab)q(oratory)l(,)e(Marc)o(h)h (1993.)75 1448 y([20])21 b(O.)c(Kr\177)-23 b(amer)16 b(and)h(H.)g(M)q(\177)-24 b(uhlen)o(b)q(ein.)26 b(Mapping)17 b(strategies)g(in)g(message{based)f(m)o(ultipro)q(cessor)168 1504 y(systems.)k Fl(Par)n(al)r(lel)15 b(Computing)p Fv(,)g(9:213{225,)d(1989.)75 1595 y([21])21 b(nCUBE)16 b(Corp)q(oration.)j Fl(nCUBE)c(2)i(Pr)n(o)n(gr)n(ammers)e(Guide,)i (r2.0)p Fv(,)f(Decem)o(b)q(er)f(1990.)75 1686 y([22])21 b(P)o(arasoft)10 b(Corp)q(oration,)g(P)o(asadena,)h(CA.)i Fl(Expr)n(ess)f(User's)f(Guide)p Fv(,)h(v)o(ersion)f(3.2.5)f(edition,)i (1992.)75 1776 y([23])21 b(P)o(aul)c(Pierce.)23 b(The)16 b(NX/2)g(op)q(erating)g(system.)22 b(In)17 b Fl(Pr)n(o)n(c)n(e)n(e)n (dings)e(of)i(the)g(Thir)n(d)g(Confer)n(enc)n(e)e(on)168 1833 y(Hyp)n(er)n(cub)n(e)g(Concurr)n(ent)f(Computers)i(and)f(Applic)n (ations)p Fv(,)e(pages)h(384{390.)d(A)o(CM)j(Press,)f(1988.)75 1923 y([24])21 b(A.)14 b(Skjellum)i(and)e(A.)g(Leung.)19 b(Zip)q(co)q(de:)h(a)14 b(p)q(ortable)h(m)o(ulticomputer)f(comm)o (unication)h(library)168 1980 y(atop)g(the)f(reactiv)o(e)h(k)o(ernel.) 20 b(In)15 b(D.)f(W.)g(W)l(alk)o(er)h(and)g(Q.)g(F.)f(Stout,)g (editors,)g Fl(Pr)n(o)n(c)n(e)n(e)n(dings)g(of)i(the)168 2036 y(Fifth)21 b(Distribute)n(d)h(Memory)f(Concurr)n(ent)g(Computing)g (Confer)n(enc)n(e)p Fv(,)e(pages)i(767{776.)d(IEEE)168 2093 y(Press,)d(1990.)75 2184 y([25])21 b(A.)12 b(Skjellum,)i(S.)e (Smith,)h(C.)e(Still,)j(A.)d(Leung,)i(and)g(M.)e(Morari.)i(The)g(Zip)q (co)q(de)g(message)f(passing)168 2240 y(system.)20 b(T)l(ec)o(hnical)d (rep)q(ort,)d(La)o(wrence)i(Liv)o(ermore)f(National)h(Lab)q(oratory)l (,)e(Septem)o(b)q(er)i(1992.)75 2331 y([26])21 b(An)o(thon)o(y)12 b(Skjellum,)i(Nathan)e(E.)g(Doss,)g(and)g(Purushotham)g(V.)g (Bangalore.)j(W)l(riting)e(Libraries)168 2387 y(in)i(MPI.)j(In)d(An)o (thon)o(y)f(Skjellum)h(and)f(Donna)g(S.)g(Reese,)h(editors,)f Fl(Pr)n(o)n(c)n(e)n(e)n(dings)f(of)i(the)g(Sc)n(alable)168 2444 y(Par)n(al)r(lel)h(Libr)n(aries)f(Confer)n(enc)n(e)p Fv(,)d(pages)j(166{173.)e(IEEE)i(Computer)g(So)q(ciet)o(y)g(Press,)g (Octob)q(er)168 2500 y(1993.)75 2591 y([27])21 b(An)o(thon)o(y)c (Skjellum,)h(Stev)o(en)f(G.)e(Smith,)j(Nathan)e(E.)g(Doss,)g(Alvin)i(P) e(.)g(Leung,)h(and)g(Manfred)168 2647 y(Morari.)32 b(The)19 b(Design)h(and)f(Ev)o(olution)h(of)f(Zip)q(co)q(de.)33 b Fl(Par)n(al)r(lel)19 b(Computing)p Fv(,)h(1994.)31 b(\(In)o(vited)168 2704 y(P)o(ap)q(er,)15 b(to)g(app)q(ear)g(in)h(Sp)q (ecial)h(Issue)f(on)f(Message)g(P)o(assing\).)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 210 218 210 217 bop 75 -100 a Fv(210)1368 b Fm(BIBLIOGRAPHY)75 45 y Fv([28])21 b(An)o(thon)o(y)d(Skjellum,)j(Stev)o(en)d(G.)g(Smith,)h (Nathan)f(E.)g(Doss,)g(Charles)h(H.)f(Still,)i(Alvin)g(P)l(.)e(Le-)168 102 y(ung,)g(and)f(Manfred)g(Morari.)24 b(Zip)q(co)q(de:)h(A)17 b(Portable)g(Comm)o(unication)h(La)o(y)o(er)e(for)h(High)g(P)o(er-)168 158 y(formance)g(Multicomputing.)28 b(T)l(ec)o(hnical)20 b(Rep)q(ort)d(UCRL-JC-106725)g(\(revised)h(9/92,)f(12/93,)168 214 y(4/94\),)f(La)o(wrence)i(Liv)o(ermore)g(National)f(Lab)q(oratory)l (,)g(Marc)o(h)f(1991.)25 b(T)l(o)17 b(app)q(ear)h(in)g(Concur-)168 271 y(rency:)j(Practice)15 b(&)h(Exp)q(erience.)75 365 y([29])21 b(D.)12 b(W)l(alk)o(er.)k(Standards)c(for)g(message)g (passing)g(in)i(a)e(distributed)i(memory)e(en)o(vironmen)o(t.)j(T)l(ec) o(h-)168 421 y(nical)i(Rep)q(ort)f(TM-12147,)d(Oak)i(Ridge)h(National)g (Lab)q(oratory)l(,)e(August)h(1992.)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 211 219 211 218 bop 75 356 a Fo(Annex)33 b(A)75 564 y Fr(Language)41 b(Binding)75 805 y Fn(A.1)59 b(Intro)r(duction)75 906 y Fv(In)15 b(this)f(section)h(w)o(e)f(summarize)g(the)h(sp)q(eci\014c)h (bindings)g(for)d(b)q(oth)h(F)l(ortran)f(and)h(C.)g(W)l(e)g(presen)o(t) g(\014rst)75 963 y(the)h(C)g(bindings,)i(then)e(the)h(F)l(ortran)e (bindings.)22 b(Listings)16 b(are)f(alphab)q(etical)i(within)f(c)o (hapter.)75 1106 y Fn(A.2)59 b(De\014ned)19 b(Constants)g(fo)n(r)i(C)f (and)f(F)n(o)n(rtran)75 1207 y Fv(These)e(are)g(required)h(de\014ned)h (constan)o(ts,)d(to)g(b)q(e)i(de\014ned)h(in)e(the)h(\014les)g Ft(mpi.h)e Fv(\(for)g(C\))h(and)g Ft(mpif.h)75 1264 y Fv(\(for)d(F)l(ortran\).)75 1370 y Ft(/*)24 b(return)f(codes)g(\(both)g (C)h(and)f(Fortran\))g(*/)75 1427 y(MPI_SUCCESS)75 1483 y(MPI_ERR_BUFFER)75 1539 y(MPI_ERR_COUNT)75 1596 y(MPI_ERR_TYPE)75 1652 y(MPI_ERR_TAG)75 1709 y(MPI_ERR_COMM)75 1765 y(MPI_ERR_RANK)75 1822 y(MPI_ERR_REQUEST)75 1878 y(MPI_ERR_ROOT)75 1935 y(MPI_ERR_GROUP)75 1991 y(MPI_ERR_OP)75 2048 y(MPI_ERR_TOPOLOGY)75 2104 y(MPI_ERR_DIMS)75 2160 y(MPI_ERR_ARG)75 2217 y(MPI_ERR_UNKNOWN)75 2273 y(MPI_ERR_TRUNCATE)75 2330 y(MPI_ERR_OTHER)75 2386 y(MPI_ERR_INTERN)75 2443 y(MPI_PENDING)75 2499 y(MPI_ERR_IN_STATUS)75 2556 y(MPI_ERR_LASTCODE)75 2669 y(/*)h(assorted)e(constants)h(\(both)g (C)h(and)f(Fortran\))g(*/)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 212 220 212 219 bop 75 -100 a Fv(212)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Ft(MPI_BOTTOM)75 102 y(MPI_PROC_NULL)75 158 y(MPI_ANY_SOURCE)75 214 y(MPI_ANY_TAG)75 271 y(MPI_UNDEFINED)75 327 y(MPI_BSEND_OVERHEAD)75 384 y(MPI_KEYVAL_INVALID)75 497 y(/*)24 b(status)f(size)g(and)g(reserved)g (index)g(values)g(\(Fortran\))g(*/)75 553 y(MPI_STATUS_SIZE)75 610 y(MPI_SOURCE)75 666 y(MPI_TAG)75 723 y(MPI_ERROR)75 835 y(/*)h(Error-handling)e(specifiers)g(\(C)i(and)f(Fortran\))g(*/)75 892 y(MPI_ERRORS_ARE_FATAL)75 948 y(MPI_ERRORS_RETURN)75 1061 y(/*)h(Maximum)e(sizes)i(for)f(strings)g(*/)75 1118 y(MPI_MAX_PROCESSOR_NAME)75 1174 y(MPI_MAX_ERROR_STRING)75 1287 y(/*)h(elementary)e(datatypes)h(\(C\))g(*/)75 1344 y(MPI_CHAR)75 1400 y(MPI_SHORT)75 1456 y(MPI_INT)75 1513 y(MPI_LONG)75 1569 y(MPI_UNSIGNED_CHAR)75 1626 y(MPI_UNSIGNED_SHORT)75 1682 y(MPI_UNSIGNED)75 1739 y(MPI_UNSIGNED_LONG)75 1795 y(MPI_FLOAT)75 1852 y(MPI_DOUBLE)75 1908 y(MPI_LONG_DOUBLE)75 1965 y(MPI_BYTE)75 2021 y(MPI_PACKED)75 2247 y(/*)h(elementary)e (datatypes)h(\(Fortran\))f(*/)75 2303 y(MPI_INTEGER)75 2360 y(MPI_REAL)75 2416 y(MPI_DOUBLE_PRECISION)75 2473 y(MPI_COMPLEX)75 2529 y(MPI_DOUBLE_COMPLEX)75 2586 y(MPI_LOGICAL)75 2642 y(MPI_CHARACTER)75 2698 y(MPI_BYTE)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 213 221 213 220 bop 75 -100 a Fm(A.2.)29 b(DEFINED)15 b(CONST)l(ANTS)h(F)o(OR)f (C)g(AND)g(F)o(OR)l(TRAN)590 b Fv(213)75 45 y Ft(MPI_PACKED)75 158 y(/*)24 b(datatypes)e(for)i(reduction)e(functions)h(\(C\))g(*/)75 214 y(MPI_FLOAT_INT)75 271 y(MPI_DOUBLE_INT)75 327 y(MPI_LONG_INT)75 384 y(MPI_2INT)75 440 y(MPI_SHORT_INT)75 497 y(MPI_LONG_DOUBLE_INT)75 610 y(/*)h(datatypes)e(for)i(reduction)e(functions)h(\(Fortran\))g(*/) 75 666 y(MPI_2REAL)75 723 y(MPI_2DOUBLE_PRECISION)75 779 y(MPI_2INTEGER)75 892 y(/*)h(optional)e(datatypes)h(\(Fortran\))g (*/)75 948 y(MPI_INTEGER1)75 1005 y(MPI_INTEGER2)75 1061 y(MPI_INTEGER4)75 1118 y(MPI_REAL2)75 1174 y(MPI_REAL4)75 1231 y(MPI_REAL8)75 1344 y(/*)h(optional)e(datatypes)h(\(C\))g(*/)75 1400 y(MPI_LONG_LONG_INT)75 1513 y(/*)h(special)e(datatypes)h(for)h (constructing)e(derived)h(datatypes)75 1569 y(MPI_UB)75 1626 y(MPI_LB)75 1739 y(/*)h(reserved)e(communicators)g(\(C)i(and)f (Fortran\))g(*/)75 1795 y(MPI_COMM_WORLD)75 1852 y(MPI_COMM_SELF)75 1965 y(/*)h(results)e(of)i(communicator)e(and)i(group)f(comparisons)f (*/)75 2077 y(MPI_IDENT)75 2134 y(MPI_CONGRUENT)75 2190 y(MPI_SIMILAR)75 2247 y(MPI_UNEQUAL)75 2360 y(/*)i(environmental)e (inquiry)h(keys)g(\(C)g(and)h(Fortran\))f(*/)75 2416 y(MPI_TAG_UB)75 2473 y(MPI_IO)75 2529 y(MPI_HOST)75 2586 y(MPI_WTIME_IS_GLOBAL)75 2698 y(/*)h(collective)e(operations)h(\(C)g (and)h(Fortran\))e(*/)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 214 222 214 221 bop 75 -100 a Fv(214)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Ft(MPI_MAX)75 102 y(MPI_MIN)75 158 y(MPI_SUM)75 214 y(MPI_PROD)75 271 y(MPI_MAXLOC)75 327 y(MPI_MINLOC)75 384 y(MPI_BAND)75 440 y(MPI_BOR)75 497 y(MPI_BXOR)75 553 y(MPI_LAND)75 610 y(MPI_LOR)75 666 y(MPI_LXOR)75 779 y(/*)24 b(Null)f(handles)g(*/)75 835 y(MPI_GROUP_NULL)75 892 y(MPI_COMM_NULL)75 948 y(MPI_DATATYPE_NULL) 75 1005 y(MPI_REQUEST_NULL)75 1061 y(MPI_OP_NULL)75 1118 y(MPI_ERRHANDLER_NULL)75 1231 y(/*)h(Empty)f(group)g(*/)75 1287 y(MPI_GROUP_EMPTY)75 1400 y(/*)h(topologies)e(\(C)i(and)f (Fortran\))g(*/)75 1456 y(MPI_GRAPH)75 1513 y(MPI_CART)166 1676 y Fv(The)15 b(follo)o(wing)h(are)f(de\014ned)i(C)e(t)o(yp)q(e)g (de\014nitions,)i(also)e(included)j(in)e(the)f(\014le)h Ft(mpi.h)p Fv(.)75 1782 y Ft(/*)24 b(opaque)f(types)g(\(C\))g(*/)75 1838 y(MPI_Aint)75 1895 y(MPI_Status)75 2008 y(/*)h(handles)e(to)i (assorted)f(structures)f(\(C\))i(*/)75 2064 y(MPI_Group)75 2121 y(MPI_Comm)75 2177 y(MPI_Datatype)75 2234 y(MPI_Request)75 2290 y(MPI_Op)75 2403 y(/*)g(prototypes)e(for)i(user-defined)e (functions)g(\(C\))i(*/)75 2459 y(typedef)f(int)47 b (MPI_Copy_function\(MPI_Comm)21 b(oldcomm,)h(int)i(keyval,)552 2516 y(void)g(*extra_state,)e(void)h(*attribute_val_in,)552 2572 y(void)h(*attribute_val_out,)d(int)i(*flag\);)75 2629 y(typedef)g(int)47 b(MPI_Delete_function\(MPI_Comm)20 b(comm,)j(int)h(keyval,)695 2685 y(void)g(*attribute_val,)e(void)h (*extra_state\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 215 223 215 222 bop 75 -100 a Fm(A.3.)34 b(C)15 b(BINDINGS)h(F)o(OR)g (POINT-TO-POINT)h(COMMUNICA)l(TION)380 b Fv(215)75 45 y Ft(typedef)23 b(void)g(MPI_Handler_function\(MPI_Com)o(m)e(*,)j(int)f (*,)h(...\);)75 102 y(typedef)f(void)g(MPI_User_function\()f(void)h (*invec,)g(void)g(*inoutvec,)g(int)g(*len,)600 158 y(MPI_Datatype)f (*datatype\);)166 262 y Fv(F)l(or)d(F)l(ortran,)g(here)i(are)e (examples)i(of)e(ho)o(w)g(eac)o(h)h(of)g(the)f(user-de\014ned)j (functions)e(should)h(b)q(e)75 318 y(declared.)166 375 y(The)15 b(user-function)i(argumen)o(t)d(to)h Fw(MPI)p 878 375 14 2 v 15 w(OP)p 955 375 V 17 w(CREA)l(TE)i Fv(should)f(b)q(e)g (declared)g(lik)o(e)g(this:)75 467 y Ft(FUNCTION)23 b(USER_FUNCTION\()f (INVEC\(*\),)g(INOUTVEC\(*\),)h(LEN,)g(TYPE\))75 523 y()g(INVEC\(LEN\),)f(INOUTVEC\(LEN\))99 579 y(INTEGER)h(LEN,)g (TYPE)166 671 y Fv(The)15 b(cop)o(y-function)h(argumen)o(t)f(to)f Fw(MPI)p 887 671 V 16 w(KEYV)l(AL)p 1073 671 V 17 w(CREA)l(TE)i Fv(should)h(b)q(e)e(declared)i(lik)o(e)f(this:)99 763 y Ft(PROCEDURE)22 b(COPY_FUNCTION\(OLDCOMM,)f(KEYVAL,)i(EXTRA_STATE,) 433 820 y(ATTRIBUTE_VAL_IN,)e(ATTRIBUTE_VAL_OUT,)h(FLAG,)h(IERR\))99 876 y(INTEGER)g(OLDCOMM,)g(KEYVAL,)f(EXTRA_STATE,)h(ATTRIBUTE_VAL_IN,) 290 933 y(ATTRIBUTE_VAL_OUT,)e(IERR)99 989 y(LOGICAL)i(FLAG)166 1081 y Fv(The)15 b(delete-function)j(argumen)o(t)c(to)h Fw(MPI)p 913 1081 V 15 w(KEYV)l(AL)p 1098 1081 V 17 w(CREA)l(TE)i Fv(should)f(b)q(e)g(declared)g(lik)o(e)g(this:)75 1173 y Ft(PROCEDURE)23 b(DELETE_FUNCTION\(COMM,)e(KEYVAL,)i(ATTRIBUTE_VAL,)e (EXTRA_STATE,)i(IERR\))99 1230 y(INTEGER)g(COMM,)g(KEYVAL,)g (ATTRIBUTE_VAL,)f(EXTRA_STATE,)g(IERR)75 1372 y Fn(A.3)59 b(C)20 b(bindings)g(fo)n(r)h(P)n(oint-to-P)n(oint)e(Communication)75 1474 y Fv(These)d(are)e(presen)o(ted)i(here)g(in)g(the)f(order)g(of)g (their)g(app)q(earance)h(in)g(the)g(c)o(hapter.)75 1530 y Ft(int)23 b(MPI)p 245 1530 15 2 v 17 w(Send\(void*)g(buf,)g(int)g (count,)h(MPI)p 979 1530 V 16 w(Datatype)f(datatype,)g(int)g(dest,)393 1587 y(int)h(tag,)f(MPI)p 683 1587 V 17 w(Comm)g(comm\))75 1673 y(int)g(MPI)p 245 1673 V 17 w(Recv\(void*)g(buf,)g(int)g(count,)h (MPI)p 979 1673 V 16 w(Datatype)f(datatype,)g(int)g(source,)393 1730 y(int)h(tag,)f(MPI)p 683 1730 V 17 w(Comm)g(comm,)g(MPI)p 1034 1730 V 17 w(Status)g(*status\))75 1816 y(int)g(MPI)p 245 1816 V 17 w(Get)p 334 1816 V 17 w(count\(MPI)p 567 1816 V 16 w(Status)g(*status,)g(MPI)p 1037 1816 V 17 w(Datatype)f(datatype,)h(int)g(*count\))75 1903 y(int)g(MPI)p 245 1903 V 17 w(Bsend\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 1903 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 1959 y(int)h(tag,)f(MPI)p 683 1959 V 17 w(Comm)g(comm\))75 2045 y(int)g(MPI)p 245 2045 V 17 w(Ssend\(void*)g(buf,)g(int)g(count,)g (MPI)p 1002 2045 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2102 y(int)h(tag,)f(MPI)p 683 2102 V 17 w(Comm)g(comm\))75 2188 y(int)g(MPI)p 245 2188 V 17 w(Rsend\(void*)g(buf,)g(int)g(count,)g (MPI)p 1002 2188 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2245 y(int)h(tag,)f(MPI)p 683 2245 V 17 w(Comm)g(comm\))75 2331 y(int)g(MPI)p 245 2331 V 17 w(Buffer)p 406 2331 V 17 w(attach\()f(void*)i(buffer,)f(int)g(size\))75 2418 y(int)g(MPI)p 245 2418 V 17 w(Buffer)p 406 2418 V 17 w(detach\()f(void*)i(buffer,)f(int*)g(size\))75 2504 y(int)g(MPI)p 245 2504 V 17 w(Isend\(void*)g(buf,)g(int)g(count,)g(MPI) p 1002 2504 V 17 w(Datatype)g(datatype,)g(int)g(dest,)393 2561 y(int)h(tag,)f(MPI)p 683 2561 V 17 w(Comm)g(comm,)g(MPI)p 1034 2561 V 17 w(Request)g(*request\))75 2647 y(int)g(MPI)p 245 2647 V 17 w(Ibsend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 2647 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 2704 y(int)g(tag,)f(MPI)p 683 2704 V 17 w(Comm)g(comm,)g(MPI)p 1034 2704 V 17 w(Request)g(*request\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 216 224 216 223 bop 75 -100 a Fv(216)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Ft(int)23 b(MPI)p 245 45 15 2 v 17 w(Issend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 45 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 102 y(int)g(tag,)f(MPI)p 683 102 V 17 w(Comm)g(comm,)g(MPI)p 1034 102 V 17 w(Request)g(*request\))75 188 y(int)g(MPI)p 245 188 V 17 w(Irsend\(void*)f(buf,)i(int)f(count,)g(MPI)p 1026 188 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 244 y(int)g(tag,)f(MPI)p 683 244 V 17 w(Comm)g(comm,)g(MPI)p 1034 244 V 17 w(Request)g(*request\))75 331 y(int)g(MPI)p 245 331 V 17 w(Irecv\(void*)g(buf,)g(int)g(count,)g(MPI)p 1002 331 V 17 w(Datatype)g(datatype,)g(int)g(source,)393 387 y(int)h(tag,)f(MPI)p 683 387 V 17 w(Comm)g(comm,)g(MPI)p 1034 387 V 17 w(Request)g(*request\))75 474 y(int)g(MPI)p 245 474 V 17 w(Wait\(MPI)p 454 474 V 16 w(Request)g(*request,)g(MPI)p 972 474 V 17 w(Status)g(*status\))75 560 y(int)g(MPI)p 245 560 V 17 w(Test\(MPI)p 454 560 V 16 w(Request)g(*request,)g(int)g (*flag,)g(MPI)p 1234 560 V 17 w(Status)g(*status\))75 647 y(int)g(MPI)p 245 647 V 17 w(Request)p 430 647 V 16 w(free\(MPI)p 638 647 V 17 w(Request)f(*request\))75 733 y(int)h(MPI)p 245 733 V 17 w(Waitany\(int)g(count,)g(MPI)p 788 733 V 16 w(Request)g(*array)p 1139 733 V 17 w(of)p 1204 733 V 17 w(requests,)f(int)i(*index,)393 790 y(MPI)p 468 790 V 17 w(Status)f(*status\))75 876 y(int)g(MPI)p 245 876 V 17 w(Testany\(int)g(count,)g(MPI)p 788 876 V 16 w(Request)g(*array)p 1139 876 V 17 w(of)p 1204 876 V 17 w(requests,)f(int)i(*index,)393 933 y(int)g(*flag,)f(MPI)p 731 933 V 16 w(Status)g(*status\))75 1019 y(int)g(MPI)p 245 1019 V 17 w(Waitall\(int)g(count,)g(MPI)p 788 1019 V 16 w(Request)g(*array)p 1139 1019 V 17 w(of)p 1204 1019 V 17 w(requests,)393 1075 y(MPI)p 468 1075 V 17 w(Status)g(*array)p 796 1075 V 16 w(of)p 860 1075 V 17 w(statuses\))75 1162 y(int)g(MPI)p 245 1162 V 17 w(Testall\(int)g (count,)g(MPI)p 788 1162 V 16 w(Request)g(*array)p 1139 1162 V 17 w(of)p 1204 1162 V 17 w(requests,)f(int)i(*flag,)393 1218 y(MPI)p 468 1218 V 17 w(Status)f(*array)p 796 1218 V 16 w(of)p 860 1218 V 17 w(statuses\))75 1305 y(int)g(MPI)p 245 1305 V 17 w(Waitsome\(int)f(incount,)h(MPI)p 859 1305 V 17 w(Request)g(*array)p 1211 1305 V 16 w(of)p 1275 1305 V 17 w(requests,)g(int)g(*outcount,)393 1361 y(int)h(*array)p 636 1361 V 16 w(of)p 700 1361 V 17 w(indices,)f(MPI)p 1004 1361 V 16 w(Status)g(*array)p 1331 1361 V 17 w(of)p 1396 1361 V 17 w(statuses\))75 1448 y(int)g(MPI)p 245 1448 V 17 w(Testsome\(int)f(incount,)h(MPI)p 859 1448 V 17 w(Request)g(*array)p 1211 1448 V 16 w(of)p 1275 1448 V 17 w(requests,)g(int)g(*outcount,)393 1504 y(int)h(*array)p 636 1504 V 16 w(of)p 700 1504 V 17 w(indices,)f(MPI)p 1004 1504 V 16 w(Status)g(*array)p 1331 1504 V 17 w(of)p 1396 1504 V 17 w(statuses\))75 1591 y(int)g(MPI)p 245 1591 V 17 w(Iprobe\(int)g(source,)g(int)g(tag,)g(MPI)p 1002 1591 V 17 w(Comm)g(comm,)h(int)f(*flag,)393 1647 y(MPI)p 468 1647 V 17 w(Status)g(*status\))75 1734 y(int)g(MPI)p 245 1734 V 17 w(Probe\(int)g(source,)g(int)g(tag,)h(MPI)p 979 1734 V 16 w(Comm)g(comm,)f(MPI)p 1330 1734 V 17 w(Status)g (*status\))75 1820 y(int)g(MPI)p 245 1820 V 17 w(Cancel\(MPI)p 502 1820 V 16 w(Request)g(*request\))75 1907 y(int)g(MPI)p 245 1907 V 17 w(Test)p 358 1907 V 17 w(cancelled\(MPI)p 687 1907 V 15 w(Status)g(*status,)g(int)h(*flag\))75 1993 y(int)f(MPI)p 245 1993 V 17 w(Send)p 358 1993 V 17 w(init\(void*)f(buf,)i(int)f(count,)g(MPI)p 1091 1993 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 2049 y(int)g(tag,)f(MPI)p 683 2049 V 17 w(Comm)g(comm,)g(MPI)p 1034 2049 V 17 w(Request)g (*request\))75 2136 y(int)g(MPI)p 245 2136 V 17 w(Bsend)p 382 2136 V 17 w(init\(void*)f(buf,)i(int)f(count,)g(MPI)p 1115 2136 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 2192 y(int)g(tag,)f(MPI)p 683 2192 V 17 w(Comm)g(comm,)g(MPI)p 1034 2192 V 17 w(Request)g(*request\))75 2279 y(int)g(MPI)p 245 2279 V 17 w(Ssend)p 382 2279 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 2279 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 2335 y(int)g(tag,)f(MPI)p 683 2335 V 17 w(Comm)g(comm,)g(MPI)p 1034 2335 V 17 w(Request)g(*request\))75 2422 y(int)g(MPI)p 245 2422 V 17 w(Rsend)p 382 2422 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1115 2422 V 17 w(Datatype)g(datatype,)f(int)i(dest,)393 2478 y(int)g(tag,)f(MPI)p 683 2478 V 17 w(Comm)g(comm,)g(MPI)p 1034 2478 V 17 w(Request)g(*request\))75 2565 y(int)g(MPI)p 245 2565 V 17 w(Recv)p 358 2565 V 17 w(init\(void*)f(buf,)i(int)f (count,)g(MPI)p 1091 2565 V 17 w(Datatype)g(datatype,)f(int)i(source,) 393 2621 y(int)g(tag,)f(MPI)p 683 2621 V 17 w(Comm)g(comm,)g(MPI)p 1034 2621 V 17 w(Request)g(*request\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 217 225 217 224 bop 75 -100 a Fm(A.3.)34 b(C)15 b(BINDINGS)h(F)o(OR)g (POINT-TO-POINT)h(COMMUNICA)l(TION)380 b Fv(217)75 45 y Ft(int)23 b(MPI)p 245 45 15 2 v 17 w(Start\(MPI)p 478 45 V 16 w(Request)g(*request\))75 132 y(int)g(MPI)p 245 132 V 17 w(Startall\(int)f(count,)h(MPI)p 811 132 V 17 w(Request)g(*array)p 1163 132 V 17 w(of)p 1228 132 V 16 w(requests\))75 218 y(int)g(MPI)p 245 218 V 17 w(Sendrecv\(void)f (*sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 218 V 17 w(Datatype)f (sendtype,)393 274 y(int)i(dest,)f(int)g(sendtag,)g(void)g(*recvbuf,)g (int)g(recvcount,)393 331 y(MPI)p 468 331 V 17 w(Datatype)g(recvtype,)f (int)i(source,)f(MPI)p 1297 331 V 16 w(Datatype)g(recvtag,)393 387 y(MPI)p 468 387 V 17 w(Comm)g(comm,)g(MPI)p 819 387 V 17 w(Status)g(*status\))75 474 y(int)g(MPI)p 245 474 V 17 w(Sendrecv)p 454 474 V 16 w(replace\(void*)f(buf,)i(int)f(count,)g (MPI)p 1258 474 V 17 w(Datatype)g(datatype,)393 530 y(int)h(dest,)f (int)g(sendtag,)g(int)g(source,)g(int)h(recvtag,)f(MPI)p 1614 530 V 16 w(Comm)h(comm,)393 587 y(MPI)p 468 587 V 17 w(Status)f(*status\))75 673 y(int)g(MPI)p 245 673 V 17 w(Type)p 358 673 V 17 w(contiguous\(int)f(count,)h(MPI)p 972 673 V 17 w(Datatype)f(oldtype,)393 730 y(MPI)p 468 730 V 17 w(Datatype)h(*newtype\))75 816 y(int)g(MPI)p 245 816 V 17 w(Type)p 358 816 V 17 w(vector\(int)f(count,)h(int)h (blocklength,)e(int)i(stride,)393 873 y(MPI)p 468 873 V 17 w(Datatype)f(oldtype,)f(MPI)p 986 873 V 17 w(Datatype)h (*newtype\))75 959 y(int)g(MPI)p 245 959 V 17 w(Type)p 358 959 V 17 w(hvector\(int)f(count,)h(int)h(blocklength,)e(MPI)p 1306 959 V 17 w(Aint)h(stride,)393 1015 y(MPI)p 468 1015 V 17 w(Datatype)g(oldtype,)f(MPI)p 986 1015 V 17 w(Datatype)h (*newtype\))75 1102 y(int)g(MPI)p 245 1102 V 17 w(Type)p 358 1102 V 17 w(indexed\(int)f(count,)h(int)h(*array)p 1068 1102 V 16 w(of)p 1132 1102 V 17 w(blocklengths,)393 1158 y(int)g(*array)p 636 1158 V 16 w(of)p 700 1158 V 17 w(displacements,)e(MPI)p 1147 1158 V 17 w(Datatype)g(oldtype,)393 1215 y(MPI)p 468 1215 V 17 w(Datatype)h(*newtype\))75 1301 y(int)g(MPI)p 245 1301 V 17 w(Type)p 358 1301 V 17 w(hindexed\(int)f(count,)h(int)h(*array)p 1092 1301 V 16 w(of)p 1156 1301 V 17 w(blocklengths,)393 1358 y(MPI)p 468 1358 V 17 w(Aint)f(*array)p 748 1358 V 17 w(of)p 813 1358 V 17 w(displacements,)e(MPI)p 1259 1358 V 17 w(Datatype)i(oldtype,)393 1414 y(MPI)p 468 1414 V 17 w(Datatype)g(*newtype\))75 1501 y(int)g(MPI)p 245 1501 V 17 w(Type)p 358 1501 V 17 w(struct\(int)f(count,)h(int)h(*array)p 1044 1501 V 16 w(of)p 1108 1501 V 17 w(blocklengths,)393 1557 y(MPI)p 468 1557 V 17 w(Aint)f(*array)p 748 1557 V 17 w(of)p 813 1557 V 17 w(displacements,)e(MPI)p 1259 1557 V 17 w(Datatype)i(*array)p 1635 1557 V 16 w(of)p 1699 1557 V 17 w(types,)393 1614 y(MPI)p 468 1614 V 17 w(Datatype)g(*newtype\))75 1700 y(int)g(MPI)p 245 1700 V 17 w(Address\(void*)f(location,)h(MPI)p 907 1700 V 17 w(Aint)g(*address\))75 1786 y(int)g(MPI)p 245 1786 V 17 w(Type)p 358 1786 V 17 w(extent\(MPI)p 615 1786 V 16 w(Datatype)f(datatype,)h(MPI)p 1156 1786 V 17 w(Aint)g(*extent\)) 75 1873 y(int)g(MPI)p 245 1873 V 17 w(Type)p 358 1873 V 17 w(size\(MPI)p 567 1873 V 16 w(Datatype)g(datatype,)f(int)i (*size\))75 1959 y(int)f(MPI)p 245 1959 V 17 w(Type)p 358 1959 V 17 w(lb\(MPI)p 519 1959 V 16 w(Datatype)g(datatype,)g(MPI)p 1061 1959 V 16 w(Aint*)h(displacement\))75 2046 y(int)f(MPI)p 245 2046 V 17 w(Type)p 358 2046 V 17 w(ub\(MPI)p 519 2046 V 16 w(Datatype)g(datatype,)g(MPI)p 1061 2046 V 16 w(Aint*)h(displacement\))75 2132 y(int)f(MPI)p 245 2132 V 17 w(Type)p 358 2132 V 17 w(commit\(MPI)p 615 2132 V 16 w(Datatype)f(*datatype\))75 2219 y(int)h(MPI)p 245 2219 V 17 w(Type)p 358 2219 V 17 w(free\(MPI)p 567 2219 V 16 w(Datatype)g(*datatype\))75 2305 y(int)g(MPI)p 245 2305 V 17 w(Get)p 334 2305 V 17 w(elements\(MPI)p 639 2305 V 16 w(Status)g(*status,)f(MPI)p 1108 2305 V 17 w(Datatype)h(datatype,)g(int)g(*count\))1937 2345 y Fx(>)75 2392 y Ft(int)g(MPI)p 245 2392 V 17 w(Pack\(void*)g(inbuf,)g (int)g(incount,)g(MPI)p 1074 2392 V 17 w(Datatype)g(datatype,)f(void)h (*outbuf,)393 2448 y(int)h(outsize,)e(int)i(*position,)e(MPI)p 1136 2448 V 17 w(Comm)h(comm\))1937 2488 y Fx(?)75 2535 y Ft(int)g(MPI)p 245 2535 V 17 w(Unpack\(void*)f(inbuf,)h(int)h (insize,)f(int)g(*position,)g(void)g(*outbuf,)393 2591 y(int)h(outcount,)e(MPI)p 802 2591 V 17 w(Datatype)h(datatype,)f(MPI)p 1344 2591 V 17 w(Comm)i(comm\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 218 226 218 225 bop 75 -100 a Fv(218)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Ft(int)23 b(MPI)p 245 45 15 2 v 17 w(Pack)p 358 45 V 17 w(size\(int)g(incount,)f(MPI)p 876 45 V 17 w(Datatype)h(datatype,)g(MPI)p 1419 45 V 16 w(Comm)h(comm,)393 102 y(int)g(*size\))75 275 y Fn(A.4)59 b(C)20 b(Bindings)g(fo)n(r)h(Collective)e(Communication)75 376 y Ft(int)k(MPI)p 245 376 V 17 w(Barrier\(MPI)p 526 376 V 16 w(Comm)g(comm)h(\))75 463 y(int)f(MPI)p 245 463 V 17 w(Bcast\(void*)g(buffer,)f(int)i(count,)f(MPI)p 1074 463 V 17 w(Datatype)g(datatype,)f(int)i(root,)393 519 y(MPI)p 468 519 V 17 w(Comm)f(comm)h(\))75 606 y(int)f(MPI)p 245 606 V 17 w(Gather\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1217 606 V 17 w(Datatype)h(sendtype,)393 662 y(void*)g(recvbuf,)g(int)h (recvcount,)e(MPI)p 1184 662 V 17 w(Datatype)h(recvtype,)f(int)i(root,) 393 719 y(MPI)p 468 719 V 17 w(Comm)f(comm\))75 805 y(int)g(MPI)p 245 805 V 17 w(Gatherv\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 805 V 17 w(Datatype)h(sendtype,)393 861 y(void*)g(recvbuf,)g(int)h (*recvcounts,)e(int)h(*displs,)393 918 y(MPI)p 468 918 V 17 w(Datatype)g(recvtype,)f(int)i(root,)f(MPI)p 1249 918 V 17 w(Comm)g(comm\))75 1004 y(int)g(MPI)p 245 1004 V 17 w(Scatter\(void*)f(sendbuf,)h(int)h(sendcount,)e(MPI)p 1241 1004 V 17 w(Datatype)h(sendtype,)393 1061 y(void*)g(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1061 V 17 w(Datatype)h(recvtype,)f(int)i (root,)393 1117 y(MPI)p 468 1117 V 17 w(Comm)f(comm\))75 1204 y(int)g(MPI)p 245 1204 V 17 w(Scatterv\(void*)f(sendbuf,)h(int)g (*sendcounts,)g(int)g(*displs,)393 1260 y(MPI)p 468 1260 V 17 w(Datatype)g(sendtype,)f(void*)i(recvbuf,)e(int)i(recvcount,)393 1317 y(MPI)p 468 1317 V 17 w(Datatype)f(recvtype,)f(int)i(root,)f(MPI)p 1249 1317 V 17 w(Comm)g(comm\))75 1403 y(int)g(MPI)p 245 1403 V 17 w(Allgather\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1289 1403 V 17 w(Datatype)f(sendtype,)393 1460 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1460 V 17 w(Datatype)h(recvtype,)393 1516 y(MPI)p 468 1516 V 17 w(Comm)g(comm\))75 1602 y(int)g(MPI)p 245 1602 V 17 w(Allgatherv\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 1602 V 16 w(Datatype)g(sendtype,)393 1659 y(void*)g(recvbuf,)g (int)h(*recvcounts,)e(int)h(*displs,)393 1715 y(MPI)p 468 1715 V 17 w(Datatype)g(recvtype,)f(MPI)p 1010 1715 V 17 w(Comm)h(comm\))75 1802 y(int)g(MPI)p 245 1802 V 17 w(Alltoall\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1802 V 17 w(Datatype)f(sendtype,)393 1858 y(void*)h(recvbuf,)g (int)h(recvcount,)e(MPI)p 1184 1858 V 17 w(Datatype)h(recvtype,)393 1915 y(MPI)p 468 1915 V 17 w(Comm)g(comm\))75 2001 y(int)g(MPI)p 245 2001 V 17 w(Alltoallv\(void*)f(sendbuf,)h(int)g(*sendcounts,)f(int) i(*sdispls,)393 2058 y(MPI)p 468 2058 V 17 w(Datatype)f(sendtype,)f (void*)i(recvbuf,)e(int)i(*recvcounts,)393 2114 y(int)g(*rdispls,)e (MPI)p 802 2114 V 17 w(Datatype)h(recvtype,)f(MPI)p 1344 2114 V 17 w(Comm)i(comm\))75 2201 y(int)f(MPI)p 245 2201 V 17 w(Reduce\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)h(count,)393 2257 y(MPI)p 468 2257 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2257 V 17 w(Op)i(op,)f(int)h(root,)f(MPI)p 1505 2257 V 17 w(Comm)g(comm\))75 2343 y(int)g(MPI)p 245 2343 V 17 w(Op)p 310 2343 V 17 w(create\(MPI)p 567 2343 V 16 w(User)p 679 2343 V 17 w(function)f(*function,)h(int)g(commute,)g (MPI)p 1555 2343 V 17 w(Op)g(*op\))75 2430 y(int)g(MPI)p 245 2430 V 17 w(Op)p 310 2430 V 17 w(free\()g(MPI)p 542 2430 V 17 w(Op)h(*op\))75 2516 y(int)f(MPI)p 245 2516 V 17 w(Allreduce\(void*)f(sendbuf,)h(void*)g(recvbuf,)g(int)g(count,) 393 2573 y(MPI)p 468 2573 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 2573 V 17 w(Op)i(op,)f(MPI)p 1266 2573 V 17 w(Comm)g(comm\))75 2659 y(int)g(MPI)p 245 2659 V 17 w(Reduce)p 406 2659 V 17 w(scatter\(void*)f(sendbuf,)g(void*)i(recvbuf,)e(int)i (*recvcounts,)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 219 227 219 226 bop 75 -100 a Fm(A.5.)34 b(C)15 b(BINDINGS)h(F)o(OR)g(GR)o (OUPS,)f(CONTEXTS,)g(AND)g(COMMUNICA)l(TORS)160 b Fv(219)393 45 y Ft(MPI)p 468 45 15 2 v 17 w(Datatype)23 b(datatype,)f(MPI)p 1010 45 V 17 w(Op)i(op,)f(MPI)p 1266 45 V 17 w(Comm)g(comm\))75 132 y(int)g(MPI)p 245 132 V 17 w(Scan\(void*)g(sendbuf,)f(void*)i (recvbuf,)e(int)i(count,)393 188 y(MPI)p 468 188 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 188 V 17 w(Op)i(op,)f(MPI)p 1266 188 V 17 w(Comm)g(comm)h(\))75 361 y Fn(A.5)59 b(C)20 b(Bindings)g(fo)n(r)h(Groups,)e(Contexts,)f(and)h(Communicato)n(rs)75 463 y Ft(int)k(MPI)p 245 463 V 17 w(Group)p 382 463 V 17 w(size\(MPI)p 591 463 V 16 w(Group)g(group,)g(int)g(*size\))75 549 y(int)g(MPI)p 245 549 V 17 w(Group)p 382 549 V 17 w(rank\(MPI)p 591 549 V 16 w(Group)g(group,)g(int)g(*rank\))75 636 y(int)g(MPI)p 245 636 V 17 w(Group)p 382 636 V 17 w(translate)p 615 636 V 16 w(ranks)g(\(MPI)p 870 636 V 16 w(Group)h(group1,)f(int)g(n,)h(int)f(*ranks1,)393 692 y(MPI)p 468 692 V 17 w(Group)g(group2,)g(int)g(*ranks2\))75 779 y(int)g(MPI)p 245 779 V 17 w(Group)p 382 779 V 17 w(compare\(MPI)p 663 779 V 15 w(Group)h(group1,MPI)p 1062 779 V 15 w(Group)g(group2,)e(int)i(*result\))75 865 y(int)f(MPI)p 245 865 V 17 w(Comm)p 358 865 V 17 w(group\(MPI)p 591 865 V 16 w(Comm)g(comm,)g(MPI)p 941 865 V 17 w(Group)g(*group\))75 951 y(int)g(MPI)p 245 951 V 17 w(Group)p 382 951 V 17 w(union\(MPI)p 615 951 V 16 w(Group)g(group1,)g(MPI)p 1037 951 V 17 w(Group)g(group2,)g(MPI)p 1460 951 V 16 w(Group)h(*newgroup\))75 1038 y(int)f(MPI)p 245 1038 V 17 w(Group)p 382 1038 V 17 w(intersection\(MPI)p 783 1038 V 15 w(Group)g(group1,)g(MPI)p 1204 1038 V 17 w(Group)g(group2,)393 1094 y(MPI)p 468 1094 V 17 w(Group)g(*newgroup\)) 75 1181 y(int)g(MPI)p 245 1181 V 17 w(Group)p 382 1181 V 17 w(difference\(MPI)p 735 1181 V 15 w(Group)g(group1,)g(MPI)p 1156 1181 V 17 w(Group)g(group2,)393 1237 y(MPI)p 468 1237 V 17 w(Group)g(*newgroup\))75 1324 y(int)g(MPI)p 245 1324 V 17 w(Group)p 382 1324 V 17 w(incl\(MPI)p 591 1324 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1324 V 17 w(Group)h(*newgroup\))75 1410 y(int)g(MPI)p 245 1410 V 17 w(Group)p 382 1410 V 17 w(excl\(MPI)p 591 1410 V 16 w(Group)g(group,)g(int)g(n,)h(int)g(*ranks,)e(MPI)p 1442 1410 V 17 w(Group)h(*newgroup\))75 1497 y(int)g(MPI)p 245 1497 V 17 w(Group)p 382 1497 V 17 w(range)p 519 1497 V 16 w(incl\(MPI)p 727 1497 V 16 w(Group)g(group,)h(int)f(n,)h(int)f (ranges[][3],)393 1553 y(MPI)p 468 1553 V 17 w(Group)g(*newgroup\))75 1640 y(int)g(MPI)p 245 1640 V 17 w(Group)p 382 1640 V 17 w(range)p 519 1640 V 16 w(excl\(MPI)p 727 1640 V 16 w(Group)g(group,)h(int)f(n,)h(int)f(ranges[][3],)393 1696 y(MPI)p 468 1696 V 17 w(Group)g(*newgroup\))75 1782 y(int)g(MPI)p 245 1782 V 17 w(Group)p 382 1782 V 17 w(free\(MPI)p 591 1782 V 16 w(Group)g(*group\))75 1869 y(int)g(MPI)p 245 1869 V 17 w(Comm)p 358 1869 V 17 w(size\(MPI)p 567 1869 V 16 w(Comm)g(comm,)g(int)h(*size\))75 1955 y(int)f(MPI)p 245 1955 V 17 w(Comm)p 358 1955 V 17 w(rank\(MPI)p 567 1955 V 16 w(Comm)g(comm,)g(int)h(*rank\))75 2042 y(int)f(MPI)p 245 2042 V 17 w(Comm)p 358 2042 V 17 w(compare\(MPI)p 639 2042 V 16 w(Comm)g(comm1,MPI)p 990 2042 V 16 w(Comm)g(comm2,)g(int) h(*result\))75 2128 y(int)f(MPI)p 245 2128 V 17 w(Comm)p 358 2128 V 17 w(dup\(MPI)p 543 2128 V 16 w(Comm)g(comm,)h(MPI)p 894 2128 V 16 w(Comm)g(*newcomm\))75 2215 y(int)f(MPI)p 245 2215 V 17 w(Comm)p 358 2215 V 17 w(create\(MPI)p 615 2215 V 16 w(Comm)g(comm,)g(MPI)p 965 2215 V 17 w(Group)g(group,)g (MPI)p 1364 2215 V 17 w(Comm)g(*newcomm\))75 2301 y(int)g(MPI)p 245 2301 V 17 w(Comm)p 358 2301 V 17 w(split\(MPI)p 591 2301 V 16 w(Comm)g(comm,)g(int)h(color,)f(int)g(key,)h(MPI)p 1419 2301 V 16 w(Comm)g(*newcomm\))75 2388 y(int)f(MPI)p 245 2388 V 17 w(Comm)p 358 2388 V 17 w(free\(MPI)p 567 2388 V 16 w(Comm)g(*comm\))75 2474 y(int)g(MPI)p 245 2474 V 17 w(Comm)p 358 2474 V 17 w(test)p 471 2474 V 16 w(inter\(MPI)p 703 2474 V 16 w(Comm)h(comm,)f(int)g(*flag\))75 2561 y(int)g(MPI)p 245 2561 V 17 w(Comm)p 358 2561 V 17 w(remote)p 519 2561 V 16 w(size\(MPI)p 727 2561 V 16 w(Comm)h(comm,)f(int)g(*size\))75 2647 y(int)g(MPI)p 245 2647 V 17 w(Comm)p 358 2647 V 17 w(remote)p 519 2647 V 16 w(group\(MPI)p 751 2647 V 16 w(Comm)g(comm,)h(MPI)p 1102 2647 V 16 w(Group)g(*group\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 220 228 220 227 bop 75 -100 a Fv(220)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Ft(int)23 b(MPI)p 245 45 15 2 v 17 w(Intercomm)p 478 45 V 16 w(create\(MPI)p 734 45 V 16 w(Comm)g(local)p 989 45 V 17 w(comm,)g(int)g(local)p 1364 45 V 17 w(leader,)393 102 y(MPI)p 468 102 V 17 w(Comm)g(peer)p 700 102 V 17 w(comm,)g(int)h(remote)p 1100 102 V 16 w(leader,)f(int)g (tag,)393 158 y(MPI)p 468 158 V 17 w(Comm)g(*newintercomm\))75 244 y(int)g(MPI)p 245 244 V 17 w(Intercomm)p 478 244 V 16 w(merge\(MPI)p 710 244 V 16 w(Comm)g(intercomm,)g(int)g(high,)393 301 y(MPI)p 468 301 V 17 w(Comm)g(*newintracomm\))75 387 y(int)g(MPI)p 245 387 V 17 w(Keyval)p 406 387 V 17 w(create\(MPI)p 663 387 V 15 w(Copy)p 774 387 V 17 w(function)g(*copy)p 1126 387 V 16 w(fn,)h(MPI)p 1310 387 V 17 w(Delete)p 1471 387 V 16 w(function)393 444 y(*delete)p 564 444 V 16 w(fn,)g(int)f(*keyval,)g(void*)g(extra)p 1249 444 V 17 w(state\))75 530 y(int)g(MPI)p 245 530 V 17 w(Keyval)p 406 530 V 17 w(free\(int)f(*keyval\))75 617 y(int)h(MPI)p 245 617 V 17 w(Attr)p 358 617 V 17 w(put\(MPI)p 543 617 V 16 w(Comm)g(comm,)h(int)f(keyval,)g(void*)g(attribute)p 1467 617 V 16 w(val\))75 703 y(int)g(MPI)p 245 703 V 17 w(Attr)p 358 703 V 17 w(get\(MPI)p 543 703 V 16 w(Comm)g(comm,)h (int)f(keyval,)g(void*)g(attribute)p 1467 703 V 16 w(val,)g(int)h (*flag\))75 790 y(int)f(MPI)p 245 790 V 17 w(Attr)p 358 790 V 17 w(delete\(MPI)p 615 790 V 16 w(Comm)g(comm,)g(int)h(keyval\)) 75 963 y Fn(A.6)59 b(C)20 b(Bindings)g(fo)n(r)h(Pro)r(cess)e(T)-5 b(op)r(ologies)75 1064 y Ft(int)23 b(MPI)p 245 1064 V 17 w(Cart)p 358 1064 V 17 w(create\(MPI)p 615 1064 V 16 w(Comm)g(comm)p 846 1064 V 17 w(old,)g(int)g(ndims,)g(int)h(*dims,)f (int)g(*periods,)393 1121 y(int)h(reorder,)e(MPI)p 778 1121 V 17 w(Comm)h(*comm)p 1034 1121 V 17 w(cart\))75 1207 y(int)g(MPI)p 245 1207 V 17 w(Dims)p 358 1207 V 17 w(create\(int)f(nnodes,)h(int)h(ndims,)f(int)g(*dims\))75 1294 y(int)g(MPI)p 245 1294 V 17 w(Graph)p 382 1294 V 17 w(create\(MPI)p 639 1294 V 16 w(Comm)g(comm)p 870 1294 V 16 w(old,)h(int)f(nnodes,)g(int)h(*index,)f(int)g(*edges,)393 1350 y(int)h(reorder,)e(MPI)p 778 1350 V 17 w(Comm)h(*comm)p 1034 1350 V 17 w(graph\))75 1437 y(int)g(MPI)p 245 1437 V 17 w(Topo)p 358 1437 V 17 w(test\(MPI)p 567 1437 V 16 w(Comm)g(comm,)g(int)h(*status\))75 1523 y(int)f(MPI)p 245 1523 V 17 w(Graphdims)p 478 1523 V 16 w(get\(MPI)p 662 1523 V 16 w(Comm)h(comm,)f(int)g(*nnodes,)g(int)h(*nedges\))75 1610 y(int)f(MPI)p 245 1610 V 17 w(Graph)p 382 1610 V 17 w(get\(MPI)p 567 1610 V 16 w(Comm)g(comm,)g(int)h(maxindex,)f(int)g (maxedges,)g(int)g(*index,)393 1666 y(int)h(*edges\))75 1752 y(int)f(MPI)p 245 1752 V 17 w(Cartdim)p 430 1752 V 16 w(get\(MPI)p 614 1752 V 17 w(Comm)g(comm,)g(int)h(*ndims\))75 1839 y(int)f(MPI)p 245 1839 V 17 w(Cart)p 358 1839 V 17 w(get\(MPI)p 543 1839 V 16 w(Comm)g(comm,)h(int)f(maxdims,)g(int)g (*dims,)g(int)h(*periods,)393 1895 y(int)g(*coords\))75 1982 y(int)f(MPI)p 245 1982 V 17 w(Cart)p 358 1982 V 17 w(rank\(MPI)p 567 1982 V 16 w(Comm)g(comm,)g(int)h(*coords,)f(int)g (*rank\))75 2068 y(int)g(MPI)p 245 2068 V 17 w(Cart)p 358 2068 V 17 w(coords\(MPI)p 615 2068 V 16 w(Comm)g(comm,)g(int)h (rank,)f(int)g(maxdims,)g(int)g(*coords\))75 2155 y(int)g(MPI)p 245 2155 V 17 w(Graph)p 382 2155 V 17 w(neighbors)p 615 2155 V 16 w(count\(MPI)p 847 2155 V 16 w(Comm)g(comm,)g(int)h(rank,)f (int)g(*nneighbors\))75 2241 y(int)g(MPI)p 245 2241 V 17 w(Graph)p 382 2241 V 17 w(neighbors\(MPI)p 711 2241 V 15 w(Comm)g(comm,)h(int)f(rank,)g(int)h(maxneighbors,)393 2298 y(int)g(*neighbors\))75 2384 y(int)f(MPI)p 245 2384 V 17 w(Cart)p 358 2384 V 17 w(shift\(MPI)p 591 2384 V 16 w(Comm)g(comm,)g(int)h(direction,)e(int)i(disp,)f(int)g(*rank)p 1681 2384 V 17 w(source,)393 2441 y(int)h(*rank)p 612 2441 V 16 w(dest\))75 2527 y(int)f(MPI)p 245 2527 V 17 w(Cart)p 358 2527 V 17 w(sub\(MPI)p 543 2527 V 16 w(Comm)g(comm,)h(int) f(*remain)p 1085 2527 V 16 w(dims,)g(MPI)p 1316 2527 V 17 w(Comm)h(*newcomm\))75 2613 y(int)f(MPI)p 245 2613 V 17 w(Cart)p 358 2613 V 17 w(map\(MPI)p 543 2613 V 16 w(Comm)g(comm,)h(int)f(ndims,)g(int)h(*dims,)f(int)g(*periods,)393 2670 y(int)h(*newrank\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 221 229 221 228 bop 75 -100 a Fm(A.7.)34 b(C)15 b(BINDINGS)h(F)o(OR)g(ENVIR)o (ONMENT)l(AL)g(INQUIR)l(Y)582 b Fv(221)75 45 y Ft(int)23 b(MPI)p 245 45 15 2 v 17 w(Graph)p 382 45 V 17 w(map\(MPI)p 567 45 V 16 w(Comm)g(comm,)g(int)h(nnodes,)f(int)g(*index,)g(int)h (*edges,)393 102 y(int)g(*newrank\))75 275 y Fn(A.7)59 b(C)20 b(bindings)g(fo)n(r)h(Environmental)d(Inquiry)75 376 y Ft(int)23 b(MPI)p 245 376 V 17 w(Get)p 334 376 V 17 w(processor)p 567 376 V 16 w(name\(char)g(*name,)g(int)g (*resultlen\))75 463 y(int)g(MPI)p 245 463 V 17 w(Errhandler)p 502 463 V 16 w(create\(MPI)p 758 463 V 16 w(Handler)p 942 463 V 16 w(function)g(*function,)393 519 y(MPI)p 468 519 V 17 w(Errhandler)f(*errhandler\))75 606 y(int)h(MPI)p 245 606 V 17 w(Errhandler)p 502 606 V 16 w(set\(MPI)p 686 606 V 16 w(Comm)h(comm,)f(MPI)p 1037 606 V 17 w(Errhandler)f (errhandler\))75 692 y(int)h(MPI)p 245 692 V 17 w(Errhandler)p 502 692 V 16 w(get\(MPI)p 686 692 V 16 w(Comm)h(comm,)f(MPI)p 1037 692 V 17 w(Errhandler)f(*errhandler\))75 779 y(int)h(MPI)p 245 779 V 17 w(Errhandler)p 502 779 V 16 w(free\(MPI)p 710 779 V 16 w(Errhandler)g(*errhandler\))75 865 y(int)g(MPI)p 245 865 V 17 w(Error)p 382 865 V 17 w(string\(int)f(errorcode,)h(char)g (*string,)g(int)g(*resultlen\))75 951 y(int)g(MPI)p 245 951 V 17 w(Error)p 382 951 V 17 w(class\(int)f(errorcode,)h(int)g (*errorclass\))75 1038 y(int)g(double)g(MPI)p 412 1038 V 17 w(Wtime\(void\))75 1124 y(int)g(double)g(MPI)p 412 1124 V 17 w(Wtick\(void\))75 1211 y(int)g(MPI)p 245 1211 V 17 w(Init\(int)g(*argc,)g(char)g(***argv\))75 1297 y(int)g(MPI)p 245 1297 V 17 w(Finalize\(void\))75 1384 y(int)g(MPI)p 245 1384 V 17 w(Initialized\(int)f(*flag\))75 1470 y(int)h(MPI)p 245 1470 V 17 w(Abort\(MPI)p 478 1470 V 16 w(Comm)g(comm,)h(int)f(errorcode\))75 1643 y Fn(A.8)59 b(C)20 b(Bindings)g(fo)n(r)h(Pro\014ling)75 1745 y Ft(int)i(MPI)p 245 1745 V 17 w(Pcontrol\(const)f(int)i(level,)f Fp(:)8 b(:)g(:)d Ft(\))75 1918 y Fn(A.9)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(P)n(oint-to-P)n(oint)f(Communication)75 2020 y Ft(MPI)p 150 2020 V 17 w(SEND\(BUF,)i(COUNT,)h(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(IERROR\))170 2076 y()g(BUF\(*\))170 2132 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 2219 y(MPI)p 150 2219 V 17 w(RECV\(BUF,)f(COUNT,)h(DATATYPE,)g(SOURCE,) g(TAG,)g(COMM,)g(STATUS,)g(IERROR\))170 2275 y()g(BUF\(*\))170 2332 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (STATUS\(MPI)p 1463 2332 V 16 w(STATUS)p 1623 2332 V 17 w(SIZE\),)170 2388 y(IERROR)75 2475 y(MPI)p 150 2475 V 17 w(GET)p 239 2475 V 17 w(COUNT\(STATUS,)f(DATATYPE,)g(COUNT,)h (IERROR\))170 2531 y(INTEGER)g(STATUS\(MPI)p 604 2531 V 16 w(STATUS)p 764 2531 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h(IERROR) 75 2618 y(MPI)p 150 2618 V 17 w(BSEND\(BUF,)f(COUNT,)h(DATATYPE,)g (DEST,)g(TAG,)g(COMM,)h(IERROR\))170 2674 y()f(BUF\(*\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 222 230 222 229 bop 75 -100 a Fv(222)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y Ft(INTEGER)23 b(COUNT,)g(DATATYPE,) g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 132 y(MPI)p 150 132 15 2 v 17 w(SSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)h (IERROR\))170 188 y()f(BUF\(*\))170 244 y(INTEGER)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 331 y(MPI)p 150 331 V 17 w(RSEND\(BUF,)f(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g(COMM,) h(IERROR\))170 387 y()f(BUF\(*\))170 444 y(INTEGER)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(IERROR)75 530 y(MPI)p 150 530 V 17 w(BUFFER)p 311 530 V 16 w(ATTACH\()g(BUFFER,)g(SIZE,)g (IERROR\))170 587 y()g(BUFFER\(*\))170 643 y(INTEGER)g(SIZE,)h (IERROR)75 730 y(MPI)p 150 730 V 17 w(BUFFER)p 311 730 V 16 w(DETACH\()f(BUFFER,)g(SIZE,)g(IERROR\))170 786 y()g(BUFFER\(*\))170 843 y(INTEGER)g(SIZE,)h(IERROR)75 929 y(MPI)p 150 929 V 17 w(ISEND\(BUF,)e(COUNT,)h(DATATYPE,)g(DEST,)g (TAG,)g(COMM,)h(REQUEST,)e(IERROR\))170 985 y()h(BUF\(*\))170 1042 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)75 1128 y(MPI)p 150 1128 V 17 w(IBSEND\(BUF,)g(COUNT,)h (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1185 y()g(BUF\(*\))170 1241 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 1328 y(MPI)p 150 1328 V 17 w(ISSEND\(BUF,)g(COUNT,)h(DATATYPE,)g(DEST,)g(TAG,)g (COMM,)g(REQUEST,)g(IERROR\))170 1384 y()g(BUF\(*\))170 1441 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)h(COMM,)f(REQUEST,)f (IERROR)75 1527 y(MPI)p 150 1527 V 17 w(IRSEND\(BUF,)g(COUNT,)h (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1584 y()g(BUF\(*\))170 1640 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 1726 y(MPI)p 150 1726 V 17 w(IRECV\(BUF,)g(COUNT,)h(DATATYPE,)g(SOURCE,)g(TAG,)g (COMM,)g(REQUEST,)g(IERROR\))170 1783 y()g(BUF\(*\))170 1839 y(INTEGER)g(COUNT,)g(DATATYPE,)g(SOURCE,)g(TAG,)g(COMM,)g (REQUEST,)g(IERROR)75 1926 y(MPI)p 150 1926 V 17 w(WAIT\(REQUEST,)f (STATUS,)h(IERROR\))170 1982 y(INTEGER)g(REQUEST,)g(STATUS\(MPI)p 819 1982 V 16 w(STATUS)p 979 1982 V 16 w(SIZE\),)g(IERROR)75 2069 y(MPI)p 150 2069 V 17 w(TEST\(REQUEST,)f(FLAG,)h(STATUS,)g (IERROR\))170 2125 y(LOGICAL)g(FLAG)170 2182 y(INTEGER)g(REQUEST,)g (STATUS\(MPI)p 819 2182 V 16 w(STATUS)p 979 2182 V 16 w(SIZE\),)g(IERROR)75 2268 y(MPI)p 150 2268 V 17 w(REQUEST)p 335 2268 V 16 w(FREE\(REQUEST,)f(IERROR\))170 2325 y(INTEGER)h (REQUEST,)g(IERROR)75 2411 y(MPI)p 150 2411 V 17 w(WAITANY\(COUNT,)f (ARRAY)p 645 2411 V 16 w(OF)p 709 2411 V 17 w(REQUESTS,)h(INDEX,)g (STATUS,)g(IERROR\))170 2468 y(INTEGER)g(COUNT,)g(ARRAY)p 651 2468 V 17 w(OF)p 716 2468 V 17 w(REQUESTS\(*\),)f(INDEX,)h (STATUS\(MPI)p 1450 2468 V 16 w(STATUS)p 1610 2468 V 16 w(SIZE\),)170 2524 y(IERROR)75 2610 y(MPI)p 150 2610 V 17 w(TESTANY\(COUNT,)f(ARRAY)p 645 2610 V 16 w(OF)p 709 2610 V 17 w(REQUESTS,)h(INDEX,)g(FLAG,)g(STATUS,)g(IERROR\))170 2667 y(LOGICAL)g(FLAG)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 223 231 223 230 bop 75 -100 a Fm(A.9.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o (OR)g(POINT-TO-POINT)h(COMMUNICA)l(TION)185 b Fv(223)170 45 y Ft(INTEGER)23 b(COUNT,)g(ARRAY)p 651 45 15 2 v 17 w(OF)p 716 45 V 17 w(REQUESTS\(*\),)f(INDEX,)h(STATUS\(MPI)p 1450 45 V 16 w(STATUS)p 1610 45 V 16 w(SIZE\),)170 102 y(IERROR)75 188 y(MPI)p 150 188 V 17 w(WAITALL\(COUNT,)f(ARRAY)p 645 188 V 16 w(OF)p 709 188 V 17 w(REQUESTS,)h(ARRAY)p 1085 188 V 16 w(OF)p 1149 188 V 17 w(STATUSES,)g(IERROR\))170 244 y(INTEGER)g(COUNT,)g(ARRAY)p 651 244 V 17 w(OF)p 716 244 V 17 w(REQUESTS\(*\),)170 301 y(ARRAY)p 293 301 V 17 w(OF)p 358 301 V 17 w(STATUSES\(MPI)p 663 301 V 15 w(STATUS)p 822 301 V 17 w(SIZE,*\),)g(IERROR)75 387 y(MPI)p 150 387 V 17 w(TESTALL\(COUNT,)f(ARRAY)p 645 387 V 16 w(OF)p 709 387 V 17 w(REQUESTS,)h(FLAG,)g(ARRAY)p 1228 387 V 16 w(OF)p 1292 387 V 17 w(STATUSES,)g(IERROR\))170 444 y(LOGICAL)g(FLAG)170 500 y(INTEGER)g(COUNT,)g(ARRAY)p 651 500 V 17 w(OF)p 716 500 V 17 w(REQUESTS\(*\),)170 557 y(ARRAY)p 293 557 V 17 w(OF)p 358 557 V 17 w(STATUSES\(MPI)p 663 557 V 15 w(STATUS)p 822 557 V 17 w(SIZE,*\),)g(IERROR)75 643 y(MPI)p 150 643 V 17 w(WAITSOME\(INCOUNT,)e(ARRAY)p 716 643 V 17 w(OF)p 781 643 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 643 V 16 w(OF)p 1459 643 V 17 w(INDICES,)393 700 y(ARRAY)p 516 700 V 17 w(OF)p 581 700 V 17 w(STATUSES,)f(IERROR\))170 756 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 756 V 17 w(OF)p 764 756 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 756 V 17 w(OF)p 1514 756 V 17 w(INDICES\(*\),)170 813 y(ARRAY)p 293 813 V 17 w(OF)p 358 813 V 17 w(STATUSES\(MPI)p 663 813 V 15 w(STATUS)p 822 813 V 17 w(SIZE,*\),)h(IERROR)75 899 y(MPI)p 150 899 V 17 w(TESTSOME\(INCOUNT,)e(ARRAY)p 716 899 V 17 w(OF)p 781 899 V 17 w(REQUESTS,)h(OUTCOUNT,)h(ARRAY)p 1395 899 V 16 w(OF)p 1459 899 V 17 w(INDICES,)393 955 y(ARRAY)p 516 955 V 17 w(OF)p 581 955 V 17 w(STATUSES,)f(IERROR\))170 1012 y(INTEGER)h(INCOUNT,)g(ARRAY)p 699 1012 V 17 w(OF)p 764 1012 V 17 w(REQUESTS\(*\),)f(OUTCOUNT,)g(ARRAY)p 1449 1012 V 17 w(OF)p 1514 1012 V 17 w(INDICES\(*\),)170 1068 y(ARRAY)p 293 1068 V 17 w(OF)p 358 1068 V 17 w(STATUSES\(MPI)p 663 1068 V 15 w(STATUS)p 822 1068 V 17 w(SIZE,*\),)h(IERROR)75 1155 y(MPI)p 150 1155 V 17 w(IPROBE\(SOURCE,)f(TAG,)h(COMM,)g(FLAG,)g (STATUS,)g(IERROR\))170 1211 y(LOGICAL)g(FLAG)170 1268 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 1268 V 16 w(STATUS)p 1218 1268 V 16 w(SIZE\),)g(IERROR)75 1354 y(MPI)p 150 1354 V 17 w(PROBE\(SOURCE,)f(TAG,)h(COMM,)g(STATUS,)g (IERROR\))170 1411 y(INTEGER)g(SOURCE,)g(TAG,)h(COMM,)f(STATUS\(MPI)p 1058 1411 V 16 w(STATUS)p 1218 1411 V 16 w(SIZE\),)g(IERROR)75 1497 y(MPI)p 150 1497 V 17 w(CANCEL\(REQUEST,)f(IERROR\))170 1554 y(INTEGER)h(REQUEST,)g(IERROR)75 1640 y(MPI)p 150 1640 V 17 w(TEST)p 263 1640 V 16 w(CANCELLED\(STATUS,)f(FLAG,)h (IERROR\))170 1696 y(LOGICAL)g(FLAG)170 1753 y(INTEGER)g(STATUS\(MPI)p 604 1753 V 16 w(STATUS)p 764 1753 V 17 w(SIZE\),)g(IERROR)75 1839 y(MPI)p 150 1839 V 17 w(SEND)p 263 1839 V 16 w(INIT\(BUF,)g (COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 1896 y()g(BUF\(*\))170 1952 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 2039 y(MPI)p 150 2039 V 17 w(BSEND)p 287 2039 V 16 w(INIT\(BUF,)g (COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2095 y()g(BUF\(*\))170 2152 y(INTEGER)g(REQUEST,)g(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 2238 y(MPI)p 150 2238 V 17 w(SSEND)p 287 2238 V 16 w(INIT\(BUF,)g (COUNT,)g(DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2295 y()g(BUF\(*\))170 2351 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 2438 y(MPI)p 150 2438 V 17 w(RSEND)p 287 2438 V 16 w(INIT\(BUF,)h(COUNT,)g (DATATYPE,)g(DEST,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2494 y()g(BUF\(*\))170 2550 y(INTEGER)g(COUNT,)g(DATATYPE,)g (DEST,)g(TAG,)h(COMM,)f(REQUEST,)f(IERROR)75 2637 y(MPI)p 150 2637 V 17 w(RECV)p 263 2637 V 16 w(INIT\(BUF,)h(COUNT,)g(DATATYPE,) g(SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR\))170 2693 y()g(BUF\(*\))-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 224 232 224 231 bop 75 -100 a Fv(224)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y Ft(INTEGER)23 b(COUNT,)g(DATATYPE,) g(SOURCE,)g(TAG,)g(COMM,)g(REQUEST,)g(IERROR)75 132 y(MPI)p 150 132 15 2 v 17 w(START\(REQUEST,)f(IERROR\))170 188 y(INTEGER)h(REQUEST,)g(IERROR)75 274 y(MPI)p 150 274 V 17 w(STARTALL\(COUNT,)f(ARRAY)p 669 274 V 16 w(OF)p 733 274 V 17 w(REQUESTS,)h(IERROR\))170 331 y(INTEGER)g(COUNT,)g(ARRAY) p 651 331 V 17 w(OF)p 716 331 V 17 w(REQUESTS\(*\),)f(IERROR)75 417 y(MPI)p 150 417 V 17 w(SENDRECV\(SENDBUF,)f(SENDCOUNT,)i(SENDTYPE,) f(DEST,)i(SENDTAG,)e(RECVBUF,)393 474 y(RECVCOUNT,)h(RECVTYPE,)f (SOURCE,)h(RECVTAG,)g(COMM,)g(STATUS,)g(IERROR\))170 530 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 587 y(INTEGER)g (SENDCOUNT,)g(SENDTYPE,)f(DEST,)i(SENDTAG,)e(RECVCOUNT,)h(RECVTYPE,)170 643 y(SOURCE,)g(RECVTAG,)g(COMM,)g(STATUS\(MPI)p 962 643 V 16 w(STATUS)p 1122 643 V 16 w(SIZE\),)h(IERROR)75 730 y(MPI)p 150 730 V 17 w(SENDRECV)p 359 730 V 16 w(REPLACE\(BUF,)e (COUNT,)h(DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g(RECVTAG,)393 786 y(COMM,)g(STATUS,)g(IERROR\))170 843 y()g(BUF\(*\))170 899 y(INTEGER)g(COUNT,)g(DATATYPE,)g(DEST,)g(SENDTAG,)g(SOURCE,)g (RECVTAG,)g(COMM,)170 955 y(STATUS\(MPI)p 413 955 V 16 w(STATUS)p 573 955 V 17 w(SIZE\),)g(IERROR)75 1042 y(MPI)p 150 1042 V 17 w(TYPE)p 263 1042 V 16 w(CONTIGUOUS\(COUNT,)f(OLDTYPE,)h (NEWTYPE,)f(IERROR\))170 1098 y(INTEGER)h(COUNT,)g(OLDTYPE,)g(NEWTYPE,) g(IERROR)75 1185 y(MPI)p 150 1185 V 17 w(TYPE)p 263 1185 V 16 w(VECTOR\(COUNT,)f(BLOCKLENGTH,)h(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h (IERROR\))170 1241 y(INTEGER)g(COUNT,)g(BLOCKLENGTH,)g(STRIDE,)g (OLDTYPE,)f(NEWTYPE,)h(IERROR)75 1328 y(MPI)p 150 1328 V 17 w(TYPE)p 263 1328 V 16 w(HVECTOR\(COUNT,)f(BLOCKLENGTH,)h(STRIDE,) f(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1384 y(INTEGER)g(COUNT,)g (BLOCKLENGTH,)g(STRIDE,)g(OLDTYPE,)f(NEWTYPE,)h(IERROR)75 1471 y(MPI)p 150 1471 V 17 w(TYPE)p 263 1471 V 16 w(INDEXED\(COUNT,)f (ARRAY)p 757 1471 V 17 w(OF)p 822 1471 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1293 1471 V 16 w(OF)p 1357 1471 V 17 w(DISPLACEMENTS,)393 1527 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1584 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1584 V 17 w(OF)p 716 1584 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1584 V 17 w(OF)p 1323 1584 V 17 w(DISPLACEMENTS\(*\),)170 1640 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 1726 y(MPI)p 150 1726 V 17 w(TYPE)p 263 1726 V 16 w(HINDEXED\(COUNT,)f(ARRAY)p 781 1726 V 17 w(OF)p 846 1726 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1317 1726 V 16 w(OF)p 1381 1726 V 17 w(DISPLACEMENTS,)393 1783 y(OLDTYPE,)h(NEWTYPE,)g(IERROR\))170 1839 y(INTEGER)g(COUNT,)g (ARRAY)p 651 1839 V 17 w(OF)p 716 1839 V 17 w(BLOCKLENGTHS\(*\),)e (ARRAY)p 1258 1839 V 17 w(OF)p 1323 1839 V 17 w(DISPLACEMENTS\(*\),)170 1896 y(OLDTYPE,)i(NEWTYPE,)g(IERROR)75 1982 y(MPI)p 150 1982 V 17 w(TYPE)p 263 1982 V 16 w(STRUCT\(COUNT,)f(ARRAY)p 733 1982 V 17 w(OF)p 798 1982 V 17 w(BLOCKLENGTHS,)g(ARRAY)p 1269 1982 V 16 w(OF)p 1333 1982 V 17 w(DISPLACEMENTS,)393 2039 y(ARRAY)p 516 2039 V 17 w(OF)p 581 2039 V 17 w(TYPES,)h(NEWTYPE,)f (IERROR\))170 2095 y(INTEGER)h(COUNT,)g(ARRAY)p 651 2095 V 17 w(OF)p 716 2095 V 17 w(BLOCKLENGTHS\(*\),)e(ARRAY)p 1258 2095 V 17 w(OF)p 1323 2095 V 17 w(DISPLACEMENTS\(*\),)170 2152 y(ARRAY)p 293 2152 V 17 w(OF)p 358 2152 V 17 w(TYPES\(*\),)h (NEWTYPE,)h(IERROR)75 2238 y(MPI)p 150 2238 V 17 w(ADDRESS\(LOCATION,)e (ADDRESS,)i(IERROR\))170 2295 y()g(LOCATION\(*\))170 2351 y(INTEGER)g(ADDRESS,)g(IERROR)75 2438 y(MPI)p 150 2438 V 17 w(TYPE)p 263 2438 V 16 w(EXTENT\(DATATYPE,)f(EXTENT,)h (IERROR\))170 2494 y(INTEGER)g(DATATYPE,)g(EXTENT,)g(IERROR)75 2580 y(MPI)p 150 2580 V 17 w(TYPE)p 263 2580 V 16 w(SIZE\(DATATYPE,)f (SIZE,)i(IERROR\))170 2637 y(INTEGER)f(DATATYPE,)g(SIZE,)g(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 225 233 225 232 bop 75 -100 a Fm(A.10.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o(OR)g(COLLECTIVE)g(COMMUNICA)l(TION)257 b Fv(225)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(TYPE)p 263 45 V 16 w(LB\()24 b(DATATYPE,)e(DISPLACEMENT,)h(IERROR\))170 102 y(INTEGER)g(DATATYPE,)g(DISPLACEMENT,)f(IERROR)75 188 y(MPI)p 150 188 V 17 w(TYPE)p 263 188 V 16 w(UB\()i(DATATYPE,)e (DISPLACEMENT,)h(IERROR\))170 244 y(INTEGER)g(DATATYPE,)g (DISPLACEMENT,)f(IERROR)75 331 y(MPI)p 150 331 V 17 w(TYPE)p 263 331 V 16 w(COMMIT\(DATATYPE,)g(IERROR\))170 387 y(INTEGER)h (DATATYPE,)g(IERROR)75 474 y(MPI)p 150 474 V 17 w(TYPE)p 263 474 V 16 w(FREE\(DATATYPE,)f(IERROR\))170 530 y(INTEGER)h (DATATYPE,)g(IERROR)75 617 y(MPI)p 150 617 V 17 w(GET)p 239 617 V 17 w(ELEMENTS\(STATUS,)e(DATATYPE,)i(COUNT,)g(IERROR\))170 673 y(INTEGER)g(STATUS\(MPI)p 604 673 V 16 w(STATUS)p 764 673 V 17 w(SIZE\),)g(DATATYPE,)f(COUNT,)h(IERROR)75 760 y(MPI)p 150 760 V 17 w(PACK\(INBUF,)f(INCOUNT,)h(DATATYPE,)g (OUTBUF,)f(OUTSIZE,)h(POSITION,)g(COMM,)g(IERROR\))170 816 y()g(INBUF\(*\),)g(OUTBUF\(*\))170 873 y(INTEGER)g(INCOUNT,)g (DATATYPE,)g(OUTSIZE,)g(POSITION,)f(COMM,)h(IERROR)75 959 y(MPI)p 150 959 V 17 w(UNPACK\(INBUF,)f(INSIZE,)h(POSITION,)f (OUTBUF,)h(OUTCOUNT,)g(DATATYPE,)g(COMM,)393 1015 y(IERROR\))170 1072 y()g(INBUF\(*\),)g(OUTBUF\(*\))170 1128 y(INTEGER)g(INSIZE,) g(POSITION,)g(OUTCOUNT,)g(DATATYPE,)f(COMM,)h(IERROR)75 1215 y(MPI)p 150 1215 V 17 w(PACK)p 263 1215 V 16 w(SIZE\(INCOUNT,)f (DATATYPE,)h(COMM,)g(SIZE,)g(IERROR\))170 1271 y(INTEGER)g(INCOUNT,)g (DATATYPE,)g(COMM,)g(SIZE,)g(IERROR)75 1445 y Fn(A.10)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Collective)e(Communication)75 1546 y Ft(MPI)p 150 1546 V 17 w(BARRIER\(COMM,)j(IERROR\))170 1602 y(INTEGER)h(COMM,)h(IERROR)75 1689 y(MPI)p 150 1689 V 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR\)) 170 1745 y()g(BUFFER\(*\))170 1802 y(INTEGER)g(COUNT,)g (DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)75 1888 y(MPI)p 150 1888 V 17 w(GATHER\(SENDBUF,)f(SENDCOUNT,)g(SENDTYPE,)h(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 1945 y(ROOT,)g(COMM,)h(IERROR\))170 2001 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 2058 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g (COMM,)g(IERROR)75 2144 y(MPI)p 150 2144 V 17 w(GATHERV\(SENDBUF,)e (SENDCOUNT,)i(SENDTYPE,)g(RECVBUF,)f(RECVCOUNTS,)h(DISPLS,)393 2201 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\))170 2257 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2313 y(INTEGER)g (SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g(DISPLS\(*\),)h(RECVTYPE,)g (ROOT,)170 2370 y(COMM,)h(IERROR)75 2456 y(MPI)p 150 2456 V 17 w(SCATTER\(SENDBUF,)d(SENDCOUNT,)i(SENDTYPE,)g(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 2513 y(ROOT,)g(COMM,)h(IERROR\))170 2569 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 2626 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g (COMM,)g(IERROR)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 226 234 226 233 bop 75 -100 a Fv(226)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(SCATTERV\(SENDBUF,)21 b(SENDCOUNTS,)i(DISPLS,)g(SENDTYPE,)f (RECVBUF,)h(RECVCOUNT,)393 102 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\)) 170 158 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 214 y(INTEGER)g(SENDCOUNTS\(*\),)f(DISPLS\(*\),)h(SENDTYPE,)f(RECVCOUNT,)h (RECVTYPE,)g(ROOT,)170 271 y(COMM,)h(IERROR)75 357 y(MPI)p 150 357 V 17 w(ALLGATHER\(SENDBUF,)d(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,) h(RECVCOUNT,)g(RECVTYPE,)393 414 y(COMM,)g(IERROR\))170 470 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 527 y(INTEGER)g (SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g(IERROR)75 613 y(MPI)p 150 613 V 17 w(ALLGATHERV\(SENDBUF,)e(SENDCOUNT,)i (SENDTYPE,)f(RECVBUF,)h(RECVCOUNTS,)f(DISPLS,)393 670 y(RECVTYPE,)h(COMM,)g(IERROR\))170 726 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 783 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f (RECVCOUNTS\(*\),)g(DISPLS\(*\),)h(RECVTYPE,)g(COMM,)170 839 y(IERROR)75 925 y(MPI)p 150 925 V 17 w(ALLTOALL\(SENDBUF,)e (SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h(RECVCOUNT,)g(RECVTYPE,)393 982 y(COMM,)g(IERROR\))170 1038 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1095 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,) h(RECVTYPE,)g(COMM,)g(IERROR)75 1181 y(MPI)p 150 1181 V 17 w(ALLTOALLV\(SENDBUF,)e(SENDCOUNTS,)i(SDISPLS,)f(SENDTYPE,)h (RECVBUF,)g(RECVCOUNTS,)393 1238 y(RDISPLS,)g(RECVTYPE,)g(COMM,)g (IERROR\))170 1294 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1351 y(INTEGER)g(SENDCOUNTS\(*\),)f(SDISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNTS\(*\),)g(RDISPLS\(*\),)170 1407 y(RECVTYPE,)h(COMM,)g (IERROR)75 1494 y(MPI)p 150 1494 V 17 w(REDUCE\(SENDBUF,)f(RECVBUF,)g (COUNT,)h(DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR\))170 1550 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1606 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR)75 1693 y(MPI)p 150 1693 V 17 w(OP)p 215 1693 V 17 w(CREATE\()g(FUNCTION,) f(COMMUTE,)h(OP,)g(IERROR\))170 1749 y(EXTERNAL)g(FUNCTION)170 1806 y(LOGICAL)g(COMMUTE)170 1862 y(INTEGER)g(OP,)h(IERROR)75 1949 y(MPI)p 150 1949 V 17 w(OP)p 215 1949 V 17 w(FREE\()f(OP,)g (IERROR\))170 2005 y(INTEGER)g(OP,)h(IERROR)75 2092 y(MPI)p 150 2092 V 17 w(ALLREDUCE\(SENDBUF,)d(RECVBUF,)i(COUNT,)g(DATATYPE,)g (OP,)g(COMM,)g(IERROR\))170 2148 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 2205 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)75 2291 y(MPI)p 150 2291 V 17 w(REDUCE)p 311 2291 V 16 w(SCATTER\(SENDBUF,)e(RECVBUF,)h(RECVCOUNTS,)f(DATATYPE,)h (OP,)g(COMM,)393 2347 y(IERROR\))170 2404 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 2460 y(INTEGER)g(RECVCOUNTS\(*\),)f(DATATYPE,)h(OP,)g (COMM,)h(IERROR)75 2547 y(MPI)p 150 2547 V 17 w(SCAN\(SENDBUF,)e (RECVBUF,)h(COUNT,)g(DATATYPE,)f(OP,)i(COMM,)f(IERROR\))170 2603 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2660 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h(IERROR)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 227 235 227 234 bop 75 -100 a Fm(A.11.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o(OR)g(GR)o(OUPS,)f(CONTEXTS,)g(ETC.)375 b Fv(227)75 45 y Fn(A.11)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i (Groups,)f(Contexts,)d(etc.)75 147 y Ft(MPI)p 150 147 15 2 v 17 w(GROUP)p 287 147 V 16 w(SIZE\(GROUP,)23 b(SIZE,)g(IERROR\)) 170 203 y(INTEGER)g(GROUP,)g(SIZE,)h(IERROR)75 289 y(MPI)p 150 289 V 17 w(GROUP)p 287 289 V 16 w(RANK\(GROUP,)f(RANK,)g(IERROR\)) 170 346 y(INTEGER)g(GROUP,)g(RANK,)h(IERROR)75 432 y(MPI)p 150 432 V 17 w(GROUP)p 287 432 V 16 w(TRANSLATE)p 519 432 V 16 w(RANKS\(GROUP1,)e(N,)i(RANKS1,)f(GROUP2,)g(RANKS2,)g (IERROR\))170 489 y(INTEGER)g(GROUP1,)g(N,)h(RANKS1\(*\),)e(GROUP2,)h (RANKS2\(*\),)g(IERROR)75 575 y(MPI)p 150 575 V 17 w(GROUP)p 287 575 V 16 w(COMPARE\(GROUP1,)f(GROUP2,)h(RESULT,)g(IERROR\))170 632 y(INTEGER)g(GROUP1,)g(GROUP2,)g(RESULT,)g(IERROR)75 718 y(MPI)p 150 718 V 17 w(COMM)p 263 718 V 16 w(GROUP\(COMM,)g(GROUP,) g(IERROR\))170 775 y(INTEGER)g(COMM,)h(GROUP,)f(IERROR)75 861 y(MPI)p 150 861 V 17 w(GROUP)p 287 861 V 16 w(UNION\(GROUP1,)f (GROUP2,)h(NEWGROUP,)g(IERROR\))170 918 y(INTEGER)g(GROUP1,)g(GROUP2,)g (NEWGROUP,)g(IERROR)75 1004 y(MPI)p 150 1004 V 17 w(GROUP)p 287 1004 V 16 w(INTERSECTION\(GROUP1,)e(GROUP2,)i(NEWGROUP,)g(IERROR\)) 170 1060 y(INTEGER)g(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 1147 y(MPI)p 150 1147 V 17 w(GROUP)p 287 1147 V 16 w (DIFFERENCE\(GROUP1,)f(GROUP2,)h(NEWGROUP,)f(IERROR\))170 1203 y(INTEGER)h(GROUP1,)g(GROUP2,)g(NEWGROUP,)g(IERROR)75 1290 y(MPI)p 150 1290 V 17 w(GROUP)p 287 1290 V 16 w(INCL\(GROUP,)g(N,) g(RANKS,)g(NEWGROUP,)g(IERROR\))170 1346 y(INTEGER)g(GROUP,)g(N,)h (RANKS\(*\),)f(NEWGROUP,)f(IERROR)75 1433 y(MPI)p 150 1433 V 17 w(GROUP)p 287 1433 V 16 w(EXCL\(GROUP,)h(N,)g(RANKS,)g (NEWGROUP,)g(IERROR\))170 1489 y(INTEGER)g(GROUP,)g(N,)h(RANKS\(*\),)f (NEWGROUP,)f(IERROR)75 1576 y(MPI)p 150 1576 V 17 w(GROUP)p 287 1576 V 16 w(RANGE)p 423 1576 V 17 w(INCL\(GROUP,)g(N,)i(RANGES,)f (NEWGROUP,)f(IERROR\))170 1632 y(INTEGER)h(GROUP,)g(N,)h (RANGES\(3,*\),)e(NEWGROUP,)h(IERROR)75 1719 y(MPI)p 150 1719 V 17 w(GROUP)p 287 1719 V 16 w(RANGE)p 423 1719 V 17 w(EXCL\(GROUP,)f(N,)i(RANGES,)f(NEWGROUP,)f(IERROR\))170 1775 y(INTEGER)h(GROUP,)g(N,)h(RANGES\(3,*\),)e(NEWGROUP,)h(IERROR)75 1862 y(MPI)p 150 1862 V 17 w(GROUP)p 287 1862 V 16 w(FREE\(GROUP,)g (IERROR\))170 1918 y(INTEGER)g(GROUP,)g(IERROR)75 2004 y(MPI)p 150 2004 V 17 w(COMM)p 263 2004 V 16 w(SIZE\(COMM,)g(SIZE,)g (IERROR\))170 2061 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 2147 y(MPI)p 150 2147 V 17 w(COMM)p 263 2147 V 16 w(RANK\(COMM,)g (RANK,)g(IERROR\))170 2204 y(INTEGER)g(COMM,)h(RANK,)f(IERROR)75 2290 y(MPI)p 150 2290 V 17 w(COMM)p 263 2290 V 16 w(COMPARE\(COMM1,)f (COMM2,)h(RESULT,)g(IERROR\))170 2347 y(INTEGER)g(COMM1,)g(COMM2,)g (RESULT,)g(IERROR)75 2433 y(MPI)p 150 2433 V 17 w(COMM)p 263 2433 V 16 w(DUP\(COMM,)g(NEWCOMM,)g(IERROR\))170 2490 y(INTEGER)g(COMM,)h(NEWCOMM,)e(IERROR)75 2576 y(MPI)p 150 2576 V 17 w(COMM)p 263 2576 V 16 w(CREATE\(COMM,)h(GROUP,)g (NEWCOMM,)f(IERROR\))170 2633 y(INTEGER)h(COMM,)h(GROUP,)f(NEWCOMM,)f (IERROR)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 228 236 228 235 bop 75 -100 a Fv(228)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)75 45 y Ft(MPI)p 150 45 15 2 v 17 w(COMM)p 263 45 V 16 w(SPLIT\(COMM,)23 b(COLOR,)g(KEY,)g (NEWCOMM,)g(IERROR\))170 102 y(INTEGER)g(COMM,)h(COLOR,)f(KEY,)g (NEWCOMM,)g(IERROR)75 188 y(MPI)p 150 188 V 17 w(COMM)p 263 188 V 16 w(FREE\(COMM,)g(IERROR\))170 244 y(INTEGER)g(COMM,)h (IERROR)75 331 y(MPI)p 150 331 V 17 w(COMM)p 263 331 V 16 w(TEST)p 375 331 V 17 w(INTER\(COMM,)e(FLAG,)i(IERROR\))170 387 y(INTEGER)f(COMM,)h(IERROR)170 444 y(LOGICAL)f(FLAG)75 530 y(MPI)p 150 530 V 17 w(COMM)p 263 530 V 16 w(REMOTE)p 423 530 V 17 w(SIZE\(COMM,)f(SIZE,)h(IERROR\))170 587 y(INTEGER)g(COMM,)h(SIZE,)f(IERROR)75 673 y(MPI)p 150 673 V 17 w(COMM)p 263 673 V 16 w(REMOTE)p 423 673 V 17 w(GROUP\(COMM,)f(GROUP,)h(IERROR\))170 730 y(INTEGER)g(COMM,)h(GROUP,)f (IERROR)75 816 y(MPI)p 150 816 V 17 w(INTERCOMM)p 383 816 V 16 w(CREATE\(LOCAL)p 687 816 V 15 w(COMM,)g(LOCAL)p 965 816 V 17 w(LEADER,)g(PEER)p 1269 816 V 16 w(COMM,)h(REMOTE)p 1573 816 V 16 w(LEADER,)f(TAG,)393 873 y(NEWINTERCOMM,)f(IERROR\))170 929 y(INTEGER)h(LOCAL)p 484 929 V 17 w(COMM,)g(LOCAL)p 764 929 V 17 w(LEADER,)f(PEER)p 1067 929 V 17 w(COMM,)h(REMOTE)p 1371 929 V 17 w(LEADER,)g(TAG,)170 985 y(NEWINTERCOMM,)g(IERROR)75 1072 y(MPI)p 150 1072 V 17 w(INTERCOMM)p 383 1072 V 16 w(MERGE\(INTERCOMM,)e(HIGH,)j(INTRACOMM,)e(IERROR\))170 1128 y(INTEGER)h(INTERCOMM,)g(INTRACOMM,)f(IERROR)170 1185 y(LOGICAL)h(HIGH)75 1271 y(MPI)p 150 1271 V 17 w(KEYVAL)p 311 1271 V 16 w(CREATE\(COPY)p 591 1271 V 16 w(FN,)g(DELETE)p 846 1271 V 17 w(FN,)g(KEYVAL,)g(EXTRA)p 1269 1271 V 16 w(STATE,)h(IERROR\))170 1328 y(EXTERNAL)f(COPY)p 484 1328 V 17 w(FN,)g(DELETE)p 740 1328 V 17 w(FN)170 1384 y(INTEGER)g(KEYVAL,)g(EXTRA)p 675 1384 V 17 w(STATE,)g(IERROR)75 1471 y(MPI)p 150 1471 V 17 w(KEYVAL)p 311 1471 V 16 w(FREE\(KEYVAL,)f (IERROR\))170 1527 y(INTEGER)h(KEYVAL,)g(IERROR)75 1614 y(MPI)p 150 1614 V 17 w(ATTR)p 263 1614 V 16 w(PUT\(COMM,)g(KEYVAL,)g (ATTRIBUTE)p 925 1614 V 16 w(VAL,)g(IERROR\))170 1670 y(INTEGER)g(COMM,)h(KEYVAL,)e(ATTRIBUTE)p 914 1670 V 16 w(VAL,)i(IERROR)75 1756 y(MPI)p 150 1756 V 17 w(ATTR)p 263 1756 V 16 w(GET\(COMM,)f(KEYVAL,)g(ATTRIBUTE)p 925 1756 V 16 w(VAL,)g(FLAG,)g(IERROR\))170 1813 y(INTEGER)g(COMM,)h (KEYVAL,)e(ATTRIBUTE)p 914 1813 V 16 w(VAL,)i(IERROR)170 1869 y(LOGICAL)f(FLAG)75 1956 y(MPI)p 150 1956 V 17 w(ATTR)p 263 1956 V 16 w(DELETE\(COMM,)g(KEYVAL,)g(IERROR\))170 2012 y(INTEGER)g(COMM,)h(KEYVAL,)e(IERROR)75 2186 y Fn(A.12)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Pro)r(cess)e(T)-5 b(op)r(ologies)75 2287 y Ft(MPI)p 150 2287 V 17 w(CART)p 263 2287 V 16 w(CREATE\(COMM)p 543 2287 V 16 w(OLD,)23 b(NDIMS,)g(DIMS,)h(PERIODS,)e(REORDER,)h(COMM)p 1514 2287 V 17 w(CART,)g(IERROR\))170 2343 y(INTEGER)g(COMM)p 460 2343 V 17 w(OLD,)g(NDIMS,)g(DIMS\(*\),)g(COMM)p 1074 2343 V 17 w(CART,)g(IERROR)170 2400 y(LOGICAL)g(PERIODS\(*\),)g (REORDER)75 2486 y(MPI)p 150 2486 V 17 w(DIMS)p 263 2486 V 16 w(CREATE\(NNODES,)f(NDIMS,)h(DIMS,)h(IERROR\))170 2543 y(INTEGER)f(NNODES,)g(NDIMS,)g(DIMS\(*\),)g(IERROR)75 2629 y(MPI)p 150 2629 V 17 w(GRAPH)p 287 2629 V 16 w(CREATE\(COMM)p 567 2629 V 16 w(OLD,)g(NNODES,)g(INDEX,)g(EDGES,)g(REORDER,)g(COMM)p 1538 2629 V 17 w(GRAPH,)393 2686 y(IERROR\))1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 229 237 229 236 bop 75 -100 a Fm(A.13.)34 b(F)o(OR)l(TRAN)16 b(BINDINGS)g(F)o(OR)g(ENVIR)o(ONMENT)l(AL)g(INQUIR)l(Y)364 b Fv(229)170 45 y Ft(INTEGER)23 b(COMM)p 460 45 15 2 v 17 w(OLD,)g(NNODES,)g(INDEX\(*\),)g(EDGES\(*\),)f(COMM)p 1360 45 V 17 w(GRAPH,)h(IERROR)170 102 y(LOGICAL)g(REORDER)75 188 y(MPI)p 150 188 V 17 w(TOPO)p 263 188 V 16 w(TEST\(COMM,)g(STATUS,) g(IERROR\))170 244 y(INTEGER)g(COMM,)h(STATUS,)e(IERROR)75 331 y(MPI)p 150 331 V 17 w(GRAPHDIMS)p 383 331 V 16 w(GET\(COMM,)g (NNODES,)h(NEDGES,)g(IERROR\))170 387 y(INTEGER)g(COMM,)h(NNODES,)e (NEDGES,)h(IERROR)75 474 y(MPI)p 150 474 V 17 w(GRAPH)p 287 474 V 16 w(GET\(COMM,)g(MAXINDEX,)g(MAXEDGES,)f(INDEX,)h(EDGES,)g (IERROR\))170 530 y(INTEGER)g(COMM,)h(MAXINDEX,)e(MAXEDGES,)h (INDEX\(*\),)f(EDGES\(*\),)h(IERROR)75 617 y(MPI)p 150 617 V 17 w(CARTDIM)p 335 617 V 16 w(GET\(COMM,)g(NDIMS,)g(IERROR\))170 673 y(INTEGER)g(COMM,)h(NDIMS,)f(IERROR)75 760 y(MPI)p 150 760 V 17 w(CART)p 263 760 V 16 w(GET\(COMM,)g(MAXDIMS,)g(DIMS,)g (PERIODS,)g(COORDS,)g(IERROR\))170 816 y(INTEGER)g(COMM,)h(MAXDIMS,)e (DIMS\(*\),)h(COORDS\(*\),)g(IERROR)170 873 y(LOGICAL)g(PERIODS\(*\))75 959 y(MPI)p 150 959 V 17 w(CART)p 263 959 V 16 w(RANK\(COMM,)g(COORDS,) g(RANK,)g(IERROR\))170 1015 y(INTEGER)g(COMM,)h(COORDS\(*\),)e(RANK,)h (IERROR)75 1102 y(MPI)p 150 1102 V 17 w(CART)p 263 1102 V 16 w(COORDS\(COMM,)g(RANK,)g(MAXDIMS,)g(COORDS,)f(IERROR\))170 1158 y(INTEGER)h(COMM,)h(RANK,)f(MAXDIMS,)f(COORDS\(*\),)h(IERROR)75 1245 y(MPI)p 150 1245 V 17 w(GRAPH)p 287 1245 V 16 w(NEIGHBORS)p 519 1245 V 16 w(COUNT\(COMM,)g(RANK,)g(NNEIGHBORS,)f(IERROR\))170 1301 y(INTEGER)h(COMM,)h(RANK,)f(NNEIGHBORS,)f(IERROR)75 1388 y(MPI)p 150 1388 V 17 w(GRAPH)p 287 1388 V 16 w(NEIGHBORS\(COMM,)g (RANK,)h(MAXNEIGHBORS,)f(NEIGHBORS,)h(IERROR\))170 1444 y(INTEGER)g(COMM,)h(RANK,)f(MAXNEIGHBORS,)f(NEIGHBORS\(*\),)g(IERROR)75 1531 y(MPI)p 150 1531 V 17 w(CART)p 263 1531 V 16 w(SHIFT\(COMM,)h (DIRECTION,)f(DISP,)h(RANK)p 1067 1531 V 17 w(SOURCE,)g(RANK)p 1371 1531 V 17 w(DEST,)g(IERROR\))170 1587 y(INTEGER)g(COMM,)h (DIRECTION,)e(DISP,)h(RANK)p 1009 1587 V 17 w(SOURCE,)g(RANK)p 1313 1587 V 16 w(DEST,)h(IERROR)75 1674 y(MPI)p 150 1674 V 17 w(CART)p 263 1674 V 16 w(SUB\(COMM,)f(REMAIN)p 662 1674 V 16 w(DIMS,)h(NEWCOMM,)e(IERROR\))170 1730 y(INTEGER)h(COMM,)h (NEWCOMM,)e(IERROR)170 1786 y(LOGICAL)h(REMAIN)p 508 1786 V 17 w(DIMS\(*\))75 1873 y(MPI)p 150 1873 V 17 w(CART)p 263 1873 V 16 w(MAP\(COMM,)g(NDIMS,)g(DIMS,)g(PERIODS,)g(NEWRANK,)g (IERROR\))170 1929 y(INTEGER)g(COMM,)h(NDIMS,)f(DIMS\(*\),)f(NEWRANK,)h (IERROR)170 1986 y(LOGICAL)g(PERIODS\(*\))75 2072 y(MPI)p 150 2072 V 17 w(GRAPH)p 287 2072 V 16 w(MAP\(COMM,)g(NNODES,)g(INDEX,)g (EDGES,)g(NEWRANK,)g(IERROR\))170 2129 y(INTEGER)g(COMM,)h(NNODES,)e (INDEX\(*\),)h(EDGES\(*\),)g(NEWRANK,)f(IERROR)75 2302 y Fn(A.13)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Environmental)d (Inquiry)75 2403 y Ft(MPI)p 150 2403 V 17 w(GET)p 239 2403 V 17 w(PROCESSOR)p 472 2403 V 15 w(NAME\(NAME,)23 b(RESULTLEN,)f(IERROR\))170 2460 y(CHARACTER*\(*\))h(NAME)170 2516 y(INTEGER)g(RESULTLEN,)g(IERROR)75 2603 y(MPI)p 150 2603 V 17 w(ERRHANDLER)p 407 2603 V 16 w(CREATE\(FUNCTION,)e (HANDLER,)i(IERROR\))170 2659 y(EXTERNAL)g(FUNCTION)-32 46 y Fs(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 230 238 230 237 bop 75 -100 a Fv(230)987 b Fm(ANNEX)15 b(A.)30 b(LANGUA)o(GE)15 b(BINDING)170 45 y Ft(INTEGER)23 b(ERRHANDLER,)g (IERROR)75 132 y(MPI)p 150 132 15 2 v 17 w(ERRHANDLER)p 407 132 V 16 w(SET\(COMM,)f(ERRHANDLER,)h(IERROR\))170 188 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)75 274 y(MPI)p 150 274 V 17 w(ERRHANDLER)p 407 274 V 16 w(GET\(COMM,)g(ERRHANDLER,)h (IERROR\))170 331 y(INTEGER)g(COMM,)h(ERRHANDLER,)e(IERROR)75 417 y(MPI)p 150 417 V 17 w(ERRHANDLER)p 407 417 V 16 w(FREE\(ERRHANDLER,)f(IERROR\))170 474 y(INTEGER)i(ERRHANDLER,)g (IERROR)75 560 y(MPI)p 150 560 V 17 w(ERROR)p 287 560 V 16 w(STRING\(ERRORCODE,)f(STRING,)h(RESULTLEN,)f(IERROR\))170 617 y(INTEGER)h(ERRORCODE,)g(RESULTLEN,)f(IERROR)170 673 y(CHARACTER*\(*\))h(STRING)75 760 y(MPI)p 150 760 V 17 w(ERROR)p 287 760 V 16 w(CLASS\(ERRORCODE,)f(ERRORCLASS,)g (IERROR\))170 816 y(INTEGER)h(ERRORCODE,)g(ERRORCLASS,)f(IERROR)75 903 y(DOUBLE)h(PRECISION)g(MPI)p 556 903 V 16 w(WTIME\(\))75 989 y(DOUBLE)g(PRECISION)g(MPI)p 556 989 V 16 w(WTICK\(\))75 1075 y(MPI)p 150 1075 V 17 w(INIT\(IERROR\))170 1132 y(INTEGER)g(IERROR)75 1218 y(MPI)p 150 1218 V 17 w(FINALIZE\(IERROR\)) 170 1275 y(INTEGER)g(IERROR)75 1361 y(MPI)p 150 1361 V 17 w(INITIALIZED\(FLAG,)e(IERROR\))170 1418 y(LOGICAL)i(FLAG)170 1474 y(INTEGER)g(IERROR)75 1561 y(MPI)p 150 1561 V 17 w(ABORT\(COMM,)f(ERRORCODE,)h(IERROR\))170 1617 y(INTEGER)g(COMM,)h (ERRORCODE,)e(IERROR)75 1790 y Fn(A.14)59 b(F)n(o)n(rtran)21 b(Bindings)e(fo)n(r)i(Pro\014ling)75 1892 y Ft(MPI)p 150 1892 V 17 w(PCONTROL\(level\))170 1948 y(INTEGER)i(LEVEL,)g Fp(:)8 b(:)g(:)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 231 239 231 238 bop 75 283 a Fr(MPI)40 b(F)-10 b(unction)40 b(Index)75 490 y Fv(MPI)p 167 490 14 2 v 16 w(ABOR)l(T,)16 b(201)75 547 y(MPI)p 167 547 V 16 w(ADDRESS,)g(69)75 603 y(MPI)p 167 603 V 16 w(ALLGA)l(THER,)g(110)75 660 y(MPI)p 167 660 V 16 w(ALLGA)l(THER)-5 b(V,)16 b(111)75 716 y(MPI)p 167 716 V 16 w(ALLREDUCE,)g(125)75 773 y(MPI)p 167 773 V 16 w(ALL)l(TO)o(ALL,)h(112)75 829 y(MPI)p 167 829 V 16 w(ALL)l(TO)o(ALL)-5 b(V,)17 b(113)75 886 y(MPI)p 167 886 V 16 w(A)l(TTR)p 312 886 V 17 w(DELETE,)e(172)75 942 y(MPI)p 167 942 V 16 w(A)l(TTR)p 312 942 V 17 w(GET,)f(172)75 999 y(MPI)p 167 999 V 16 w(A)l(TTR)p 312 999 V 17 w(PUT,)h(171)75 1055 y(MPI)p 167 1055 V 16 w(BARRIER,)i(95)75 1111 y(MPI)p 167 1111 V 16 w(BCAST,)e(95)75 1168 y(MPI)p 167 1168 V 16 w(BSEND,)g(28)75 1224 y(MPI)p 167 1224 V 16 w(BSEND)p 340 1224 V 17 w(INIT,)g(56)75 1281 y(MPI)p 167 1281 V 16 w(BUFFER)p 373 1281 V 17 w(A)l(TT)l(A)o(CH,)f(34)75 1337 y(MPI)p 167 1337 V 16 w(BUFFER)p 373 1337 V 17 w(DET)l(A)o(CH,)g (34)75 1394 y(MPI)p 167 1394 V 16 w(CANCEL,)h(54)75 1450 y(MPI)p 167 1450 V 16 w(CAR)l(T)p 312 1450 V 17 w(COORDS,)h(185)75 1507 y(MPI)p 167 1507 V 16 w(CAR)l(T)p 312 1507 V 17 w(CREA)l(TE,)f(180)75 1563 y(MPI)p 167 1563 V 16 w(CAR)l(T)p 312 1563 V 17 w(GET,)f(184)75 1620 y(MPI)p 167 1620 V 16 w(CAR)l(T)p 312 1620 V 17 w(MAP)l(,)h(189)75 1676 y(MPI)p 167 1676 V 16 w(CAR)l(T)p 312 1676 V 17 w(RANK,)h(185)75 1732 y(MPI)p 167 1732 V 16 w(CAR)l(T)p 312 1732 V 17 w(SHIFT,)f(187)75 1789 y(MPI)p 167 1789 V 16 w(CAR)l(T)p 312 1789 V 17 w(SUB,)g(188)75 1845 y(MPI)p 167 1845 V 16 w(CAR)l(TDIM)p 405 1845 V 17 w(GET,)f(184)75 1902 y(MPI)p 167 1902 V 16 w(COMM)p 335 1902 V 16 w(COMP)l(ARE,)i(145)75 1958 y(MPI)p 167 1958 V 16 w(COMM)p 335 1958 V 16 w(CREA)l(TE,)f(146)75 2015 y(MPI)p 167 2015 V 16 w(COMM)p 335 2015 V 16 w(DUP)l(,)g(146)75 2071 y(MPI)p 167 2071 V 16 w(COMM)p 335 2071 V 16 w(FREE,)g(148)75 2128 y(MPI)p 167 2128 V 16 w(COMM)p 335 2128 V 16 w(GR)o(OUP)l(,)g(140) 75 2184 y(MPI)p 167 2184 V 16 w(COMM)p 335 2184 V 16 w(RANK,)h(145)75 2241 y(MPI)p 167 2241 V 16 w(COMM)p 335 2241 V 16 w(REMOTE)p 556 2241 V 17 w(GR)o(OUP)l(,)f(159)75 2297 y(MPI)p 167 2297 V 16 w(COMM)p 335 2297 V 16 w(REMOTE)p 556 2297 V 17 w(SIZE,)g(158)75 2353 y(MPI)p 167 2353 V 16 w(COMM)p 335 2353 V 16 w(SIZE,)g(144)75 2410 y(MPI)p 167 2410 V 16 w(COMM)p 335 2410 V 16 w(SPLIT,)h(147)75 2466 y(MPI)p 167 2466 V 16 w(COMM)p 335 2466 V 16 w(TEST)p 473 2466 V 16 w(INTER,)g(158)75 2523 y(MPI)p 167 2523 V 16 w(DIMS)p 301 2523 V 17 w(CREA)l(TE,)f(180)75 2579 y(MPI)p 167 2579 V 16 w(ERRHANDLER)p 509 2579 V 18 w(CREA)l(TE,)g(195) 75 2636 y(MPI)p 167 2636 V 16 w(ERRHANDLER)p 509 2636 V 18 w(FREE,)g(197)75 2692 y(MPI)p 167 2692 V 16 w(ERRHANDLER)p 509 2692 V 18 w(GET,)g(196)1048 490 y(MPI)p 1140 490 V 16 w(ERRHANDLER)p 1482 490 V 18 w(SET,)g(196)1048 547 y(MPI)p 1140 547 V 16 w(ERR)o(OR)p 1320 547 V 18 w(CLASS,)g(198)1048 603 y(MPI)p 1140 603 V 16 w(ERR)o(OR)p 1320 603 V 18 w(STRING,)g(197)1048 660 y(MPI)p 1140 660 V 16 w(FINALIZE,)h(200)1048 716 y(MPI)p 1140 716 V 16 w(GA)l(THER,)f(96)1048 773 y(MPI)p 1140 773 V 16 w(GA)l(THER)-5 b(V,)15 b(98)1048 829 y(MPI)p 1140 829 V 16 w(GET)p 1256 829 V 16 w(COUNT,)g(22)1048 886 y(MPI)p 1140 886 V 16 w(GET)p 1256 886 V 16 w(ELEMENTS,)g(75)1048 942 y(MPI)p 1140 942 V 16 w(GET)p 1256 942 V 16 w(PR)o(OCESSOR)p 1552 942 V 18 w(NAME,)f(194)1048 999 y(MPI)p 1140 999 V 16 w(GRAPH)p 1324 999 V 17 w(CREA)l(TE,)h(181)1048 1055 y(MPI)p 1140 1055 V 16 w(GRAPH)p 1324 1055 V 17 w(GET,)f(184)1048 1111 y(MPI)p 1140 1111 V 16 w(GRAPH)p 1324 1111 V 17 w(MAP)l(,)g(190)1048 1168 y(MPI)p 1140 1168 V 16 w(GRAPH)p 1324 1168 V 17 w(NEIGHBORS,)i(186)1048 1224 y(MPI)p 1140 1224 V 16 w(GRAPH)p 1324 1224 V 17 w(NEIGHBORS)p 1617 1224 V 17 w(COUNT,)10 b(186)1048 1281 y(MPI)p 1140 1281 V 16 w(GRAPHDIMS)p 1442 1281 V 17 w(GET,)k(183)1048 1337 y(MPI)p 1140 1337 V 16 w(GR)o(OUP)p 1324 1337 V 17 w(COMP)l(ARE,)h(139)1048 1394 y(MPI)p 1140 1394 V 16 w(GR)o(OUP)p 1324 1394 V 17 w(DIFFERENCE,)f(141)1048 1450 y(MPI)p 1140 1450 V 16 w(GR)o(OUP)p 1324 1450 V 17 w(EX)o(CL,)g(142)1048 1507 y(MPI)p 1140 1507 V 16 w(GR)o(OUP)p 1324 1507 V 17 w(FREE,)g(143)1048 1563 y(MPI)p 1140 1563 V 16 w(GR)o(OUP)p 1324 1563 V 17 w(INCL,)h(141)1048 1620 y(MPI)p 1140 1620 V 16 w(GR)o(OUP)p 1324 1620 V 17 w(INTERSECTION,)h(140)1048 1676 y(MPI)p 1140 1676 V 16 w(GR)o(OUP)p 1324 1676 V 17 w(RANGE)p 1509 1676 V 16 w(EX)o(CL,)f(143)1048 1732 y(MPI)p 1140 1732 V 16 w(GR)o(OUP)p 1324 1732 V 17 w(RANGE)p 1509 1732 V 16 w(INCL,)h(142)1048 1789 y(MPI)p 1140 1789 V 16 w(GR)o(OUP)p 1324 1789 V 17 w(RANK,)f(138)1048 1845 y(MPI)p 1140 1845 V 16 w(GR)o(OUP)p 1324 1845 V 17 w(SIZE,)g(138)1048 1902 y(MPI)p 1140 1902 V 16 w(GR)o(OUP)p 1324 1902 V 17 w(TRANSLA)l(TE)p 1622 1902 V 17 w(RANKS,)c(139)1048 1958 y(MPI)p 1140 1958 V 16 w(GR)o(OUP)p 1324 1958 V 17 w(UNION,)k(140)1048 2015 y(MPI)p 1140 2015 V 16 w(IBSEND,)g(39)1048 2071 y(MPI)p 1140 2071 V 16 w(INIT,)h(200)1048 2128 y(MPI)p 1140 2128 V 16 w(INITIALIZED,)g(201)1048 2184 y(MPI)p 1140 2184 V 16 w(INTER)o(COMM)p 1454 2184 V 16 w(CREA)l(TE,)f(160)1048 2241 y(MPI)p 1140 2241 V 16 w(INTER)o(COMM)p 1454 2241 V 16 w(MER)o(GE,)g(160)1048 2297 y(MPI)p 1140 2297 V 16 w(IPR)o(OBE,)h(51)1048 2353 y(MPI)p 1140 2353 V 16 w(IRECV,)g(40)1048 2410 y(MPI)p 1140 2410 V 16 w(IRSEND,)g(40)1048 2466 y(MPI)p 1140 2466 V 16 w(ISEND,)f(38)1048 2523 y(MPI)p 1140 2523 V 16 w(ISSEND,)g(39)1048 2579 y(MPI)p 1140 2579 V 16 w(KEYV)-5 b(AL)p 1347 2579 V 17 w(CREA)l(TE,)15 b(169)1048 2636 y(MPI)p 1140 2636 V 16 w(KEYV)-5 b(AL)p 1347 2636 V 17 w(FREE,)15 b(171)1048 2692 y(MPI)p 1140 2692 V 16 w(OP)p 1222 2692 V 17 w(CREA)l(TE,)g(121)941 2828 y(231)p eop %%Page: 232 240 232 239 bop 75 -100 a Fv(232)1326 b Fm(MPI)15 b(F)l(unction)h(Index)75 45 y Fv(MPI)p 167 45 14 2 v 16 w(OP)p 249 45 V 17 w(FREE,)f(123)75 102 y(MPI)p 167 102 V 16 w(P)l(A)o(CK,)g(85)75 158 y(MPI)p 167 158 V 16 w(P)l(A)o(CK)p 311 158 V 17 w(SIZE,)g(87)75 214 y(MPI)p 167 214 V 16 w(PCONTR)o(OL,)h(203)75 271 y(MPI)p 167 271 V 16 w(PR)o(OBE,)g(52)75 327 y(MPI)p 167 327 V 16 w(RECV,)f(20)75 384 y(MPI)p 167 384 V 16 w(RECV)p 314 384 V 17 w(INIT,)h(57)75 440 y(MPI)p 167 440 V 16 w(REDUCE,)f(114)75 497 y(MPI)p 167 497 V 16 w(REDUCE)p 380 497 V 17 w(SCA)l(TTER,)g(126)75 553 y(MPI)p 167 553 V 16 w(REQUEST)p 405 553 V 18 w(FREE,)f(43)75 610 y(MPI)p 167 610 V 16 w(RSEND,)i(29)75 666 y(MPI)p 167 666 V 16 w(RSEND)p 341 666 V 17 w(INIT,)g(57)75 723 y(MPI)p 167 723 V 16 w(SCAN,)f(127)75 779 y(MPI)p 167 779 V 16 w(SCA)l(TTER,)h(105)75 835 y(MPI)p 167 835 V 16 w(SCA)l(TTER)-5 b(V,)16 b(106)75 892 y(MPI)p 167 892 V 16 w(SEND,)f(17)75 948 y(MPI)p 167 948 V 16 w(SEND)p 308 948 V 17 w(INIT,)g(55)75 1005 y(MPI)p 167 1005 V 16 w(SENDRECV,)g(59)75 1061 y(MPI)p 167 1061 V 16 w(SENDRECV)p 439 1061 V 17 w(REPLA)o(CE,)g(60)75 1118 y(MPI)p 167 1118 V 16 w(SSEND,)g(29)75 1174 y(MPI)p 167 1174 V 16 w(SSEND)p 333 1174 V 17 w(INIT,)h(56)75 1231 y(MPI)p 167 1231 V 16 w(ST)l(AR)l(T,)g(58)75 1287 y(MPI)p 167 1287 V 16 w(ST)l(AR)l(T)l(ALL,)h(58)75 1344 y(MPI)p 167 1344 V 16 w(TEST,)e(42)75 1400 y(MPI)p 167 1400 V 16 w(TEST)p 305 1400 V 17 w(CANCELLED,)g(54)75 1456 y(MPI)p 167 1456 V 16 w(TEST)l(ALL,)h(48)75 1513 y(MPI)p 167 1513 V 16 w(TEST)l(ANY,)f(46)75 1569 y(MPI)p 167 1569 V 16 w(TESTSOME,)g(49)75 1626 y(MPI)p 167 1626 V 16 w(TOPO)p 317 1626 V 17 w(TEST,)g(183)75 1682 y(MPI)p 167 1682 V 16 w(TYPE)p 312 1682 V 17 w(COMMIT,)f(72)75 1739 y(MPI)p 167 1739 V 16 w(TYPE)p 312 1739 V 17 w(CONTIGUOUS,)h(63)75 1795 y(MPI)p 167 1795 V 16 w(TYPE)p 312 1795 V 17 w(EXTENT,)f(70)75 1852 y(MPI)p 167 1852 V 16 w(TYPE)p 312 1852 V 17 w(FREE,)g(72)75 1908 y(MPI)p 167 1908 V 16 w(TYPE)p 312 1908 V 17 w(HINDEXED,)g(67)75 1965 y(MPI)p 167 1965 V 16 w(TYPE)p 312 1965 V 17 w(HVECTOR,)h(65)75 2021 y(MPI)p 167 2021 V 16 w(TYPE)p 312 2021 V 17 w(INDEXED,)f(66)75 2077 y(MPI)p 167 2077 V 16 w(TYPE)p 312 2077 V 17 w(LB,)h(71)75 2134 y(MPI)p 167 2134 V 16 w(TYPE)p 312 2134 V 17 w(SIZE,)g(70)75 2190 y(MPI)p 167 2190 V 16 w(TYPE)p 312 2190 V 17 w(STR)o(UCT,)f(68)75 2247 y(MPI)p 167 2247 V 16 w(TYPE)p 312 2247 V 17 w(UB,)h(72)75 2303 y(MPI)p 167 2303 V 16 w(TYPE)p 312 2303 V 17 w(VECTOR,)g(63)75 2360 y(MPI)p 167 2360 V 16 w(UNP)l(A)o(CK,)g(86)75 2416 y(MPI)p 167 2416 V 16 w(W)-5 b(AIT,)15 b(41)75 2473 y(MPI)p 167 2473 V 16 w(W)-5 b(AIT)l(ALL,)16 b(47)75 2529 y(MPI)p 167 2529 V 16 w(W)-5 b(AIT)l(ANY,)16 b(45)75 2586 y(MPI)p 167 2586 V 16 w(W)-5 b(AITSOME,)15 b(48)75 2642 y(MPI)p 167 2642 V 16 w(WTICK,)g(199)75 2698 y(MPI)p 167 2698 V 16 w(WTIME,)g(199)1967 46 y Fs(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF .