%!PS-Adobe-2.0 %%Creator: dvips 5.528 Copyright 1986, 1994 Radical Eye Software %%Title: temp.dvi %%CreationDate: Tue May 30 10:23:32 1995 %%Pages: 43 %%PageOrder: Ascend %%BoundingBox: 0 0 612 792 %%EndComments %DVIPSCommandLine: dvips -o temp.ps temp %DVIPSParameters: dpi=300, comments removed %DVIPSSource: TeX output 1995.05.30:1023 %%BeginProcSet: tex.pro /TeXDict 250 dict def TeXDict begin /N{def}def /B{bind def}N /S{exch}N /X{S N}B /TR{translate}N /isls false N /vsize 11 72 mul N /hsize 8.5 72 mul N /landplus90{false}def /@rigin{isls{[0 landplus90{1 -1}{-1 1} ifelse 0 0 0]concat}if 72 Resolution div 72 VResolution div neg scale isls{landplus90{VResolution 72 div vsize mul 0 exch}{Resolution -72 div hsize mul 0}ifelse TR}if Resolution VResolution vsize -72 div 1 add mul TR matrix currentmatrix dup dup 4 get round 4 exch put dup dup 5 get round 5 exch put setmatrix}N /@landscape{/isls true N}B /@manualfeed{ statusdict /manualfeed true put}B /@copies{/#copies X}B /FMat[1 0 0 -1 0 0]N /FBB[0 0 0 0]N /nn 0 N /IE 0 N /ctr 0 N /df-tail{/nn 8 dict N nn begin /FontType 3 N /FontMatrix fntrx N /FontBBox FBB N string /base X array /BitMaps X /BuildChar{CharBuilder}N /Encoding IE N end dup{/foo setfont}2 array copy cvx N load 0 nn put /ctr 0 N[}B /df{/sf 1 N /fntrx FMat N df-tail}B /dfs{div /sf X /fntrx[sf 0 0 sf neg 0 0]N df-tail}B /E{ pop nn dup definefont setfont}B /ch-width{ch-data dup length 5 sub get} B /ch-height{ch-data dup length 4 sub get}B /ch-xoff{128 ch-data dup length 3 sub get sub}B /ch-yoff{ch-data dup length 2 sub get 127 sub}B /ch-dx{ch-data dup length 1 sub get}B /ch-image{ch-data dup type /stringtype ne{ctr get /ctr ctr 1 add N}if}B /id 0 N /rw 0 N /rc 0 N /gp 0 N /cp 0 N /G 0 N /sf 0 N /CharBuilder{save 3 1 roll S dup /base get 2 index get S /BitMaps get S get /ch-data X pop /ctr 0 N ch-dx 0 ch-xoff ch-yoff ch-height sub ch-xoff ch-width add ch-yoff setcachedevice ch-width ch-height true[1 0 0 -1 -.1 ch-xoff sub ch-yoff .1 add]{ ch-image}imagemask restore}B /D{/cc X dup type /stringtype ne{]}if nn /base get cc ctr put nn /BitMaps get S ctr S sf 1 ne{dup dup length 1 sub dup 2 index S get sf div put}if put /ctr ctr 1 add N}B /I{cc 1 add D }B /bop{userdict /bop-hook known{bop-hook}if /SI save N @rigin 0 0 moveto /V matrix currentmatrix dup 1 get dup mul exch 0 get dup mul add .99 lt{/QV}{/RV}ifelse load def pop pop}N /eop{SI restore showpage userdict /eop-hook known{eop-hook}if}N /@start{userdict /start-hook known{start-hook}if pop /VResolution X /Resolution X 1000 div /DVImag X /IE 256 array N 0 1 255{IE S 1 string dup 0 3 index put cvn put}for 65781.76 div /vsize X 65781.76 div /hsize X}N /p{show}N /RMat[1 0 0 -1 0 0]N /BDot 260 string N /rulex 0 N /ruley 0 N /v{/ruley X /rulex X V}B /V {}B /RV statusdict begin /product where{pop product dup length 7 ge{0 7 getinterval dup(Display)eq exch 0 4 getinterval(NeXT)eq or}{pop false} ifelse}{false}ifelse end{{gsave TR -.1 -.1 TR 1 1 scale rulex ruley false RMat{BDot}imagemask grestore}}{{gsave TR -.1 -.1 TR rulex ruley scale 1 1 false RMat{BDot}imagemask grestore}}ifelse B /QV{gsave transform round exch round exch itransform moveto rulex 0 rlineto 0 ruley neg rlineto rulex neg 0 rlineto fill grestore}B /a{moveto}B /delta 0 N /tail{dup /delta X 0 rmoveto}B /M{S p delta add tail}B /b{S p tail} B /c{-4 M}B /d{-3 M}B /e{-2 M}B /f{-1 M}B /g{0 M}B /h{1 M}B /i{2 M}B /j{ 3 M}B /k{4 M}B /w{0 rmoveto}B /l{p -4 w}B /m{p -3 w}B /n{p -2 w}B /o{p -1 w}B /q{p 1 w}B /r{p 2 w}B /s{p 3 w}B /t{p 4 w}B /x{0 S rmoveto}B /y{ 3 2 roll p a}B /bos{/SS save N}B /eos{SS restore}B end %%EndProcSet %%BeginProcSet: special.pro TeXDict begin /SDict 200 dict N SDict begin /@SpecialDefaults{/hs 612 N /vs 792 N /ho 0 N /vo 0 N /hsc 1 N /vsc 1 N /ang 0 N /CLIP 0 N /rwiSeen false N /rhiSeen false N /letter{}N /note{}N /a4{}N /legal{}N}B /@scaleunit 100 N /@hscale{@scaleunit div /hsc X}B /@vscale{@scaleunit div /vsc X}B /@hsize{/hs X /CLIP 1 N}B /@vsize{/vs X /CLIP 1 N}B /@clip{ /CLIP 2 N}B /@hoffset{/ho X}B /@voffset{/vo X}B /@angle{/ang X}B /@rwi{ 10 div /rwi X /rwiSeen true N}B /@rhi{10 div /rhi X /rhiSeen true N}B /@llx{/llx X}B /@lly{/lly X}B /@urx{/urx X}B /@ury{/ury X}B /magscale true def end /@MacSetUp{userdict /md known{userdict /md get type /dicttype eq{userdict begin md length 10 add md maxlength ge{/md md dup length 20 add dict copy def}if end md begin /letter{}N /note{}N /legal{} N /od{txpose 1 0 mtx defaultmatrix dtransform S atan/pa X newpath clippath mark{transform{itransform moveto}}{transform{itransform lineto} }{6 -2 roll transform 6 -2 roll transform 6 -2 roll transform{ itransform 6 2 roll itransform 6 2 roll itransform 6 2 roll curveto}}{{ closepath}}pathforall newpath counttomark array astore /gc xdf pop ct 39 0 put 10 fz 0 fs 2 F/|______Courier fnt invertflag{PaintBlack}if}N /txpose{pxs pys scale ppr aload pop por{noflips{pop S neg S TR pop 1 -1 scale}if xflip yflip and{pop S neg S TR 180 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{pop S neg S TR pop 180 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{ppr 1 get neg ppr 0 get neg TR}if}{noflips{TR pop pop 270 rotate 1 -1 scale}if xflip yflip and{TR pop pop 90 rotate 1 -1 scale ppr 3 get ppr 1 get neg sub neg ppr 2 get ppr 0 get neg sub neg TR}if xflip yflip not and{TR pop pop 90 rotate ppr 3 get ppr 1 get neg sub neg 0 TR}if yflip xflip not and{TR pop pop 270 rotate ppr 2 get ppr 0 get neg sub neg 0 S TR}if}ifelse scaleby96{ppr aload pop 4 -1 roll add 2 div 3 1 roll add 2 div 2 copy TR .96 dup scale neg S neg S TR}if}N /cp {pop pop showpage pm restore}N end}if}if}N /normalscale{Resolution 72 div VResolution 72 div neg scale magscale{DVImag dup scale}if 0 setgray} N /psfts{S 65781.76 div N}N /startTexFig{/psf$SavedState save N userdict maxlength dict begin /magscale false def normalscale currentpoint TR /psf$ury psfts /psf$urx psfts /psf$lly psfts /psf$llx psfts /psf$y psfts /psf$x psfts currentpoint /psf$cy X /psf$cx X /psf$sx psf$x psf$urx psf$llx sub div N /psf$sy psf$y psf$ury psf$lly sub div N psf$sx psf$sy scale psf$cx psf$sx div psf$llx sub psf$cy psf$sy div psf$ury sub TR /showpage{}N /erasepage{}N /copypage{}N /p 3 def @MacSetUp}N /doclip{ psf$llx psf$lly psf$urx psf$ury currentpoint 6 2 roll newpath 4 copy 4 2 roll moveto 6 -1 roll S lineto S lineto S lineto closepath clip newpath moveto}N /endTexFig{end psf$SavedState restore}N /@beginspecial{SDict begin /SpecialSave save N gsave normalscale currentpoint TR @SpecialDefaults count /ocount X /dcount countdictstack N}N /@setspecial {CLIP 1 eq{newpath 0 0 moveto hs 0 rlineto 0 vs rlineto hs neg 0 rlineto closepath clip}if ho vo TR hsc vsc scale ang rotate rwiSeen{rwi urx llx sub div rhiSeen{rhi ury lly sub div}{dup}ifelse scale llx neg lly neg TR }{rhiSeen{rhi ury lly sub div dup scale llx neg lly neg TR}if}ifelse CLIP 2 eq{newpath llx lly moveto urx lly lineto urx ury lineto llx ury lineto closepath clip}if /showpage{}N /erasepage{}N /copypage{}N newpath }N /@endspecial{count ocount sub{pop}repeat countdictstack dcount sub{ end}repeat grestore SpecialSave restore end}N /@defspecial{SDict begin} N /@fedspecial{end}B /li{lineto}B /rl{rlineto}B /rc{rcurveto}B /np{ /SaveX currentpoint /SaveY X N 1 setlinecap newpath}N /st{stroke SaveX SaveY moveto}N /fil{fill SaveX SaveY moveto}N /ellipse{/endangle X /startangle X /yrad X /xrad X /savematrix matrix currentmatrix N TR xrad yrad scale 0 0 1 startangle endangle arc savematrix setmatrix}N end %%EndProcSet TeXDict begin 40258431 52099146 1000 300 300 (/tmp_mnt/Net/charlotte/ftp/pub/mpi/mpi-1.jun95/temp.dvi) @start /Fa 1 1 df0 D E /Fb 2 111 df<0300038003000000000000000000000000001C002400460046008C000C001800 1800180031003100320032001C0009177F960C>105 D<383C0044C6004702004602008E 06000C06000C06000C0C00180C00180C40181840181880300880300F00120E7F8D15> 110 D E /Fc 8 81 df<0000180000300000600000E00000C00001800003800007000006 00000E00000C00001C0000380000380000700000700000E00000E00001E00001C00001C0 000380000380000380000780000700000700000F00000E00000E00001E00001E00001E00 001C00001C00003C00003C00003C00003C00003800007800007800007800007800007800 00780000780000780000700000F00000F00000F00000F00000F00000F00000F00000F000 00F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F00000F000 00F00000F00000F00000F00000F00000F000007000007800007800007800007800007800 007800007800007800003800003C00003C00003C00003C00001C00001C00001E00001E00 001E00000E00000E00000F000007000007000007800003800003800003800001C00001C0 0001E00000E00000E000007000007000003800003800001C00000C00000E000006000007 000003800001800000C00000E0000060000030000018157C768121>32 DI<00001C00003C0000F80001E00003C000 0780000F00000E00001E00003C00003C00003C0000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000F00000F00000F00001 E00001E00003C0000380000700000E00001C0000780000E00000E000007800001C00000E 000007000003800003C00001E00001E00000F00000F00000F00000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 780000780000780000780000780000780000780000780000780000780000780000780000 7800007800007800007800007800007800007800007800007800007800007800003C0000 3C00003C00001E00000E00000F000007800003C00001E00000F800003C00001C167C7B81 21>40 D<0018007800F001E003C007800F001F001E003E003C007C007C007800F800F800 F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800F800 F800F800F8000D25707E25>56 D58 D<007C007C007C007C007C007C 007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C 00F800F800F800F001F001E003E003C0078007000E001C003800F000C000F00038001C00 0E000700078003C003E001E001F000F000F800F800F8007C007C007C007C007C007C007C 007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C007C0E4D 798025>60 D62 D80 D E /Fd 42 122 df44 D<07C01FF03878701C60 1EE00EC00F400F400700070007000F000E001E001C003C007800F001E001C0038007000E 001C0038007000FFFFFFFF101C7E9B15>50 D58 D<001C0000003E0000003E0000002E0000006700000067000000E780 0000C7800000C3800001C3C0000183C0000181C0000381E0000381E0000700F0000700F0 000600F0000E0078000FFFF8000FFFF8001C003C001C003C0018003C0038001E0038001E 0070001F0070000F0070000F00E0000780191D7F9C1C>65 DI<003FC000FFF003C0F00780300F00001E00003C00003C000078 0000780000780000F00000F00000F00000F00000F00000F00000F00000F00000F0000078 00007800007800003C00003C00001E00000F000807801803C07800FFF0003F80151F7D9D 1B>IIII<003F80 01FFF003C0F80780380F00181E00003C00003C0000780000780000780000F00000F00000 F00000F00000F00000F00000F007F8F007F8F000387800387800387800383C00383C0038 1E00380F003807803803C0F801FFF0003F80151F7D9D1C>III76 DII<003F000001FFE00003FFF00007C0F8000F807C001E001E003E001F003C00 0F00780007807800078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F000 03C0F00003C0F00003C0F80007C078000780780007807C000F803C000F003E001F001F00 3E000F807C0007C0F80003FFF00001FFE000003F00001A1F7E9D1F>II 82 D<03F8000FFE001C0F00380700700300600000E00000E00000E00000E00000F00000 7800007F00003FE0001FFC0007FE0001FF00001F800007800003C00003C00001C00001C0 0001C00001C0C00180E00380F007007C0E001FFC0007F000121F7E9D17>III<78 000E007C001E003C003C001E0038000F0070000F00F0000781E00003C1C00001C3C00001 E7800000F70000007E0000003E0000003C0000003C0000007E00000077000000E7800001 E3800003C1C0000381E0000700F0000F00F8000E0078001C003C003C003E0078001F0070 000F00F0000F80191D7F9C1C>88 D<0FC03FF07FF87038401C001C001C00FC0FFC3FFC78 1CE01CE01CE01CF07C7FFC7FDC3F1C0E127E9114>97 DI<07E00FF81FFC3C1C70047000E000E000E000E000 E000E000700070043C1C1FFC0FF807E00E127E9112>I<000E000E000E000E000E000E00 0E000E000E000E000E0F8E1FEE3FFE7C3E700E700EE00EE00EE00EE00EE00EE00EF00E70 1E7C3E3FFE1FEE0F8E0F1D7E9C15>I<07C01FE03FF078787018601CFFFCFFFCFFFCE000 E000E000700070043C1C3FFC1FF807E00E127E9112>I<00FC01FC03FC07000E000E000E 000E000E000E000E00FFE0FFE00E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E1D809C0D>I<03C3C00FFFC01FFFC01C3800381C00381C00381C 00381C00381C001C38001FF8001FF0003BC0003800003800001FFC001FFF003FFF807007 80E001C0E001C0E001C0F003C07C0F803FFF001FFE0007F800121B7F9115>I105 D 108 DII<03F0000FFC001FFE003C0F00780780700380E0 01C0E001C0E001C0E001C0E001C0F003C07003807807803C0F001FFE000FFC0003F00012 127F9115>II 114 D<1FC03FF07FF0F030E000E000F0007F003FC01FE000F0003800388038F078FFF07F E01FC00D127F9110>I<1C001C001C001C001C001C00FFE0FFE01C001C001C001C001C00 1C001C001C001C001C001C001C001C201FF00FF007C00C187F970F>II< E007E0077006700E700E381C381C381C1C381C381C380E700E700660076007E003C003C0 10127F9113>I<7003807807003C0E001C1C000E1C0007380003F00001E00001C00001E0 0003F0000738000E18000E1C001C0E00380700700380F003C01212809113>120 DI E /Fe 10 118 df<1FE0003FF0007FF800783C00300E00000E00000E0003FE001FFE003E0E0070 0E00E00E00E00E00E00E00783E007FFFE03FE7E00F83E013127E9116>97 D<03E00FF81FFC3C1E780E7007E007FFFFFFFFFFFFE000E000700778073C0F1FFE0FFC03 F010127D9116>101 D<001F00007F8000FF8001E78001C30001C00001C0007FFF00FFFF 00FFFF0001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0 0001C0003FFE007FFF003FFE0011197F9816>I<018003C003C001800000000000000000 7FC07FC07FC001C001C001C001C001C001C001C001C001C001C001C001C07FFFFFFF7FFF 101A7D9916>105 D<003000780078003000000000000000001FF81FF81FF80038003800 3800380038003800380038003800380038003800380038003800380038003800386070F0 F0FFE07FC03F800D237E9916>I108 D114 D<0FEC3FFC7FFCF03CE01CE01C70007F801FF007 F8003C600EE00EF00EF81EFFFCFFF8C7E00F127D9116>I<030000070000070000070000 0700007FFF00FFFF00FFFF00070000070000070000070000070000070000070000070100 07038007038007038007870003FE0001FC0000F80011177F9616>I<7E1F80FE3F807E1F 800E03800E03800E03800E03800E03800E03800E03800E03800E03800E03800E03800E0F 800FFFF007FBF803E3F01512809116>I E /Ff 47 121 df<000078007C7800FC7801FC 7803C000038000078000078000078000078000078000078000078000078000FFFC78FFFC 78FFFC780780780780780780780780780780780780780780780780780780780780780780 78078078078078078078078078078078078078078078078078152480A31A>12 D<787878781830306060E0050A7C830E>44 DII<00C001C007C0FFC0FFC0FBC003C003C003C003C003C003C003C003C003 C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C003C0FFFFFF FFFFFF10227CA118>49 D<03F0000FFC001FFE003C1F003007807007C06003C0E003E0C0 01E04001E04001E00001E00001E00001E00003C00003C0000780000780000F00001E0000 3C0000780000F00001E00001C0000380000700000E00001C0000380000700000FFFFE0FF FFE0FFFFE013227EA118>I<01F00007FC001FFF003E0F003807807003C02003C02003C0 0003C00003C00003C0000780000780000F00001E0003FC0003F80003FE00000F00000780 0003C00003C00001E00001E00001E00001E00001E08001E0C003C0E003C07007803C0F80 1FFF000FFC0003F00013237EA118>I<001F00001F00002F00002F00006F0000EF0000CF 0001CF0001CF00038F00038F00078F00070F000F0F000E0F001E0F003C0F003C0F00780F 00780F00F00F00FFFFF8FFFFF8FFFFF8000F00000F00000F00000F00000F00000F00000F 00000F00000F0015217FA018>I<3FFF803FFF803FFF803C00003C00003C00003C00003C 00003C00003C00003C00003C00003CF8003FFE003FFF003F0F803E07803C03C03803C000 01E00001E00001E00001E00001E00001E00001E04003C04003C0E003C07007807C1F003F FE000FFC0003F00013227EA018>I<007E0001FF0003FF0007C1000F00001E00001E0000 3C00003C000078000078000078F800F1FE00F7FF00FF0F80FC0780F803C0F803C0F801C0 F001E0F001E0F001E0F001E0F001E07001E07001E07801E07803C03803C03C03801C0780 1F0F000FFE0007FC0001F00013237EA118>II<01F00007FC000FFE001E0F003C070038078078 03807803C0F003C0F001C0F001E0F001E0F001E0F001E0F001E0F001E07003E07803E078 03E03C07E03E1FE01FFDE00FF1E003E3C00003C00003C0000380000780000700000F0010 1E00183C003FF8001FF00007C00013237EA118>57 D<001F0000001F0000003F8000003F 8000003B8000007BC0000073C0000071C00000F1E00000F1E00000E0E00001E0F00001E0 F00001C0F00003C0780003C078000380780007803C0007803C0007003C000F001E000F00 1E000FFFFE001FFFFF001FFFFF001C000F003C0007803C00078038000780780003C07800 03C0700003C0F00001E0F00001E0E00001E01B237EA220>65 DI<000FF000007F FC0000FFFF0001F01F0003C00700078000000F0000001E0000003E0000003C0000003C00 0000780000007800000078000000F0000000F0000000F0000000F0000000F0000000F000 0000F0000000F0000000F00000007800000078000000780000003C0000003C0000003E00 00001E0000000F0000000780008003C0038001F00F8000FFFF00007FFC00000FF0001925 7DA31F>III<000FF000003FFE0000FFFF8001F80F8003E0038007800000 0F0000001E0000001E0000003C0000003C000000780000007800000078000000F0000000 F0000000F0000000F0000000F0000000F0000000F000FFC0F000FFC0F000FFC0780003C0 780003C0780003C03C0003C03C0003C01E0003C01E0003C00F0003C0078003C003E003C0 01F807C000FFFFC0003FFF00000FF8001A257DA321>71 DII76 DII<001FC000007FF00001FFFC0003F07E0007C01F000F800F801F00 07C01E0003C03C0001E03C0001E0780000F0780000F0780000F070000070F0000078F000 0078F0000078F0000078F0000078F0000078F0000078F0000078F0000078780000F07800 00F0780000F07C0001F03C0001E03E0003E01E0003C01F0007C00F800F8007C01F0003F0 7E0001FFFC00007FF000001FC0001D257DA324>II82 D<00FE0003FFC007FFE00F81E01E00603C00003C00007800007800007800007800007800 007C00003C00003F00001FC0000FFC0007FF0001FF80003FC00007E00001F00000F00000 F8000078000078000078000078000078000078C000F0E000F0F801E07E07C03FFF800FFF 0001FC0015257EA31B>IIII<780003E07C0003C03E0007C0 1E000F800F000F000F801F0007C03E0003C03C0003E07C0001F0780000F0F0000079F000 007DE000003FC000001FC000001F8000000F0000001F8000001FC000003FC000007DE000 0079F00000F0F00001F0780001E03C0003C03E0007C01E0007800F000F000F801F000780 1E0003C03C0003E07C0001F0780000F0F80000F81D237FA220>88 D<07E01FF83FFC381E201E000F000F000F000F00FF07FF1FFF3E0F780FF00FF00FF00FF0 0FF83F7FFF3FEF1F8F10167E9517>97 D<01FC0007FF000FFF801F03803C018078000078 0000700000F00000F00000F00000F00000F00000F000007800007800007800003C00401F 03C00FFFC007FF8001FC0012167E9516>99 D<0003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C003E3C00FFBC01FFFC03F0FC03C07 C07803C07803C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C07803C07803 C03C07C03E0FC01FFFC00FFBC003E3C012237EA219>I<03F00007FC001FFE003E0F003C 0780780380780380F001C0FFFFC0FFFFC0FFFFC0F00000F00000F0000070000078000078 00003C00801F07800FFF8007FF0001F80012167E9516>I<01F07807FFF80FFFF81F1F00 1E0F003C07803C07803C07803C07803C07801E0F001F1F000FFE001FFC0019F000380000 3800003C00001FFE001FFFC01FFFE03FFFF07801F07800F8F00078F00078F00078F00078 7800F03E03E01FFFC00FFF8001FC0015217F9518>103 D105 D108 DII<01FC0007FF000FFF801F07C03C01E07800F07800F0700070F0 0078F00078F00078F00078F00078F000787800F07800F07C01F03E03E01F07C00FFF8007 FF0001FC0015167F9518>II114 D<07F01FFC3FFE3C0E78067800 78007C003F003FF01FF80FFC01FE001F000F000F000FC00FF81EFFFE3FFC0FF010167F95 13>I<0F000F000F000F000F000F00FFF8FFF8FFF80F000F000F000F000F000F000F000F 000F000F000F000F000F000F000F080F1C07FC07F803E00E1C7F9B12>II<7801F07C01E03E03C01E07C00F0780078F0007DE0003FC0001FC00 00F80000700000F80001FC0003DC00039E00078F000F07801E07801E03C03C01E07800F0 F800F81516809516>120 D E /Fg 78 126 df<70F8F8F8F8F8F8F8F8F8F8F8F8F8F8F8 F870000000000070F8F8F870051C779B18>33 D<4010E038F078E038E038E038E038E038 E038E038E038E038E03860300D0E7B9C18>I<030600078F00078F00078F00078F00078F 00078F007FFFC0FFFFE0FFFFE07FFFC00F1E000F1E000F1E000F1E000F1E000F1E007FFF C0FFFFE0FFFFE07FFFC01E3C001E3C001E3C001E3C001E3C001E3C000C1800131C7E9B18 >I<3803007C07807C0780EE0F80EE0F00EE0F00EE1F00EE1E00EE1E00EE3E007C3C007C 3C00387C0000780000780000F80000F00001F00001E00001E00003E00003C00003C00007 C0000783800787C00F87C00F0EE00F0EE01F0EE01E0EE01E0EE03E0EE03C07C03C07C018 038013247E9F18>37 D<01C00007E0000FF0000E70001C38001C38001C38001C38001C73 F01C73F01CE3F00FE3800FC7000F87000F07001F0E003F0E007B8E0073DC00E1DC00E0F8 00E0F800E07070E0787070FC707FFFE03FCFE00F03C0141C7F9B18>I<007000F001E003 C007800F001E001C00380038007000700070007000E000E000E000E000E000E000E000E0 007000700070007000380038001C001E000F00078003C001F000F000700C24799F18>40 D<6000F00078003C001E000F000780038001C001C000E000E000E000E000700070007000 70007000700070007000E000E000E000E001C001C0038007800F001E003C007800F00060 000C247C9F18>I<01C00001C00001C00001C000C1C180F1C780F9CF807FFF001FFC0007 F00007F0001FFC007FFF00F9CF80F1C780C1C18001C00001C00001C00001C00011147D97 18>I<00600000F00000F00000F00000F00000F00000F00000F0007FFFC0FFFFE0FFFFE0 7FFFC000F00000F00000F00000F00000F00000F00000F00000600013147E9718>I<1C3E 7E7F3F1F070E1E7CF860080C788518>I<7FFF00FFFF80FFFF807FFF0011047D8F18>I<30 78FCFC78300606778518>I<000300000780000780000F80000F00001F00001E00001E00 003E00003C00007C0000780000780000F80000F00001F00001E00003E00003C00003C000 07C0000780000F80000F00000F00001F00001E00003E00003C00003C00007C0000780000 F80000F00000F0000060000011247D9F18>I<01F00007FC000FFE001F1F001C07003803 807803C07001C07001C0E000E0E000E0E000E0E000E0E000E0E000E0E000E0E000E0E000 E0F001E07001C07001C07803C03803801C07001F1F000FFE0007FC0001F000131C7E9B18 >I<01800380038007800F803F80FF80FB80438003800380038003800380038003800380 038003800380038003800380038003807FFCFFFE7FFC0F1C7B9B18>I<03F0000FFE003F FF007C0F807003C0E001C0F000E0F000E06000E00000E00000E00001C00001C00003C000 0780000F00001E00003C0000780000F00001E00007C0000F80001E00E03C00E07FFFE0FF FFE07FFFE0131C7E9B18>I<07F8001FFE003FFF007807807803C07801C03001C00001C0 0003C0000380000F0003FF0003FE0003FF000007800003C00001C00000E00000E00000E0 F000E0F000E0F001C0F003C07C07803FFF001FFE0003F800131C7E9B18>I<1FFF803FFF 803FFF803800003800003800003800003800003800003800003800003BF8003FFE003FFF 003C07801803C00001C00000E00000E06000E0F000E0F000E0E001C07003C07C0F803FFF 001FFC0003F000131C7E9B18>53 D<3078FCFC783000000000000000003078FCFC783006 14779318>58 D<183C7E7E3C180000000000000000183C7E7E3E1E0E1C3C78F060071A78 9318>I<000300000780001F80003F00007E0001FC0003F00007E0001FC0003F00007E00 00FC0000FC00007E00003F00001FC00007E00003F00001FC00007E00003F00001F800007 8000030011187D9918>I<7FFFC0FFFFE0FFFFE0FFFFE0000000000000000000000000FF FFE0FFFFE0FFFFE07FFFC0130C7E9318>I<600000F00000FC00007E00003F00001FC000 07E00003F00001FC00007E00003F00001F80001F80003F00007E0001FC0003F00007E000 1FC0003F00007E0000FC0000F0000060000011187D9918>I<00700000F80000F80000D8 0000D80001DC0001DC0001DC00018C00038E00038E00038E00038E000306000707000707 000707000707000FFF800FFF800FFF800E03800E03801C01C01C01C07F07F0FF8FF87F07 F0151C7F9B18>65 DI<00F8E003FEE007FF E00F07E01E03E03C01E03800E07000E07000E0700000E00000E00000E00000E00000E000 00E00000E00000E000007000007000E07000E03800E03C00E01E01C00F07C007FF8003FE 0000F800131C7E9B18>I<7FF800FFFE007FFF001C0F801C03C01C03C01C01E01C00E01C 00E01C00F01C00701C00701C00701C00701C00701C00701C00701C00701C00F01C00E01C 00E01C01E01C01C01C03C01C0F807FFF00FFFE007FF800141C7F9B18>III<01F1C003 FDC00FFFC01F0FC01C03C03803C03801C07001C07001C0700000E00000E00000E00000E0 0000E00000E00FF0E01FF0E00FF07001C07001C07003C03803C03803C01C07C01F0FC00F FFC003FDC001F1C0141C7E9B18>I<7F07F0FF8FF87F07F01C01C01C01C01C01C01C01C0 1C01C01C01C01C01C01C01C01C01C01FFFC01FFFC01FFFC01C01C01C01C01C01C01C01C0 1C01C01C01C01C01C01C01C01C01C01C01C07F07F0FF8FF87F07F0151C7F9B18>I<7FFF 00FFFF807FFF0001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0 0001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0 007FFF00FFFF807FFF00111C7D9B18>I<7F07F0FF87F87F07F01C03C01C07801C07001C 0E001C1E001C3C001C38001C70001CF0001DF0001DF0001FB8001FB8001F1C001E1C001C 0E001C0E001C07001C07001C03801C03801C01C07F03F0FF87F87F03F0151C7F9B18>75 D<7FE000FFE0007FE0000E00000E00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E00000E00000E00000E00000E00700E00700E00700E00 700E00707FFFF0FFFFF07FFFF0141C7F9B18>II<7E07F0FF0FF87F07F01D81C01D81C01D81C01DC1C01CC1C01CC1C01CE1C01CE1C0 1CE1C01C61C01C71C01C71C01C31C01C39C01C39C01C39C01C19C01C19C01C1DC01C0DC0 1C0DC01C0DC07F07C0FF87C07F03C0151C7F9B18>I<0FF8003FFE007FFF00780F007007 00F00780E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380E003 80E00380E00380E00380E00380E00380F00780700700780F007FFF003FFE000FF800111C 7D9B18>II<0FF8003FFE007FFF00780F00 700700F00780E00380E00380E00380E00380E00380E00380E00380E00380E00380E00380 E00380E00380E00380E00380E1E380E1E380F0E78070F700787F007FFF003FFE000FFC00 001C00001E00000E00000F0000070000070011227D9B18>I<7FF800FFFE007FFF001C0F 801C03801C03C01C01C01C01C01C01C01C03C01C03801C0F801FFF001FFE001FFE001C0F 001C07001C03801C03801C03801C03801C03801C039C1C039C1C039C7F01F8FF81F87F00 F0161C7F9B18>I<03F3801FFF803FFF807C0F80700780E00380E00380E00380E0000070 00007800003F00001FF00007FE0000FF00000F800003C00001C00000E00000E06000E0E0 00E0E001E0F001C0F80780FFFF80FFFE00E7F800131C7E9B18>I<7FFFF8FFFFF8FFFFF8 E07038E07038E07038E07038007000007000007000007000007000007000007000007000 00700000700000700000700000700000700000700000700000700000700007FF0007FF00 07FF00151C7F9B18>IIII<7F8FE07F9F E07F8FE00E07000F0700070E00078E00039C0003DC0001F80001F80000F00000F0000070 0000F00000F80001F80001DC00039E00038E00070F000707000E07800E03801E03C07F07 F0FF8FF87F07F0151C7F9B18>II91 D93 D<7FFF00FFFF80FFFF807FFF0011047D7F18>95 D<1FE0003FF8007FFC00781E00300E0000070000070000FF0007FF001FFF007F07007807 00E00700E00700E00700F00F00781F003FFFF01FFBF007E1F014147D9318>97 D<7E0000FE00007E00000E00000E00000E00000E00000E00000E3E000EFF800FFFC00FC1 E00F80E00F00700E00700E00380E00380E00380E00380E00380E00380F00700F00700F80 E00FC1E00FFFC00EFF80063E00151C809B18>I<01FE0007FF001FFF803E078038030070 0000700000E00000E00000E00000E00000E00000E000007000007001C03801C03E03C01F FF8007FF0001FC0012147D9318>I<001F80003F80001F80000380000380000380000380 00038003E3800FFB801FFF803C1F80380F80700780700380E00380E00380E00380E00380 E00380E00380700780700780380F803C1F801FFFF00FFBF803E3F0151C7E9B18>I<01F0 0007FC001FFE003E0F00380780700380700380E001C0E001C0FFFFC0FFFFC0FFFFC0E000 007000007001C03801C03E03C01FFF8007FF0001FC0012147D9318>I<001F80007FC000 FFE000E1E001C0C001C00001C00001C0007FFFC0FFFFC0FFFFC001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C00001C0007FFF007F FF007FFF00131C7F9B18>I<01E1F007FFF80FFFF81E1E301C0E00380700380700380700 3807003807001C0E001E1E001FFC001FF80039E0003800001C00001FFE001FFFC03FFFE0 7801F0700070E00038E00038E00038E000387800F07E03F01FFFC00FFF8001FC00151F7F 9318>I<7E0000FE00007E00000E00000E00000E00000E00000E00000E3E000EFF800FFF C00FC1C00F80E00F00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00E00 E00E00E00E00E07FC3FCFFE7FE7FC3FC171C809B18>I<03800007C00007C00007C00003 80000000000000000000000000007FC000FFC0007FC00001C00001C00001C00001C00001 C00001C00001C00001C00001C00001C00001C00001C00001C00001C000FFFF00FFFF80FF FF00111D7C9C18>I<0038007C007C007C003800000000000000000FFC1FFC0FFC001C00 1C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C001C00 1C001C001C6038F078FFF07FE03F800E277E9C18>II<7FE000FFE0007FE00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00000E00000E0007FFFC0FFFFE07FFFC0131C7E9B18>I<7CE0E000FFFBF8007FFFF800 1F1F1C001E1E1C001E1E1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C001C1C1C00 1C1C1C001C1C1C001C1C1C001C1C1C001C1C1C007F1F1F00FFBFBF807F1F1F0019148193 18>I<7E3E00FEFF807FFFC00FC1C00F80E00F00E00E00E00E00E00E00E00E00E00E00E0 0E00E00E00E00E00E00E00E00E00E00E00E07FC3FCFFE7FE7FC3FC1714809318>I<01F0 000FFE001FFF003E0F803803807001C07001C0E000E0E000E0E000E0E000E0E000E0F001 E07001C07803C03C07803E0F801FFF000FFE0001F00013147E9318>I<7E3E00FEFF807F FFC00FC1E00F80E00F00700E00700E00380E00380E00380E00380E00380E00380F00700F 00700F80E00FC1E00FFFC00EFF800E3E000E00000E00000E00000E00000E00000E00000E 00007FC000FFE0007FC000151E809318>I<01E38007FB801FFF803E1F80380F80700780 700780E00380E00380E00380E00380E00380E00380700780700780380F803C1F801FFF80 0FFB8003E380000380000380000380000380000380000380000380003FF8003FF8003FF8 151E7E9318>I<7F87E0FF9FF07FBFF803F87803F03003E00003C00003C0000380000380 000380000380000380000380000380000380000380007FFE00FFFF007FFE0015147F9318 >I<07F7003FFF007FFF00780F00E00700E00700E007007C00007FE0001FFC0003FE0000 1F00600780E00380E00380F00380F80F00FFFF00FFFC00E7F00011147D9318>I<018000 0380000380000380000380007FFFC0FFFFC0FFFFC0038000038000038000038000038000 0380000380000380000380000380400380E00380E00380E001C1C001FFC000FF80003E00 13197F9818>I<7E07E0FE0FE07E07E00E00E00E00E00E00E00E00E00E00E00E00E00E00 E00E00E00E00E00E00E00E00E00E00E00E01E00F03E007FFFC03FFFE01FCFC1714809318 >I<7F8FF0FF8FF87F8FF01E03C00E03800E03800E0380070700070700070700038E0003 8E00038E00038E0001DC0001DC0001DC0000F80000F80000700015147F9318>II<7F8FF07F9FF07F8F F0070700078E00039E0001DC0001F80000F80000700000F00000F80001DC00039E00038E 000707000F07807F8FF0FF8FF87F8FF015147F9318>I<7F8FF0FF8FF87F8FF00E01C00E 03800E0380070380070700070700038700038600038E0001CE0001CE0000CC0000CC0000 DC0000780000780000780000700000700000700000F00000E00079E0007BC0007F80003F 00001E0000151E7F9318>I<3FFFF07FFFF07FFFF07001E07003C0700780000F00001E00 003C0000F80001F00003C0000780000F00701E00703C0070780070FFFFF0FFFFF0FFFFF0 14147F9318>I<0007E0001FE0007FE000780000E00000E00000E00000E00000E00000E0 0000E00000E00000E00000E00000E00001E0007FC000FF8000FF80007FC00001E00000E0 0000E00000E00000E00000E00000E00000E00000E00000E00000E00000E000007800007F E0001FE00007E013247E9F18>I<7C0000FF0000FFC00003C00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E00000E00000F000007FC0003FE0003FE000 7FC000F00000E00000E00000E00000E00000E00000E00000E00000E00000E00000E00000 E00003C000FFC000FF00007C000013247E9F18>125 D E /Fh 39 123 df<007E1F0001C1B1800303E3C00703C3C00E03C1800E01C0000E01C0000E01C000 0E01C0000E01C0000E01C000FFFFFC000E01C0000E01C0000E01C0000E01C0000E01C000 0E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C0000E01C000 0E01C0000E01C0007F87FC001A1D809C18>11 D<007E0001C1800301800703C00E03C00E 01800E00000E00000E00000E00000E0000FFFFC00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C07F87F815 1D809C17>I<004000800100020006000C000C0018001800300030007000600060006000 E000E000E000E000E000E000E000E000E000E000E000E000600060006000700030003000 180018000C000C00060002000100008000400A2A7D9E10>40 D<80004000200010001800 0C000C000600060003000300038001800180018001C001C001C001C001C001C001C001C0 01C001C001C001C0018001800180038003000300060006000C000C001800100020004000 80000A2A7E9E10>I<60F0F0701010101020204080040C7C830C>44 DI<60F0F06004047C830C>I<60F0F06000000000000000000000 60F0F0701010101020204080041A7C910C>59 D<000600000006000000060000000F0000 000F0000000F00000017800000178000001780000023C0000023C0000023C0000041E000 0041E0000041E0000080F0000080F0000180F8000100780001FFF80003007C0002003C00 02003C0006003E0004001E0004001E000C001F001E001F00FF80FFF01C1D7F9C1F>65 D69 D73 D78 D<003F800000E0E000038038 0007001C000E000E001C0007003C00078038000380780003C0780003C0700001C0F00001 E0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0F00001E0700001C0780003 C0780003C0380003803C0007801C0007000E000E0007001C000380380000E0E000003F80 001B1E7E9C20>I<7FFFFFC0700F01C0600F00C0400F0040400F0040C00F0020800F0020 800F0020800F0020000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000 000F0000001F800003FFFC001B1C7F9B1E>84 DI<1F C000307000783800781C00301C00001C00001C0001FC000F1C00381C00701C00601C00E0 1C40E01C40E01C40603C40304E801F870012127E9115>97 DI<07E00C301878307870306000E000E000E000E000E000E00060 007004300418080C3007C00E127E9112>I<003F00000700000700000700000700000700 00070000070000070000070000070003E7000C1700180F00300700700700600700E00700 E00700E00700E00700E00700E00700600700700700300700180F000C370007C7E0131D7E 9C17>I<03E00C301818300C700E6006E006FFFEE000E000E000E0006000700230021804 0C1803E00F127F9112>I<00F8018C071E061E0E0C0E000E000E000E000E000E00FFE00E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E007FE00F1D80 9C0D>I<00038003C4C00C38C01C3880181800381C00381C00381C00381C001818001C38 000C300013C0001000003000001800001FF8001FFF001FFF803003806001C0C000C0C000 C0C000C06001803003001C0E0007F800121C7F9215>II<18003C003C0018000000000000000000000000000000FC001C001C 001C001C001C001C001C001C001C001C001C001C001C001C001C001C00FF80091D7F9C0C >I107 DIII<03F000 0E1C00180600300300700380600180E001C0E001C0E001C0E001C0E001C0E001C0600180 7003803003001806000E1C0003F00012127F9115>II<03 C1000C3300180B00300F00700700700700E00700E00700E00700E00700E00700E0070060 0700700700300F00180F000C370007C70000070000070000070000070000070000070000 0700003FE0131A7E9116>II<1F9030704030C010C010E010F8007F803F E00FF000F880388018C018C018E010D0608FC00D127F9110>I<04000400040004000C00 0C001C003C00FFE01C001C001C001C001C001C001C001C001C001C101C101C101C101C10 0C100E2003C00C1A7F9910>IIII121 D<7FFC70386038407040F040E041C003C0038007000F040E 041C043C0C380870087038FFF80E127F9112>I E /Fi 9 57 df<07C018303018701C60 0C600CE00EE00EE00EE00EE00EE00EE00EE00EE00E600C600C701C30181C7007C00F157F 9412>48 D<03000700FF0007000700070007000700070007000700070007000700070007 0007000700070007007FF00C157E9412>I<0F8030E040708030C038E038403800380070 0070006000C00180030006000C08080810183FF07FF0FFF00D157E9412>I<0FE0303060 18701C701C001C00180038006007E000300018000C000E000EE00EE00EC00C401830300F E00F157F9412>I<00300030007000F001F001700270047008701870107020704070C070 FFFE0070007000700070007003FE0F157F9412>I<20303FE03FC0240020002000200020 002F8030E020700030003800384038E038E0388030406020C01F000D157E9412>I<01F0 0608080C181C301C70006000E000E3E0EC30F018F00CE00EE00EE00E600E600E300C3018 183007C00F157F9412>I<40007FFE7FFC7FF8C008801080200040008000800100010003 000200060006000E000E000E000E000E0004000F167E9512>I<07E018302018600C600C 700C78183E101F600FC00FF018F8607C601EC00EC006C006C004600C38300FE00F157F94 12>I E /Fj 22 120 df<70F8F8F87005057C840D>58 D<70F8FCFC7404040408081010 2040060E7C840D>I<000001C00000078000001E00000078000001E00000078000000E00 000038000000F0000003C000000F0000003C000000F0000000F00000003C0000000F0000 0003C0000000F0000000380000000E0000000780000001E0000000780000001E00000007 80000001C01A1A7C9723>I62 D<000002000000060000000E0000000E0000001E00 00001F0000002F0000002F0000004F0000008F0000008F0000010F0000010F0000020F00 00040F0000040F0000080F8000080780001007800020078000200780007FFF8000400780 00800780018007800100078002000780020007C0040003C00C0003C01E0007C0FF807FFC 1E207E9F22>65 D<00F1800389C00707800E03801C03803C038038070078070078070078 0700F00E00F00E00F00E00F00E10F01C20F01C20703C20705C40308C400F078014147E93 18>97 D<007C01C207010E0F1E0F1C0E3C04780078007800F000F000F000F000F0007001 7002300418380FC010147E9314>99 D<0000780003F80000700000700000700000700000 E00000E00000E00000E00001C00001C000F1C00389C00707800E03801C03803C03803807 00780700780700780700F00E00F00E00F00E00F00E10F01C20F01C20703C20705C40308C 400F078015207E9F18>I<007C01C207010E011C013C013802780C7BF07C00F000F000F0 00F0007000700170023004183807C010147E9315>I<001E3000713800E0F001C0700380 700780700700E00F00E00F00E00F00E01E01C01E01C01E01C01E01C01E03801E03800E07 800E0B8006170001E700000700000700000E00000E00300E00781C00F038006070003FC0 00151D809316>103 D<00E001E001E000C000000000000000000000000000000E001300 23804380438043808700070007000E000E001C001C001C20384038403840388019000E00 0B1F7E9E10>105 D<0000C00001E00001E00001C0000000000000000000000000000000 000000000000001E00006300004380008380010380010380020700000700000700000700 000E00000E00000E00000E00001C00001C00001C00001C00003800003800003800003800 00700000700030700078E000F1C0006380003E00001328819E13>I<01E0000FE00001C0 0001C00001C00001C0000380000380000380000380000700000700000701E00706100E08 700E10F00E20F00E40601C80001D00001E00001FC000387000383800383800381C207038 40703840703840701880E01880600F0014207E9F18>I<03C01FC0038003800380038007 000700070007000E000E000E000E001C001C001C001C0038003800380038007000700070 007100E200E200E200E200640038000A207E9F0E>I<1E07802318C023A06043C0704380 704380708700E00700E00700E00700E00E01C00E01C00E01C00E03821C03841C07041C07 081C03083803101801E017147E931B>110 D<007C0001C3000301800E01C01E01C01C01 E03C01E07801E07801E07801E0F003C0F003C0F003C0F00780F00700700F00700E003018 0018700007C00013147E9316>I<1E1E0023210023C38043C78043878043830087000007 00000700000700000E00000E00000E00000E00001C00001C00001C00001C000038000018 000011147E9315>114 D<007C018203010603060706060E00078007F803FC01FE001F00 077007F006F006E004400820301FC010147E9315>I<00C000E001C001C001C001C00380 0380FFF8038007000700070007000E000E000E000E001C001C001C001C10382038203820 384018800F000D1C7F9B10>I<0F00601180702180E021C0E041C0E04380E08381C00701 C00701C00701C00E03800E03800E03800E03840E07080C07080C07080E0F1006131003E1 E016147E931A>I<0F01801183C02183E021C1E041C0E043806083804007004007004007 00400E00800E00800E00800E01000E01000C02000E04000E040006180001E00013147E93 16>I<0F006060118070F02180E0F821C0E07841C0E0384380E0188381C0100701C01007 01C0100701C0100E0380200E0380200E0380200E0380400E0380400E0380800E07808006 0781000709860001F078001D147E9321>I E /Fk 30 88 df45 D<3078F8787005057C840D>I<00000400000C00000C0000180000180000300000300000 600000600000C00000C0000180000180000300000300000600000600000C00000C000018 0000180000300000300000600000600000C00000C0000180000180000300000300000600 000600000600000C00000C0000180000180000300000300000600000600000C00000C000 00800000162D7EA117>I<007E0001C3000301800701C00E00C00E00E01C00E01C00E03C 01E03801E07801E07801E07801E07801E07801E0F003C0F003C0F003C0F003C0F003C0F0 0380F00780E00780E00700E00700E00E00600E00701C003038003870000FC000131F7C9D 17>I<000C001C00FC0F380038003800380038003800700070007000700070007000E000 E000E000E000E000E001C001C001C001C001C001C0038003C0FFFE0F1E7C9D17>I<003F 8000C1E00100F00200780400780400780F007C0F807C0F807C0F00780600780000F80000 F00001E00001C0000380000700000E00001C0000380000600000C0000180000300200600 200800401000403FFFC07FFF80FFFF80161E7E9D17>I<01803001FFE003FFC003FF0003 FC00020000020000020000040000040000040000047C000587000603800C01800801C000 01C00001E00001E00001E00001E07003C0F803C0F003C0E00380800780400700400E0020 1C0018700007C000141F7D9D17>53 D<000F8000704000C0200180E00301E00701E00E00 C01E00001C00003C000038000078F800790E007A07007C0300F80380F80380F003C0F003 C0F003C0F003C0F00780E00780E00780E00700E00F00600E00701C0030180018700007C0 00131F7C9D17>I<2000003FFFE07FFFC07FFF80400100C0020080020080040000080000 10000020000040000040000080000180000300000300000700000600000E00000E00001E 00001C00001C00003C00003C00003C0000780000780000780000300000131F799D17>I< 003F0000C1C00100600200600400300C00300C00300C00300C00600E00600F80C00FC180 07F60003FC0001FC0001FF00063F800C0F801007C03003C06001C06000C0C000C0C000C0 C000C0C00080C0010060030030040018180007E000141F7D9D17>I<007E0001C3000301 800601C00E01C01C00C03C00E03C00E03C01E07801E07801E07801E07801E07803E07803 E03803C03807C01C0BC00C13C003E380000780000780000700000E00600E00F01C00F018 00E0300080600041C0003F0000131F7C9D17>I<00001000000018000000380000003800 00007800000078000000FC000001BC0000013C0000033C0000023C0000063C0000043E00 00081E0000081E0000101E0000101E0000201E0000200F0000400F0000400F0000FFFF00 00800F0001000F8001000780020007800200078004000780040007800C0007C03E0007C0 FF807FFC1E207E9F22>65 D<07FFFF00007C01C0003C01E0003C00F0007800F8007800F8 007800F8007800F8007800F8007800F000F001F000F001E000F003C000F00F8000FFFE00 00F00F0001E007C001E003C001E003E001E001E001E001E001E001E003C001E003C003E0 03C003E003C003C003C007C003C00F8007800F0007803E00FFFFF0001D1F7E9E20>I<00 01F808000E061800380138007000F801E0007803C0007007800030078000300F0000301F 0000301E0000303E0000203C0000007C0000007C0000007C0000007C000000F8000000F8 000000F8000000F8000000F80000007800004078000080780000803C0000803C0001001C 0002000E00020006000C000300100001C0E000003F00001D217B9F21>I<07FFFF00007C 01E0003C00F0003C00780078003C0078003C0078001E0078001E0078001E0078001F00F0 001F00F0001F00F0001F00F0001F00F0001F00F0001F01E0001E01E0003E01E0003E01E0 003E01E0003C01E0007C03C0007803C000F003C000F003C001E003C003C003C007800780 0F0007803C00FFFFE000201F7E9E23>I<07FFFFF8007C0078003C0038003C0018007800 18007800080078000800780008007800080078080800F0100000F0100000F0100000F030 0000FFF00000F0700001E0200001E0200001E0200001E0200001E0000801E0001003C000 1003C0001003C0002003C0002003C0006003C000C0078001C0078007C0FFFFFF801D1F7E 9E1F>I<0001FC04000F030C003C009C0070007C00E0003C01C000380380001807800018 0F0000181F0000181E0000183E0000103C0000007C0000007C0000007C0000007C000000 F8000000F8000000F8007FFCF80003E0780001E0780001E0780003C0780003C03C0003C0 3C0003C01C0003C00E0007C007000B800380118001E06080003F80001E217B9F24>71 D<07FFC7FFC0007C00F800003C007800003C007800007800F000007800F000007800F000 007800F000007800F000007800F00000F001E00000F001E00000F001E00000F001E00000 FFFFE00000F001E00001E003C00001E003C00001E003C00001E003C00001E003C00001E0 03C00003C007800003C007800003C007800003C007800003C007800003C007800007800F 000007C00F8000FFF8FFF800221F7E9E22>I<07FFE0007C00003C00003C000078000078 0000780000780000780000780000F00000F00000F00000F00000F00000F00001E00001E0 0001E00001E00001E00001E00003C00003C00003C00003C00003C00003C00007800007C0 00FFFC00131F7F9E10>I<07FFF000007E0000003C0000003C0000007800000078000000 78000000780000007800000078000000F0000000F0000000F0000000F0000000F0000000 F0000001E0000001E0000001E0000001E0000001E0008001E0010003C0010003C0010003 C0030003C0020003C0060003C0060007801E0007807C00FFFFFC00191F7E9E1C>76 D<07FC0000FFC0007C0000F800003C00017800003C00017800004E0002F000004E0002F0 00004E0004F000004E0004F000004E0008F000004E0008F00000870011E00000870011E0 0000870021E00000870021E00000870041E00000838041E00001038083C00001038083C0 0001038103C00001038203C0000101C203C0000101C403C0000201C40780000201C80780 000201C80780000201D00780000200F00780000600E00780000600E00F00000F00C00F80 00FFE0C1FFF8002A1F7E9E2A>I<07FC01FFC0003E003E00003E001800003E001800004F 001000004F001000004780100000478010000043C010000043C010000083C020000081E0 20000081E020000080F020000080F020000080782000010078400001007C400001003C40 0001003C400001001E400001001E400002000F800002000F800002000F80000200078000 0200078000060003800006000300000F00010000FFE0010000221F7E9E22>I<0003F800 001E0E000038070000E0038001C001C003C001E0078000E00F0000F00F0000F01E0000F0 1E0000F83E0000F83C0000F87C0000F87C0000F87C0000F87C0000F8F80001F0F80001F0 F80001F0F80001F0F80003E0780003E0780003C0780007C07C0007803C000F003C001E00 1E001C000E0038000700F00003C3C00000FE00001D217B9F23>I<07FFFF00007C03C000 3C01E0003C00F0007800F0007800F8007800F8007800F8007800F8007800F000F001F000 F001E000F003C000F0078000F00F0000FFF80001E0000001E0000001E0000001E0000001 E0000001E0000003C0000003C0000003C0000003C0000003C0000003C000000780000007 C00000FFFC00001D1F7E9E1F>I<07FFFC00007C0700003C03C0003C01E0007801E00078 01F0007801F0007801F0007801F0007801E000F003E000F003C000F0078000F00F0000F0 3C0000FFF00001E0300001E0380001E01C0001E01C0001E01C0001E01E0003C03E0003C0 3E0003C03E0003C03E0003C03E0003C03E0207803E0407C01F04FFFC0F18000003E01F20 7E9E21>82 D<003F040060CC01803C03801C03001C0700180600080E00080E00080E0008 0E00000F00000F80000FE00007FE0003FF8001FFC0007FE00007E00001E00000E00000F0 0000F04000E04000E04000E04000E06000C0600180E00380F80300C60C0081F80016217D 9F19>I<3FFFFFF03C0780F03007803060078030400F0010400F0010C00F0010800F0010 800F0010800F0010001E0000001E0000001E0000001E0000001E0000001E0000003C0000 003C0000003C0000003C0000003C0000003C000000780000007800000078000000780000 007800000078000000F0000001F800007FFFE0001C1F7A9E21>IIII E /Fl 10 58 df<1F00318060C04040C060C060C060C060C060C060C060C060404060C0 31801F000B107F8F0F>48 D<0C003C00CC000C000C000C000C000C000C000C000C000C00 0C000C000C00FF8009107E8F0F>I<1F00618040C08060C0600060006000C00180030006 000C00102020207FC0FFC00B107F8F0F>I<1F00218060C060C000C0008001800F000080 00400060C060C060804060801F000B107F8F0F>I<0300030007000F000B001300330023 004300C300FFE003000300030003001FE00B107F8F0F>I<20803F002C00200020002000 2F0030802040006000600060C06080C061801F000B107F8F0F>I<0780184030C060C060 00C000CF00F080E040C060C060C060406060C030801F000B107F8F0F>I<40007FE07FC0 8080808001000200040004000C0008000800180018001800180018000B117E900F>I<1F 00318060C060C060C071803F000F00338061C0C060C060C060404060801F000B107F8F0F >I<1F00318060C0C040C060C060C06040E021E01E600060004060C0608043003E000B10 7F8F0F>I E /Fm 24 122 df<70F8F8F0E005057B840E>46 D<00000200000006000000 060000000E0000001E0000001E0000003F0000002F0000004F0000004F0000008F000001 0F0000010F0000020F0000020F0000040F00000C0F0000080F0000100F0000100F000020 0F80003FFF800040078000C0078000800780010007800100078002000780020007800600 07801E000F80FF807FF81D207E9F22>65 D<01FFFFFE001E001C001E000C001E0004001E 0004003C0004003C0004003C0004003C00040078080800780800007808000078180000F0 300000FFF00000F0300000F0300001E0200001E0200001E0200001E0001003C0002003C0 002003C0004003C00040078000800780018007800100078007000F001F00FFFFFE001F1F 7D9E1F>69 D<01FFFF00001E03C0001E00E0001E0070001E0078003C0078003C0078003C 0078003C0078007800F0007800F0007801E0007801C000F0070000F01E0000FFF00000F0 380001E01C0001E01E0001E00E0001E00F0003C01E0003C01E0003C01E0003C01E000780 3C0007803C0807803C0807803C100F801C10FFF00C20000007C01D207D9E21>82 D<00F1800389C00707800E03801C03803C0380380700780700780700780700F00E00F00E 00F00E00F00E20F01C40F01C40703C40705C40308C800F070013147C9317>97 D<007E0001C1000300800E07801E07801C07003C0200780000780000780000F00000F000 00F00000F00000F0000070010070020030040018380007C00011147C9315>99 D<0000780003F80000700000700000700000700000E00000E00000E00000E00001C00001 C000F1C00389C00707800E03801C03803C0380380700780700780700780700F00E00F00E 00F00E00F00E20F01C40F01C40703C40705C40308C800F070015207C9F17>I<007C01C2 07010E011C013C013802780C7BF07C00F000F000F000F0007000700170023804183807C0 10147C9315>I<00007800019C00033C00033C000718000700000700000E00000E00000E 00000E00000E0001FFE0001C00001C00001C00001C000038000038000038000038000038 0000700000700000700000700000700000700000E00000E00000E00000E00000C00001C0 0001C0000180003180007B0000F300006600003C00001629829F0E>I<003C6000E27001 C1E00380E00700E00F00E00E01C01E01C01E01C01E01C03C03803C03803C03803C03803C 07003C07001C0F001C17000C2E0003CE00000E00000E00001C00001C00301C00783800F0 700060E0003F8000141D7E9315>I<01E0000FE00001C00001C00001C00001C000038000 038000038000038000070000070000071E000763000E81800F01C00E01C00E01C01C0380 1C03801C03801C0380380700380700380700380E10700E20700C20701C20700C40E00CC0 60070014207D9F17>I<00C001E001E001C000000000000000000000000000000E003300 230043804300470087000E000E000E001C001C001C003840388030807080310033001C00 0B1F7C9E0E>I<03C01FC0038003800380038007000700070007000E000E000E000E001C 001C001C001C0038003800380038007000700070007100E200E200E200E200640038000A 207C9F0C>108 D<1C0F80F0002630C318004740640C004780680E004700700E00470070 0E008E00E01C000E00E01C000E00E01C000E00E01C001C01C038001C01C038001C01C038 001C01C0708038038071003803806100380380E100380380620070070066003003003800 21147C9325>I<1C0F802630C04740604780604700704700708E00E00E00E00E00E00E00 E01C01C01C01C01C01C01C03843803883803083807083803107003303001C016147C931A >I<007C0001C3000301800E01C01E01C01C01E03C01E07801E07801E07801E0F003C0F0 03C0F003C0F00780F00700700F00700E0030180018700007C00013147C9317>I<01C1E0 02621804741C04781C04701E04701E08E01E00E01E00E01E00E01E01C03C01C03C01C03C 01C0380380780380700380E003C1C0072380071E000700000700000E00000E00000E0000 0E00001C00001C0000FFC000171D809317>I<1C1E002661004783804787804707804703 008E00000E00000E00000E00001C00001C00001C00001C00003800003800003800003800 0070000030000011147C9313>114 D<00FC030206010C030C070C060C000F800FF007F8 03FC003E000E700EF00CF00CE008401020601F8010147D9313>I<018001C00380038003 80038007000700FFF007000E000E000E000E001C001C001C001C00380038003800382070 4070407080708031001E000C1C7C9B0F>I<0E00C03300E02301C04381C04301C04701C0 8703800E03800E03800E03801C07001C07001C07001C07101C0E20180E20180E201C1E20 0C264007C38014147C9318>I<0E03803307802307C04383C04301C04700C08700800E00 800E00800E00801C01001C01001C01001C02001C02001C04001C04001C08000E300003C0 0012147C9315>I<0383800CC4401068E01071E02071E02070C040E00000E00000E00000 E00001C00001C00001C00001C040638080F38080F38100E5810084C60078780013147D93 15>120 D<0E00C03300E02301C04381C04301C04701C08703800E03800E03800E03801C 07001C07001C07001C07001C0E00180E00180E001C1E000C3C0007DC00001C00001C0000 3800F03800F07000E06000C0C0004380003E0000131D7C9316>I E /Fn 19 121 df<387CFEFEFE7C3807077C860F>46 D<01FC0007FF001F07C01E03C03E 03E07C01F07C01F07C01F0FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC01F8FC 01F8FC01F8FC01F8FC01F8FC01F87C01F07C01F07C01F03E03E01E03C01F8FC007FF0001 FC00151D7E9C1A>48 D<00E00001E0000FE000FFE000F3E00003E00003E00003E00003E0 0003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E0 0003E00003E00003E00003E00003E00003E000FFFF80FFFF80111D7C9C1A>I<07F0001F FE00383F007C1F80FE0FC0FE0FC0FE0FE0FE07E07C07E03807E0000FE0000FC0000FC000 1F80001F00003E0000780000F00000E00001C0000380600700600E00601C00E01FFFC03F FFC07FFFC0FFFFC0FFFFC0131D7D9C1A>I<01FC0007FF000E0F801E0FC03F07E03F07E0 3F07E03F07E01E0FC0000FC0000F80001F0001FC0001FC00000F800007C00003E00003F0 0003F83803F87C03F8FE03F8FE03F8FE03F0FC03F07807E03C0FC01FFF8003FC00151D7E 9C1A>I<0001C00003C00007C00007C0000FC0001FC0003BC00073C00063C000C3C00183 C00383C00703C00E03C00C03C01803C03803C07003C0E003C0FFFFFEFFFFFE0007C00007 C00007C00007C00007C00007C000FFFE00FFFE171D7F9C1A>I<3803803FFF803FFF003F FE003FFC003FF0003F800030000030000030000030000033F80037FE003C1F00380F8010 07C00007C00007E00007E07807E0FC07E0FC07E0FC07E0FC07C0780FC0600F80381F001F FC0007F000131D7D9C1A>I<003F0001FFC007E0E00F81E01F03F01E03F03E03F07C03F0 7C01E07C0000FC1000FCFF00FDFFC0FD03E0FE01F0FE01F0FC01F8FC01F8FC01F8FC01F8 7C01F87C01F87C01F83C01F03E01F01E03E00F07C007FF8001FE00151D7E9C1A>I<6000 007FFFF87FFFF87FFFF07FFFE07FFFC0E00180C00300C00300C00600000C000018000038 0000380000780000700000F00000F00001F00001F00001F00001F00003F00003F00003F0 0003F00003F00003F00001E00000C000151E7D9D1A>I<01FC0007FF000F07801E03C01C 01E03C01E03C01E03E01E03F01E03FC3C01FE3801FFF000FFE0007FF8007FFC01FFFE03C 3FF0780FF07803F8F001F8F000F8F00078F00078F000707800707C00E03E03C00FFF8003 FC00151D7E9C1A>I<01FC000FFF001F07803E03C07C03E07C01E0FC01F0FC01F0FC01F0 FC01F8FC01F8FC01F8FC01F87C03F87C03F83E05F81FFDF807F9F80041F80001F03C01F0 7E01F07E03E07E03E07E07C03C0780381F001FFC0007F000151D7E9C1A>I<07F8001FFE 00381F80780F80FC0FC0FC0FC0FC0FC0780FC0301F80001F00003E00007C0000700000E0 0000E00000C00000C00000C00000C00000C00000C00000000000000000000000000001C0 0003E00007F00007F00007F00003E00001C00012207D9F19>63 D69 D<07FC001FFF003F0F803F07C03F03E03F03E00C03E0 0003E0007FE007FBE01F03E03C03E07C03E0F803E0F803E0F803E0FC05E07E0DE03FF8FE 0FE07E17147F9319>97 D<01FE0007FF800F83C01E01E03E00F07C00F07C00F8FC00F8FF FFF8FFFFF8FC0000FC0000FC00007C00007C00003E00181E00180F807007FFE000FF8015 147F9318>101 D108 DI112 D120 D E /Fo 11 104 df0 D<70F8F8F87005057C8D0D>I<400004C0000C6000183000301800600C00C006 018003030001860000CC0000780000300000300000780000CC000186000303000601800C 00C0180060300030600018C0000C40000416187A9623>I<03C00FF01C38300C60066006 C003C003C003C003C003C00360066006300C1C380FF003C010127D9317>14 D<03C00FF01FF83FFC7FFE7FFEFFFFFFFFFFFFFFFFFFFFFFFF7FFE7FFE3FFC1FF80FF003 C010127D9317>I21 D<00000C00000C0000180000180000300000300000 600000600000C00000C0000180000180000180000300000300000600000600000C00000C 0000180000180000300000300000600000600000C00000C0000180000180000300000300 000600000600000600000C00000C0000180000180000300000300000600000600000C000 00400000162C7AA000>54 D62 D<00040000000C0000000C0000 000C0000000C0000000C0000000C0000000C0000000C0000000C0000000C0000000C0000 000C0000000C0000000C0000000C0000000C0000000C0000000C0000000C0000000C0000 000C0000000C0000000C0000000C0000000C0000FFFFFFE0FFFFFFE01B1C7C9B23>I<00 0F0038006000E001C001C001C001C001C001C001C001C001C001C001C001C001C001C001 C0038007001E00F8001E000700038001C001C001C001C001C001C001C001C001C001C001 C001C001C001C001C000E000600038000F102D7DA117>102 DI E /Fp 58 122 df<007F07F001FF1FF003FF3FF007 807800070070000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F000FF F8FF80FFF8FF80FFF8FF800F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F 00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F00F0000F 00F0000F00F0001C20809F1B>11 D<0000F000F8F001F8F003F8F00780000700000F0000 0F00000F00000F00000F00000F00000F0000FFF8F0FFF8F0FFF8F00F00F00F00F00F00F0 0F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F0 0F00F00F00F0142180A018>I<00F8F001F8F003F8F00780F00700F00F00F00F00F00F00 F00F00F00F00F00F00F00F00F0FFF8F0FFF8F0FFF8F00F00F00F00F00F00F00F00F00F00 F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00F00 F01420809F18>I<007000E001C00380078007000E001E001E003C003C003C0078007800 780078007000F000F000F000F000F000F000F000F000F000F000F000F000700078007800 780078003C003C003C001E001E000E0007000780038001C000E000700C2E7EA112>40 DI<7878787818303060 60E0050A7D830D>44 DII<03F000 07F8001FFE001E1E003C0F00380700780780780780700380F003C0F003C0F003C0F003C0 F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0780780780780 7807803807003C0F001E1E001FFE000FFC0003F00012207E9E17>48 D<00C001C00FC0FFC0FFC0F3C003C003C003C003C003C003C003C003C003C003C003C003 C003C003C003C003C003C003C003C003C003C003C0FFFEFFFEFFFE0F1F7C9E17>I<07F0 000FFC001FFE00383F00700F00600780E00780E003C04003C04003C00003C00003C00003 C0000780000780000F00000E00001C00003C0000780000E00001C0000380000700000E00 001C0000380000700000FFFFC0FFFFC0FFFFC0121F7E9E17>I61 D<001F0000001F0000003F8000003B8000003B8000007BC0000073 C0000071C00000F1E00000E1E00000E0E00001E0F00001E0F00001C0F00003C0780003C0 78000380780007803C0007803C0007003C000FFFFE000FFFFE000FFFFE001E000F001E00 0F003C000F803C0007803C000780780007C0780003C0780003C0F00003E01B207F9F1E> 65 DI<001F C000FFF801FFFC03E03C07800C0F00001E00003E00003C00007C00007800007800007800 00F00000F00000F00000F00000F00000F00000F00000F000007800007800007800007C00 003C00003E00001E00000F000207800E03E03E01FFFC00FFF0001FC017227DA01D>IIII<001FE000FFF801FFFE03E03E07800E0F 00001E00003E00003C00007C0000780000780000780000F00000F00000F00000F00000F0 0000F00000F003FEF003FE7803FE78001E78001E7C001E3C001E3E001E1E001E0F001E07 801E03E03E01FFFE00FFF8001FC017227DA01E>III76 DI I<003F000000FFC00003FFF00007E1F8000F807C001F003E001E001E003C000F003C000F 00780007807800078078000780F00003C0F00003C0F00003C0F00003C0F00003C0F00003 C0F00003C0F00003C0F00003C0F80007C07800078078000780780007803C000F003C000F 001E001E001F003E000F807C0007E1F80003FFF00000FFC000003F00001A227DA021>I< FFF800FFFF00FFFF80F00FC0F003E0F001E0F000F0F000F0F000F0F000F0F000F0F000F0 F000F0F001E0F003E0F00FC0FFFF80FFFF00FFF800F00000F00000F00000F00000F00000 F00000F00000F00000F00000F00000F00000F00000F0000014207B9F1D>I82 D<01FC0007FF800FFFC0 1F03C03C00C03C00007800007800007800007800007800007C00003C00003F00001FE000 0FFC0007FE0001FF00003F800007C00003C00003E00001E00001E00001E00001E00001E0 0001C0C003C0F007C0FC0F807FFF001FFE0003F80013227EA019>IIII<780007807C000F003E001F001E001E000F003C000F807C0007807800 03C0F00003E1F00001F1E00000F3C000007FC000007F8000003F0000001F0000001E0000 003F0000007F8000007FC00000F3C00001F1E00001E0F00003C0F80007C0780007803C00 0F003E001F001E001E000F003C000F807C0007C0780003C0F00003E01B207F9F1E>88 DI91 D93 D<07E03FF87FFC701E401F 000F000F000F003F07FF1FFF7E0FF80FF00FF00FF00FF83F7FFF3FEF1F8F10147E9316> 97 DI<03F0 0FFC1FFE3E0E3C0278007800F000F000F000F000F000F000780078003C013E0F1FFF0FFE 03F010147E9314>I<000780000780000780000780000780000780000780000780000780 00078000078000078007C7800FF7801FFF803E1F807C0780780780F80780F00780F00780 F00780F00780F00780F00780F00780780780780F803E1F801FFF800FF78007C78011207E 9F17>I<03F0000FFC001FFE003E1F003C0700780700700380FFFF80FFFF80FFFF80F000 00F00000F000007000007800003C01003E07001FFF0007FE0001F80011147F9314>I<00 7E01FE03FE078007000F000F000F000F000F000F000F00FFF0FFF0FFF00F000F000F000F 000F000F000F000F000F000F000F000F000F000F000F000F000F000F20809F0E>I<03E0 F00FFFF01FFFF03E3E003C1E00780F00780F00780F00780F00780F003C1E003E3E001FFC 003FF80033E0003000003800003FFE003FFF801FFFC03FFFE07803F0F000F0F000F0F000 F0F801F07E07E03FFFC00FFF0003FC00141E7F9317>III<01E001E001E001E0000000 0000000000000000000000000001E001E001E001E001E001E001E001E001E001E001E001 E001E001E001E001E001E001E001E001E001E001E001E001E001E0C3C0FFC0FF803F000B 29839F0C>I108 DI I<01F80007FE001FFF803F0FC03C03C07801E07801E0F000F0F000F0F000F0F000F0F000 F0F000F07801E07801E03C03C03F0FC01FFF8007FE0001F80014147F9317>II114 D<07F01FFC3FFC780C7800780078007C003FC01FF00FF803F8007C003C003CC03CF07CFF F87FF00FC00E147F9311>I<1E001E001E001E001E001E00FFF0FFF0FFF01E001E001E00 1E001E001E001E001E001E001E001E001E001E001E201FF00FF007C00C1A7F9910>IIII<7801E07C03C03E07801E0F000F0F0007 9E0003FC0003F80001F80000F00001F00001F80003FC00079E000F0F000E0F001E07803C 03C07801E0F801F01414809315>I I E /Fq 42 123 df45 DI<00 000600000E00000E00001C00001C00001C0000380000380000380000700000700000E000 00E00000E00001C00001C00001C0000380000380000380000700000700000700000E0000 0E00000E00001C00001C00001C0000380000380000700000700000700000E00000E00000 E00001C00001C00001C0000380000380000380000700000700000700000E00000E00000E 00001C00001C0000380000380000380000700000700000700000E00000E00000C0000017 3C7DAC1E>I<007F000001FFC00007FFF0000FFFF8000FC1F8001F007C003F007E003E00 3E003C001E007C001F007C001F007C001F0078000F00F8000F80F8000F80F8000F80F800 0F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F800 0F80F8000F80F8000F8078000F007C001F007C001F007C001F003E003E003E003E003F00 7E001F80FC000FC1F8000FFFF80007FFF00001FFC000007F000019297EA71E>I<001800 00380000F80007F800FFF800FFF800F8F80000F80000F80000F80000F80000F80000F800 00F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F800 00F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F80000F800 7FFFF07FFFF07FFFF014287BA71E>I<00FE0003FFC007FFE00FFFF01F03F83C00FC3800 7E78003E70003EF0001FF0001F60001F20001F00001F00001F00001F00003E00003E0000 7C00007C0000F80001F00001E00003C0000780000F00001E00003C0000780000F00001E0 0003C0000780000F00001E00003C00007FFFFF7FFFFF7FFFFF7FFFFF18287EA71E>I<00 7F000001FFC00007FFF0000FFFF8001FC1F8003E007C003C003E0078003E0038003E0010 003E0000003E0000003E0000003C0000007C000000FC000001F8000007F00000FFE00000 FFC00000FFE00000FFF0000001FC0000007C0000003E0000001F0000001F0000000F8000 000F8000000F8000000F8000000F8040000F8060001F00F0001F00F8003F007E007E003F 81FC001FFFF8000FFFF00003FFE000007F000019297EA71E>I<0003F0000007F0000005 F000000DF000000DF000001DF0000039F0000039F0000079F0000079F00000F1F00000F1 F00001E1F00003E1F00003E1F00007C1F00007C1F0000F81F0000F81F0001F01F0001F01 F0003E01F0007C01F0007C01F000F801F000FFFFFF80FFFFFF80FFFFFF80FFFFFF800001 F0000001F0000001F0000001F0000001F0000001F0000001F0000001F0000001F0000001 F00019277EA61E>I<3FFFFC3FFFFC3FFFFC3FFFFC3E00003E00003E00003E00003E0000 3E00003E00003E00003E00003E00003E3F003EFFC03FFFE03FFFF03FE1F83F807C3F003E 3E003E00003E00001F00001F00001F00001F00001F00001F00001F20001F60003E70003E F8007C7C00FC3F03F81FFFF00FFFE007FF8000FE0018287EA61E>I<000FF000003FFC00 00FFFC0001FFFC0003F80C0007E000000FC000000F8000001F0000001E0000003E000000 3C0000007C0000007C0000007C3FE000F8FFF000F9FFF800FBFFFC00FF807E00FF003E00 FE003F00FC001F00FC001F00FC000F80F8000F80F8000F80F8000F80F8000F8078000F80 7C000F807C000F807C000F003E001F003E001F001F003E001F807C000FC1FC0007FFF800 03FFF00001FFC000007F000019297EA71E>II<007F000001FFC00007FFF0000FFFF8001FC1FC003F007E003E003E007E003F007C00 1F007C001F007C001F007C001F007C001F003E003E003E003E001F007C000FC1F80007FF F00003FFE00003FFE0000FFFF8001FC1FC003F007E003E003E007C001F007C001F00F800 0F80F8000F80F8000F80F8000F80F8000F80F8000F807C001F007C001F007E003F003F00 7E001FC1FC000FFFF80007FFF00003FFE000007F000019297EA71E>I<007F000001FFC0 0003FFE0000FFFF0000FC1F8001F007C003E007C007C003E007C001E007C001F00F8001F 00F8001F00F8000F00F8000F80F8000F80F8000F80F8000F80F8001F807C001F807C001F 807E003F803E007F803F00FF801FFFEF800FFFCF8007FF8F8003FE1F0000001F0000001F 0000001E0000003E0000003E0000007C0000007C000000F8001801F0001E07E0003FFFC0 001FFF80000FFE000003F8000019297EA71E>I<0001F000000003F800000003F8000000 07FC00000007BC00000007BC0000000F3E0000000F1E0000000F1E0000001F1F0000001E 1F0000001E0F0000003E0F8000003C0F8000003C078000007C07C000007807C00000F803 E00000F803E00000F003E00001F001F00001F001F00001E001F00003E000F80003E000F8 0003C000F80007FFFFFC0007FFFFFC000FFFFFFE000F80003E000F80003E001F00003F00 1F00001F001E00001F003E00000F803E00000F803C00000F807C000007C07C000007C078 000007C0F8000003E0F8000003E0232A7EA928>65 DI<0001FF00000FFFE0003FFFF8007FFFF800FE01 F801F8003003F0001007C000000F8000001F8000001F0000003E0000003E0000007E0000 007C0000007C0000007C000000F8000000F8000000F8000000F8000000F8000000F80000 00F8000000F8000000F8000000F80000007C0000007C0000007C0000007E0000003E0000 003E0000001F0000001F8000000F80000007C0000003F0000401F8001C00FE00FC007FFF FC003FFFF8000FFFE00001FF001E2C7CAA26>I<0001FF00000FFFE0003FFFFC007FFFFE 00FF01FE01F8003E03F0000C07C000000FC000001F8000001F0000003F0000003E000000 7E0000007C0000007C0000007C000000F8000000F8000000F8000000F8000000F8000000 F8000000F8000000F8001FFEF8001FFEF8001FFE7C001FFE7C00003E7C00003E7E00003E 3E00003E3F00003E1F00003E1F80003E0FC0003E07C0003E03F0003E01F8003E00FF00FE 007FFFFE003FFFFC000FFFE00001FF001F2C7CAA28>71 D 73 D<0001FC0000000FFF8000003FFFE000007FFFF00001FE03FC0003F800FE0007E000 3F0007C0001F000F80000F801F000007C01F000007C03E000003E03E000003E07C000001 F07C000001F07C000001F078000000F0F8000000F8F8000000F8F8000000F8F8000000F8 F8000000F8F8000000F8F8000000F8F8000000F8F8000000F8F8000000F87C000001F07C 000001F07C000001F07E000003F03E000003E03F000007E01F000007C01F80000FC00FC0 001F8007E0003F0007F0007F0003F800FE0001FE03FC0000FFFFF800003FFFE000000FFF 80000001FC0000252C7DAA2C>79 D82 D<007FC00001FFF80007FFFE000FFFFF001FC07F003F000F007E 0006007C0000007C000000F8000000F8000000F8000000F8000000F8000000FC0000007E 0000007F0000003F8000001FF800000FFF800007FFE00003FFF80000FFFC00000FFE0000 00FF0000003F0000001F8000000F8000000FC0000007C0000007C0000007C0000007C000 0007C0000007C000000F8060000F80F0001F00FC003F00FF80FE007FFFFC001FFFF80007 FFE00000FF80001A2C7DAA21>I<01FE000FFF803FFFC03FFFE03C03F03001F00001F800 00F80000F80000F80000F80000F8007FF807FFF81FFFF83FE0F87F00F8FC00F8F800F8F8 00F8F800F8FC01F87E07F87FFFF83FFFF81FFCF80FE0F8151B7E9A1D>97 DI<007FC001FFF007FFFC0F FFFC1FC07C1F00083E00007C00007C00007C0000F80000F80000F80000F80000F80000F8 0000F800007C00007C00007E00003E00001F000C1FC07C0FFFFC07FFFC01FFF0007F8016 1B7E9A1B>I<00003E00003E00003E00003E00003E00003E00003E00003E00003E00003E 00003E00003E00003E00003E00003E00FC3E03FF3E07FFFE0FFFFE1FC1FE3F007E3E003E 7C003E7C003EFC003EF8003EF8003EF8003EF8003EF8003EF8003EF8003EFC003E7C003E 7C003E3E007E3F00FE1FC1FE0FFFFE07FFBE03FF3E00FC3E172A7EA91F>I<007E0003FF 8007FFC00FFFE01F83F03F00F03E00787C00787C003878003CFFFFFCFFFFFCFFFFFCFFFF FCF80000F80000F800007800007C00007C00003E00003F000C1FC07C0FFFFC07FFFC01FF F0007F80161B7E9A1B>I<00F8078003FE7FC00FFFFFC01FFFFFC01F07C0003E03E0003E 03E0007C01F0007C01F0007C01F0007C01F0007C01F0007C01F0003E03E0003E03E0001F 07C0001FFFC0003FFF80003BFE000038F8000078000000780000003C0000003FFFC0003F FFF8001FFFFC001FFFFE003FFFFF007C007F00F8001F80F8000F80F8000F80F8000F80FC 001F807E003F003F80FE003FFFFE000FFFF80007FFF00000FF80001A287E9A1E>103 DII108 DII<007F0000 01FFC00007FFF0000FFFF8001FC1FC003F007E003E003E007C001F007C001F0078000F00 F8000F80F8000F80F8000F80F8000F80F8000F80F8000F80F8000F807C001F007C001F00 7E003F003E003E003F007E001FC1FC000FFFF80007FFF00001FFC000007F0000191B7E9A 1E>II114 D<03FC001FFF803FFFC07FFFC07C07C0F80080F80000F80000F800 00FC00007F80007FF8003FFE001FFF0007FF8000FFC0000FE00007E00003E00003E04003 E0E007E0FC0FC0FFFFC07FFF801FFE0003F800131B7E9A17>I<07C00007C00007C00007 C00007C00007C00007C000FFFFC0FFFFC0FFFFC007C00007C00007C00007C00007C00007 C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007C00007 C00007C04007E1C003FFE003FFE001FF8000FC0013227FA116>IIII121 DI E /Fr 12 119 df<000000003FFE00000E0000 000FFFFFC0001E0000007FFFFFF8003E000003FFFFFFFE00FE00000FFFFFFFFF81FE0000 3FFFF800FFC3FE0000FFFF80000FF7FE0001FFFC000003FFFE0007FFF0000001FFFE000F FFC00000007FFE001FFF800000003FFE003FFF000000001FFE007FFE000000000FFE00FF FC0000000007FE01FFF80000000007FE03FFF00000000003FE03FFF00000000001FE07FF E00000000001FE07FFE00000000000FE0FFFC00000000000FE0FFFC000000000007E1FFF C000000000007E1FFF8000000000007E3FFF8000000000007E3FFF8000000000003E3FFF 8000000000003E7FFF8000000000003E7FFF0000000000003E7FFF000000000000007FFF 00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF 00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF 00000000000000FFFF00000000000000FFFF00000000000000FFFF00000000000000FFFF 00000000000000FFFF000000000000007FFF000000000000007FFF000000000000007FFF 000000000000007FFF8000000000003E3FFF8000000000003E3FFF8000000000003E3FFF 8000000000003E1FFF8000000000003E1FFFC000000000003E0FFFC000000000007C0FFF C000000000007C07FFE000000000007C07FFE00000000000F803FFF00000000000F803FF F00000000001F801FFF80000000001F000FFFC0000000003E0007FFE0000000007E0003F FF000000000FC0001FFF800000001F80000FFFC00000003F000007FFF0000000FE000001 FFFC000001FC000000FFFF80000FF80000003FFFF8007FF00000000FFFFFFFFFC0000000 03FFFFFFFF00000000007FFFFFFC00000000000FFFFFE00000000000003FFE0000004749 79C756>67 D<0007FFFC000000007FFFFFC0000001FFFFFFF8000003FFFFFFFE000007FE 001FFF000007FF0003FFC0000FFF8001FFE0000FFF8000FFF0000FFF80007FF0000FFF80 007FF8000FFF80007FF80007FF00003FFC0007FF00003FFC0003FE00003FFC0000F80000 3FFC00000000003FFC00000000003FFC00000000003FFC00000000003FFC00000007FFFF FC000000FFFFFFFC000007FFFFFFFC00003FFFE03FFC0000FFFE003FFC0003FFF0003FFC 0007FFC0003FFC000FFF00003FFC001FFE00003FFC003FFC00003FFC007FF800003FFC00 7FF800003FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FFF000003FFC00FF F000003FFC00FFF000007FFC007FF80000FFFC007FF80001EFFC003FFC0003EFFC003FFF 0007CFFF000FFFC03F8FFFF807FFFFFF07FFFC01FFFFFC03FFFC007FFFF001FFFC0003FF 80007FF8362E7DAD3A>97 D<00001FFFC0000000FFFFF8000007FFFFFE00001FFFFFFF80 007FFC00FFC000FFE001FFC001FFC003FFE003FF8003FFE007FF0003FFE00FFE0003FFE0 0FFE0003FFE01FFC0001FFC01FFC0001FFC03FFC0000FF803FFC00003E007FF800000000 7FF8000000007FF800000000FFF800000000FFF800000000FFF800000000FFF800000000 FFF800000000FFF800000000FFF800000000FFF800000000FFF800000000FFF800000000 7FF8000000007FF8000000007FFC000000003FFC000000003FFC000000001FFC000000F8 1FFE000000F80FFE000000F80FFF000001F007FF800003F003FFC00007E001FFE0000FC0 00FFF0001F80007FFE00FF00001FFFFFFE000007FFFFF8000000FFFFE00000001FFE0000 2D2E7CAD35>99 D<00001FFE00000001FFFFE0000007FFFFF800001FFFFFFE00007FFC07 FF0000FFE001FF8001FFC0007FC003FF80003FE007FF00003FF00FFE00001FF01FFE0000 0FF81FFC00000FF83FFC00000FFC3FFC000007FC7FFC000007FC7FF8000007FC7FF80000 07FE7FF8000007FEFFF8000007FEFFF8000007FEFFFFFFFFFFFEFFFFFFFFFFFEFFFFFFFF FFFEFFFFFFFFFFFCFFF800000000FFF800000000FFF800000000FFF8000000007FF80000 00007FF8000000007FFC000000003FFC000000003FFC000000003FFC0000001C1FFE0000 003E0FFE0000003E07FF0000007E07FF000000FC03FF800001F801FFC00003F0007FF000 1FE0003FFE00FFC0001FFFFFFF800007FFFFFE000000FFFFF80000000FFF80002F2E7DAD 36>101 D<00FC0001FE0003FF0007FF800FFFC01FFFE01FFFE01FFFE01FFFE01FFFE01F FFE00FFFC007FF8003FF0001FE0000FC0000000000000000000000000000000000000000 0000000000000000000000000000007FC0FFFFC0FFFFC0FFFFC0FFFFC0FFFFC003FFC001 FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001 FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001 FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC001FFC0FFFFFFFFFFFFFF FFFFFFFFFFFFFFFF18497CC820>105 D<007FC000FFFFC000FFFFC000FFFFC000FFFFC0 00FFFFC00003FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC0 0001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC00001FFC000FFFFFF 80FFFFFF80FFFFFF80FFFFFF80FFFFFF8019487CC720>108 D<007FC001FFC00000FFE0 0000FFFFC00FFFF80007FFFC0000FFFFC03FFFFE001FFFFF0000FFFFC0FFFFFF007FFFFF 8000FFFFC1FC07FF80FE03FFC000FFFFC3E003FFC1F001FFE00003FFC7C001FFC3E000FF E00001FFCF0001FFE78000FFF00001FFDE0000FFEF00007FF00001FFDC0000FFEE00007F F00001FFFC0000FFFE00007FF80001FFF80000FFFC00007FF80001FFF00000FFF800007F F80001FFF00000FFF800007FF80001FFF00000FFF800007FF80001FFE00000FFF000007F F80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007F F80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007F F80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007F F80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007F F80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007F F80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007F F80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007F F80001FFE00000FFF000007FF80001FFE00000FFF000007FF80001FFE00000FFF000007F F80001FFE00000FFF000007FF800FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFF FFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFFFFF0FFFFFFC07FFFFFE03FFF FFF05C2E7CAD63>I<007FC001FFC00000FFFFC00FFFF80000FFFFC03FFFFE0000FFFFC0 FFFFFF0000FFFFC1FC07FF8000FFFFC3E003FFC00003FFC7C001FFC00001FFCF0001FFE0 0001FFDE0000FFE00001FFDC0000FFE00001FFFC0000FFF00001FFF80000FFF00001FFF0 0000FFF00001FFF00000FFF00001FFF00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF0 0001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE0 0000FFF00001FFE00000FFF000FFFFFFC07FFFFFE0FFFFFFC07FFFFFE0FFFFFFC07FFFFF E0FFFFFFC07FFFFFE0FFFFFFC07FFFFFE03B2E7CAD42>I<00000FFF0000000000FFFFF0 00000007FFFFFE0000001FFFFFFF8000003FFC03FFC00000FFE0007FF00001FF80001FF8 0003FF00000FFC0007FE000007FE000FFE000007FF000FFC000003FF001FFC000003FF80 3FFC000003FFC03FF8000001FFC03FF8000001FFC07FF8000001FFE07FF8000001FFE07F F8000001FFE0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8 000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF8000001FFF0FFF800 0001FFF07FF8000001FFE07FF8000001FFE07FF8000001FFE07FF8000001FFE03FFC0000 03FFC03FFC000003FFC01FFC000003FF801FFE000007FF800FFE000007FF0007FF00000F FE0003FF80001FFC0001FFC0003FF80000FFE0007FF000007FFC03FFE000001FFFFFFF80 000007FFFFFE00000000FFFFF0000000000FFF000000342E7DAD3B>I<0001F000000001 F000000001F000000001F000000001F000000001F000000003F000000003F000000003F0 00000007F000000007F000000007F00000000FF00000000FF00000001FF00000003FF000 00003FF00000007FF0000001FFF0000003FFF000000FFFFFFFC0FFFFFFFFC0FFFFFFFFC0 FFFFFFFFC0FFFFFFFFC000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000 FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FF F0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0000000FFF0 000000FFF0000000FFF0000000FFF0000000FFF001F000FFF001F000FFF001F000FFF001 F000FFF001F000FFF001F000FFF001F000FFF001F000FFF001F0007FF001E0007FF803E0 003FF803E0003FFC07C0001FFE0F80000FFFFF800007FFFE000001FFFC0000001FF00024 427EC12E>116 D<007FE000003FF000FFFFE0007FFFF000FFFFE0007FFFF000FFFFE000 7FFFF000FFFFE0007FFFF000FFFFE0007FFFF00003FFE00001FFF00001FFE00000FFF000 01FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE000 00FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF000 01FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE000 00FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF000 01FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE00000FFF00001FFE000 00FFF00001FFE00000FFF00001FFE00000FFF00001FFE00001FFF00001FFE00001FFF000 01FFE00001FFF00001FFE00003FFF00000FFE00007FFF00000FFE0000F7FF000007FE000 1F7FF000007FF0003E7FF800003FFC00FC7FFFE0001FFFFFF87FFFE00007FFFFE07FFFE0 0001FFFF807FFFE000003FFE007FFFE03B2E7CAD42>II E /Fs 8 117 df<00001E000000003E00000000FE00000003FE0000003FFE0000FFFFFE 0000FFFFFE0000FFFFFE0000FFCFFE0000000FFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000 000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE000000 0FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000F FE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE 0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE0000000FFE00 00000FFE0000000FFE0000000FFE00007FFFFFFFC07FFFFFFFC07FFFFFFFC07FFFFFFFC0 223879B731>49 D<0000001FFF000030000001FFFFE000F000000FFFFFFC01F000007FFF FFFE03F00001FFFE007F87F00003FFE0000FCFF0000FFF000003FFF0001FFC000001FFF0 003FF80000007FF0007FF00000003FF000FFC00000003FF001FFC00000001FF003FF8000 00000FF007FF000000000FF00FFF0000000007F00FFE0000000007F01FFE0000000003F0 1FFE0000000003F03FFC0000000003F03FFC0000000001F03FFC0000000001F07FFC0000 000001F07FF80000000001F07FF80000000000007FF8000000000000FFF8000000000000 FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000000000FFF80000 00000000FFF8000000000000FFF8000000000000FFF8000000000000FFF8000000000000 FFF80000000000007FF80000000000007FF80000000000007FF80000000000007FFC0000 000000F03FFC0000000000F03FFC0000000000F03FFC0000000000F01FFE0000000000F0 1FFE0000000001E00FFE0000000001E00FFF0000000001E007FF0000000003C003FF8000 000003C001FFC0000000078000FFE00000000F00007FF00000001F00003FF80000003E00 001FFC0000007C00000FFF000001F8000003FFE00007F0000001FFFE003FC00000007FFF FFFF000000000FFFFFFC0000000001FFFFF000000000001FFF0000003C3D7BBB47>67 D<001FFF00000001FFFFF0000003FFFFFC000007F007FE00000FF801FF00001FFC00FF80 001FFC007FC0001FFC007FE0001FFC003FE0000FF8003FF0000FF8003FF00007F0003FF0 0001C0003FF0000000003FF0000000003FF0000000003FF0000000FFFFF000000FFFFFF0 00007FF83FF00001FF803FF00007FE003FF0000FF8003FF0001FF0003FF0003FE0003FF0 007FE0003FF0007FE0003FF000FFC0003FF000FFC0003FF000FFC0003FF000FFC0003FF0 00FFC0007FF0007FE0007FF0007FE000DFF0003FF0039FF8001FFC0F0FFFF007FFFE0FFF F001FFFC07FFF0003FE000FFF02C267DA530>97 D<0001FFC000000FFFF800003FFFFE00 00FF80FF0001FE003F8007FC001FC00FF8000FE00FF8000FF01FF00007F03FF00007F83F F00007F87FE00007F87FE00003FC7FE00003FC7FE00003FCFFE00003FCFFFFFFFFFCFFFF FFFFFCFFFFFFFFFCFFE0000000FFE0000000FFE0000000FFE00000007FE00000007FE000 00007FE00000003FE00000003FF000003C1FF000003C1FF000003C0FF800007807FC0000 F803FE0001F001FF0007E000FFC03FC0003FFFFF000007FFFC000000FFE00026267DA52D >101 D<00FF00000000FFFF00000000FFFF00000000FFFF00000000FFFF0000000007FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF 0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF007FC00003FF 01FFF80003FF07FFFC0003FF0F03FE0003FF1C01FF0003FF3001FF8003FF6000FF8003FF E000FFC003FFC000FFC003FF8000FFC003FF8000FFC003FF8000FFC003FF0000FFC003FF 0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF 0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF 0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF0000FFC003FF 0000FFC003FF0000FFC003FF0000FFC0FFFFFC3FFFFFFFFFFC3FFFFFFFFFFC3FFFFFFFFF FC3FFFFF303C7CBB37>104 D<00FF01FF8000FFFF0FFFF000FFFF3FFFFC00FFFFFE03FF 00FFFFF000FF8003FFC0007FC003FF80003FE003FF00003FF003FF00001FF803FF00001F FC03FF00000FFC03FF00000FFE03FF00000FFE03FF000007FE03FF000007FF03FF000007 FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007FF03FF000007 FF03FF000007FF03FF000007FE03FF000007FE03FF00000FFE03FF00000FFC03FF00000F FC03FF00001FF803FF00001FF803FF00003FF003FF80003FE003FFC0007FC003FFF001FF 8003FFFC07FF0003FF3FFFFC0003FF0FFFF00003FF01FF000003FF0000000003FF000000 0003FF0000000003FF0000000003FF0000000003FF0000000003FF0000000003FF000000 0003FF0000000003FF0000000003FF0000000003FF0000000003FF00000000FFFFFC0000 00FFFFFC000000FFFFFC000000FFFFFC00000030377DA537>112 D<00FE03F000FFFE0FFE00FFFE1FFF00FFFE3C3F80FFFE707FC007FE60FFE003FEE0FFE0 03FEC0FFE003FFC0FFE003FF807FC003FF807FC003FF803F8003FF800E0003FF00000003 FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF 00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00000003FF00 000003FF00000003FF00000003FF00000003FF00000003FF000000FFFFFE0000FFFFFE00 00FFFFFE0000FFFFFE000023267DA529>114 D<00078000000780000007800000078000 00078000000F8000000F8000000F8000000F8000001F8000001F8000003F8000003F8000 007F800000FF800001FF800007FF80001FFFFFF0FFFFFFF0FFFFFFF0FFFFFFF001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF800001FF8000 01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C01FF803C00FF8078 00FFC078007FC070003FE0E0001FFFC00007FF800001FF001E377EB626>116 D E /Ft 76 124 df<001F83E000F06E3001C078780380F8780300F03007007000070070 000700700007007000070070000700700007007000FFFFFF800700700007007000070070 000700700007007000070070000700700007007000070070000700700007007000070070 000700700007007000070070000700700007007000070070007FE3FF001D20809F1B>11 D<003F0000E0C001C0C00381E00701E00701E00700000700000700000700000700000700 00FFFFE00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700 E00700E00700E00700E00700E00700E00700E00700E07FC3FE1720809F19>I<003FE000 E0E001C1E00381E00700E00700E00700E00700E00700E00700E00700E00700E0FFFFE007 00E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E007 00E00700E00700E00700E00700E00700E07FE7FE1720809F19>I<001F81F80000F04F04 0001C07C06000380F80F000300F00F000700F00F00070070000007007000000700700000 070070000007007000000700700000FFFFFFFF0007007007000700700700070070070007 007007000700700700070070070007007007000700700700070070070007007007000700 700700070070070007007007000700700700070070070007007007000700700700070070 07007FE3FE3FF02420809F26>I<7038F87CFC7EFC7E743A040204020402080408041008 1008201040200F0E7E9F17>34 D<70F8FCFC74040404080810102040060E7C9F0D>39 D<0020004000800100020006000C000C00180018003000300030007000600060006000E0 00E000E000E000E000E000E000E000E000E000E000E00060006000600070003000300030 00180018000C000C000600020001000080004000200B2E7DA112>I<8000400020001000 08000C00060006000300030001800180018001C000C000C000C000E000E000E000E000E0 00E000E000E000E000E000E000E000C000C000C001C00180018001800300030006000600 0C00080010002000400080000B2E7DA112>I<0006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000000600000006000000 06000000060000FFFFFFF0FFFFFFF0000600000006000000060000000600000006000000 060000000600000006000000060000000600000006000000060000000600000006000000 0600001C207D9A23>43 D<70F8FCFC74040404080810102040060E7C840D>II<70F8F8F87005057C840D>I<000100030003000600060006000C000C000C 00180018001800300030003000600060006000C000C000C0018001800180030003000300 0600060006000C000C000C00180018001800300030003000600060006000C000C000C000 102D7DA117>I<03F0000E1C001C0E00180600380700700380700380700380700380F003 C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003C0F003 C07003807003807003807807803807001806001C0E000E1C0003F000121F7E9D17>I<01 8003800F80F3800380038003800380038003800380038003800380038003800380038003 8003800380038003800380038003800380038007C0FFFE0F1E7C9D17>I<03F0000C1C00 100E00200700400780800780F007C0F803C0F803C0F803C02007C00007C0000780000780 000F00000E00001C0000380000700000600000C0000180000300000600400C0040180040 1000803FFF807FFF80FFFF80121E7E9D17>I<03F0000C1C00100E00200F00780F807807 80780780380F80000F80000F00000F00000E00001C0000380003F000003C00000E00000F 000007800007800007C02007C0F807C0F807C0F807C0F00780400780400F00200E001C3C 0003F000121F7E9D17>I<000600000600000E00000E00001E00002E00002E00004E0000 8E00008E00010E00020E00020E00040E00080E00080E00100E00200E00200E00400E00C0 0E00FFFFF0000E00000E00000E00000E00000E00000E00000E0000FFE0141E7F9D17>I< 1803001FFE001FFC001FF8001FE00010000010000010000010000010000010000011F000 161C00180E001007001007800003800003800003C00003C00003C07003C0F003C0F003C0 E00380400380400700200600100E000C380003E000121F7E9D17>I<007C000182000701 000E03800C07801C0780380300380000780000700000700000F1F000F21C00F40600F807 00F80380F80380F003C0F003C0F003C0F003C0F003C07003C07003C07003803803803807 001807000C0E00061C0001F000121F7E9D17>I<4000007FFFC07FFF807FFF8040010080 020080020080040000080000080000100000200000200000400000400000C00000C00001 C00001800003800003800003800003800007800007800007800007800007800007800007 8000030000121F7D9D17>I<03F0000C0C00100600300300200180600180600180600180 7001807803003E03003F06001FC8000FF00003F80007FC000C7E00103F00300F80600380 4001C0C001C0C000C0C000C0C000C0C000806001802001001002000C0C0003F000121F7E 9D17>I<03F0000E18001C0C00380600380700700700700380F00380F00380F003C0F003 C0F003C0F003C0F003C07007C07007C03807C0180BC00E13C003E3C00003800003800003 80000700300700780600780E00700C002018001070000FC000121F7E9D17>I<70F8F8F8 700000000000000000000070F8F8F87005147C930D>I<70F8F8F8700000000000000000 000070F0F8F878080808101010202040051D7C930D>I<7FFFFFE0FFFFFFF00000000000 000000000000000000000000000000000000000000000000000000FFFFFFF07FFFFFE01C 0C7D9023>61 D<000100000003800000038000000380000007C0000007C0000007C00000 09E0000009E0000009E0000010F0000010F0000010F00000207800002078000020780000 403C0000403C0000403C0000801E0000801E0000FFFE0001000F0001000F0001000F0002 0007800200078002000780040003C00E0003C01F0007E0FFC03FFE1F207F9F22>65 DI<000FC040007030C001C009C0 038005C0070003C00E0001C01E0000C01C0000C03C0000C07C0000407C00004078000040 F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8000000 780000007C0000407C0000403C0000401C0000401E0000800E0000800700010003800200 01C0040000703800000FC0001A217D9F21>IIII<000FE0200078186000E004E0038002E0070001E0 0F0000E01E0000601E0000603C0000603C0000207C00002078000020F8000000F8000000 F8000000F8000000F8000000F8000000F8000000F8007FFCF80003E0780001E07C0001E0 3C0001E03C0001E01E0001E01E0001E00F0001E0070001E0038002E000E0046000781820 000FE0001E217D9F24>III76 DII<001F800000F0F0 0001C0380007801E000F000F000E0007001E0007803C0003C03C0003C07C0003E0780001 E0780001E0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001F0F80001 F0F80001F0780001E07C0003E07C0003E03C0003C03C0003C01E0007800E0007000F000F 0007801E0001C0380000F0F000001F80001C217D9F23>II82 D<07E0800C1980100780300380600180600180E00180E00080E00080E00080F00000 F000007800007F00003FF0001FFC000FFE0003FF00001F800007800003C00003C00001C0 8001C08001C08001C08001C0C00180C00380E00300F00600CE0C0081F80012217D9F19> I<7FFFFFE0780F01E0600F0060400F0020400F0020C00F0030800F0010800F0010800F00 10800F0010000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F00 00000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F0000000F00 00000F0000000F0000001F800007FFFE001C1F7E9E21>II87 D<7FF83FF80FE00FC007C0070003C0020001E0040001F00C0000F0080000781000007C10 00003C2000003E4000001E4000000F8000000F8000000780000003C0000007E0000005E0 000009F0000018F8000010780000207C0000603C0000401E0000801F0001800F00010007 80020007C0070003C01F8007E0FFE01FFE1F1F7F9E22>I91 D<080410082010201040204020804080408040B85CFC7EFC7E7C3E381C0F0E7B 9F17>II<1FE000303000781800781C00300E0000 0E00000E00000E0000FE00078E001E0E00380E00780E00F00E10F00E10F00E10F01E1078 1E103867200F83C014147E9317>97 D<0E0000FE00000E00000E00000E00000E00000E00 000E00000E00000E00000E00000E00000E3E000EC3800F01C00F00E00E00E00E00700E00 700E00780E00780E00780E00780E00780E00780E00700E00700E00E00F00E00D01C00CC3 00083E0015207F9F19>I<03F80E0C1C1E381E380C70007000F000F000F000F000F000F0 0070007000380138011C020E0C03F010147E9314>I<000380003F800003800003800003 8000038000038000038000038000038000038000038003E380061B801C07803803803803 80700380700380F00380F00380F00380F00380F00380F003807003807003803803803807 801C07800E1B8003E3F815207E9F19>I<03F0000E1C001C0E0038070038070070070070 0380F00380F00380FFFF80F00000F00000F000007000007000003800801800800C010007 060001F80011147F9314>I<007C00C6018F038F07060700070007000700070007000700 FFF007000700070007000700070007000700070007000700070007000700070007000700 07007FF01020809F0E>I<0000E003E3300E3C301C1C30380E00780F00780F00780F0078 0F00780F00380E001C1C001E380033E0002000002000003000003000003FFE001FFF800F FFC03001E0600070C00030C00030C00030C000306000603000C01C038003FC00141F7F94 17>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00000E00000E0000 0E00000E3E000E43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16207F9F19>I<1C00 1E003E001E001C000000000000000000000000000E007E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E00FFC00A1F809E0C>I<00E001F001 F001F000E0000000000000000000000000007007F000F000700070007000700070007000 70007000700070007000700070007000700070007000700070007000706070F060F0C061 803F000C28829E0E>I<0E0000FE00000E00000E00000E00000E00000E00000E00000E00 000E00000E00000E00000E0FF00E03C00E03000E02000E04000E08000E10000E30000E70 000EF8000F38000E1C000E1E000E0E000E07000E07800E03800E03C00E03E0FFCFF81520 7F9F18>I<0E00FE000E000E000E000E000E000E000E000E000E000E000E000E000E000E 000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00FFE00B2080 9F0C>I<0E1F01F000FE618618000E81C81C000F00F00E000F00F00E000E00E00E000E00 E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E000E00E0 0E000E00E00E000E00E00E000E00E00E000E00E00E000E00E00E00FFE7FE7FE023147F93 26>I<0E3E00FE43000E81800F01C00F01C00E01C00E01C00E01C00E01C00E01C00E01C0 0E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C0FFE7FC16147F9319>I<01F8 00070E001C03803801C03801C07000E07000E0F000F0F000F0F000F0F000F0F000F0F000 F07000E07000E03801C03801C01C0380070E0001F80014147F9317>I<0E3E00FEC3800F 01C00F00E00E00E00E00F00E00700E00780E00780E00780E00780E00780E00780E00700E 00F00E00E00F01E00F01C00EC3000E3E000E00000E00000E00000E00000E00000E00000E 00000E0000FFE000151D7F9319>I<03E0800619801C05803C0780380380780380700380 F00380F00380F00380F00380F00380F003807003807803803803803807801C0B800E1380 03E380000380000380000380000380000380000380000380000380003FF8151D7E9318> I<0E78FE8C0F1E0F1E0F0C0E000E000E000E000E000E000E000E000E000E000E000E000E 000E00FFE00F147F9312>I<1F9030704030C010C010C010E00078007F803FE00FF00070 803880188018C018C018E030D0608F800D147E9312>I<020002000200060006000E000E 003E00FFF80E000E000E000E000E000E000E000E000E000E000E000E080E080E080E080E 080610031001E00D1C7F9B12>I<0E01C0FE1FC00E01C00E01C00E01C00E01C00E01C00E 01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E01C00E03C00603C0030DC001 F1FC16147F9319>III<7FC3FC0F01E00701C007018003810001C2 0000E40000EC00007800003800003C00007C00004E000087000107000303800201C00601 E01E01E0FF07FE1714809318>II< 3FFF380E200E201C40384078407000E001E001C00380078007010E011E011C0338027006 700EFFFE10147F9314>II E /Fu 45 122 df<70F8FCFC7404040404080810102040060F7C840E>44 DI<70 F8F8F87005057C840E>I<01F000071C000C06001803003803803803807001C07001C070 01C07001C0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F001E0F0 01E0F001E0F001E0F001E07001C07001C07001C07803C03803803803801C07000C060007 1C0001F00013227EA018>48 D<008003800F80F380038003800380038003800380038003 800380038003800380038003800380038003800380038003800380038003800380038003 80038007C0FFFE0F217CA018>I<03F8000C1E001007002007804007C07807C07803C078 07C03807C0000780000780000700000F00000E0000380003F000001C00000F0000078000 07800003C00003C00003E02003E07003E0F803E0F803E0F003C04003C040078020078010 0F000C1C0003F00013227EA018>51 D<000200000600000E00000E00001E00001E00002E 00004E00004E00008E00008E00010E00020E00020E00040E00040E00080E00100E00100E 00200E00200E00400E00800E00FFFFF8000E00000E00000E00000E00000E00000E00000E 00001F0001FFF015217FA018>I<1000801E07001FFF001FFE001FF80013E00010000010 000010000010000010000010000010F800130E001407001803801003800001C00001C000 01E00001E00001E00001E07001E0F001E0F001E0E001C08001C04003C040038020070010 06000C1C0003F00013227EA018>I<007E0001C1000300800601C00E03C01C03C0180180 380000380000780000700000700000F0F800F30C00F40600F40300F80380F801C0F001C0 F001E0F001E0F001E0F001E0F001E07001E07001E07001E03801C03801C01803801C0300 0C0600070C0001F00013227EA018>I<01F800060E000803001001802001802000C06000 C06000C06000C07000C07801803E01003F02001FC4000FF80003F80003FC00067F00083F 80100F803007C06001C06000E0C000E0C00060C00060C00060C000606000406000C03000 801803000E0E0003F00013227EA018>56 D<01F000060C000C0600180700380380700380 700380F001C0F001C0F001C0F001E0F001E0F001E0F001E0F001E07001E07003E03803E0 1805E00C05E00619E003E1E00001C00001C00001C0000380000380300300780700780600 700C002018001030000FC00013227EA018>I<0001800000018000000180000003C00000 03C0000003C0000005E0000005E000000DF0000008F0000008F0000010F8000010780000 10780000203C0000203C0000203C0000401E0000401E0000401E0000800F0000800F0000 FFFF000100078001000780030007C0020003C0020003C0040003E0040001E0040001E00C 0000F00C0000F03E0001F8FF800FFF20237EA225>65 D<0007E0100038183000E0063001 C00170038000F0070000F00E0000701E0000701C0000303C0000303C0000307C00001078 00001078000010F8000000F8000000F8000000F8000000F8000000F8000000F8000000F8 00000078000000780000107C0000103C0000103C0000101C0000201E0000200E00004007 0000400380008001C0010000E0020000381C000007E0001C247DA223>67 D69 DI< FFFC0FC00780078007800780078007800780078007800780078007800780078007800780 078007800780078007800780078007800780078007800780078007800FC0FFFC0E227EA1 12>73 D 77 DI 80 D82 D<03F0200C0C601802603001E07000E0600060E00060E00060E00020E00020 E00020F00000F000007800007F00003FF0001FFE000FFF0003FF80003FC00007E00001E0 0000F00000F0000070800070800070800070800070C00060C00060E000C0F000C0C80180 C6070081FC0014247DA21B>I<7FFFFFF878078078600780184007800840078008400780 08C007800C80078004800780048007800480078004000780000007800000078000000780 000007800000078000000780000007800000078000000780000007800000078000000780 000007800000078000000780000007800000078000000780000007800000078000000FC0 0003FFFF001E227EA123>I<0FE0001838003C0C003C0E00180700000700000700000700 00FF0007C7001E07003C0700780700700700F00708F00708F00708F00F087817083C2390 0FC1E015157E9418>97 D<0E0000FE00001E00000E00000E00000E00000E00000E00000E 00000E00000E00000E00000E00000E00000E1F000E61C00E80600F00300E00380E003C0E 001C0E001E0E001E0E001E0E001E0E001E0E001E0E001E0E001C0E003C0E00380F00700C 80600C41C0083F0017237FA21B>I<01FE000703000C07801C0780380300780000700000 F00000F00000F00000F00000F00000F00000F000007000007800403800401C00800C0100 07060001F80012157E9416>I<0000E0000FE00001E00000E00000E00000E00000E00000 E00000E00000E00000E00000E00000E00000E001F8E00704E00C02E01C01E03800E07800 E07000E0F000E0F000E0F000E0F000E0F000E0F000E0F000E07000E07800E03800E01801 E00C02E0070CF001F0FE17237EA21B>I<01FC000707000C03801C01C03801C07801E070 00E0F000E0FFFFE0F00000F00000F00000F00000F000007000007800203800201C00400E 008007030000FC0013157F9416>I<003C00C6018F038F030F0700070007000700070007 00070007000700FFF8070007000700070007000700070007000700070007000700070007 00070007000700070007807FF8102380A20F>I<00007001F198071E180E0E181C07001C 07003C07803C07803C07803C07801C07001C07000E0E000F1C0019F00010000010000018 00001800001FFE000FFFC00FFFE03800F0600030400018C00018C00018C0001860003060 00303800E00E038003FE0015217F9518>I<0E0000FE00001E00000E00000E00000E0000 0E00000E00000E00000E00000E00000E00000E00000E00000E1F800E60C00E80E00F0070 0F00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E0070 0E00700E00700E00700E0070FFE7FF18237FA21B>I<1C001E003E001E001C0000000000 0000000000000000000000000E00FE001E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E00FFC00A227FA10E>I<01C003E003E003E001C000 00000000000000000000000000000001E00FE001E000E000E000E000E000E000E000E000 E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E060E0F0C0F1 8061803E000B2C82A10F>I<0E0000FE00001E00000E00000E00000E00000E00000E0000 0E00000E00000E00000E00000E00000E00000E03FC0E01F00E01C00E01800E02000E0400 0E08000E10000E38000EF8000F1C000E1E000E0E000E07000E07800E03C00E01C00E01E0 0E00F00E00F8FFE3FE17237FA21A>I<0E00FE001E000E000E000E000E000E000E000E00 0E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E000E00 0E000E000E000E000E000E00FFE00B237FA20E>I<0E1FC07F00FE60E183801E807201C0 0F003C00E00F003C00E00E003800E00E003800E00E003800E00E003800E00E003800E00E 003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E003800E00E00 3800E00E003800E00E003800E0FFE3FF8FFE27157F942A>I<0E1F80FE60C01E80E00F00 700F00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00700E00 700E00700E00700E00700E0070FFE7FF18157F941B>I<01FC000707000C01801800C038 00E0700070700070F00078F00078F00078F00078F00078F00078F000787000707800F038 00E01C01C00E038007070001FC0015157F9418>I<0E1F00FE61C00E80600F00700E0038 0E003C0E001C0E001E0E001E0E001E0E001E0E001E0E001E0E001E0E003C0E003C0E0038 0F00700E80E00E41C00E3F000E00000E00000E00000E00000E00000E00000E00000E0000 0E0000FFE000171F7F941B>I<0E3CFE461E8F0F0F0F060F000E000E000E000E000E000E 000E000E000E000E000E000E000E000F00FFF010157F9413>114 D<0F8830786018C018C008C008E008F0007F803FE00FF001F8003C801C800C800CC00CC0 08E018D0308FC00E157E9413>I<02000200020002000600060006000E001E003E00FFF8 0E000E000E000E000E000E000E000E000E000E000E000E040E040E040E040E040E040708 030801F00E1F7F9E13>I<0E0070FE07F01E00F00E00700E00700E00700E00700E00700E 00700E00700E00700E00700E00700E00700E00700E00700E00F00E00F006017003827800 FC7F18157F941B>III121 D E /Fv 20 118 df45 D68 D73 D77 D 80 D<007F802001FFE02007C078600F001C601E0006E03C0003E0380001E0780000E070 0000E070000060F0000060F0000060F0000020F0000020F0000020F8000020F80000007C 0000007E0000003F0000003FC000001FF800000FFF800007FFF80003FFFC0000FFFF0000 0FFF800000FFC000001FE0000007E0000003F0000001F0000000F0000000F8000000F880 00007880000078800000788000007880000078C0000078C0000070E00000F0E00000E0F0 0000E0F80001C0EC000380C7000700C1F01E00807FFC00800FF0001D337CB125>83 D<00FE00000303C0000C00E00010007000100038003C003C003E001C003E001E003E001E 0008001E0000001E0000001E0000001E00000FFE0000FC1E0003E01E000F801E001F001E 003E001E003C001E007C001E00F8001E04F8001E04F8001E04F8003E04F8003E0478003E 047C005E043E008F080F0307F003FC03E01E1F7D9E21>97 D<003F8000E0600380180700 040F00041E001E1C003E3C003E7C003E7C0008780000F80000F80000F80000F80000F800 00F80000F80000F80000F800007800007C00007C00003C00011E00011E00020F00020700 0403801800E060003F80181F7D9E1D>99 D<000001E000003FE000003FE0000003E00000 01E0000001E0000001E0000001E0000001E0000001E0000001E0000001E0000001E00000 01E0000001E0000001E0000001E0000001E0000001E0001F81E000F061E001C019E00780 05E00F0003E00E0003E01E0001E03C0001E03C0001E07C0001E0780001E0F80001E0F800 01E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0F80001E0780001E07800 01E03C0001E03C0001E01C0001E01E0003E00E0005E0070009E0038011F000E061FF003F 81FF20327DB125>I<003F800000E0E0000380380007003C000E001E001E001E001C000F 003C000F007C000F0078000F8078000780F8000780F8000780FFFFFF80F8000000F80000 00F8000000F8000000F8000000F8000000780000007C0000003C0000003C0000801E0000 800E0001000F0002000780020001C00C0000F03000001FC000191F7E9E1D>I<0007E000 1C1000383800707C00E07C01E07C01C03803C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C000FFFFC0FFFFC003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C00003C00003C00003C00003C00007E0007FFF007F FF0016327FB114>I<000000F0007F030801C1C41C0380E81C070070080F0078001E003C 001E003C003E003E003E003E003E003E003E003E003E003E003E003E001E003C001E003C 000F007800070070000780E00009C1C000087F0000180000001800000018000000180000 00180000001C0000000E0000000FFFF80007FFFF0003FFFF800E000FC0180001E0300000 F070000070E0000038E0000038E0000038E0000038E00000387000007070000070380000 E01C0001C00700070001C01C00003FE0001E2F7E9F21>I<07000F801F801F800F800700 000000000000000000000000000000000000000000000780FF80FF800F80078007800780 078007800780078007800780078007800780078007800780078007800780078007800780 0780078007800FC0FFF8FFF80D307EAF12>105 D<0780FE001FC000FF83078060F000FF 8C03C18078000F9001E2003C0007A001E4003C0007A000F4001E0007C000F8001E0007C0 00F8001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000 F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0 001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F000 1E00078000F0001E00078000F0001E00078000F0001E00078000F0001E00078000F0001E 000FC001F8003F00FFFC1FFF83FFF0FFFC1FFF83FFF0341F7E9E38>109 D<0780FE0000FF83078000FF8C03C0000F9001E00007A001E00007A000F00007C000F000 07C000F000078000F000078000F000078000F000078000F000078000F000078000F00007 8000F000078000F000078000F000078000F000078000F000078000F000078000F0000780 00F000078000F000078000F000078000F000078000F000078000F000078000F0000FC001 F800FFFC1FFF80FFFC1FFF80211F7E9E25>I<001FC00000F0780001C01C00070007000F 0007801E0003C01C0001C03C0001E03C0001E0780000F0780000F0780000F0F80000F8F8 0000F8F80000F8F80000F8F80000F8F80000F8F80000F8F80000F8780000F07C0001F03C 0001E03C0001E01E0003C01E0003C00F00078007800F0001C01C0000F07800001FC0001D 1F7E9E21>I<0783E0FF8C18FF907C0F907C07A07C07C03807C00007C00007C000078000 078000078000078000078000078000078000078000078000078000078000078000078000 0780000780000780000780000780000780000FC000FFFE00FFFE00161F7E9E19>114 D<01FC100E03301800F0300070600030E00030E00010E00010E00010F00010F800007E00 003FF0001FFF000FFFC003FFE0003FF00001F80000F880003C80003C80001CC0001CC000 1CE0001CE00018F00038F00030CC0060C301C080FE00161F7E9E1A>I<00400000400000 400000400000400000C00000C00000C00001C00001C00003C00007C0000FC0001FFFE0FF FFE003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003C00003 C00003C00003C00003C00003C00003C01003C01003C01003C01003C01003C01003C01003 C01001C02001E02000E0400078C0001F00142C7FAB19>I<078000F000FF801FF000FF80 1FF0000F8001F000078000F000078000F000078000F000078000F000078000F000078000 F000078000F000078000F000078000F000078000F000078000F000078000F000078000F0 00078000F000078000F000078000F000078000F000078000F000078000F000078001F000 078001F000078001F000038002F00003C004F00001C008F800007030FF80001FC0FF8021 1F7E9E25>I E /Fw 5 85 df<00000000C00000000000E00000000001E00000000003E0 0000000003E00000000007E00000000007E0000000000FE0000000000FE0000000001FE0 000000001FE00000000037E00000000067E00000000067E000000000C7E000000000C7F0 0000000183F00000000183F00000000303F00000000703F00000000603F00000000C03F0 0000000C03F00000001803F00000001803F00000003003F00000003003F00000006003F0 000000C003F0000000C003F00000018003F00000018003F8000003FFFFF8000003FFFFF8 0000060001F800000E0001F800000C0001F80000180001F80000180001F80000300001F8 0000300001F80000600001F80000E00001F80000C00001F80001C00001F80001C00001F8 0007C00001FC001FC00003FC00FFF8007FFFE0FFF8007FFFE02B327BB135>65 D<000FFFFFFE0000000FFFFFFF800000007F000FE00000007E0003F00000007E0000F800 00007E0000FC0000007E00007C000000FC00003E000000FC00003E000000FC00003F0000 00FC00001F000001F800001F000001F800001F800001F800001F800001F800001F800003 F000001F800003F000001F800003F000001F800003F000001F800007E000003F800007E0 00003F800007E000003F800007E000003F80000FC000003F00000FC000007F00000FC000 007F00000FC000007F00001F8000007E00001F800000FE00001F800000FE00001F800000 FC00003F000001FC00003F000001F800003F000001F800003F000003F000007E000003E0 00007E000007E000007E00000FC000007E00000F800000FC00001F800000FC00003F0000 00FC00007E000000FC0000FC000001F80001F0000001F80003E0000001F8000FC0000003 F8007F000000FFFFFFFC000000FFFFFFE000000031317BB036>68 D<000FFFFFFFFC000FFFFFFFFC00007F0001FC00007E00007C00007E00003C00007E0000 3C00007E0000180000FC0000180000FC0000180000FC0000180000FC0000180001F80000 180001F80000180001F80000180001F80000180003F00080100003F00180000003F00180 000003F00180000007E00300000007E00300000007E00700000007E01F0000000FFFFE00 00000FFFFE0000000FC01E0000000FC00E0000001F800C0000001F800C0000001F800C00 00001F800C0000003F00180000003F00080000003F00000000003F00000000007E000000 00007E00000000007E00000000007E0000000000FC0000000000FC0000000000FC000000 0000FC0000000001F80000000001F80000000001F80000000003F800000000FFFFF00000 00FFFFF00000002E317BB02F>70 D<000FFFFFF000000FFFFFFE0000007F003F8000007E 000FC000007E0007E000007E0003F000007E0001F80000FC0001F80000FC0001F80000FC 0001F80000FC0001F80001F80003F80001F80003F80001F80003F80001F80003F00003F0 0007F00003F00007E00003F0000FC00003F0000FC00007E0001F000007E0007E000007E0 00FC000007E007F000000FFFFFC000000FFFFF0000000FC00F8000000FC003C000001F80 03E000001F8001F000001F8001F000001F8001F800003F0001F800003F0001F800003F00 01F800003F0001F800007E0003F800007E0003F800007E0003F000007E0003F00000FC00 07F00000FC0007F00000FC0007F00800FC0007F00C01F80007F01801F80007F01801F800 03F03003F80003F030FFFFE001F0E0FFFFE000FFC0000000003F002E327BB034>82 D<07FFFFFFFFF00FFFFFFFFFF00FC00FE003F01E000FC000F01C000FC000E018000FC000 E038000FC0006030001F8000E030001F8000E060001F8000C060001F8000C060003F0000 C0C0003F0000C0C0003F0000C0C0003F0000C080007E00008000007E00000000007E0000 0000007E0000000000FC0000000000FC0000000000FC0000000000FC0000000001F80000 000001F80000000001F80000000001F80000000003F00000000003F00000000003F00000 000003F00000000007E00000000007E00000000007E00000000007E0000000000FC00000 00000FC0000000000FC0000000000FC0000000001F80000000001F80000000001F800000 00001F80000000003F00000000003F00000000003F0000000000FF00000000FFFFFF0000 00FFFFFF0000002C3173B033>84 D E end %%EndProlog %%BeginSetup %%Feature: *Resolution 300dpi TeXDict begin %%EndSetup %%Page: 0 1 0 0 bop 795 908 a Fw(D)26 b(R)g(A)f(F)h(T)225 999 y Fv(Do)r(cumen)n(t) 20 b(for)i(a)f(Standard)g(Message-P)n(assing)f(In)n(terface)621 1194 y Fu(Message)c(P)o(assing)h(In)o(terface)e(F)l(orum)832 1320 y(Ma)o(y)h(30,)g(1995)77 1378 y(This)h(w)o(ork)f(w)o(as)h(supp)q (orted)g(in)f(part)h(b)o(y)e(ARP)l(A)h(and)h(NSF)e(under)h(gran)o(t)h (ASC-9310330,)i(the)192 1436 y(National)d(Science)f(F)l(oundation)i (Science)e(and)i(T)l(ec)o(hnology)f(Cen)o(ter)f(Co)q(op)q(erativ)o(e)76 1494 y(Agreemen)o(t)e(No.)22 b(CCR-8809615,)d(and)e(b)o(y)e(the)h (Commission)e(of)j(the)f(Europ)q(ean)i(Comm)o(unit)n(y)654 1552 y(through)f(Esprit)f(pro)s(ject)g(P6643.)p eop %%Page: 1 2 1 1 bop 166 45 a Ft(This)20 b(is)h(the)f(result)g(of)f(a)h(LaT)l(eX)g (run)g(of)g(a)f(draft)g(of)h(a)f(single)j(c)o(hapter)d(of)h(the)g(MPIF) f(Final)75 102 y(Rep)q(ort)d(do)q(cumen)o(t.)969 2828 y(i)p eop %%Page: 1 3 1 2 bop 75 356 a Fs(Chapter)34 b(1)75 564 y Fr(Collecti)q(v)m(e)42 b(Comm)m(unication)75 805 y Fq(1.1)59 b(Intro)r(duction)18 b(and)i(Overview)75 906 y Ft(Collectiv)o(e)e(comm)o(unication)g(is)f (de\014ned)h(as)e(comm)o(unication)i(that)e(in)o(v)o(olv)o(es)h(a)f (group)h(of)f(pro)q(cesses.)75 963 y(The)f(functions)h(of)f(this)h(t)o (yp)q(e)f(pro)o(vided)h(b)o(y)f Fp(MPI)g Ft(are)g(the)g(follo)o(wing:) 143 1056 y Fo(\017)23 b Ft(Barrier)15 b(sync)o(hronization)h(across)f (all)h(group)f(mem)o(b)q(ers)g(\(Sec.)20 b(1.3\).)143 1150 y Fo(\017)j Ft(Broadcast)15 b(from)h(one)h(mem)o(b)q(er)g(to)f (all)h(mem)o(b)q(ers)g(of)f(a)h(group)f(\(Sec.)25 b(1.4\).)d(This)c(is) f(sho)o(wn)f(in)189 1207 y(\014gure)f(1.1.)143 1301 y Fo(\017)23 b Ft(Gather)18 b(data)h(from)f(all)j(group)e(mem)o(b)q(ers)g (to)g(one)g(mem)o(b)q(er)g(\(Sec.)33 b(1.5\).)e(This)19 b(is)h(sho)o(wn)f(in)189 1357 y(\014gure)c(1.1.)143 1451 y Fo(\017)23 b Ft(Scatter)16 b(data)g(from)h(one)g(mem)o(b)q(er)g(to)f (all)i(mem)o(b)q(ers)g(of)e(a)h(group)g(\(Sec.)25 b(1.6\).)f(This)18 b(is)f(sho)o(wn)189 1507 y(in)f(\014gure)f(1.1.)143 1601 y Fo(\017)23 b Ft(A)16 b(v)m(ariation)i(on)e(Gather)g(where)h(all)h (mem)o(b)q(ers)f(of)f(the)h(group)f(receiv)o(e)i(the)f(result)g(\(Sec.) 25 b(1.7\).)189 1658 y(This)15 b(is)h(sho)o(wn)f(as)g(\\allgather")g (in)h(\014gure)f(1.1.)143 1751 y Fo(\017)23 b Ft(Scatter/Gather)10 b(data)h(from)h(all)g(mem)o(b)q(ers)g(to)g(all)g(mem)o(b)q(ers)g(of)g (a)g(group)f(\(also)h(called)h(complete)189 1808 y(exc)o(hange)i(or)g (all-to-all\))h(\(Sec.)k(1.8\).)f(This)d(is)g(sho)o(wn)e(as)h (\\alltoall")h(in)g(\014gure)g(1.1.)143 1902 y Fo(\017)23 b Ft(Global)15 b(reduction)h(op)q(erations)g(suc)o(h)f(as)g(sum,)g (max,)f(min,)i(or)e(user-de\014ned)j(functions,)f(where)189 1958 y(the)11 b(result)h(is)g(returned)g(to)e(all)j(group)e(mem)o(b)q (ers)g(and)h(a)f(v)m(ariation)h(where)f(the)h(result)g(is)g(returned) 189 2015 y(to)i(only)i(one)f(mem)o(b)q(er)h(\(Sec.)k(1.9\).)143 2108 y Fo(\017)j Ft(A)15 b(com)o(bined)h(reduction)g(and)g(scatter)e (op)q(eration)h(\(Sec.)21 b(1.10\).)143 2202 y Fo(\017)i Ft(Scan)15 b(across)g(all)h(mem)o(b)q(ers)f(of)g(a)g(group)g(\(also)g (called)h(pre\014x\))g(\(Sec.)k(1.11\).)166 2296 y(A)e(collectiv)o(e)i (op)q(eration)e(is)g(executed)h(b)o(y)f(ha)o(ving)g(all)h(pro)q(cesses) f(in)h(the)f(group)g(call)h(the)f(com-)75 2352 y(m)o(unication)f (routine,)f(with)g(matc)o(hing)g(argumen)o(ts.)21 b(The)16 b(syn)o(tax)f(and)h(seman)o(tics)g(of)f(the)h(collectiv)o(e)75 2409 y(op)q(erations)e(are)g(de\014ned)i(to)e(b)q(e)g(consisten)o(t)h (with)f(the)h(syn)o(tax)e(and)i(seman)o(tics)f(of)g(the)g(p)q(oin)o (t-to-p)q(oin)o(t)75 2465 y(op)q(erations.)22 b(Th)o(us,)16 b(general)g(datat)o(yp)q(es)g(are)f(allo)o(w)o(ed)i(and)f(m)o(ust)f (matc)o(h)g(b)q(et)o(w)o(een)i(sending)g(and)f(re-)75 2522 y(ceiving)g(pro)q(cesses)f(as)f(sp)q(eci\014ed)j(in)e(Chapter)28 b Fn(??)p Ft(.)19 b(One)d(of)e(the)g(k)o(ey)h(argumen)o(ts)e(is)i(a)f (comm)o(unicator)75 2578 y(that)h(de\014nes)i(the)f(group)g(of)g (participating)h(pro)q(cesses)f(and)g(pro)o(vides)h(a)e(con)o(text)h (for)f(the)h(op)q(eration.)75 2635 y(Sev)o(eral)c(collectiv)o(e)g (routines)g(suc)o(h)f(as)g(broadcast)f(and)h(gather)g(ha)o(v)o(e)f(a)h (single)h(originating)g(or)e(receiving)75 2691 y(pro)q(cess.)20 b(Suc)o(h)c(pro)q(cesses)f(are)g(called)i(the)e Fm(r)n(o)n(ot)p Ft(.)k(Some)d(argumen)o(ts)e(in)i(the)f(collectiv)o(e)h(functions)g (are)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 2 4 2 3 bop 75 -100 a Ft(2)726 b Fk(CHAPTER)15 b(1.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 2444 y @beginspecial @setspecial %%BeginDocument: coll-fig1.ps % Cookbook Example Program from First Printing, Revised 7 Jan 1985 % Program: Drawing Arrows Number: 8 %----------------------------------------------------------------------------- % /arrowdict 13 dict def % Local storage for the procedure % ``arrow.'' /arrow % The procedure ``arrow'' adds an { arrowdict begin % arrow shape to the current path. /headlength exch def % It takes seven arguments: the x /halfheadthickness exch 2 div def % and y coordinates of the tail /halfthickness exch 2 div def % (imagine that a line has been /tipy exch def /tipx exch def % drawn down the center of the /taily exch def /tailx exch def % arrow from the tip to the tail, % then x and y lie on this line), % the x and y coordinates of the % tip of the arrow, the thickness % of the arrow in the tail % portion, the thickness of the % arrow at the widest part of the % arrowhead and the length of the % arrowhead. /dx tipx tailx sub def % Compute the differences in x and /dy tipy taily sub def % y for the tip and tail. These /arrowlength dx dx mul dy dy mul add % will be used to compute the sqrt def % length of the arrow and to /angle dy dx atan def % compute the angle of direction % that the arrow is facing with % respect to the current user % coordinate system origin. /base arrowlength headlength sub def % Compute where the base of the % arrowhead will be. /savematrix matrix currentmatrix def % Save the current user coordinate % system. We are using the same % strategy to localize the effect % of transformations as was used % in the program to draw an % ellipse. tailx taily translate % Translate to the starting point % of the tail. angle rotate % Rotate the x-axis to correspond % with the center line of the % arrow. 0 halfthickness neg moveto % Add the arrow shape to the % current path. base halfthickness neg lineto base halfheadthickness neg lineto arrowlength 0 lineto base halfheadthickness lineto base halfthickness lineto 0 halfthickness lineto closepath savematrix setmatrix % Restore the current user % coordinate system. end } def /Box { /height exch def /length exch def length 0 rlineto 0 height rlineto length neg 0 rlineto closepath } def /Gdict 200 dict def /Grid { Gdict begin /ny exch def /nx exch def /dely exch def /delx exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def 0 1 ny { pop x y moveto lenx 0 rlineto stroke /y y dely add def} for /y ypos def /x xpos def 0 1 nx { pop x y moveto 0 leny rlineto stroke /x x delx add def} for end } def /GLdict 300 dict def /GridLabels { GLdict begin /shift exch def /raise exch def /yoff exch def /xoff exch def /p1 exch def /p2 exch def /ny exch def /nx exch def /dely exch def /delx exch def /Darray exch def /leny { ny dely mul} def /lenx { nx delx mul} def currentpoint /ypos exch def /xpos exch def /y ypos def /x xpos def /dx3 delx 3 div def /dy3 dely 3 div def /ix -1 def /iy ny 1 sub def Darray{ aload pop /Subc exch def /Text exch def /ix ix 1 add def ix nx ge { /ix 0 def /iy iy 1 sub def} if /x xpos delx ix 0.5 add mul add /Helvetica findfont p1 scalefont setfont Text stringwidth pop /Helvetica findfont p2 scalefont setfont Subc stringwidth pop add xoff add 2 div sub shift add def /y ypos dely iy 0.5 add mul add raise add def x y moveto /Helvetica findfont p1 scalefont Text show xoff yoff rmoveto /Helvetica findfont p2 scalefont Subc show } forall end clear } def 2 setlinecap 6.5 72 mul 320 sub 2 div 0 translate 0 300 moveto 20 20 6 6 Grid 0 300 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 380 200 380 12 24 18 arrow stroke 200 330 140 330 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (scatter) dup stringwidth pop 170 exch 2 div sub 400 moveto show (gather) dup stringwidth pop 170 exch 2 div sub 350 moveto show 220 300 moveto 20 20 6 6 Grid 220 300 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(1)] [()()] [()()] [()()] [()()] [()()] [(A)(2)] [()()] [()()] [()()] [()()] [()()] [(A)(3)] [()()] [()()] [()()] [()()] [()()] [(A)(4)] [()()] [()()] [()()] [()()] [()()] [(A)(5)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 0 moveto 20 20 6 6 Grid 0 0 moveto [ [(A)(0)] [(A)(1)] [(A)(2)] [(A)(3)] [(A)(4)] [(A)(5)] [(B)(0)] [(B)(1)] [(B)(2)] [(B)(3)] [(B)(4)] [(B)(5)] [(C)(0)] [(C)(1)] [(C)(2)] [(C)(3)] [(C)(4)] [(C)(5)] [(D)(0)] [(D)(1)] [(D)(2)] [(D)(3)] [(D)(4)] [(D)(5)] [(E)(0)] [(E)(1)] [(E)(2)] [(E)(3)] [(E)(4)] [(E)(5)] [(F)(0)] [(F)(1)] [(F)(2)] [(F)(3)] [(F)(4)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 220 0 moveto 20 20 6 6 Grid 220 0 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(1)] [(B)(1)] [(C)(1)] [(D)(1)] [(E)(1)] [(F)(1)] [(A)(2)] [(B)(2)] [(C)(2)] [(D)(2)] [(E)(2)] [(F)(2)] [(A)(3)] [(B)(3)] [(C)(3)] [(D)(3)] [(E)(3)] [(F)(3)] [(A)(4)] [(B)(4)] [(C)(4)] [(D)(4)] [(E)(4)] [(F)(4)] [(A)(5)] [(B)(5)] [(C)(5)] [(D)(5)] [(E)(5)] [(F)(5)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 60 200 60 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (alltoall) dup stringwidth pop 170 exch 2 div sub 77 moveto show 0 150 moveto 20 20 6 6 Grid 0 150 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(B)(0)] [()()] [()()] [()()] [()()] [()()] [(C)(0)] [()()] [()()] [()()] [()()] [()()] [(D)(0)] [()()] [()()] [()()] [()()] [()()] [(E)(0)] [()()] [()()] [()()] [()()] [()()] [(F)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 200 200 200 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont (allgather) dup stringwidth pop 170 exch 2 div sub 220 moveto show 220 150 moveto 20 20 6 6 Grid 220 150 moveto [ [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] [(A)(0)] [(B)(0)] [(C)(0)] [(D)(0)] [(E)(0)] [(F)(0)] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 0 450 moveto 20 20 6 6 Grid 0 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels 140 510 200 510 12 24 18 arrow stroke /Helvetica findfont 12 scalefont setfont 0 575 moveto (data) show (data) stringwidth pop 4 add 575 4 add (data) stringwidth pop 33 add 575 4 add 1 4 5 arrow fill (broadcast) dup stringwidth pop 170 exch 2 div sub 527 moveto show gsave 0 570 (processes) stringwidth pop sub translate 90 rotate 0 5 moveto (processes) show -4 8 -33 8 1 4 5 arrow fill grestore 220 450 moveto 20 20 6 6 Grid 220 450 moveto [ [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] [(A)(0)] [()()] [()()] [()()] [()()] [()()] ] 20 20 6 6 9 12 1 -5 -2 2 GridLabels showpage %%EndDocument @endspecial 98 x Ft(Figure)f(1.1:)k(Collectiv)o(e)e(mo)o(v)o(e)d (functions)i(illustrated)h(for)e(a)f(group)h(of)g(six)h(pro)q(cesses.)k (In)c(eac)o(h)f(case,)75 2598 y(eac)o(h)f(ro)o(w)e(of)i(b)q(o)o(xes)f (represen)o(ts)h(data)f(lo)q(cations)i(in)f(one)g(pro)q(cess.)20 b(Th)o(us,)13 b(in)i(the)e(broadcast,)g(initially)75 2655 y(just)i(the)g(\014rst)g(pro)q(cess)g(con)o(tains)g(the)g(data)g Fj(A)889 2662 y Fi(0)909 2655 y Ft(,)f(but)h(after)g(the)g(broadcast)f (all)i(pro)q(cesses)g(con)o(tain)f(it.)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 3 5 3 4 bop 75 -100 a Fk(1.1.)34 b(INTR)o(ODUCTION)16 b(AND)f(O)o(VER)-5 b(VIEW)912 b Ft(3)75 45 y(sp)q(eci\014ed)18 b(as)d(\\signi\014can)o(t)i (only)f(at)f(ro)q(ot,")f(and)i(are)g(ignored)g(for)f(all)i(participan)o (ts)f(except)g(the)g(ro)q(ot.)75 102 y(The)j(reader)g(is)g(referred)g (to)f(Chapter)g Fn(??)30 b Ft(for)18 b(information)h(concerning)h(comm) o(unication)f(bu\013ers,)75 158 y(general)e(datat)o(yp)q(es)f(and)g(t)o (yp)q(e)h(matc)o(hing)f(rules,)h(and)g(to)f(Chapter)g Fn(??)23 b Ft(for)16 b(information)g(on)h(ho)o(w)f(to)75 214 y(de\014ne)g(groups)f(and)h(create)f(comm)o(unicators.)166 271 y(The)d(t)o(yp)q(e-matc)o(hing)h(conditions)g(for)e(the)i (collectiv)o(e)g(op)q(erations)g(are)e(more)h(strict)g(than)g(the)g (cor-)75 327 y(resp)q(onding)j(conditions)g(b)q(et)o(w)o(een)f(sender)h (and)f(receiv)o(er)h(in)f(p)q(oin)o(t-to-p)q(oin)o(t.)20 b(Namely)l(,)15 b(for)e(collectiv)o(e)75 384 y(op)q(erations,)18 b(the)g(amoun)o(t)f(of)g(data)g(sen)o(t)g(m)o(ust)g(exactly)h(matc)o(h) f(the)h(amoun)o(t)f(of)g(data)g(sp)q(eci\014ed)j(b)o(y)75 440 y(the)c(receiv)o(er.)24 b(Distinct)17 b(t)o(yp)q(e)f(maps)g(\(the)g (la)o(y)o(out)g(in)h(memory)l(,)f(see)g(Sec.)24 b Fn(??)p Ft(\))e(b)q(et)o(w)o(een)17 b(sender)g(and)75 497 y(receiv)o(er)f(are)f (still)h(allo)o(w)o(ed.)166 553 y(Collectiv)o(e)g(routine)f(calls)h (can)f(\(but)f(are)g(not)h(required)g(to\))f(return)h(as)f(so)q(on)h (as)f(their)h(participa-)75 610 y(tion)d(in)h(the)f(collectiv)o(e)i (comm)o(unication)e(is)h(complete.)19 b(The)12 b(completion)h(of)f(a)f (call)i(indicates)h(that)d(the)75 666 y(caller)k(is)g(no)o(w)f(free)g (to)g(access)g(lo)q(cations)h(in)g(the)g(comm)o(unication)g(bu\013er.)k (It)14 b(do)q(es)h(not)f(indicate)i(that)75 723 y(other)c(pro)q(cesses) h(in)h(the)f(group)f(ha)o(v)o(e)h(completed)g(or)f(ev)o(en)i(started)d (the)i(op)q(eration)g(\(unless)h(otherwise)75 779 y(indicated)k(in)f (the)f(description)i(of)e(the)g(op)q(eration\).)23 b(Th)o(us,)16 b(a)g(collectiv)o(e)i(comm)o(unication)f(call)h(ma)o(y)l(,)75 835 y(or)c(ma)o(y)h(not,)f(ha)o(v)o(e)h(the)g(e\013ect)f(of)h(sync)o (hronizing)h(all)g(calling)h(pro)q(cesses.)j(This)c(statemen)o(t)e (excludes,)75 892 y(of)h(course,)g(the)g(barrier)g(function.)166 948 y(Collectiv)o(e)24 b(comm)o(unication)f(calls)g(ma)o(y)f(use)g(the) g(same)g(comm)o(unicators)g(as)g(p)q(oin)o(t-to-p)q(oin)o(t)75 1005 y(comm)o(unication;)16 b Fp(MPI)f Ft(guaran)o(tees)g(that)g (messages)g(generated)h(on)g(b)q(ehalf)h(of)e(collectiv)o(e)i(comm)o (uni-)75 1061 y(cation)c(calls)h(will)g(not)f(b)q(e)g(confused)h(with)f (messages)f(generated)h(b)o(y)g(p)q(oin)o(t-to-p)q(oin)o(t)h(comm)o (unication.)75 1118 y(A)h(more)g(detailed)i(discussion)g(of)d(correct)h (use)h(of)e(collectiv)o(e)j(routines)f(is)g(found)f(in)h(Sec.)21 b(1.12.)189 1195 y Fm(R)n(ationale.)50 b Ft(The)18 b(equal-data)f (restriction)g(\(on)g(t)o(yp)q(e)g(matc)o(hing\))g(w)o(as)f(made)h(so)f (as)h(to)f(a)o(v)o(oid)189 1252 y(the)e(complexit)o(y)h(of)f(pro)o (viding)h(a)f(facilit)o(y)h(analogous)f(to)g(the)g(status)f(argumen)o (t)h(of)f Fp(MPI)p 1746 1252 14 2 v 16 w(RECV)189 1308 y Ft(for)f(disco)o(v)o(ering)j(the)e(amoun)o(t)g(of)g(data)f(sen)o(t.) 19 b(Some)14 b(of)f(the)g(collectiv)o(e)j(routines)d(w)o(ould)h (require)189 1365 y(an)h(arra)o(y)f(of)g(status)h(v)m(alues.)189 1436 y(The)e(statemen)o(ts)g(ab)q(out)g(sync)o(hronization)i(are)e (made)h(so)f(as)g(to)g(allo)o(w)h(a)f(v)m(ariet)o(y)h(of)f(implemen-) 189 1492 y(tations)h(of)h(the)g(collectiv)o(e)i(functions.)189 1563 y(The)c(collectiv)o(e)i(op)q(erations)e(do)g(not)g(accept)g(a)g (message)f(tag)g(argumen)o(t.)19 b(If)13 b(future)g(revisions)h(of)189 1619 y Fp(MPI)g Ft(de\014ne)j(non-blo)q(c)o(king)g(collectiv)o(e)g (functions,)f(then)f(tags)g(\(or)f(a)h(similar)i(mec)o(hanism\))e(will) 189 1676 y(need)i(to)g(b)q(e)g(added)h(so)f(as)f(to)h(allo)o(w)g(the)g (dis-am)o(biguation)h(of)e(m)o(ultiple,)j(p)q(ending,)g(collectiv)o(e) 189 1732 y(op)q(erations.)h(\()p Fm(End)15 b(of)i(r)n(ationale.)p Ft(\))189 1810 y Fm(A)n(dvic)n(e)h(to)h(users.)59 b Ft(It)19 b(is)f(dangerous)h(to)e(rely)i(on)f(sync)o(hronization)i (side-e\013ects)f(of)f(the)g(col-)189 1866 y(lectiv)o(e)j(op)q (erations)f(for)f(program)g(correctness.)34 b(F)l(or)20 b(example,)h(ev)o(en)g(though)f(a)f(particular)189 1923 y(implemen)o(tation)i(ma)o(y)e(pro)o(vide)i(a)e(broadcast)g(routine)i (with)f(a)g(side-e\013ect)h(of)e(sync)o(hroniza-)189 1979 y(tion,)14 b(the)g(standard)g(do)q(es)h(not)f(require)h(this,)g (and)f(a)g(program)f(that)h(relies)h(on)g(this)f(will)i(not)e(b)q(e)189 2036 y(p)q(ortable.)189 2107 y(On)d(the)g(other)f(hand,)i(a)f(correct,) g(p)q(ortable)g(program)f(m)o(ust)g(allo)o(w)h(for)f(the)h(fact)g(that) f(a)g(collectiv)o(e)189 2163 y(call)j Fm(may)f Ft(b)q(e)g(sync)o (hronizing.)21 b(Though)11 b(one)h(cannot)g(rely)g(on)g(an)o(y)g(sync)o (hronization)g(side-e\013ect,)189 2220 y(one)17 b(m)o(ust)f(program)f (so)i(as)f(to)g(allo)o(w)h(it.)25 b(These)18 b(issues)f(are)g (discussed)h(further)f(in)g(Sec.)26 b(1.12.)189 2276 y(\()p Fm(End)15 b(of)i(advic)n(e)f(to)g(users.)p Ft(\))189 2353 y Fm(A)n(dvic)n(e)21 b(to)h(implementors.)77 b Ft(While)22 b(v)o(endors)g(ma)o(y)e(write)i(optimized)g(collectiv)o(e)i(routines) 189 2410 y(matc)o(hed)c(to)f(their)i(arc)o(hitectures,)g(a)f(complete)h (library)f(of)g(the)g(collectiv)o(e)i(comm)o(unication)189 2466 y(routines)d(can)h(b)q(e)f(written)h(en)o(tirely)g(using)g(the)f Fp(MPI)f Ft(p)q(oin)o(t-to-p)q(oin)o(t)i(comm)o(unication)g(func-)189 2523 y(tions)11 b(and)g(a)f(few)h(auxiliary)i(functions.)19 b(If)11 b(implemen)o(ting)i(on)e(top)f(of)h(p)q(oin)o(t-to-p)q(oin)o (t,)h(a)f(hidden,)189 2579 y(sp)q(ecial)j(comm)o(unicator)e(m)o(ust)g (b)q(e)h(created)g(for)f(the)g(collectiv)o(e)i(op)q(eration)f(so)f(as)g (to)g(a)o(v)o(oid)h(in)o(ter-)189 2636 y(ference)j(with)f(an)o(y)g (on-going)h(p)q(oin)o(t-to-p)q(oin)o(t)f(comm)o(unication)h(at)f(the)g (time)h(of)f(the)g(collectiv)o(e)189 2692 y(call.)21 b(This)15 b(is)h(discussed)h(further)e(in)h(Sec.)k(1.12.)f(\()p Fm(End)d(of)g(advic)n(e)g(to)h(implementors.)p Ft(\))-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 4 6 4 5 bop 75 -100 a Ft(4)726 b Fk(CHAPTER)15 b(1.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fq(1.2)59 b(Communicato)n(r)19 b(a)n(rgument)75 147 y Ft(The)c(k)o(ey)h(concept)f(of)g(the)g (collectiv)o(e)i(functions)f(is)g(to)f(ha)o(v)o(e)f(a)h(\\group")g(of)g (participating)h(pro)q(cesses.)75 203 y(The)d(routines)f(do)h(not)e(ha) o(v)o(e)h(a)g(group)g(iden)o(ti\014er)i(as)e(an)g(explicit)j(argumen)o (t.)j(Instead,)13 b(there)f(is)h(a)f(com-)75 259 y(m)o(unicator)k (argumen)o(t.)21 b(F)l(or)15 b(the)h(purp)q(oses)h(of)e(this)i(c)o (hapter,)e(a)h(comm)o(unicator)g(can)g(b)q(e)g(though)o(t)f(of)75 316 y(as)d(a)g(group)g(iden)o(ti\014er)h(link)o(ed)h(with)f(a)f(con)o (text.)18 b(An)12 b(in)o(ter-comm)o(unicator,)h(that)e(is,)i(a)f(comm)o (unicator)75 372 y(that)i(spans)i(t)o(w)o(o)e(groups,)g(is)i Fm(not)f Ft(allo)o(w)o(ed)g(as)g(an)g(argumen)o(t)g(to)f(a)h(collectiv) o(e)i(function.)75 516 y Fq(1.3)59 b(Ba)n(rrier)21 b(synchronization)75 712 y Fp(MPI)p 160 712 14 2 v 16 w(BARRIER\()16 b(comm)d(\))117 789 y Fh(IN)155 b Fp(comm)470 b Fh(comm)o(unicator)11 b(\(handle\))75 913 y Fg(int)23 b(MPI)p 245 913 15 2 v 17 w(Barrier\(MPI)p 526 913 V 16 w(Comm)g(comm)h(\))75 1000 y(MPI)p 150 1000 V 17 w(BARRIER\(COMM,)e(IERROR\))170 1056 y(INTEGER)h(COMM,)h(IERROR)166 1143 y Fp(MPI)p 251 1143 14 2 v 16 w(BARRIER)13 b Ft(blo)q(c)o(ks)g(the)f(caller)i(un)o (til)f(all)g(group)f(mem)o(b)q(ers)h(ha)o(v)o(e)e(called)j(it.)19 b(The)13 b(call)g(returns)75 1199 y(at)i(an)o(y)f(pro)q(cess)i(only)g (after)e(all)i(group)f(mem)o(b)q(ers)g(ha)o(v)o(e)g(en)o(tered)h(the)f (call.)75 1342 y Fq(1.4)59 b(Broadcast)90 1538 y Fp(MPI)p 175 1538 V 16 w(BCAST\()16 b(bu\013er,)f(count,)h(datat)o(yp)q(e,)g(ro) q(ot,)f(comm)e(\))117 1615 y Fh(INOUT)78 b Fp(bu\013er)462 b Fh(starting)14 b(address)h(of)f(bu\013er)g(\(c)o(hoice\))117 1691 y(IN)171 b Fp(count)466 b Fh(n)o(um)o(b)q(er)13 b(of)h(en)o(tries)h(in)e(bu\013er)i(\(in)o(teger\))117 1766 y(IN)171 b Fp(datat)o(yp)q(e)408 b Fh(data)14 b(t)o(yp)q(e)g(of)f (bu\013er)i(\(handle\))117 1841 y(IN)171 b Fp(ro)q(ot)492 b Fh(rank)14 b(of)f(broadcast)i(ro)q(ot)f(\(in)o(teger\))117 1916 y(IN)171 b Fp(comm)454 b Fh(comm)o(unicator)11 b(\(handle\))75 2040 y Fg(int)23 b(MPI)p 245 2040 15 2 v 17 w(Bcast\(void*)g(buffer,)f (int)i(count,)f(MPI)p 1074 2040 V 17 w(Datatype)g(datatype,)f(int)i (root,)393 2097 y(MPI)p 468 2097 V 17 w(Comm)f(comm)h(\))75 2183 y(MPI)p 150 2183 V 17 w(BCAST\(BUFFER,)e(COUNT,)h(DATATYPE,)g (ROOT,)g(COMM,)g(IERROR\))170 2240 y()g(BUFFER\(*\))170 2296 y(INTEGER)g(COUNT,)g(DATATYPE,)g(ROOT,)g(COMM,)g(IERROR)166 2383 y Fp(MPI)p 251 2383 14 2 v 16 w(BCAST)17 b Ft(broadcasts)e(a)h (message)g(from)g(the)g(pro)q(cess)g(with)h(rank)f Fp(ro)q(ot)g Ft(to)g(all)h(pro)q(cesses)f(of)75 2439 y(the)h(group,)g(itself)h (included.)27 b(It)17 b(is)h(called)g(b)o(y)f(all)h(mem)o(b)q(ers)f(of) g(group)f(using)i(the)f(same)g(argumen)o(ts)75 2496 y(for)d Fp(comm,)f(ro)q(ot)p Ft(.)20 b(On)15 b(return,)g(the)g(con)o(ten)o(ts)f (of)h Fp(ro)q(ot)p Ft('s)f(comm)o(unication)h(bu\013er)g(has)g(b)q(een) h(copied)h(to)75 2552 y(all)f(pro)q(cesses.)166 2609 y(General,)24 b(deriv)o(ed)f(datat)o(yp)q(es)e(are)h(allo)o(w)o(ed)g (for)g Fp(datat)o(yp)q(e)p Ft(.)42 b(The)22 b(t)o(yp)q(e)g(signature)g (of)g Fp(count,)75 2665 y(datat)o(yp)q(e)16 b Ft(on)f(an)o(y)f(pro)q (cess)h(m)o(ust)f(b)q(e)h(equal)g(to)f(the)h(t)o(yp)q(e)g(signature)g (of)f Fp(count,)h(datat)o(yp)q(e)i Ft(at)c(the)i(ro)q(ot.)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 5 7 5 6 bop 75 -100 a Fk(1.5.)34 b(GA)l(THER)1474 b Ft(5)75 45 y(This)16 b(implies)i(that)d(the)h(amoun)o(t)e(of)i(data)f(sen)o(t)g (m)o(ust)g(b)q(e)h(equal)h(to)e(the)h(amoun)o(t)e(receiv)o(ed,)j (pairwise)75 102 y(b)q(et)o(w)o(een)i(eac)o(h)h(pro)q(cess)f(and)g(the) g(ro)q(ot.)31 b Fp(MPI)p 909 102 14 2 v 16 w(BCAST)20 b Ft(and)f(all)h(other)f(data-mo)o(v)o(emen)o(t)e(collectiv)o(e)75 158 y(routines)j(mak)o(e)g(this)g(restriction.)34 b(Distinct)21 b(t)o(yp)q(e)f(maps)g(b)q(et)o(w)o(een)g(sender)g(and)g(receiv)o(er)h (are)f(still)75 214 y(allo)o(w)o(ed.)75 336 y Ff(1.4.1)49 b(Example)17 b(using)g(MPI)p 622 336 15 2 v 18 w(BCAST)75 422 y Fn(Example)h(1.1)k Ft(Broadcast)15 b(100)f(in)o(ts)h(from)g(pro)q (cess)g Fg(0)g Ft(to)g(ev)o(ery)g(pro)q(cess)g(in)i(the)e(group.)170 528 y Fg(MPI_Comm)23 b(comm;)170 585 y(int)h(array[100];)170 641 y(int)g(root=0;)170 698 y(...)170 754 y(MPI_Bcast\()f(array,)g (100,)g(MPI_INT,)g(root,)g(comm\);)75 860 y Ft(As)15 b(in)h(man)o(y)f(of)g(our)g(example)h(co)q(de)g(fragmen)o(ts,)e(w)o(e)h (assume)g(that)g(some)g(of)g(the)g(v)m(ariables)i(\(suc)o(h)e(as)75 917 y Fg(comm)g Ft(in)h(the)f(ab)q(o)o(v)o(e\))f(ha)o(v)o(e)h(b)q(een)i (assigned)e(appropriate)h(v)m(alues.)75 1060 y Fq(1.5)59 b(Gather)75 1256 y Fp(MPI)p 160 1256 14 2 v 16 w(GA)l(THER\()16 b(sendbuf,)h(sendcount,)h(sendt)o(yp)q(e,)f(recvbuf,)f(recvcount,)g (recvt)o(yp)q(e,)g(ro)q(ot,)e(comm\))117 1333 y Fh(IN)171 b Fp(sendbuf)428 b Fh(starting)14 b(address)h(of)f(send)g(bu\013er)h (\(c)o(hoice\))117 1408 y(IN)171 b Fp(sendcount)383 b Fh(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in) o(teger\))117 1483 y(IN)171 b Fp(sendt)o(yp)q(e)408 b Fh(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 1559 y(OUT)124 b Fp(recvbuf)434 b Fh(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i(signi\014can)o(t)d(only)g (at)905 1615 y(ro)q(ot\))117 1690 y(IN)171 b Fp(recvcount)389 b Fh(n)o(um)o(b)q(er)11 b(of)g(elemen)o(ts)g(for)g(an)o(y)g(single)g (receiv)o(e)i(\(in)o(teger,)f(sig-)905 1747 y(ni\014can)o(t)i(only)f (at)h(ro)q(ot\))117 1822 y(IN)171 b Fp(recvt)o(yp)q(e)414 b Fh(data)16 b(t)o(yp)q(e)g(of)f(recv)i(bu\013er)g(elemen)o(ts)f (\(signi\014can)o(t)f(only)g(at)905 1878 y(ro)q(ot\))f(\(handle\))117 1953 y(IN)171 b Fp(ro)q(ot)492 b Fh(rank)14 b(of)f(receiving)i(pro)q (cess)g(\(in)o(teger\))117 2029 y(IN)171 b Fp(comm)454 b Fh(comm)o(unicator)11 b(\(handle\))75 2153 y Fg(int)23 b(MPI)p 245 2153 15 2 v 17 w(Gather\(void*)f(sendbuf,)h(int)h (sendcount,)e(MPI)p 1217 2153 V 17 w(Datatype)h(sendtype,)393 2209 y(void*)g(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 2209 V 17 w(Datatype)h(recvtype,)f(int)i(root,)393 2266 y(MPI)p 468 2266 V 17 w(Comm)f(comm\))75 2352 y(MPI)p 150 2352 V 17 w(GATHER\(SENDBUF,)f(SENDCOUNT,)g(SENDTYPE,)h(RECVBUF,)f (RECVCOUNT,)h(RECVTYPE,)393 2409 y(ROOT,)g(COMM,)h(IERROR\))170 2465 y()f(SENDBUF\(*\),)g(RECVBUF\(*\))170 2522 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(ROOT,)g (COMM,)g(IERROR)166 2608 y Ft(Eac)o(h)16 b(pro)q(cess)h(\(ro)q(ot)f (pro)q(cess)h(included\))h(sends)g(the)e(con)o(ten)o(ts)g(of)g(its)h (send)h(bu\013er)e(to)g(the)h(ro)q(ot)75 2665 y(pro)q(cess.)i(The)11 b(ro)q(ot)f(pro)q(cess)i(receiv)o(es)g(the)f(messages)f(and)i(stores)e (them)h(in)h(rank)f(order.)18 b(The)11 b(outcome)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 6 8 6 7 bop 75 -100 a Ft(6)726 b Fk(CHAPTER)15 b(1.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(is)e Fm(as)h(if)f Ft(eac)o(h)g(of)f(the)h Fg(n)g Ft(pro)q(cesses)g(in)h(the)f(group)f(\(including)k(the)d(ro)q (ot)f(pro)q(cess\))g(had)h(executed)h(a)f(call)75 102 y(to)189 195 y Fg(MPI)p 264 195 15 2 v 16 w(Send)p Ft(\()p Fg(sendbuf)p Fj(;)8 b Fg(sendcount)p Fj(;)f Fg(se)o(ndtype)p Fj(;)g Fg(roo)o(t)p Fj(;)h(:)o(::)o Ft(\))p Fj(;)75 289 y Ft(and)15 b(the)h(ro)q(ot)e(had)h(executed)i Fg(n)e Ft(calls)h(to)189 383 y Fg(MPI)p 264 383 V 16 w(Recv)p Ft(\()p Fg(recvbuf)9 b Ft(+)h Fg(i)g Fo(\001)g Fg(recvcount)f Fo(\001)g Fg(extent)p Ft(\()p Fg(recvtype)p Ft(\))p Fj(;)f Fg(recvc)o(ount)p Fj(;)f Fg(recv)o(type)p Fj(;)g Fg(i)p Fj(;)g(::)o(:)p Ft(\))p Fj(;)75 477 y Ft(where)15 b Fg (extent\(recvtype\))e Ft(is)j(the)f(t)o(yp)q(e)h(exten)o(t)f(obtained)g (from)g(a)g(call)h(to)f Fg(MPI)p 1515 477 V 16 w(Type)p 1627 477 V 17 w(extent\(\))p Ft(.)166 533 y(An)j(alternativ)o(e)h (description)h(is)e(that)g(the)g Fg(n)g Ft(messages)g(sen)o(t)g(b)o(y)g (the)g(pro)q(cesses)h(in)g(the)f(group)75 590 y(are)c(concatenated)h (in)g(rank)f(order,)g(and)h(the)g(resulting)g(message)f(is)h(receiv)o (ed)h(b)o(y)e(the)h(ro)q(ot)f(as)g(if)h(b)o(y)f(a)75 646 y(call)i(to)f Fp(MPI)p 299 646 14 2 v 16 w(RECV\(recvbuf,)h (recvcount)p Fo(\001)p Fp(n,)g(recvt)o(yp)q(e,)g(...\))p Ft(.)166 703 y(The)f(receiv)o(e)h(bu\013er)g(is)f(ignored)h(for)f(all)h (non-ro)q(ot)f(pro)q(cesses.)166 759 y(General,)i(deriv)o(ed)h(datat)o (yp)q(es)e(are)g(allo)o(w)o(ed)h(for)f(b)q(oth)h Fp(sendt)o(yp)q(e)i Ft(and)e Fp(recvt)o(yp)q(e)p Ft(.)25 b(The)17 b(t)o(yp)q(e)g(sig-)75 816 y(nature)c(of)g Fp(sendcount,)j(sendt)o(yp)q(e)f Ft(on)e(pro)q(cess)h Fg(i)e Ft(m)o(ust)h(b)q(e)h(equal)g(to)e(the)h(t)o (yp)q(e)g(signature)h(of)e Fp(recvcount,)75 872 y(recvt)o(yp)q(e)h Ft(at)e(the)i(ro)q(ot.)18 b(This)12 b(implies)i(that)e(the)g(amoun)o(t) f(of)h(data)f(sen)o(t)h(m)o(ust)g(b)q(e)g(equal)h(to)f(the)g(amoun)o(t) 75 928 y(of)j(data)g(receiv)o(ed,)i(pairwise)f(b)q(et)o(w)o(een)g(eac)o (h)g(pro)q(cess)g(and)g(the)g(ro)q(ot.)k(Distinct)c(t)o(yp)q(e)g(maps)g (b)q(et)o(w)o(een)75 985 y(sender)g(and)f(receiv)o(er)h(are)f(still)i (allo)o(w)o(ed.)166 1041 y(All)e(argumen)o(ts)f(to)f(the)h(function)h (are)f(signi\014can)o(t)h(on)f(pro)q(cess)h Fp(ro)q(ot)p Ft(,)e(while)j(on)e(other)g(pro)q(cesses,)75 1098 y(only)19 b(argumen)o(ts)f Fp(sendbuf,)k(sendcount,)g(sendt)o(yp)q(e,)g(ro)q(ot,) d(comm)e Ft(are)i(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 1154 y Fp(ro)q(ot)c Ft(and)g Fp(comm)f Ft(m)o(ust)g(ha)o(v)o(e)h(iden)o (tical)i(v)m(alues)f(on)f(all)i(pro)q(cesses.)166 1211 y(The)f(sp)q(eci\014cation)h(of)f(coun)o(ts)f(and)h(t)o(yp)q(es)f (should)i(not)e(cause)h(an)o(y)g(lo)q(cation)g(on)g(the)f(ro)q(ot)g(to) g(b)q(e)75 1267 y(written)g(more)g(than)g(once.)20 b(Suc)o(h)c(a)f (call)h(is)g(erroneous.)166 1324 y(Note)e(that)f(the)h Fp(recvcount)i Ft(argumen)o(t)d(at)h(the)g(ro)q(ot)f(indicates)j(the)e (n)o(um)o(b)q(er)h(of)e(items)i(it)f(receiv)o(es)75 1380 y(from)g Fm(e)n(ach)i Ft(pro)q(cess,)f(not)f(the)i(total)e(n)o(um)o(b)q (er)i(of)f(items)g(it)h(receiv)o(es.)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 7 9 7 8 bop 75 -100 a Fk(1.5.)34 b(GA)l(THER)1474 b Ft(7)75 45 y Fp(MPI)p 160 45 14 2 v 16 w(GA)l(THERV\()24 b(sendbuf,)j (sendcount,)g(sendt)o(yp)q(e,)g(recvbuf,)e(recvcounts,)h(displs,)g (recvt)o(yp)q(e,)g(ro)q(ot,)75 102 y(comm\))117 179 y Fh(IN)171 b Fp(sendbuf)428 b Fh(starting)14 b(address)h(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 254 y(IN)171 b Fp(sendcount)383 b Fh(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 329 y(IN)171 b Fp(sendt)o(yp)q(e)408 b Fh(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 404 y(OUT)124 b Fp(recvbuf)434 b Fh(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i(signi\014can)o(t)d(only)g (at)905 461 y(ro)q(ot\))117 536 y(IN)171 b Fp(recvcounts)372 b Fh(in)o(teger)10 b(arra)o(y)f(\(of)g(length)g(group)g(size\))h(con)o (taining)e(the)i(n)o(um-)905 592 y(b)q(er)20 b(of)e(elemen)o(ts)h(that) f(are)i(receiv)o(ed)g(from)d(eac)o(h)i(pro)q(cess)905 649 y(\(signi\014can)o(t)14 b(only)f(at)h(ro)q(ot\))117 724 y(IN)171 b Fp(displs)469 b Fh(in)o(teger)15 b(arra)o(y)f(\(of)g (length)g(group)g(size\).)20 b(En)o(try)15 b Fe(i)f Fh(sp)q(eci\014es) 905 780 y(the)h(displacemen)o(t)e(relativ)o(e)g(to)h Fd(recvbuf)i Fh(at)d(whic)o(h)h(to)g(place)905 837 y(the)h(incoming)e (data)h(from)f(pro)q(cess)k Fe(i)d Fh(\(signi\014can)o(t)g(only)g(at) 905 893 y(ro)q(ot\))117 968 y(IN)171 b Fp(recvt)o(yp)q(e)414 b Fh(data)16 b(t)o(yp)q(e)g(of)f(recv)i(bu\013er)g(elemen)o(ts)f (\(signi\014can)o(t)f(only)g(at)905 1025 y(ro)q(ot\))f(\(handle\))117 1100 y(IN)171 b Fp(ro)q(ot)492 b Fh(rank)14 b(of)f(receiving)i(pro)q (cess)g(\(in)o(teger\))117 1175 y(IN)171 b Fp(comm)454 b Fh(comm)o(unicator)11 b(\(handle\))75 1299 y Fg(int)23 b(MPI)p 245 1299 15 2 v 17 w(Gatherv\(void*)f(sendbuf,)h(int)h (sendcount,)e(MPI)p 1241 1299 V 17 w(Datatype)h(sendtype,)393 1356 y(void*)g(recvbuf,)g(int)h(*recvcounts,)e(int)h(*displs,)393 1412 y(MPI)p 468 1412 V 17 w(Datatype)g(recvtype,)f(int)i(root,)f(MPI)p 1249 1412 V 17 w(Comm)g(comm\))75 1499 y(MPI)p 150 1499 V 17 w(GATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)g(RECVBUF,)f (RECVCOUNTS,)h(DISPLS,)393 1555 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\)) 170 1612 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1668 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g (DISPLS\(*\),)h(RECVTYPE,)g(ROOT,)170 1725 y(COMM,)h(IERROR)166 1811 y Fp(MPI)p 251 1811 14 2 v 16 w(GA)l(THERV)15 b Ft(extends)g(the)f(functionalit)o(y)h(of)e Fp(MPI)p 1126 1811 V 16 w(GA)l(THER)i Ft(b)o(y)f(allo)o(wing)h(a)f(v)m(arying)g(coun) o(t)75 1868 y(of)i(data)g(from)g(eac)o(h)h(pro)q(cess,)g(since)h Fp(recvcounts)g Ft(is)f(no)o(w)f(an)h(arra)o(y)l(.)23 b(It)17 b(also)f(allo)o(ws)h(more)f(\015exibili)q(t)o(y)75 1924 y(as)f(to)f(where)i(the)f(data)g(is)g(placed)i(on)e(the)g(ro)q (ot,)f(b)o(y)h(pro)o(viding)h(the)g(new)f(argumen)o(t,)f Fp(displs)p Ft(.)166 1980 y(The)i(outcome)g(is)h Fm(as)g(if)f Ft(eac)o(h)h(pro)q(cess,)f(including)j(the)e(ro)q(ot)e(pro)q(cess,)i (sends)f(a)g(message)g(to)g(the)75 2037 y(ro)q(ot,)189 2131 y Fg(MPI)p 264 2131 15 2 v 16 w(Send)p Ft(\()p Fg(sendbuf)p Fj(;)8 b Fg(sendcount)p Fj(;)f Fg(se)o(ndtype)p Fj(;)g Fg(roo)o(t)p Fj(;)h(:)o(::)o Ft(\))p Fj(;)75 2225 y Ft(and)15 b(the)h(ro)q(ot)e(executes)i Fg(n)f Ft(receiv)o(es,)189 2318 y Fg(MPI)p 264 2318 V 16 w(Recv)p Ft(\()p Fg(recvbuf)9 b Ft(+)h Fg(disp)p Ft([)p Fg(i)p Ft(])f Fo(\001)h Fg(extent)p Ft(\()p Fg(recvtype)p Ft(\))p Fj(;)d Fg(recvc)o(ounts)p Ft([)o Fg(i)p Ft(])p Fj(;)g Fg(recvt)o(ype)p Fj(;)g Fg(i)p Fj(;)g(:::)o Ft(\))p Fj(:)166 2412 y Ft(Messages)14 b(are)g(placed)i (in)f(the)g(receiv)o(e)g(bu\013er)g(of)f(the)g(ro)q(ot)g(pro)q(cess)h (in)g(rank)f(order,)g(that)g(is,)h(the)75 2469 y(data)e(sen)o(t)g(from) f(pro)q(cess)i Fg(j)f Ft(is)g(placed)i(in)f(the)f Fg(j)p Ft(th)g(p)q(ortion)h(of)f(the)g(receiv)o(e)h(bu\013er)g Fp(recvbuf)f Ft(on)h(pro)q(cess)75 2525 y Fp(ro)q(ot)p Ft(.)19 b(The)13 b Fg(j)p Ft(th)g(p)q(ortion)h(of)f Fp(recvbuf)h Ft(b)q(egins)g(at)f(o\013set)f Fp(displs[j])j Ft(elemen)o(ts)f(\(in)g (terms)e(of)h Fp(recvt)o(yp)q(e)p Ft(\))h(in)o(to)75 2582 y Fp(recvbuf)p Ft(.)166 2638 y(The)h(receiv)o(e)h(bu\013er)g(is)f (ignored)h(for)f(all)h(non-ro)q(ot)f(pro)q(cesses.)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 8 10 8 9 bop 75 -100 a Ft(8)731 b Fk(CHAPTER)15 b(1.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)458 4 y 16577003 6465030 0 0 33943388 13419479 startTexFig 458 4 a %%BeginDocument: mycoll-fig2.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 369.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth % Polyline n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr % Polyline n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr % Polyline n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr % Polyline n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr % Polyline n 89 244 m 119 294 l gs 0.50 setgray fill gr gs col-1 s gr n 116.599 286.111 m 119.000 294.000 l 113.169 288.169 l gs 2 setlinejoin col-1 s gr % Polyline n 189 244 m 189 294 l gs 0.50 setgray fill gr gs col-1 s gr n 191.000 286.000 m 189.000 294.000 l 187.000 286.000 l gs 2 setlinejoin col-1 s gr % Polyline n 289 244 m 259 294 l gs 0.50 setgray fill gr gs col-1 s gr n 264.831 288.169 m 259.000 294.000 l 261.401 286.111 l gs 2 setlinejoin col-1 s gr % Polyline n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 409 m gs 1 -1 scale (rbuf) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr $F2psEnd %%EndDocument endTexFig 186 511 a Ft(Figure)g(1.2:)j(The)c(ro)q(ot)g(pro)q(cess)g(gathers)f (100)h Fg(int)p Ft(s)f(from)h(eac)o(h)g(pro)q(cess)h(in)g(the)f(group.) 166 644 y(The)i(t)o(yp)q(e)f(signature)h(implied)i(b)o(y)d Fp(sendcount,)j(sendt)o(yp)q(e)g Ft(on)e(pro)q(cess)g Fg(i)f Ft(m)o(ust)g(b)q(e)h(equal)g(to)f(the)75 701 y(t)o(yp)q(e)f (signature)h(implied)i(b)o(y)d Fp(recvcounts[i],)i(recvt)o(yp)q(e)f Ft(at)f(the)g(ro)q(ot.)20 b(This)c(implies)i(that)c(the)i(amoun)o(t)75 757 y(of)g(data)f(sen)o(t)h(m)o(ust)g(b)q(e)h(equal)g(to)e(the)i(amoun) o(t)e(of)h(data)f(receiv)o(ed,)j(pairwise)f(b)q(et)o(w)o(een)f(eac)o(h) h(pro)q(cess)75 814 y(and)12 b(the)g(ro)q(ot.)17 b(Distinct)c(t)o(yp)q (e)f(maps)f(b)q(et)o(w)o(een)h(sender)g(and)g(receiv)o(er)g(are)g (still)h(allo)o(w)o(ed,)f(as)g(illustrated)75 870 y(in)k(Example)g (1.6.)166 927 y(All)f(argumen)o(ts)f(to)f(the)h(function)h(are)f (signi\014can)o(t)h(on)f(pro)q(cess)h Fp(ro)q(ot)p Ft(,)e(while)j(on)e (other)g(pro)q(cesses,)75 983 y(only)19 b(argumen)o(ts)f Fp(sendbuf,)k(sendcount,)g(sendt)o(yp)q(e,)g(ro)q(ot,)d(comm)e Ft(are)i(signi\014can)o(t.)32 b(The)19 b(argumen)o(ts)75 1040 y Fp(ro)q(ot)c Ft(and)g Fp(comm)f Ft(m)o(ust)g(ha)o(v)o(e)h(iden)o (tical)i(v)m(alues)f(on)f(all)i(pro)q(cesses.)166 1096 y(The)d(sp)q(eci\014cation)i(of)d(coun)o(ts,)h(t)o(yp)q(es,)g(and)g (displacemen)o(ts)h(should)g(not)f(cause)g(an)o(y)f(lo)q(cation)i(on)75 1153 y(the)g(ro)q(ot)g(to)f(b)q(e)i(written)f(more)g(than)g(once.)20 b(Suc)o(h)c(a)f(call)h(is)g(erroneous.)75 1274 y Ff(1.5.1)49 b(Examples)16 b(using)h(MPI)p 640 1274 15 2 v 18 w(GA)l(THER,)g(MPI)p 963 1274 V 18 w(GA)l(THERV)75 1360 y Fn(Example)h(1.2)k Ft(Gather)15 b(100)f(in)o(ts)i(from)e(ev)o(ery)h(pro)q(cess)h(in)g (group)e(to)h(ro)q(ot.)k(See)d(\014gure)f(1.2.)170 1466 y Fg(MPI_Comm)23 b(comm;)170 1523 y(int)h(gsize,sendarray[100];)170 1579 y(int)g(root,)f(*rbuf;)170 1636 y(...)170 1692 y(MPI_Comm_size\()f (comm,)i(&gsize\);)170 1749 y(rbuf)g(=)f(\(int)h (*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 1805 y(MPI_Gather\()f (sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(100,)f(MPI_INT,)g(root,)g (comm\);)75 1911 y Fn(Example)18 b(1.3)k Ft(Previous)15 b(example)g(mo)q(di\014ed)g({)f(only)h(the)f(ro)q(ot)f(allo)q(cates)i (memory)e(for)h(the)g(receiv)o(e)75 1968 y(bu\013er.)170 2074 y Fg(MPI_Comm)23 b(comm;)170 2130 y(int)h(gsize,sendarray[100];) 170 2187 y(int)g(root,)f(myrank,)g(*rbuf;)170 2243 y(...)170 2300 y(MPI_Comm_rank\()f(comm,)i(myrank\);)170 2356 y(if)g(\()g(myrank) f(==)g(root\))h({)242 2413 y(MPI_Comm_size\()e(comm,)h(&gsize\);)242 2469 y(rbuf)g(=)h(\(int)f(*\)malloc\(gsize*100*sizeof\(int)o(\)\);)242 2526 y(})170 2582 y(MPI_Gather\()g(sendarray,)f(100,)i(MPI_INT,)e (rbuf,)i(100,)f(MPI_INT,)g(root,)g(comm\);)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 9 11 9 10 bop 75 -100 a Fk(1.5.)29 b(GA)l(THER)1479 b Ft(9)75 45 y Fn(Example)18 b(1.4)k Ft(Do)d(the)h(same)f(as)g(the)h(previous)g (example,)h(but)f(use)g(a)f(deriv)o(ed)i(datat)o(yp)q(e.)32 b(Note)75 102 y(that)17 b(the)h(t)o(yp)q(e)g(cannot)g(b)q(e)h(the)f(en) o(tire)g(set)g(of)g Fg(gsize*100)k(int)p Ft(s)c(since)h(t)o(yp)q(e)f (matc)o(hing)g(is)h(de\014ned)75 158 y(pairwise)d(b)q(et)o(w)o(een)g (the)f(ro)q(ot)f(and)i(eac)o(h)f(pro)q(cess)g(in)h(the)g(gather.)170 264 y Fg(MPI_Comm)23 b(comm;)170 320 y(int)h(gsize,sendarray[100];)170 376 y(int)g(root,)f(*rbuf;)170 433 y(MPI_Datatype)g(rtype;)170 489 y(...)170 546 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 602 y(MPI_Type_contiguous\()e(100,)h(MPI_INT,)g(&rtype)g(\);)170 659 y(MPI_Type_commit\()f(&rtype)h(\);)170 715 y(rbuf)h(=)f(\(int)h (*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 772 y(MPI_Gather\()f (sendarray,)f(100,)i(MPI_INT,)e(rbuf,)i(1,)f(rtype,)g(root,)g(comm\);) 75 877 y Fn(Example)18 b(1.5)k Ft(No)o(w)d(ha)o(v)o(e)g(eac)o(h)g(pro)q (cess)h(send)f(100)g(in)o(ts)g(to)g(ro)q(ot,)g(but)g(place)h(eac)o(h)g (set)f(\(of)f(100\))75 934 y Fm(stride)f Ft(in)o(ts)f(apart)g(at)g (receiving)i(end.)25 b(Use)17 b Fp(MPI)p 945 934 14 2 v 16 w(GA)l(THERV)h Ft(and)f(the)f Fp(displs)j Ft(argumen)o(t)c(to)h (ac)o(hiev)o(e)75 990 y(this)g(e\013ect.)j(Assume)d Fj(str)q(ide)c Fo(\025)h Ft(100.)19 b(See)d(\014gure)f(1.3.)170 1096 y Fg(MPI_Comm)23 b(comm;)170 1152 y(int)h(gsize,sendarray[100];)170 1209 y(int)g(root,)f(*rbuf,)g(stride;)170 1265 y(int)h (*displs,i,*rcounts;)170 1378 y(...)170 1491 y(MPI_Comm_size\()e(comm,) i(&gsize\);)170 1547 y(rbuf)g(=)f(\(int)h(*\)malloc\(gsize*stride*siz)o (eof\(int)o(\)\);)170 1604 y(displs)f(=)h(\(int)g (*\)malloc\(gsize*sizeof\(i)o(nt\)\);)170 1660 y(rcounts)f(=)h(\(int)f (*\)malloc\(gsize*sizeof\(int\)\);)170 1717 y(for)h(\(i=0;)f(i)23 b(SENDBUF\(*\),)g(RECVBUF\(*\))170 102 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g (ROOT,)g(COMM,)g(IERROR)166 188 y Fp(MPI)p 251 188 14 2 v 16 w(SCA)l(TTER)16 b Ft(is)g(the)f(in)o(v)o(erse)h(op)q(eration)f (to)g Fp(MPI)p 1097 188 V 15 w(GA)l(THER)p Ft(.)166 244 y(The)g(outcome)g(is)h Fm(as)g(if)f Ft(the)g(ro)q(ot)g(executed)h Fg(n)f Ft(send)h(op)q(erations,)189 320 y Fg(MPI)p 264 320 15 2 v 16 w(Send)p Ft(\()p Fg(sendbuf)9 b Ft(+)h Fg(i)g Fo(\001)g Fg(sendcount)f Fo(\001)g Fg(extent)p Ft(\()p Fg(sendtype)p Ft(\))p Fj(;)f Fg(sendc)o(ount)p Fj(;)f Fg(send)o(type)p Fj(;)g Fg(i)p Fj(;)g(::)o(:)p Ft(\))p Fj(;)75 395 y Ft(and)15 b(eac)o(h)h(pro)q(cess)f(executed)h(a)f (receiv)o(e,)189 471 y Fg(MPI)p 264 471 V 16 w(Recv)p Ft(\()p Fg(recvbuf)p Fj(;)8 b Fg(recvcount)p Fj(;)f Fg(re)o(cvtype)p Fj(;)g Fg(i)p Fj(;)g(::)o(:)p Ft(\))o Fj(:)166 546 y Ft(An)20 b(alternativ)o(e)g(description)h(is)g(that)e(the)h(ro)q(ot)f (sends)h(a)f(message)h(with)g Fp(MPI)p 1594 546 14 2 v 16 w(Send\(sendbuf,)75 602 y(sendcount)p Fo(\001)p Fp(n,)f(sendt)o(yp)q(e,)h(...\))p Ft(.)i(This)17 b(message)f(is)h (split)h(in)o(to)e Fg(n)g Ft(equal)i(segmen)o(ts,)e(the)g Fj(i)p Ft(th)g(segmen)o(t)g(is)75 659 y(sen)o(t)f(to)g(the)g Fj(i)p Ft(th)g(pro)q(cess)g(in)h(the)f(group,)g(and)g(eac)o(h)g(pro)q (cess)h(receiv)o(es)g(this)g(message)e(as)h(ab)q(o)o(v)o(e.)166 715 y(The)g(send)h(bu\013er)f(is)h(ignored)g(for)e(all)i(non-ro)q(ot)f (pro)q(cesses.)166 772 y(The)j(t)o(yp)q(e)f(signature)h(asso)q(ciated)g (with)f Fp(sendcount,)k(sendt)o(yp)q(e)f Ft(at)d(the)h(ro)q(ot)e(m)o (ust)h(b)q(e)h(equal)h(to)75 828 y(the)f(t)o(yp)q(e)g(signature)g(asso) q(ciated)g(with)g Fp(recvcount,)h(recvt)o(yp)q(e)g Ft(at)e(all)i(pro)q (cesses)f(\(ho)o(w)o(ev)o(er,)f(the)h(t)o(yp)q(e)75 885 y(maps)f(ma)o(y)g(b)q(e)h(di\013eren)o(t\).)27 b(This)18 b(implies)i(that)d(the)g(amoun)o(t)g(of)g(data)g(sen)o(t)g(m)o(ust)g(b) q(e)i(equal)f(to)f(the)75 941 y(amoun)o(t)f(of)f(data)h(receiv)o(ed,)i (pairwise)f(b)q(et)o(w)o(een)f(eac)o(h)h(pro)q(cess)f(and)h(the)f(ro)q (ot.)23 b(Distinct)17 b(t)o(yp)q(e)f(maps)75 998 y(b)q(et)o(w)o(een)g (sender)f(and)h(receiv)o(er)g(are)e(still)j(allo)o(w)o(ed.)166 1054 y(All)e(argumen)o(ts)f(to)f(the)h(function)h(are)f(signi\014can)o (t)h(on)f(pro)q(cess)h Fp(ro)q(ot)p Ft(,)e(while)j(on)e(other)g(pro)q (cesses,)75 1110 y(only)f(argumen)o(ts)f Fp(recvbuf,)i(recvcount,)g (recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)f Ft(are)h(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fp(ro)q(ot)75 1167 y Ft(and)j Fp(comm)f Ft(m)o(ust)g(ha)o(v)o(e)h(iden)o(tical)i(v)m(alues)g(on)e (all)h(pro)q(cesses.)166 1223 y(The)g(sp)q(eci\014cation)h(of)f(coun)o (ts)f(and)h(t)o(yp)q(es)f(should)i(not)e(cause)h(an)o(y)g(lo)q(cation)g (on)g(the)f(ro)q(ot)g(to)g(b)q(e)75 1280 y(read)g(more)g(than)g(once.) 189 1360 y Fm(R)n(ationale.)79 b Ft(Though)22 b(not)g(needed,)i(the)e (last)g(restriction)g(is)g(imp)q(osed)h(so)f(as)f(to)g(ac)o(hiev)o(e) 189 1417 y(symmetry)16 b(with)h Fp(MPI)p 593 1417 V 16 w(GA)l(THER)p Ft(,)h(where)f(the)g(corresp)q(onding)h(restriction)g (\(a)e(m)o(ultiple-write)189 1473 y(restriction\))f(is)h(necessary)l(.) k(\()p Fm(End)c(of)g(r)n(ationale.)p Ft(\))75 1648 y Fp(MPI)p 160 1648 V 16 w(SCA)l(TTERV\()21 b(sendbuf,)j(sendcounts)q(,)g (displs,)f(sendt)o(yp)q(e,)h(recvbuf,)f(recvcount,)f(recvt)o(yp)q(e,)h (ro)q(ot,)75 1705 y(comm\))117 1782 y Fh(IN)171 b Fp(sendbuf)428 b Fh(address)13 b(of)d(send)i(bu\013er)h(\(c)o(hoice,)f(signi\014can)o (t)e(only)h(at)g(ro)q(ot\))117 1853 y(IN)171 b Fp(sendcounts)366 b Fh(in)o(teger)10 b(arra)o(y)f(\(of)g(length)g(group)g(size\))h(sp)q (ecifying)g(the)g(n)o(um-)905 1910 y(b)q(er)15 b(of)e(elemen)o(ts)h(to) g(send)h(to)e(eac)o(h)i(pro)q(cessor)117 1981 y(IN)171 b Fp(displs)469 b Fh(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g (size\).)20 b(En)o(try)15 b Fe(i)f Fh(sp)q(eci\014es)905 2038 y(the)20 b(displacemen)o(t)e(\(relativ)o(e)h(to)f Fd(sendbuf)j Fh(from)c(whic)o(h)i(to)905 2094 y(tak)o(e)14 b(the)h(outgoing)d(data)i(to)g(pro)q(cess)h Fe(i)117 2165 y Fh(IN)171 b Fp(sendt)o(yp)q(e)408 b Fh(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 2237 y(OUT)124 b Fp(recvbuf)434 b Fh(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 2308 y(IN)171 b Fp(recvcount)389 b Fh(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(receiv)o(e)h (bu\013er)g(\(in)o(teger\))117 2380 y(IN)171 b Fp(recvt)o(yp)q(e)414 b Fh(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e (\(handle\))117 2451 y(IN)171 b Fp(ro)q(ot)492 b Fh(rank)14 b(of)f(sending)h(pro)q(cess)i(\(in)o(teger\))117 2523 y(IN)171 b Fp(comm)454 b Fh(comm)o(unicator)11 b(\(handle\))75 2647 y Fg(int)23 b(MPI)p 245 2647 15 2 v 17 w(Scatterv\(void*)f (sendbuf,)h(int)g(*sendcounts,)g(int)g(*displs,)393 2704 y(MPI)p 468 2704 V 17 w(Datatype)g(sendtype,)f(void*)i(recvbuf,)e(int)i (recvcount,)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 16 18 16 17 bop 75 -100 a Ft(16)708 b Fk(CHAPTER)15 b(1.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)393 45 y Fg(MPI)p 468 45 15 2 v 17 w(Datatype)23 b(recvtype,)f(int)i(root,)f(MPI)p 1249 45 V 17 w(Comm)g(comm\))75 132 y(MPI)p 150 132 V 17 w(SCATTERV\(SENDBUF,)e(SENDCOUNTS,)i(DISPLS,)g(SENDTYPE,)f(RECVBUF,) h(RECVCOUNT,)393 188 y(RECVTYPE,)g(ROOT,)g(COMM,)g(IERROR\))170 244 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 301 y(INTEGER)g (SENDCOUNTS\(*\),)f(DISPLS\(*\),)h(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g (ROOT,)170 357 y(COMM,)h(IERROR)166 444 y Fp(MPI)p 251 444 14 2 v 16 w(SCA)l(TTERV)16 b Ft(is)g(the)f(in)o(v)o(erse)h(op)q (eration)f(to)g Fp(MPI)p 1127 444 V 16 w(GA)l(THERV)p Ft(.)166 500 y Fp(MPI)p 251 500 V 16 w(SCA)l(TTERV)24 b Ft(extends)f(the)g(functionalit)o(y)g(of)g Fp(MPI)p 1193 500 V 15 w(SCA)l(TTER)h Ft(b)o(y)f(allo)o(wing)g(a)f(v)m(arying)75 557 y(coun)o(t)17 b(of)h(data)f(to)g(b)q(e)h(sen)o(t)f(to)g(eac)o(h)h (pro)q(cess,)g(since)h Fp(sendcounts)i Ft(is)d(no)o(w)f(an)g(arra)o(y)l (.)27 b(It)17 b(also)h(allo)o(ws)75 613 y(more)k(\015exibilit)o(y)i(as) e(to)f(where)h(the)g(data)g(is)g(tak)o(en)g(from)f(on)h(the)g(ro)q(ot,) h(b)o(y)f(pro)o(viding)h(the)f(new)75 670 y(argumen)o(t,)14 b Fp(displs)p Ft(.)166 726 y(The)h(outcome)g(is)h(as)f(if)g(the)h(ro)q (ot)e(executed)i Fg(n)f Ft(send)h(op)q(erations,)189 820 y Fg(MPI)p 264 820 15 2 v 16 w(Send)p Ft(\()p Fg(sendbuf)9 b Ft(+)h Fg(displs)p Ft([)p Fg(i)p Ft(])f Fo(\001)g Fg(extent)p Ft(\()p Fg(sendtype)p Ft(\))p Fj(;)f Fg(sen)o(dcounts)p Ft([)o Fg(i)p Ft(])p Fj(;)f Fg(sen)o(dtype)p Fj(;)g Fg(i)p Fj(;)g(::)o(:)p Ft(\))p Fj(;)75 914 y Ft(and)15 b(eac)o(h)h(pro)q(cess) f(executed)h(a)f(receiv)o(e,)189 1008 y Fg(MPI)p 264 1008 V 16 w(Recv)p Ft(\()p Fg(recvbuf)p Fj(;)8 b Fg(recvcount)p Fj(;)f Fg(re)o(cvtype)p Fj(;)g Fg(i)p Fj(;)g(::)o(:)p Ft(\))o Fj(:)166 1101 y Ft(The)15 b(send)h(bu\013er)f(is)h(ignored)g (for)e(all)i(non-ro)q(ot)f(pro)q(cesses.)166 1158 y(The)h(t)o(yp)q(e)g (signature)g(implied)i(b)o(y)e Fp(sendcount[i],)j(sendt)o(yp)q(e)g Ft(at)c(the)h(ro)q(ot)f(m)o(ust)g(b)q(e)i(equal)g(to)e(the)75 1214 y(t)o(yp)q(e)f(signature)h(implied)i(b)o(y)d Fp(recvcount,)i (recvt)o(yp)q(e)f Ft(at)f(pro)q(cess)g Fg(i)g Ft(\(ho)o(w)o(ev)o(er,)f (the)i(t)o(yp)q(e)f(maps)h(ma)o(y)e(b)q(e)75 1271 y(di\013eren)o(t\).) 20 b(This)c(implies)h(that)e(the)g(amoun)o(t)f(of)h(data)g(sen)o(t)g(m) o(ust)f(b)q(e)i(equal)g(to)f(the)g(amoun)o(t)g(of)g(data)75 1327 y(receiv)o(ed,)j(pairwise)f(b)q(et)o(w)o(een)g(eac)o(h)g(pro)q (cess)g(and)g(the)g(ro)q(ot.)23 b(Distinct)17 b(t)o(yp)q(e)g(maps)f(b)q (et)o(w)o(een)h(sender)75 1384 y(and)e(receiv)o(er)h(are)f(still)i (allo)o(w)o(ed.)166 1440 y(All)e(argumen)o(ts)f(to)f(the)h(function)h (are)f(signi\014can)o(t)h(on)f(pro)q(cess)h Fp(ro)q(ot)p Ft(,)e(while)j(on)e(other)g(pro)q(cesses,)75 1497 y(only)f(argumen)o (ts)f Fp(recvbuf,)i(recvcount,)g(recvt)o(yp)q(e,)g(ro)q(ot,)e(comm)f Ft(are)h(signi\014can)o(t.)20 b(The)13 b(argumen)o(ts)f Fp(ro)q(ot)75 1553 y Ft(and)j Fp(comm)f Ft(m)o(ust)g(ha)o(v)o(e)h(iden) o(tical)i(v)m(alues)g(on)e(all)h(pro)q(cesses.)166 1609 y(The)e(sp)q(eci\014cation)i(of)d(coun)o(ts,)h(t)o(yp)q(es,)g(and)g (displacemen)o(ts)h(should)g(not)f(cause)g(an)o(y)f(lo)q(cation)i(on)75 1666 y(the)g(ro)q(ot)g(to)f(b)q(e)i(read)f(more)g(than)g(once.)75 1788 y Ff(1.6.1)49 b(Examples)16 b(using)h(MPI)p 640 1788 V 18 w(SCA)l(TTER,)h(MPI)p 988 1788 V 18 w(SCA)l(TTERV)75 1873 y Fn(Example)g(1.11)23 b Ft(The)15 b(rev)o(erse)g(of)f(Example)i (1.2.)j(Scatter)14 b(sets)h(of)g(100)f(in)o(ts)h(from)f(the)h(ro)q(ot)f (to)h(eac)o(h)75 1930 y(pro)q(cess)g(in)h(the)g(group.)j(See)d (\014gure)g(1.7.)170 2036 y Fg(MPI_Comm)23 b(comm;)170 2093 y(int)h(gsize,*sendbuf;)170 2149 y(int)g(root,)f(rbuf[100];)170 2206 y(...)170 2262 y(MPI_Comm_size\()f(comm,)i(&gsize\);)170 2318 y(sendbuf)f(=)h(\(int)f(*\)malloc\(gsize*100*sizeof\(int)o(\)\);) 170 2375 y(...)170 2431 y(MPI_Scatter\()g(sendbuf,)f(100,)i(MPI_INT,)f (rbuf,)g(100,)g(MPI_INT,)g(root,)g(comm\);)75 2587 y Fn(Example)18 b(1.12)23 b Ft(The)16 b(rev)o(erse)h(of)f(Example)i(1.5.) 23 b(The)17 b(ro)q(ot)f(pro)q(cess)h(scatters)f(sets)g(of)h(100)f(in)o (ts)h(to)75 2644 y(the)d(other)g(pro)q(cesses,)g(but)g(the)g(sets)g(of) g(100)f(are)h Fm(stride)f Ft(in)o(ts)i(apart)e(in)i(the)f(sending)h (bu\013er.)20 b(Requires)75 2700 y(use)c(of)e Fp(MPI)p 290 2700 14 2 v 16 w(SCA)l(TTERV)p Ft(.)i(Assume)g Fj(str)q(ide)c Fo(\025)h Ft(100.)19 b(See)d(\014gure)g(1.8.)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 17 19 17 18 bop 75 -100 a Fk(1.6.)29 b(SCA)l(TTER)1436 b Ft(17)458 4 y 16577003 6630800 0 0 33943388 13682606 startTexFig 458 4 a %%BeginDocument: mycoll-fig7.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 373.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth % Polyline n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr % Polyline n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr % Polyline n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 219 359 m 219 319 l 159 319 l 159 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr % Polyline n 279 359 m 279 319 l 219 319 l 219 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Roman findfont 18.00 scalefont setfont 99 414 m gs 1 -1 scale (sendbuf) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 234 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr % Polyline n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr % Polyline n 119 299 m 89 244 l gs col-1 s gr n 91.075 251.981 m 89.000 244.000 l 94.587 250.065 l gs 2 setlinejoin col-1 s gr % Polyline n 189 299 m 189 244 l gs col-1 s gr n 187.000 252.000 m 189.000 244.000 l 191.000 252.000 l gs 2 setlinejoin col-1 s gr % Polyline n 249 299 m 289 244 l gs col-1 s gr n 282.677 249.294 m 289.000 244.000 l 285.912 251.646 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr $F2psEnd %%EndDocument endTexFig 138 522 a Ft(Figure)16 b(1.7:)i(The)e(ro)q(ot)e(pro)q(cess)i(scatters) e(sets)h(of)g(100)f Fg(int)p Ft(s)h(to)f(eac)o(h)i(pro)q(cess)f(in)h (the)f(group.)458 580 y 16577003 6796570 0 0 33943388 14011514 startTexFig 458 580 a %%BeginDocument: mycoll-fig8.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -54.0 378.0 translate 0.900 -0.900 scale /Times-Italic findfont 16.00 scalefont setfont 74 194 m gs 1 -1 scale (100) col-1 show gr 0.500 setlinewidth % Polyline n 219 239 m 219 199 l 159 199 l 159 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 174 194 m gs 1 -1 scale (100) col-1 show gr % Polyline n 319 239 m 319 199 l 259 199 l 259 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 194 m gs 1 -1 scale (100) col-1 show gr % Polyline n 159 359 m 159 319 l 99 319 l 99 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 114 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 239 359 m 239 319 l 179 319 l 179 359 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 194 314 m gs 1 -1 scale (100) col-1 show gr % Polyline n 319 359 m 319 319 l 259 319 l 259 359 l clp gs 0.50 setgray fill gr gs col-1 s gr % Polyline n 119 239 m 119 199 l 59 199 l 59 239 l clp gs 0.50 setgray fill gr gs col-1 s gr /Times-Italic findfont 16.00 scalefont setfont 274 314 m gs 1 -1 scale (100) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 99 419 m gs 1 -1 scale (sendbuf) col-1 show gr % Polyline n 479 359 m 479 319 l 99 319 l 99 359 l clp gs col-1 s gr % Polyline n 99 394 m 99 359 l gs 0.50 setgray fill gr gs col-1 s gr n 97.000 367.000 m 99.000 359.000 l 101.000 367.000 l gs 2 setlinejoin col-1 s gr n 187.000 371.000 m 179.000 369.000 l 187.000 367.000 l gs 2 setlinejoin col-1 s gr % Polyline n 179 369 m 259 369 l gs col-1 s gr n 251.000 367.000 m 259.000 369.000 l 251.000 371.000 l gs 2 setlinejoin col-1 s gr % Polyline n 119 299 m 84 244 l gs col-1 s gr n 86.608 251.823 m 84.000 244.000 l 89.982 249.676 l gs 2 setlinejoin col-1 s gr % Polyline n 204 304 m 189 244 l gs col-1 s gr n 189.000 252.246 m 189.000 244.000 l 192.881 251.276 l gs 2 setlinejoin col-1 s gr % Polyline n 284 304 m 289 244 l gs col-1 s gr n 286.343 251.806 m 289.000 244.000 l 290.329 252.138 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 524 344 m gs 1 -1 scale (at root) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 519 229 m gs 1 -1 scale (all processes) col-1 show gr /Times-Roman findfont 18.00 scalefont setfont 199 394 m gs 1 -1 scale (stride) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 1109 a Ft(Figure)g(1.8:)j(The)d(ro)q(ot)e(pro)q(cess)i(scatters)f (sets)g(of)g(100)f Fg(int)p Ft(s,)h(mo)o(ving)g(b)o(y)h Fg(stride)e Ft(in)o(ts)i(from)f(send)h(to)75 1165 y(send)h(in)g(the)f (scatter.)170 1290 y Fg(MPI_Comm)23 b(comm;)170 1346 y(int)h(gsize,*sendbuf;)170 1403 y(int)g(root,)f(rbuf[100],)g(i,)g (*displs,)g(*scounts;)170 1515 y(...)170 1628 y(MPI_Comm_size\()f (comm,)i(&gsize\);)170 1685 y(sendbuf)f(=)h(\(int)f (*\)malloc\(gsize*stride*sizeof\()o(int\)\);)170 1741 y(...)170 1798 y(displs)g(=)h(\(int)g(*\)malloc\(gsize*sizeof\(i)o (nt\)\);)170 1854 y(scounts)f(=)h(\(int)f (*\)malloc\(gsize*sizeof\(int\)\);)170 1911 y(for)h(\(i=0;)f(i)g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1319 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)166 1405 y Fp(MPI)p 251 1405 14 2 v 16 w(ALLGA)l(THER)16 b Ft(can)g(b)q(e)h(though)o(t)e(of)g(as)h Fp(MPI)p 1047 1405 V 15 w(GA)l(THER)p Ft(,)h(but)f(where)g(all)h(pro)q(cesses)f (receiv)o(e)75 1462 y(the)e(result,)f(instead)i(of)e(just)g(the)g(ro)q (ot.)19 b(The)13 b Fg(j)p Ft(th)h(blo)q(c)o(k)g(of)f(data)g(sen)o(t)g (from)g(eac)o(h)g(pro)q(cess)h(is)g(receiv)o(ed)75 1518 y(b)o(y)h(ev)o(ery)g(pro)q(cess)h(and)f(placed)h(in)g(the)g Fg(j)p Ft(th)f(blo)q(c)o(k)h(of)e(the)i(bu\013er)f Fp(recvbuf)p Ft(.)166 1575 y(The)g(t)o(yp)q(e)g(signature)g(asso)q(ciated)g(with)h Fp(sendcount,)h(sendt)o(yp)q(e)p Ft(,)h(at)c(a)h(pro)q(cess)g(m)o(ust)f (b)q(e)i(equal)g(to)75 1631 y(the)f(t)o(yp)q(e)h(signature)f(asso)q (ciated)g(with)h Fp(recvcount,)g(recvt)o(yp)q(e)g Ft(at)f(an)o(y)f (other)h(pro)q(cess.)166 1687 y(The)f(outcome)g(of)g(a)g(call)h(to)f Fp(MPI)p 749 1687 V 16 w(ALLGA)l(THER\(...\))19 b Ft(is)c(as)f(if)g (all)h(pro)q(cesses)g(executed)g Fg(n)f Ft(calls)i(to)147 1781 y Fg(MPI_GATHER\(sendbuf,send)o(count,s)o(endtype,)o(recvbuf)o (,recvco)o(unt,)1244 1838 y(recvtype,root,comm\),)75 1932 y Ft(for)11 b Fg(root)23 b(=)h(0)g(,)f(...,)h(n-1)p Ft(.)18 b(The)12 b(rules)g(for)f(correct)g(usage)g(of)h Fp(MPI)p 1289 1932 V 15 w(ALLGA)l(THER)h Ft(are)e(easily)i(found)75 1988 y(from)h(the)i(corresp)q(onding)g(rules)g(for)e Fp(MPI)p 816 1988 V 16 w(GA)l(THER)p Ft(.)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 20 22 20 21 bop 75 -100 a Ft(20)708 b Fk(CHAPTER)15 b(1.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fp(MPI)p 160 45 14 2 v 16 w(ALLGA)l(THERV\()11 b(sendbuf,)i(sendcount,)h(sendt)o(yp) q(e,)f(recvbuf,)f(recvcounts,)g(displs,)g(recvt)o(yp)q(e,)g(comm\))117 179 y Fh(IN)171 b Fp(sendbuf)428 b Fh(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 254 y(IN)171 b Fp(sendcount)383 b Fh(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f (\(in)o(teger\))117 329 y(IN)171 b Fp(sendt)o(yp)q(e)408 b Fh(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f (\(handle\))117 404 y(OUT)124 b Fp(recvbuf)434 b Fh(address)15 b(of)f(receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 479 y(IN)171 b Fp(recvcounts)372 b Fh(in)o(teger)10 b(arra)o(y)f(\(of)g (length)g(group)g(size\))h(con)o(taining)e(the)i(n)o(um-)905 536 y(b)q(er)15 b(of)e(elemen)o(ts)h(that)g(are)g(receiv)o(ed)h(from)e (eac)o(h)h(pro)q(cess)117 611 y(IN)171 b Fp(displs)469 b Fh(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group)g(size\).)20 b(En)o(try)15 b Fe(i)f Fh(sp)q(eci\014es)905 667 y(the)d(displacemen)o (t)e(\(relativ)o(e)g(to)h Fd(recvbuf)p Fh(\))i(at)d(whic)o(h)h(to)g (place)905 724 y(the)15 b(incoming)c(data)j(from)e(pro)q(cess)k Fe(i)117 799 y Fh(IN)171 b Fp(recvt)o(yp)q(e)414 b Fh(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j(bu\013er)f(elemen)o(ts)e(\(handle\)) 117 874 y(IN)171 b Fp(comm)454 b Fh(comm)o(unicator)11 b(\(handle\))75 999 y Fg(int)23 b(MPI)p 245 999 15 2 v 17 w(Allgatherv\(void*)f(sendbuf,)h(int)g(sendcount,)g(MPI)p 1313 999 V 16 w(Datatype)g(sendtype,)393 1055 y(void*)g(recvbuf,)g(int) h(*recvcounts,)e(int)h(*displs,)393 1111 y(MPI)p 468 1111 V 17 w(Datatype)g(recvtype,)f(MPI)p 1010 1111 V 17 w(Comm)h(comm\))75 1198 y(MPI)p 150 1198 V 17 w (ALLGATHERV\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h (RECVCOUNTS,)f(DISPLS,)393 1254 y(RECVTYPE,)h(COMM,)g(IERROR\))170 1311 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1367 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNTS\(*\),)g(DISPLS\(*\),)h (RECVTYPE,)g(COMM,)170 1424 y(IERROR)166 1510 y Fp(MPI)p 251 1510 14 2 v 16 w(ALLGA)l(THERV)11 b Ft(can)g(b)q(e)g(though)o(t)e (of)h(as)g Fp(MPI)p 1044 1510 V 16 w(GA)l(THERV)p Ft(,)h(but)g(where)f (all)h(pro)q(cesses)g(receiv)o(e)75 1567 y(the)j(result,)f(instead)i (of)e(just)g(the)g(ro)q(ot.)19 b(The)13 b Fg(j)p Ft(th)h(blo)q(c)o(k)g (of)f(data)g(sen)o(t)g(from)g(eac)o(h)g(pro)q(cess)h(is)g(receiv)o(ed) 75 1623 y(b)o(y)i(ev)o(ery)h(pro)q(cess)g(and)f(placed)i(in)f(the)g Fg(j)p Ft(th)f(blo)q(c)o(k)h(of)f(the)h(bu\013er)f Fp(recvbuf)p Ft(.)25 b(These)17 b(blo)q(c)o(ks)g(need)g(not)75 1680 y(all)f(b)q(e)g(the)f(same)g(size.)166 1736 y(The)g(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)g Fp(sendcount,)j(sendt)o(yp)q(e)p Ft(,)f(at)d(pro)q(cess)h Fg(j)g Ft(m)o(ust)f(b)q(e)i(equal)g(to)75 1792 y(the)f(t)o(yp)q(e)h(signature)f(asso)q(ciated)g(with)h Fp(recvcounts[j],)h(recvt)o(yp)q(e)f Ft(at)e(an)o(y)h(other)g(pro)q (cess.)166 1849 y(The)g(outcome)g(is)h(as)f(if)g(all)h(pro)q(cesses)g (executed)g(calls)g(to)170 1943 y Fg(MPI_GATHERV\(sendbuf,sendcount)o (,sendtyp)o(e,recvb)o(uf,recv)o(counts,d)o(ispls,)1292 1999 y(recvtype,root,comm\),)75 2093 y Ft(for)k Fg(root)j(=)h(0)f(,)h (...,)f(n-1)p Ft(.)35 b(The)20 b(rules)h(for)e(correct)h(usage)g(of)g Fp(MPI)p 1365 2093 V 16 w(ALLGA)l(THERV)h Ft(are)f(easily)75 2149 y(found)c(from)e(the)h(corresp)q(onding)h(rules)g(for)f Fp(MPI)p 944 2149 V 16 w(GA)l(THERV)p Ft(.)75 2271 y Ff(1.7.1)49 b(Examples)16 b(using)h(MPI)p 640 2271 15 2 v 18 w(ALLGA)l(THER,)h(MPI)p 1048 2271 V 18 w(ALLGA)l(THERV)75 2357 y Fn(Example)g(1.14)23 b Ft(The)18 b(all-gather)h(v)o(ersion)g(of) f(Example)i(1.2.)29 b(Using)19 b Fp(MPI)p 1434 2357 14 2 v 16 w(ALLGA)l(THER)p Ft(,)g(w)o(e)f(will)75 2413 y(gather)d(100)f (in)o(ts)h(from)g(ev)o(ery)g(pro)q(cess)g(in)h(the)g(group)e(to)h(ev)o (ery)g(pro)q(cess.)170 2520 y Fg(MPI_Comm)23 b(comm;)170 2576 y(int)h(gsize,sendarray[100];)170 2633 y(int)g(*rbuf;)170 2689 y(...)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 21 23 21 22 bop 75 -100 a Fk(1.8.)34 b(ALL-TO-ALL)18 b(SCA)l(TTER/GA)l(THER) 915 b Ft(21)170 45 y Fg(MPI_Comm_size\()22 b(comm,)i(&gsize\);)170 102 y(rbuf)g(=)f(\(int)h(*\)malloc\(gsize*100*sizeof)o(\(int\)\);)170 158 y(MPI_Allgather\()e(sendarray,)h(100,)g(MPI_INT,)g(rbuf,)g(100,)g (MPI_INT,)g(comm\);)166 264 y Ft(After)15 b(the)g(call,)h(ev)o(ery)f (pro)q(cess)h(has)f(the)g(group-wide)h(concatenation)f(of)g(the)g(sets) g(of)g(data.)75 407 y Fq(1.8)59 b(All-to-All)20 b(Scatter/Gather)75 603 y Fp(MPI)p 160 603 14 2 v 16 w(ALL)l(TO)o(ALL\(sendbuf,)c (sendcount,)i(sendt)o(yp)q(e,)f(recvbuf,)f(recvcount,)g(recvt)o(yp)q (e,)g(comm\))117 681 y Fh(IN)171 b Fp(sendbuf)428 b Fh(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 756 y(IN)171 b Fp(sendcount)383 b Fh(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(sen)o(t)i(to)f(eac)o(h)g(pro)q(cess)i(\(in)o (teger\))117 831 y(IN)171 b Fp(sendt)o(yp)q(e)408 b Fh(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 906 y(OUT)124 b Fp(recvbuf)434 b Fh(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 981 y(IN)171 b Fp(recvcount)389 b Fh(n)o(um)o(b)q(er)17 b(of)f(elemen)o(ts)h(receiv)o(ed)i(from)c(an)o (y)i(pro)q(cess)i(\(in)o(te-)905 1038 y(ger\))117 1113 y(IN)171 b Fp(recvt)o(yp)q(e)414 b Fh(data)14 b(t)o(yp)q(e)g(of)f (receiv)o(e)j(bu\013er)f(elemen)o(ts)e(\(handle\))117 1188 y(IN)171 b Fp(comm)454 b Fh(comm)o(unicator)11 b(\(handle\))75 1312 y Fg(int)23 b(MPI)p 245 1312 15 2 v 17 w(Alltoall\(void*)f (sendbuf,)h(int)g(sendcount,)g(MPI)p 1265 1312 V 17 w(Datatype)f (sendtype,)393 1369 y(void*)h(recvbuf,)g(int)h(recvcount,)e(MPI)p 1184 1369 V 17 w(Datatype)h(recvtype,)393 1425 y(MPI)p 468 1425 V 17 w(Comm)g(comm\))75 1512 y(MPI)p 150 1512 V 17 w(ALLTOALL\(SENDBUF,)e(SENDCOUNT,)i(SENDTYPE,)f(RECVBUF,)h (RECVCOUNT,)g(RECVTYPE,)393 1568 y(COMM,)g(IERROR\))170 1625 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1681 y(INTEGER)g(SENDCOUNT,)g(SENDTYPE,)f(RECVCOUNT,)h(RECVTYPE,)g(COMM,)g (IERROR)166 1768 y Fp(MPI)p 251 1768 14 2 v 16 w(ALL)l(TO)o(ALL)18 b Ft(is)h(an)g(extension)h(of)e Fp(MPI)p 962 1768 V 16 w(ALLGA)l(THER)i Ft(to)e(the)h(case)g(where)g(eac)o(h)g(pro)q(cess)75 1824 y(sends)e(distinct)g(data)f(to)g(eac)o(h)g(of)g(the)g(receiv)o (ers.)24 b(The)17 b Fg(j)p Ft(th)f(blo)q(c)o(k)h(sen)o(t)f(from)f(pro)q (cess)i Fg(i)f Ft(is)h(receiv)o(ed)75 1880 y(b)o(y)e(pro)q(cess)h Fg(j)f Ft(and)g(is)h(placed)g(in)g(the)f Fg(i)p Ft(th)g(blo)q(c)o(k)h (of)f Fp(recvbuf)p Ft(.)166 1937 y(The)g(t)o(yp)q(e)g(signature)g(asso) q(ciated)g(with)h Fp(sendcount,)h(sendt)o(yp)q(e)p Ft(,)h(at)c(a)h(pro) q(cess)g(m)o(ust)f(b)q(e)i(equal)g(to)75 1993 y(the)i(t)o(yp)q(e)g (signature)g(asso)q(ciated)g(with)g Fp(recvcount,)i(recvt)o(yp)q(e)f Ft(at)e(an)o(y)h(other)f(pro)q(cess.)28 b(This)19 b(implies)75 2050 y(that)10 b(the)h(amoun)o(t)g(of)f(data)h(sen)o(t)f(m)o(ust)h(b)q (e)h(equal)f(to)g(the)g(amoun)o(t)f(of)h(data)f(receiv)o(ed,)j (pairwise)f(b)q(et)o(w)o(een)75 2106 y(ev)o(ery)j(pair)h(of)e(pro)q (cesses.)21 b(As)15 b(usual,)g(ho)o(w)o(ev)o(er,)f(the)i(t)o(yp)q(e)f (maps)g(ma)o(y)f(b)q(e)i(di\013eren)o(t.)166 2163 y(The)d(outcome)g(is) h(as)e(if)i(eac)o(h)f(pro)q(cess)g(executed)h(a)f(send)h(to)e(eac)o(h)h (pro)q(cess)h(\(itself)f(included\))j(with)75 2219 y(a)f(call)h(to,)189 2313 y Fg(MPI)p 264 2313 15 2 v 16 w(Send)p Ft(\()p Fg(sendbuf)9 b Ft(+)h Fg(i)g Fo(\001)g Fg(sendcount)f Fo(\001)g Fg(extent)p Ft(\()p Fg(sendtype)p Ft(\))p Fj(;)f Fg(sendc)o(ount)p Fj(;)f Fg(send)o(type)p Fj(;)g Fg(i)p Fj(;)g(::)o(:)p Ft(\))p Fj(;)75 2407 y Ft(and)15 b(a)g(receiv)o(e)h(from)f(ev)o(ery)g (other)g(pro)q(cess)g(with)h(a)f(call)h(to,)189 2501 y Fg(MPI)p 264 2501 V 16 w(Recv)p Ft(\()p Fg(recvbuf)9 b Ft(+)h Fg(i)g Fo(\001)g Fg(recvcount)f Fo(\001)g Fg(extent)p Ft(\()p Fg(recvtype)p Ft(\))p Fj(;)f Fg(recvc)o(ount)p Fj(;)f Fg(i)p Fj(;)g(::)o(:)p Ft(\))p Fj(:)166 2594 y Ft(All)14 b(argumen)o(ts)d(on)h(all)i(pro)q(cesses)f(are)f (signi\014can)o(t.)20 b(The)12 b(argumen)o(t)g Fp(comm)e Ft(m)o(ust)i(ha)o(v)o(e)g(iden)o(tical)75 2651 y(v)m(alues)k(on)f(all)i (pro)q(cesses.)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 22 24 22 23 bop 75 -100 a Ft(22)703 b Fk(CHAPTER)15 b(1.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fp(MPI)p 160 45 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V\(sendbuf,)13 b(sendcounts)q(,)h (sdispls,)g(sendt)o(yp)q(e,)g(recvbuf,)e(recvcounts,)i(rdispls,)e (recvt)o(yp)q(e,)75 102 y(comm\))117 179 y Fh(IN)171 b Fp(sendbuf)428 b Fh(starting)14 b(address)h(of)f(send)g(bu\013er)h (\(c)o(hoice\))117 252 y(IN)171 b Fp(sendcounts)366 b Fh(in)o(teger)20 b(arra)o(y)f(equal)h(to)f(the)h(group)g(size)g(sp)q (ecifying)f(the)905 309 y(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(to)h (send)h(to)f(eac)o(h)g(pro)q(cessor)117 382 y(IN)171 b Fp(sdispls)452 b Fh(in)o(teger)15 b(arra)o(y)f(\(of)g(length)g(group) g(size\).)20 b(En)o(try)15 b Fe(j)f Fh(sp)q(eci\014es)905 439 y(the)20 b(displacemen)o(t)e(\(relativ)o(e)h(to)f Fd(sendbuf)j Fh(from)c(whic)o(h)i(to)905 495 y(tak)o(e)14 b(the)h(outgoing)d(data)i(destined)h(for)e(pro)q(cess)j Fe(j)117 568 y Fh(IN)171 b Fp(sendt)o(yp)q(e)408 b Fh(data)14 b(t)o(yp)q(e)g(of)f(send)i(bu\013er)g(elemen)o(ts)f(\(handle\))117 642 y(OUT)124 b Fp(recvbuf)434 b Fh(address)15 b(of)f(receiv)o(e)h (bu\013er)g(\(c)o(hoice\))117 715 y(IN)171 b Fp(recvcounts)372 b Fh(in)o(teger)20 b(arra)o(y)f(equal)h(to)f(the)h(group)g(size)g(sp)q (ecifying)f(the)905 772 y(n)o(um)o(b)q(er)g(of)f(elemen)o(ts)h(that)g (can)h(b)q(e)g(receiv)o(ed)g(from)d(eac)o(h)905 828 y(pro)q(cessor)117 902 y(IN)171 b Fp(rdispls)453 b Fh(in)o(teger)15 b(arra)o(y)f(\(of)g (length)g(group)g(size\).)20 b(En)o(try)15 b Fe(i)f Fh(sp)q(eci\014es) 905 958 y(the)f(displacemen)o(t)e(\(relativ)o(e)g(to)h Fd(recvbuf)i Fh(at)d(whic)o(h)h(to)g(place)905 1015 y(the)j(incoming)c (data)j(from)e(pro)q(cess)k Fe(i)117 1088 y Fh(IN)171 b Fp(recvt)o(yp)q(e)414 b Fh(data)14 b(t)o(yp)q(e)g(of)f(receiv)o(e)j (bu\013er)f(elemen)o(ts)e(\(handle\))117 1162 y(IN)171 b Fp(comm)454 b Fh(comm)o(unicator)11 b(\(handle\))75 1286 y Fg(int)23 b(MPI)p 245 1286 15 2 v 17 w(Alltoallv\(void*)f (sendbuf,)h(int)g(*sendcounts,)f(int)i(*sdispls,)393 1342 y(MPI)p 468 1342 V 17 w(Datatype)f(sendtype,)f(void*)i(recvbuf,)e (int)i(*recvcounts,)393 1399 y(int)g(*rdispls,)e(MPI)p 802 1399 V 17 w(Datatype)h(recvtype,)f(MPI)p 1344 1399 V 17 w(Comm)i(comm\))75 1485 y(MPI)p 150 1485 V 17 w (ALLTOALLV\(SENDBUF,)d(SENDCOUNTS,)i(SDISPLS,)f(SENDTYPE,)h(RECVBUF,)g (RECVCOUNTS,)393 1542 y(RDISPLS,)g(RECVTYPE,)g(COMM,)g(IERROR\))170 1598 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1655 y(INTEGER)g(SENDCOUNTS\(*\),)f(SDISPLS\(*\),)h(SENDTYPE,)f (RECVCOUNTS\(*\),)g(RDISPLS\(*\),)170 1711 y(RECVTYPE,)h(COMM,)g (IERROR)166 1798 y Fp(MPI)p 251 1798 14 2 v 16 w(ALL)l(TO)o(ALL)-5 b(V)12 b Ft(adds)g(\015exibili)q(t)o(y)j(to)c Fp(MPI)p 959 1798 V 16 w(ALL)l(TO)o(ALL)h Ft(in)h(that)f(the)h(lo)q(cation)g(of) f(data)g(for)g(the)75 1854 y(send)j(is)f(sp)q(eci\014ed)j(b)o(y)d Fp(sdispls)i Ft(and)e(the)g(lo)q(cation)h(of)f(the)g(placemen)o(t)h(of) f(the)g(data)g(on)g(the)g(receiv)o(e)h(side)75 1911 y(is)h(sp)q (eci\014ed)h(b)o(y)e Fp(rdispls)p Ft(.)166 1967 y(The)k Fg(j)p Ft(th)f(blo)q(c)o(k)i(sen)o(t)e(from)g(pro)q(cess)h Fg(i)g Ft(is)g(receiv)o(ed)h(b)o(y)e(pro)q(cess)h Fg(j)g Ft(and)g(is)g(placed)h(in)f(the)g Fg(i)p Ft(th)75 2023 y(blo)q(c)o(k)d(of)f Fp(recvbuf)p Ft(.)20 b(These)c(blo)q(c)o(ks)g (need)g(not)f(all)h(ha)o(v)o(e)f(the)g(same)g(size.)166 2080 y(The)i(t)o(yp)q(e)h(signature)f(asso)q(ciated)h(with)f Fp(sendcount[)q(j],)j(sendt)o(yp)q(e)g Ft(at)d(pro)q(cess)g Fg(i)g Ft(m)o(ust)g(b)q(e)h(equal)75 2136 y(to)d(the)i(t)o(yp)q(e)f (signature)g(asso)q(ciated)g(with)h Fp(recvcount[i],)g(recvt)o(yp)q(e)g Ft(at)e(pro)q(cess)i Fg(j)p Ft(.)22 b(This)17 b(implies)h(that)75 2193 y(the)f(amoun)o(t)g(of)g(data)g(sen)o(t)g(m)o(ust)f(b)q(e)i(equal) g(to)f(the)h(amoun)o(t)e(of)h(data)g(receiv)o(ed,)h(pairwise)h(b)q(et)o (w)o(een)75 2249 y(ev)o(ery)c(pair)h(of)e(pro)q(cesses.)21 b(Distinct)16 b(t)o(yp)q(e)f(maps)g(b)q(et)o(w)o(een)g(sender)h(and)f (receiv)o(er)h(are)f(still)i(allo)o(w)o(ed.)166 2306 y(The)e(outcome)g(is)h(as)f(if)g(eac)o(h)h(pro)q(cess)f(sen)o(t)g(a)g (message)g(to)f(ev)o(ery)h(other)g(pro)q(cess)h(with,)189 2391 y Fg(MPI)p 264 2391 15 2 v 16 w(Send)p Ft(\()p Fg(sendbuf)9 b Ft(+)h Fg(displs)p Ft([)p Fg(i)p Ft(])f Fo(\001)g Fg(extent)p Ft(\()p Fg(sendtype)p Ft(\))p Fj(;)f Fg(sen)o(dcounts)p Ft([)o Fg(i)p Ft(])p Fj(;)f Fg(sen)o(dtype)p Fj(;)g Fg(i)p Fj(;)g(::)o(:)p Ft(\))p Fj(;)75 2476 y Ft(and)15 b(receiv)o(ed)i(a)e (message)f(from)h(ev)o(ery)g(other)g(pro)q(cess)g(with)h(a)f(call)h(to) 189 2562 y Fg(MPI)p 264 2562 V 16 w(Recv)p Ft(\()p Fg(recvbuf)9 b Ft(+)h Fg(displs)p Ft([)p Fg(i)p Ft(])f Fo(\001)g Fg(extent)p Ft(\()p Fg(recvtype)p Ft(\))p Fj(;)f Fg(rec)o(vcounts)p Ft([)o Fg(i)p Ft(])p Fj(;)f Fg(rec)o(vtype)p Fj(;)g Fg(i)p Fj(;)g(::)o(:)p Ft(\))p Fj(:)166 2647 y Ft(All)14 b(argumen)o(ts)d(on)h (all)i(pro)q(cesses)f(are)f(signi\014can)o(t.)20 b(The)12 b(argumen)o(t)g Fp(comm)e Ft(m)o(ust)i(ha)o(v)o(e)g(iden)o(tical)75 2704 y(v)m(alues)k(on)f(all)i(pro)q(cesses.)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 23 25 23 24 bop 75 -100 a Fk(1.9.)34 b(GLOBAL)16 b(REDUCTION)g(OPERA)l(TIONS) 837 b Ft(23)189 45 y Fm(R)n(ationale.)60 b Ft(The)19 b(de\014nitions)h(of)e Fp(MPI)p 911 45 14 2 v 16 w(ALL)l(TO)o(ALL)g Ft(and)h Fp(MPI)p 1338 45 V 16 w(ALL)l(TO)o(ALL)-5 b(V)18 b Ft(giv)o(e)h(as)f(m)o(uc)o(h)189 102 y(\015exibilit)o(y)e(as)d(one)h (w)o(ould)g(ac)o(hiev)o(e)g(b)o(y)f(sp)q(ecifying)j Fg(n)d Ft(indep)q(enden)o(t,)j(p)q(oin)o(t-to-p)q(oin)o(t)e(comm)o(uni-)189 158 y(cations,)i(with)h(t)o(w)o(o)f(exceptions:)23 b(all)18 b(messages)e(use)h(the)g(same)f(datat)o(yp)q(e,)g(and)h(messages)f(are) 189 214 y(scattered)e(from)h(\(or)f(gathered)h(to\))f(sequen)o(tial)j (storage.)h(\()p Fm(End)e(of)g(r)n(ationale.)p Ft(\))189 318 y Fm(A)n(dvic)n(e)k(to)h(implementors.)70 b Ft(Although)21 b(the)f(discussion)i(of)e(collectiv)o(e)i(comm)o(unication)f(in)189 375 y(terms)f(of)g(p)q(oin)o(t-to-p)q(oin)o(t)h(op)q(eration)g(implies) i(that)d(eac)o(h)g(message)g(is)i(transferred)e(directly)189 431 y(from)i(sender)h(to)f(receiv)o(er,)j(implemen)o(tations)f(ma)o(y)e (use)h(a)g(tree)f(comm)o(unication)i(pattern.)189 488 y(Messages)12 b(can)h(b)q(e)h(forw)o(arded)e(b)o(y)h(in)o(termediate)h (no)q(des)f(where)h(they)f(are)f(split)j(\(for)d(scatter\))f(or)189 544 y(concatenated)k(\(for)f(gather\),)g(if)i(this)f(is)h(more)f (e\016cien)o(t.)20 b(\()p Fm(End)c(of)g(advic)n(e)g(to)h(implementors.) p Ft(\))75 687 y Fq(1.9)59 b(Global)20 b(Reduction)e(Op)r(erations)75 788 y Ft(The)e(functions)g(in)g(this)g(section)g(p)q(erform)f(a)g (global)h(reduce)g(op)q(eration)g(\(suc)o(h)f(as)g(sum,)g(max,)g (logical)75 845 y(AND,)d(etc.\))19 b(across)12 b(all)i(the)e(mem)o(b)q (ers)h(of)f(a)h(group.)18 b(The)13 b(reduction)h(op)q(eration)f(can)g (b)q(e)g(either)h(one)e(of)75 901 y(a)j(prede\014ned)i(list)f(of)f(op)q (erations,)g(or)g(a)g(user-de\014ned)j(op)q(eration.)i(The)c(global)g (reduction)g(functions)75 958 y(come)h(in)h(sev)o(eral)g(\015a)o(v)o (ors:)23 b(a)17 b(reduce)h(that)f(returns)g(the)g(result)h(of)f(the)g (reduction)i(at)d(one)i(no)q(de,)g(an)75 1014 y(all-reduce)i(that)e (returns)g(this)g(result)h(at)e(all)i(no)q(des,)g(and)g(a)f(scan)g (\(parallel)h(pre\014x\))f(op)q(eration.)29 b(In)75 1071 y(addition,)15 b(a)f(reduce-scatter)g(op)q(eration)h(com)o(bines)g(the) f(functionalit)o(y)h(of)f(a)g(reduce)h(and)f(of)g(a)g(scatter)75 1127 y(op)q(eration.)75 1248 y Ff(1.9.1)49 b(Reduce)75 1429 y Fp(MPI)p 160 1429 V 16 w(REDUCE\()15 b(sendbuf,)j(recvbuf,)d (count,)h(datat)o(yp)q(e,)h(op,)e(ro)q(ot,)f(comm\))117 1506 y Fh(IN)171 b Fp(sendbuf)428 b Fh(address)15 b(of)f(send)g (bu\013er)h(\(c)o(hoice\))117 1581 y(OUT)124 b Fp(recvbuf)434 b Fh(address)21 b(of)e(receiv)o(e)j(bu\013er)e(\(c)o(hoice,)i (signi\014can)o(t)d(only)g(at)905 1637 y(ro)q(ot\))117 1712 y(IN)171 b Fp(count)466 b Fh(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 1787 y(IN)171 b Fp(datat)o(yp)q(e)408 b Fh(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(send)i(bu\013er)g(\(handle\))117 1861 y(IN)171 b Fp(op)525 b Fh(reduce)16 b(op)q(eration)e(\(handle\)) 117 1936 y(IN)171 b Fp(ro)q(ot)492 b Fh(rank)14 b(of)f(ro)q(ot)h(pro)q (cess)i(\(in)o(teger\))117 2011 y(IN)171 b Fp(comm)454 b Fh(comm)o(unicator)11 b(\(handle\))75 2135 y Fg(int)23 b(MPI)p 245 2135 15 2 v 17 w(Reduce\(void*)f(sendbuf,)h(void*)g (recvbuf,)g(int)h(count,)393 2192 y(MPI)p 468 2192 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 2192 V 17 w(Op)i(op,)f(int)h (root,)f(MPI)p 1505 2192 V 17 w(Comm)g(comm\))75 2278 y(MPI)p 150 2278 V 17 w(REDUCE\(SENDBUF,)f(RECVBUF,)g(COUNT,)h (DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR\))170 2335 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 2391 y(INTEGER)g(COUNT,)g (DATATYPE,)g(OP,)g(ROOT,)h(COMM,)f(IERROR)166 2478 y Fp(MPI)p 251 2478 14 2 v 16 w(REDUCE)c Ft(com)o(bines)g(the)g(elemen)o (ts)g(pro)o(vided)g(in)g(the)g(input)g(bu\013er)g(of)f(eac)o(h)g(pro)q (cess)h(in)75 2534 y(the)d(group,)g(using)h(the)f(op)q(eration)g Fp(op)p Ft(,)g(and)h(returns)f(the)g(com)o(bined)h(v)m(alue)g(in)g(the) f(output)g(bu\013er)g(of)75 2591 y(the)j(pro)q(cess)f(with)h(rank)f Fp(ro)q(ot)p Ft(.)30 b(The)19 b(input)g(bu\013er)g(is)g(de\014ned)h(b)o (y)e(the)h(argumen)o(ts)e Fp(sendbuf)p Ft(,)22 b Fp(count)75 2647 y Ft(and)d Fp(datat)o(yp)q(e)p Ft(;)i(the)d(output)h(bu\013er)f (is)h(de\014ned)h(b)o(y)e(the)g(argumen)o(ts)g Fp(recvbuf)p Ft(,)h Fp(count)h Ft(and)f Fp(datat)o(yp)q(e)p Ft(;)75 2704 y(b)q(oth)e(ha)o(v)o(e)f(the)h(same)g(n)o(um)o(b)q(er)g(of)f (elemen)o(ts,)h(with)h(the)e(same)h(t)o(yp)q(e.)24 b(The)17 b(routine)h(is)f(called)h(b)o(y)f(all)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 24 26 24 25 bop 75 -100 a Ft(24)703 b Fk(CHAPTER)15 b(1.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ft(group)d(mem)o(b)q(ers)f (using)i(the)f(same)f(argumen)o(ts)g(for)h Fp(count,)h(datat)o(yp)q(e,) g(op,)f(ro)q(ot)g Ft(and)g Fp(comm)p Ft(.)k(Th)o(us,)c(all)75 102 y(pro)q(cesses)i(pro)o(vide)f(input)i(bu\013ers)e(and)g(output)g (bu\013ers)h(of)e(the)i(same)f(length,)h(with)f(elemen)o(ts)h(of)f(the) 75 158 y(same)g(t)o(yp)q(e.)20 b(Eac)o(h)14 b(pro)q(cess)h(can)f(pro)o (vide)h(one)g(elemen)o(t,)g(or)f(a)g(sequence)h(of)g(elemen)o(ts,)f(in) i(whic)o(h)f(case)75 214 y(the)d(com)o(bine)h(op)q(eration)g(is)f (executed)h(elemen)o(t-wise)h(on)e(eac)o(h)g(en)o(try)g(of)g(the)g (sequence.)20 b(F)l(or)11 b(example,)75 271 y(if)h(the)f(op)q(eration)h (is)g Fd(MPI)p 506 271 13 2 v 14 w(MAX)f Ft(and)g(the)h(send)g (bu\013er)f(con)o(tains)h(t)o(w)o(o)e(elemen)o(ts)i(that)e(are)h (\015oating)g(p)q(oin)o(t)75 327 y(n)o(um)o(b)q(ers)h(\()p Fp(count)h Ft(=)f(2)g(and)g Fp(datat)o(yp)q(e)i Ft(=)e Fp(MPI)p 867 327 14 2 v 16 w(FLO)o(A)l(T)p Ft(\),)f(then)h Fp(recvbuf)s Ft(\()p Fp(1)p Ft(\))h(=)g(global)8 b(max\()p Fp(sendbuf)s Ft(\()p Fp(1)p Ft(\)\))75 384 y(and)15 b Fp(recvbuf)s Ft(\()p Fp(2)p Ft(\))e(=)g(global)8 b(max\()p Fp(sendbuf)s Ft(\()p Fp(2)p Ft(\)\).)166 440 y(Sec.)31 b(1.9.2,)17 b(lists)j(the)e(set)h(of)f(prede\014ned)i(op)q(erations)f (pro)o(vided)g(b)o(y)g Fp(MPI)p Ft(.)f(That)g(section)h(also)75 497 y(en)o(umerates)d(the)g(datat)o(yp)q(es)f(eac)o(h)i(op)q(eration)f (can)g(b)q(e)h(applied)h(to.)k(In)16 b(addition,)h(users)f(ma)o(y)g (de\014ne)75 553 y(their)i(o)o(wn)e(op)q(erations)i(that)e(can)h(b)q(e) h(o)o(v)o(erloaded)f(to)g(op)q(erate)f(on)i(sev)o(eral)f(datat)o(yp)q (es,)f(either)i(basic)75 610 y(or)d(deriv)o(ed.)21 b(This)15 b(is)h(further)f(explained)i(in)f(Sec.)21 b(1.9.4.)166 666 y(The)11 b(op)q(eration)g Fp(op)h Ft(is)f(alw)o(a)o(ys)f(assumed)h (to)g(b)q(e)g(asso)q(ciativ)o(e.)19 b(All)13 b(prede\014ned)f(op)q (erations)f(are)g(also)75 723 y(assumed)f(to)g(b)q(e)h(comm)o(utativ)o (e.)17 b(Users)10 b(ma)o(y)g(de\014ne)h(op)q(erations)g(that)e(are)h (assumed)h(to)e(b)q(e)i(asso)q(ciativ)o(e,)75 779 y(but)i(not)f(comm)o (utativ)o(e.)19 b(The)13 b(\\canonical")g(ev)m(aluation)h(order)e(of)h (a)f(reduction)i(is)f(determined)h(b)o(y)f(the)75 835 y(ranks)18 b(of)f(the)h(pro)q(cesses)g(in)h(the)f(group.)28 b(Ho)o(w)o(ev)o(er,)17 b(the)h(implemen)o(tation)h(can)f(tak)o(e)f(adv) m(an)o(tage)g(of)75 892 y(asso)q(ciativit)o(y)l(,)d(or)f(asso)q (ciativit)o(y)g(and)h(comm)o(utativit)o(y)f(in)h(order)f(to)f(c)o (hange)i(the)f(order)g(of)g(ev)m(aluation.)75 948 y(This)j(ma)o(y)e(c)o (hange)h(the)g(result)g(of)g(the)g(reduction)h(for)e(op)q(erations)i (that)e(are)h(not)f(strictly)i(asso)q(ciativ)o(e)75 1005 y(and)f(comm)o(utativ)o(e,)f(suc)o(h)i(as)f(\015oating)g(p)q(oin)o(t)h (addition.)189 1111 y Fm(A)n(dvic)n(e)h(to)i(implementors.)53 b Ft(It)17 b(is)h(strongly)f(recommended)i(that)d Fp(MPI)p 1480 1111 V 16 w(REDUCE)i Ft(b)q(e)g(imple-)189 1168 y(men)o(ted)e(so)g(that)g(the)g(same)h(result)f(b)q(e)i(obtained)f (whenev)o(er)f(the)h(function)g(is)g(applied)i(on)d(the)189 1224 y(same)f(argumen)o(ts,)f(app)q(earing)i(in)g(the)f(same)g(order.) 20 b(Note)14 b(that)h(this)h(ma)o(y)e(prev)o(en)o(t)h(optimiza-)189 1280 y(tions)k(that)f(tak)o(e)h(adv)m(an)o(tage)f(of)h(the)g(ph)o (ysical)h(lo)q(cation)g(of)f(pro)q(cessors.)31 b(\()p Fm(End)19 b(of)h(advic)n(e)g(to)189 1337 y(implementors.)p Ft(\))166 1443 y(The)c Fp(datat)o(yp)q(e)i Ft(argumen)o(t)e(of)f Fp(MPI)p 783 1443 V 16 w(REDUCE)i Ft(m)o(ust)f(b)q(e)g(compatible)i (with)e Fp(op)p Ft(.)23 b(Prede\014ned)18 b(op-)75 1500 y(erators)h(w)o(ork)g(only)i(with)f(the)g Fp(MPI)g Ft(t)o(yp)q(es)g (listed)h(in)g(Sec.)35 b(1.9.2)18 b(and)j(Sec.)35 b(1.9.3.)d (User-de\014ned)75 1556 y(op)q(erators)14 b(ma)o(y)g(op)q(erate)h(on)g (general,)g(deriv)o(ed)h(datat)o(yp)q(es.)k(In)15 b(this)h(case,)e(eac) o(h)h(argumen)o(t)g(that)f(the)75 1613 y(reduce)21 b(op)q(eration)g(is) g(applied)h(to)e(is)h(one)f(elemen)o(t)h(describ)q(ed)i(b)o(y)d(suc)o (h)h(a)f(datat)o(yp)q(e,)g(whic)o(h)i(ma)o(y)75 1669 y(con)o(tain)15 b(sev)o(eral)h(basic)g(v)m(alues.)21 b(This)15 b(is)h(further)f(explained)i(in)f(Section)h(1.9.4.)75 1791 y Ff(1.9.2)49 b(Prede\014ned)15 b(reduce)g(op)q(erations)75 1877 y Ft(The)k(follo)o(wing)h(prede\014ned)h(op)q(erations)e(are)f (supplied)k(for)c Fp(MPI)p 1238 1877 V 16 w(REDUCE)i Ft(and)f(related)g(functions)75 1933 y Fp(MPI)p 160 1933 V 16 w(ALLREDUCE)p Ft(,)14 b Fp(MPI)p 541 1933 V 15 w(REDUCE)p 732 1933 V 17 w(SCA)l(TTER)p Ft(,)h(and)f Fp(MPI)p 1144 1933 V 16 w(SCAN)p Ft(.)g(These)g(op)q(erations)g(are)g(in)o(v)o(ok)o (ed)75 1989 y(b)o(y)h(placing)i(the)e(follo)o(wing)h(in)g Fp(op)p Ft(.)117 2150 y Fd(Name)729 b Ft(Meaning)117 2258 y Fd(MPI)p 194 2258 13 2 v 14 w(MAX)649 b Ft(maxim)o(um)117 2313 y Fd(MPI)p 194 2313 V 14 w(MIN)664 b Ft(minim)o(um)117 2367 y Fd(MPI)p 194 2367 V 14 w(SUM)653 b Ft(sum)117 2422 y Fd(MPI)p 194 2422 V 14 w(PROD)626 b Ft(pro)q(duct)117 2476 y Fd(MPI)p 194 2476 V 14 w(LAND)632 b Ft(logical)16 b(and)117 2530 y Fd(MPI)p 194 2530 V 14 w(BAND)626 b Ft(bit-wise)16 b(and)117 2585 y Fd(MPI)p 194 2585 V 14 w(LOR)661 b Ft(logical)16 b(or)117 2639 y Fd(MPI)p 194 2639 V 14 w(BOR)655 b Ft(bit-wise)16 b(or)117 2693 y Fd(MPI)p 194 2693 V 14 w(LX)o(OR)634 b Ft(logical)16 b(xor)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 25 27 25 26 bop 75 -100 a Fk(1.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l(TIONS) 842 b Ft(25)117 45 y Fd(MPI)p 194 45 13 2 v 14 w(BX)o(OR)628 b Ft(bit-wise)16 b(xor)117 97 y Fd(MPI)p 194 97 V 14 w(MAXLOC)569 b Ft(max)14 b(v)m(alue)j(and)e(lo)q(cation)117 150 y Fd(MPI)p 194 150 V 14 w(MINLOC)584 b Ft(min)16 b(v)m(alue)g(and)f(lo)q(cation)166 233 y(The)22 b(t)o(w)o(o)e(op)q (erations)h Fd(MPI)p 662 233 V 14 w(MINLOC)h Ft(and)f Fd(MPI)p 1024 233 V 14 w(MAXLOC)g Ft(are)g(discussed)i(separately)e(in) i(Sec.)75 289 y(1.9.3.)g(F)l(or)17 b(the)g(other)f(prede\014ned)j(op)q (erations,)e(w)o(e)g(en)o(umerate)f(b)q(elo)o(w)i(the)f(allo)o(w)o(ed)g (com)o(binations)75 346 y(of)e Fp(op)g Ft(and)g Fp(datat)o(yp)q(e)i Ft(argumen)o(ts.)i(First,)14 b(de\014ne)j(groups)d(of)h Fp(MPI)g Ft(basic)h(datat)o(yp)q(es)e(in)i(the)f(follo)o(wing)75 402 y(w)o(a)o(y)l(.)117 562 y Fd(C)e(integer:)661 b Fp(MPI)p 1032 562 14 2 v 16 w(INT,)124 b(MPI)p 1343 562 V 15 w(LONG,)h(MPI)p 1698 562 V 16 w(SHORT,)905 619 y(MPI)p 990 619 V 16 w(UNSIGNED)p 1229 619 V 17 w(SHORT,)135 b(MPI)p 1626 619 V 16 w(UNSIGNED,)905 675 y(MPI)p 990 675 V 16 w(UNSIGNED)p 1229 675 V 17 w(LONG)117 727 y Fd(F)o(o)o(rtran)13 b(integer:)561 b Fp(MPI)p 1032 727 V 16 w(INTEGER)117 780 y Fd(Floating)13 b(p)q(oint:)574 b Fp(MPI)p 1032 780 V 16 w(FLO)o(A)l(T,)82 b(MPI)p 1365 780 V 16 w(DOUBLE,)h(MPI)p 1738 780 V 16 w(REAL,)905 836 y(MPI)p 990 836 V 16 w(DOUBLE)p 1185 836 V 17 w(PRECISION,)16 b(MPI)p 1543 836 V 15 w(LONG)p 1678 836 V 17 w(DOUBLE)117 889 y Fd(Logical:)696 b Fp(MPI)p 1032 889 V 16 w(LOGICAL)117 941 y Fd(Complex:)668 b Fp(MPI)p 1032 941 V 16 w(COMPLEX)117 993 y Fd(Byte:)738 b Fp(MPI)p 1032 993 V 16 w(BYTE)166 1076 y Ft(No)o(w,)14 b(the)h(v)m(alid)i(datat)o(yp)q(es)e(for)f(eac)o (h)i(option)f(is)h(sp)q(eci\014ed)h(b)q(elo)o(w.)117 1236 y Fd(Op)778 b Ft(Allo)o(w)o(ed)16 b(T)o(yp)q(es)1937 1224 y Fo(>)117 1341 y Fd(MPI)p 194 1341 13 2 v 14 w(MAX,)d(MPI)p 400 1341 V 14 w(MIN)458 b(C)13 b(integer,)i(F)o(o)o(rtran)e(integer,)i (Floating)e(p)q(oint)278 b Fo(?)117 1393 y Fd(MPI)p 194 1393 V 14 w(SUM,)13 b(MPI)p 396 1393 V 14 w(PROD)424 b(C)13 b(integer,)i(F)o(o)o(rtran)e(integer,)i(Floating)e(p)q(oint,)h (Complex)117 1446 y(MPI)p 194 1446 V 14 w(LAND,)g(MPI)p 418 1446 V 14 w(LOR,)f(MPI)p 612 1446 V 15 w(LX)o(OR)215 b(C)13 b(integer,)i(Logical)117 1498 y(MPI)p 194 1498 V 14 w(BAND,)e(MPI)p 423 1498 V 14 w(BOR,)g(MPI)p 623 1498 V 14 w(BX)o(OR)199 b(C)13 b(integer,)i(F)o(o)o(rtran)e(integer,)i (Byte)75 1616 y Fn(Example)j(1.15)23 b Ft(A)10 b(routine)i(that)e (computes)h(the)g(dot)f(pro)q(duct)i(of)e(t)o(w)o(o)g(v)o(ectors)g (that)g(are)g(distributed)75 1673 y(across)k(a)h(group)g(of)g(pro)q (cesses)h(and)f(returns)g(the)g(answ)o(er)g(at)g(no)q(de)g(zero.)75 1765 y Fg(SUBROUTINE)22 b(PAR_BLAS1\(m,)h(a,)g(b,)h(c,)f(comm\))75 1821 y(REAL)g(a\(m\),)g(b\(m\))167 b(!)24 b(local)f(slice)g(of)h(array) 75 1878 y(REAL)f(c)382 b(!)24 b(result)f(\(at)g(node)h(zero\))75 1934 y(REAL)f(sum)75 1991 y(INTEGER)g(m,)g(comm,)h(i,)f(ierr)75 2104 y(!)h(local)f(sum)75 2160 y(sum)g(=)h(0.0)75 2216 y(DO)g(i)f(=)h(1,)g(m)147 2273 y(sum)f(=)h(sum)f(+)h(a\(i\)*b\(i\))75 2329 y(END)f(DO)75 2442 y(!)h(global)f(sum)75 2499 y(CALL)g (MPI_REDUCE\(sum,)f(c,)i(1,)f(MPI_REAL,)g(MPI_SUM,)g(0,)g(comm,)g (ierr\))75 2555 y(RETURN)75 2647 y Fn(Example)18 b(1.16)23 b Ft(A)c(routine)h(that)f(computes)h(the)f(pro)q(duct)h(of)g(a)f(v)o (ector)g(and)h(an)f(arra)o(y)f(that)h(are)75 2704 y(distributed)e (across)d(a)h(group)g(of)g(pro)q(cesses)g(and)h(returns)f(the)g(answ)o (er)g(at)f(no)q(de)i(zero.)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 26 28 26 27 bop 75 -100 a Ft(26)703 b Fk(CHAPTER)15 b(1.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fg(SUBROUTINE)22 b(PAR_BLAS2\(m,)h(n,)g(a,)h(b,)f(c,)h(comm\))75 102 y(REAL)f(a\(m\),)g (b\(m,n\))95 b(!)24 b(local)f(slice)g(of)h(array)75 158 y(REAL)f(c\(n\))286 b(!)24 b(result)75 214 y(REAL)f(sum\(n\))75 271 y(INTEGER)g(n,)g(comm,)h(i,)f(j,)h(ierr)75 384 y(!)g(local)f(sum)75 440 y(DO)h(j=)f(1,)h(n)123 497 y(sum\(j\))f(=)g(0.0)123 553 y(DO)g(i)h(=)g(1,)f(m)170 610 y(sum\(j\))g(=)h(sum\(j\))f(+)h (a\(i\)*b\(i,j\))123 666 y(END)f(DO)75 723 y(END)g(DO)75 835 y(!)h(global)f(sum)75 892 y(CALL)g(MPI_REDUCE\(sum,)f(c,)i(n,)f (MPI_REAL,)g(MPI_SUM,)g(0,)g(comm,)g(ierr\))75 1005 y(!)h(return)f (result)g(at)g(node)h(zero)f(\(and)g(garbage)g(at)h(the)f(other)g (nodes\))75 1061 y(RETURN)75 1183 y Ff(1.9.3)49 b(MINLOC)18 b(and)e(MAXLOC)75 1269 y Ft(The)d(op)q(erator)e Fd(MPI)p 423 1269 13 2 v 15 w(MINLOC)h Ft(is)h(used)g(to)f(compute)g(a)h(global) g(minim)o(um)g(and)g(also)f(an)h(index)h(attac)o(hed)75 1325 y(to)e(the)g(minim)o(um)i(v)m(alue.)20 b Fd(MPI)p 615 1325 V 14 w(MAXLOC)12 b Ft(similarly)i(computes)f(a)f(global)h (maxim)o(um)f(and)h(index.)20 b(One)75 1382 y(application)13 b(of)e(these)h(is)g(to)f(compute)g(a)g(global)h(minim)o(um)h(\(maxim)o (um\))e(and)g(the)h(rank)f(of)g(the)h(pro)q(cess)75 1438 y(con)o(taining)k(this)g(v)m(alue.)166 1495 y(The)f(op)q(eration)h (that)e(de\014nes)i Fd(MPI)p 787 1495 V 15 w(MAXLOC)e Ft(is:)189 1559 y Fc( )242 1603 y Fj(u)247 1659 y(i)289 1559 y Fc(!)332 1631 y Fo(\016)365 1559 y Fc( )418 1603 y Fj(v)420 1659 y(j)463 1559 y Fc(!)508 1631 y Ft(=)556 1559 y Fc( )610 1603 y Fj(w)614 1659 y(k)664 1559 y Fc(!)75 1768 y Ft(where)189 1861 y Fj(w)f Ft(=)g(max\()p Fj(u;)8 b(v)r Ft(\))75 1955 y(and)189 2095 y Fj(k)13 b Ft(=)274 1997 y Fc(8)274 2034 y(>)274 2046 y(<)274 2121 y(>)274 2134 y(:)332 2039 y Fj(i)194 b Ft(if)15 b Fj(u)e(>)g(v)332 2095 y Ft(min)q(\()p Fj(i;)8 b(j)s Ft(\))39 b(if)15 b Fj(u)e Ft(=)g Fj(v)332 2152 y(j)191 b Ft(if)15 b Fj(u)e(<)g(v)166 2244 y Fd(MPI)p 243 2244 V 14 w(MINLOC)i Ft(is)h(de\014ned)g (similarly:)189 2309 y Fc( )242 2352 y Fj(u)247 2409 y(i)289 2309 y Fc(!)332 2381 y Fo(\016)365 2309 y Fc( )418 2352 y Fj(v)420 2409 y(j)463 2309 y Fc(!)508 2381 y Ft(=)556 2309 y Fc( )610 2352 y Fj(w)614 2409 y(k)664 2309 y Fc(!)75 2517 y Ft(where)189 2611 y Fj(w)d Ft(=)g(min)q(\()p Fj(u;)8 b(v)r Ft(\))1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 27 29 27 28 bop 75 -100 a Fk(1.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l(TIONS) 842 b Ft(27)75 45 y(and)189 185 y Fj(k)13 b Ft(=)274 87 y Fc(8)274 124 y(>)274 136 y(<)274 211 y(>)274 224 y(:)332 129 y Fj(i)194 b Ft(if)15 b Fj(u)e(<)g(v)332 185 y Ft(min)q(\()p Fj(i;)8 b(j)s Ft(\))39 b(if)15 b Fj(u)e Ft(=)g Fj(v)332 242 y(j)191 b Ft(if)15 b Fj(u)e(>)g(v)166 334 y Ft(Both)g(op)q(erations)g(are)g(asso)q(ciativ)o(e)h(and)f(comm)o (utativ)o(e.)19 b(Note)12 b(that)h(if)h Fd(MPI)p 1491 334 13 2 v 14 w(MAXLOC)e Ft(is)i(applied)75 390 y(to)k(reduce)i(a)e (sequence)i(of)e(pairs)h(\()p Fj(u)729 397 y Fi(0)748 390 y Fj(;)8 b Ft(0\))p Fj(;)g Ft(\()p Fj(u)875 397 y Fi(1)893 390 y Fj(;)g Ft(1\))p Fj(;)g(:)g(:)f(:)t(;)h Ft(\()p Fj(u)1099 397 y Fb(n)p Fa(\000)p Fi(1)1167 390 y Fj(;)g(n)j Fo(\000)i Ft(1\),)19 b(then)g(the)f(v)m(alue)i(returned)f (is)75 447 y(\()p Fj(u;)8 b(r)q Ft(\),)15 b(where)i Fj(u)f Ft(=)f(max)518 454 y Fb(i)540 447 y Fj(u)566 454 y Fb(i)597 447 y Ft(and)i Fj(r)g Ft(is)g(the)g(index)i(of)d(the)h(\014rst)f (global)i(maxim)o(um)f(in)g(the)g(sequence.)75 503 y(Th)o(us,)12 b(if)h(eac)o(h)f(pro)q(cess)g(supplies)i(a)e(v)m(alue)h(and)f(its)h (rank)e(within)j(the)e(group,)g(then)g(a)g(reduce)h(op)q(eration)75 560 y(with)18 b Fp(op)g Ft(=)g Fd(MPI)p 375 560 V 14 w(MAXLOC)f Ft(will)i(return)f(the)g(maxim)o(um)f(v)m(alue)i(and)f(the)g (rank)f(of)h(the)f(\014rst)h(pro)q(cess)75 616 y(with)g(that)f(v)m (alue.)29 b(Similarly)l(,)21 b Fd(MPI)p 708 616 V 14 w(MINLOC)d Ft(can)g(b)q(e)g(used)h(to)e(return)h(a)f(minim)o(um)i(and)f (its)g(index.)75 673 y(More)10 b(generally)l(,)i Fd(MPI)p 463 673 V 14 w(MINLOC)e Ft(computes)h(a)f Fm(lexic)n(o)n(gr)n(aphic)h (minimum)p Ft(,)g(where)g(elemen)o(ts)g(are)f(ordered)75 729 y(according)k(to)g(the)g(\014rst)g(comp)q(onen)o(t)g(of)f(eac)o(h)h (pair,)h(and)f(ties)g(are)g(resolv)o(ed)g(according)h(to)e(the)h (second)75 786 y(comp)q(onen)o(t.)166 842 y(The)i(reduce)h(op)q (eration)f(is)h(de\014ned)g(to)e(op)q(erate)h(on)g(argumen)o(ts)f(that) g(consist)i(of)e(a)h(pair:)22 b(v)m(alue)75 899 y(and)12 b(index.)20 b(F)l(or)12 b(b)q(oth)g(F)l(ortran)f(and)h(C,)g(t)o(yp)q (es)g(are)g(pro)o(vided)h(to)e(describ)q(e)j(the)e(pair.)19 b(The)13 b(p)q(oten)o(tially)75 955 y(mixed-t)o(yp)q(e)g(nature)e(of)g (suc)o(h)h(argumen)o(ts)f(is)h(a)f(problem)i(in)f(F)l(ortran.)17 b(The)12 b(problem)h(is)f(circum)o(v)o(en)o(ted,)75 1011 y(for)k(F)l(ortran,)g(b)o(y)h(ha)o(ving)h(the)f Fp(MPI)p Ft(-pro)o(vided)g(t)o(yp)q(e)g(consist)g(of)g(a)g(pair)g(of)g(the)g (same)f(t)o(yp)q(e)h(as)g(v)m(alue,)75 1068 y(and)j(co)q(ercing)g(the)g (index)g(to)f(this)h(t)o(yp)q(e)f(also.)33 b(In)20 b(C,)f(the)g Fp(MPI)p Ft(-pro)o(vided)h(pair)g(t)o(yp)q(e)f(has)h(distinct)75 1124 y(t)o(yp)q(es)15 b(and)h(the)f(index)h(is)g(an)f Fp(int)p Ft(.)166 1181 y(In)f(order)g(to)f(use)h Fd(MPI)p 547 1181 V 14 w(MINLOC)f Ft(and)h Fd(MPI)p 893 1181 V 15 w(MAXLOC)e Ft(in)j(a)e(reduce)i(op)q(eration,)f(one)f(m)o(ust)g(pro) o(vide)75 1237 y(a)19 b Fp(datat)o(yp)q(e)j Ft(argumen)o(t)d(that)g (represen)o(ts)h(a)f(pair)h(\(v)m(alue)h(and)f(index\).)34 b Fp(MPI)19 b Ft(pro)o(vides)i(sev)o(en)f(suc)o(h)75 1294 y(prede\014ned)14 b(datat)o(yp)q(es.)19 b(The)13 b(op)q(erations)g Fd(MPI)p 905 1294 V 14 w(MAXLOC)f Ft(and)h Fd(MPI)p 1264 1294 V 14 w(MINLOC)f Ft(can)h(b)q(e)h(used)f(with)g(eac)o (h)75 1350 y(of)i(the)g(follo)o(wing)h(datat)o(yp)q(es.)1333 b Fo(>)117 1454 y Fd(F)o(o)o(rtran:)117 1508 y(Name)729 b Ft(Description)117 1563 y Fd(MPI)p 194 1563 V 14 w(2REAL)618 b Ft(pair)15 b(of)g Fg(REAL)p Ft(s)117 1617 y Fd(MPI)p 194 1617 V 14 w(2DOUBLE)p 394 1617 V 14 w(PRECISION)328 b Ft(pair)15 b(of)g Fg(DOUBLE)23 b(PRECISION)14 b Ft(v)m(ariables)117 1671 y Fd(MPI)p 194 1671 V 14 w(2INTEGER)546 b Ft(pair)15 b(of)g Fg(INTEGER)p Ft(s)1937 1802 y Fo(?)117 1858 y Fd(C:)117 1912 y(Name)729 b Ft(Description)117 1967 y Fd(MPI)p 194 1967 V 14 w(FLO)o(A)m(T)p 337 1967 V 14 w(INT)529 b Fp(\015oat)15 b Ft(and)h Fp(int)117 2021 y Fd(MPI)p 194 2021 V 14 w(DOUBLE)p 373 2021 V 14 w(INT)493 b Fp(double)16 b Ft(and)g Fp(int)117 2076 y Fd(MPI)p 194 2076 V 14 w(LONG)p 318 2076 V 15 w(INT)547 b Fp(long)15 b Ft(and)g Fp(int)117 2130 y Fd(MPI)p 194 2130 V 14 w(2INT)651 b Ft(pair)15 b(of)g Fp(int)117 2184 y Fd(MPI)p 194 2184 V 14 w(SHORT)p 346 2184 V 15 w(INT)519 b Fp(sho)o(rt)15 b Ft(and)h Fp(int)117 2239 y Fd(MPI)p 194 2239 V 14 w(LONG)p 318 2239 V 15 w(DOUBLE)p 498 2239 V 14 w(INT)368 b Fp(long)15 b(double)h Ft(and)f Fp(int)166 2322 y Ft(The)g(datat)o(yp)q(e)g Fd(MPI)p 527 2322 V 14 w(2REAL)g Ft(is)h Fm(as)g(if)f Ft(de\014ned)h(b)o(y)g(the)f(follo)o(wing)h(\(see)f(Section)h Fn(??)p Ft(\).)75 2428 y Fg(MPI_TYPE_CONTIGUOUS\(2,)21 b(MPI_REAL,)h(MPI_2REAL\))166 2534 y Ft(Similar)17 b(statemen)o(ts)d (apply)i(for)e Fd(MPI)p 823 2534 V 15 w(2INTEGER)p Ft(,)f Fd(MPI)p 1134 2534 V 15 w(2DOUBLE)p 1335 2534 V 13 w(PRECISION)p Ft(,)g(and)i Fd(MPI)p 1750 2534 V 15 w(2INT)p Ft(.)166 2591 y(The)f(datat)o(yp)q(e)g Fd(MPI)p 525 2591 V 14 w(FLO)o(A)m(T)p 668 2591 V 14 w(INT)g Ft(is)h Fm(as)g(if)e Ft(de\014ned)j(b)o(y)e(the)g(follo)o(wing)h(sequence)g(of)e (instructions.)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 28 30 28 29 bop 75 -100 a Ft(28)708 b Fk(CHAPTER)15 b(1.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fg(type[0])23 b(=)h(MPI_FLOAT)75 102 y(type[1])f(=)h(MPI_INT)75 158 y(disp[0])f(=)h(0)75 214 y(disp[1])f(=)h(sizeof\(float\))75 271 y(block[0])f(=)g(1)75 327 y(block[1])g(=)g(1)75 384 y(MPI_TYPE_STRUCT\(2,)e(block,)i(disp,)h(type,)f(MPI_FLOAT_INT\))75 477 y Ft(Similar)17 b(statemen)o(ts)d(apply)i(for)e Fd(MPI)p 732 477 13 2 v 15 w(LONG)p 857 477 V 15 w(INT)h Ft(and)g Fd(MPI)p 1119 477 V 15 w(DOUBLE)p 1299 477 V 14 w(INT)p Ft(.)75 582 y Fn(Example)j(1.17)23 b Ft(Eac)o(h)12 b(pro)q(cess)g(has)g (an)g(arra)o(y)f(of)h(30)g Fg(double)p Ft(s,)f(in)i(C.)f(F)l(or)g(eac)o (h)g(of)g(the)g(30)g(lo)q(cations,)75 638 y(compute)j(the)h(v)m(alue)g (and)f(rank)g(of)g(the)g(pro)q(cess)h(con)o(taining)g(the)f(largest)g (v)m(alue.)170 743 y Fg(...)170 800 y(/*)24 b(each)f(process)g(has)h (an)f(array)g(of)h(30)g(double:)f(ain[30])194 856 y(*/)170 913 y(double)g(ain[30],)g(aout[30];)170 969 y(int)48 b(ind[30];)170 1026 y(struct)23 b({)266 1082 y(double)g(val;)266 1139 y(int)71 b(rank;)170 1195 y(})24 b(in[30],)f(out[30];)170 1251 y(int)h(i,)f(myrank,)g(root;)170 1364 y (MPI_Comm_rank\(MPI_COMM_WORLD,)d(&myrank\);)170 1421 y(for)k(\(i=0;)f(i<30;)g(++i\))h({)266 1477 y(in[i].val)f(=)g(ain[i];) 266 1534 y(in[i].rank)f(=)i(myrank;)170 1590 y(})170 1647 y(MPI_Reduce\()f(in,)g(out,)h(30,)f(MPI_DOUBLE_INT,)f(MPI_MAXLOC,) g(root,)h(comm)h(\);)170 1703 y(/*)g(At)g(this)f(point,)g(the)g(answer) g(resides)g(on)h(process)f(root)194 1760 y(*/)170 1816 y(if)h(\(myrank)f(==)h(root\))f({)266 1872 y(/*)g(read)h(ranks)f(out) 290 1929 y(*/)266 1985 y(for)g(\(i=0;)h(i<30;)f(++i\))g({)361 2042 y(aout[i])g(=)h(out[i].val;)361 2098 y(ind[i])f(=)h(out[i].rank;) 266 2155 y(})170 2211 y(})75 2316 y Fn(Example)18 b(1.18)23 b Ft(Same)15 b(example,)g(in)h(F)l(ortran.)170 2421 y Fg(...)170 2478 y(!)24 b(each)f(process)g(has)h(an)f(array)h(of)f(30)h (double:)f(ain\(30\))170 2591 y(DOUBLE)g(PRECISION)g(ain\(30\),)g (aout\(30\))170 2647 y(INTEGER)g(ind\(30\);)170 2704 y(DOUBLE)g(PRECISION)g(in\(2,30\),)g(out\(2,30\))1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 29 31 29 30 bop 75 -100 a Fk(1.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l(TIONS) 842 b Ft(29)170 45 y Fg(INTEGER)23 b(i,)h(myrank,)f(root,)g(ierr;)170 158 y(MPI_COMM_RANK\(MPI_COMM_WORLD,)d(myrank\);)170 214 y(DO)k(I=1,)f(30)266 271 y(in\(1,i\))g(=)h(ain\(i\))266 327 y(in\(2,i\))f(=)h(myrank)94 b(!)24 b(myrank)f(is)h(coerced)e(to)i (a)g(double)170 384 y(END)g(DO)170 497 y(MPI_REDUCE\()f(in,)g(out,)h (30,)f(MPI_2DOUBLE_PRECISION,)e(MPI_MAXLOC,)h(root,)1459 553 y(comm,)h(ierr)h(\);)170 610 y(!)g(At)g(this)f(point,)g(the)h (answer)f(resides)g(on)g(process)g(root)170 723 y(IF)h(\(myrank)f(.EQ.) g(root\))g(THEN)266 779 y(!)h(read)f(ranks)g(out)266 835 y(DO)g(I=)h(1,)g(30)361 892 y(aout\(i\))f(=)h(out\(1,i\))361 948 y(ind\(i\))f(=)h(out\(2,i\))47 b(!)23 b(rank)h(is)f(coerced)g(back) h(to)f(an)h(integer)266 1005 y(END)f(DO)170 1061 y(END)h(IF)75 1168 y Fn(Example)18 b(1.19)23 b Ft(Eac)o(h)16 b(pro)q(cess)h(has)f(a)h (non-empt)o(y)f(arra)o(y)g(of)g(v)m(alues.)25 b(Find)18 b(the)f(minim)o(um)g(global)75 1224 y(v)m(alue,)f(the)f(rank)g(of)g (the)g(pro)q(cess)h(that)e(holds)i(it)g(and)f(its)g(index)i(on)e(this)h (pro)q(cess.)75 1330 y Fg(#define)47 b(LEN)71 b(1000)75 1443 y(float)23 b(val[LEN];)190 b(/*)23 b(local)h(array)f(of)g(values)g (*/)75 1500 y(int)g(count;)310 b(/*)23 b(local)h(number)f(of)g(values)g (*/)75 1556 y(int)g(myrank,)g(minrank,)g(minindex;)75 1613 y(float)g(minval;)75 1725 y(struct)g({)170 1782 y(float)h(value;)170 1838 y(int)72 b(index;)75 1895 y(})24 b(in,)f(out;)170 2008 y(/*)h(local)f(minloc)g(*/)75 2064 y(in.value)g(=)g(val[0];)75 2121 y(in.index)g(=)g(0;)75 2177 y(for)g(\(i=1;)h(i)f(<)h(count;)f(i++\))170 2234 y(if)h(\(in.value)f(>)g(val[i]\))g({)266 2290 y(in.value)g(=)g(val[i];) 266 2346 y(in.index)g(=)g(i;)170 2403 y(})170 2516 y(/*)h(global)f (minloc)g(*/)75 2572 y(MPI_Comm_rank\(MPI_COMM_WOR)o(LD,)e(&myrank\);) 75 2629 y(in.index)i(=)g(myrank*LEN)g(+)h(in.index;)75 2685 y(MPI_Reduce\()e(in,)i(out,)f(1,)h(MPI_FLOAT_INT,)e(MPI_MINLOC,)g (root,)h(comm)h(\);)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 30 32 30 31 bop 75 -100 a Ft(30)708 b Fk(CHAPTER)15 b(1.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)170 45 y Fg(/*)24 b(At)g(this)f(point,)g(the)g(answer)g(resides)g(on)h(process)f(root)194 102 y(*/)75 158 y(if)h(\(myrank)e(==)i(root\))f({)170 214 y(/*)h(read)f(answer)g(out)194 271 y(*/)170 327 y(minval)g(=)h (out.value;)170 384 y(minrank)f(=)h(out.index)f(/)g(LEN;)170 440 y(minindex)g(=)h(out.index)f(\045)g(LEN;)75 497 y(})189 603 y Fm(R)n(ationale.)84 b Ft(The)23 b(de\014nition)h(of)e Fd(MPI)p 921 603 13 2 v 14 w(MINLOC)h Ft(and)f Fd(MPI)p 1285 603 V 15 w(MAXLOC)f Ft(giv)o(en)i(here)g(has)g(the)189 659 y(adv)m(an)o(tage)14 b(that)h(it)h(do)q(es)f(not)g(require)h(an)o (y)f(sp)q(ecial-case)j(handling)f(of)e(these)g(t)o(w)o(o)f(op)q (erations:)189 716 y(they)i(are)g(handled)i(lik)o(e)g(an)o(y)e(other)g (reduce)i(op)q(eration.)24 b(A)16 b(programmer)g(can)g(pro)o(vide)h (his)g(or)189 772 y(her)e(o)o(wn)g(de\014nition)i(of)f Fd(MPI)p 694 772 V 14 w(MAXLOC)e Ft(and)i Fd(MPI)p 1058 772 V 14 w(MINLOC)p Ft(,)f(if)h(so)f(desired.)22 b(The)16 b(disadv)m(an)o(tage)189 829 y(is)d(that)f(v)m(alues)h(and)g(indices)i (ha)o(v)o(e)d(to)g(b)q(e)h(\014rst)f(in)o(terlea)o(v)o(ed,)i(and)e (that)g(indices)j(and)e(v)m(alues)g(ha)o(v)o(e)189 885 y(to)h(b)q(e)i(co)q(erced)g(to)f(the)g(same)g(t)o(yp)q(e,)g(in)h(F)l (ortran.)i(\()p Fm(End)e(of)g(r)n(ationale.)p Ft(\))75 1007 y Ff(1.9.4)49 b(User-De\014ned)15 b(Op)q(erations)75 1187 y Fp(MPI)p 160 1187 14 2 v 16 w(OP)p 238 1187 V 17 w(CREA)l(TE\()h(function,)g(commute,)e(op\))117 1265 y Fh(IN)155 b Fp(function)434 b Fh(user)15 b(de\014ned)g(function)f (\(function\))117 1340 y(IN)155 b Fp(commute)411 b Fe(true)13 b Fh(if)g(comm)o(utativ)o(e;)e Fe(false)i Fh(otherwise.)117 1415 y(OUT)108 b Fp(op)541 b Fh(op)q(eration)14 b(\(handle\))75 1539 y Fg(int)23 b(MPI)p 245 1539 15 2 v 17 w(Op)p 310 1539 V 17 w(create\(MPI)p 567 1539 V 16 w(User)p 679 1539 V 17 w(function)f(*function,)h(int)g(commute,)g(MPI)p 1555 1539 V 17 w(Op)g(*op\))75 1626 y(MPI)p 150 1626 V 17 w(OP)p 215 1626 V 17 w(CREATE\()g(FUNCTION,)f(COMMUTE,)h(OP,)g (IERROR\))170 1682 y(EXTERNAL)g(FUNCTION)170 1739 y(LOGICAL)g(COMMUTE) 170 1795 y(INTEGER)g(OP,)h(IERROR)166 1882 y Fp(MPI)p 251 1882 14 2 v 16 w(OP)p 329 1882 V 17 w(CREA)l(TE)h Ft(binds)h(a)f(user-de\014ned)h(global)f(op)q(eration)g(to)f(an)h Fp(op)g Ft(handle)h(that)e(can)75 1938 y(subsequen)o(tly)e(b)q(e)f (used)g(in)g Fp(MPI)p 669 1938 V 16 w(REDUCE)p Ft(,)g Fp(MPI)p 977 1938 V 15 w(ALLREDUCE)p Ft(,)g Fp(MPI)p 1364 1938 V 15 w(REDUCE)p 1555 1938 V 18 w(SCA)l(TTER)p Ft(,)g(and)75 1994 y Fp(MPI)p 160 1994 V 16 w(SCAN)p Ft(.)c(The)g(user-de\014ned)i(op)q(eration)e(is)g(assumed)g(to)g(b)q(e) g(asso)q(ciativ)o(e.)25 b(If)18 b Fp(commute)e Ft(=)h Fg(true)p Ft(,)75 2051 y(then)k(the)g(op)q(eration)g(should)h(b)q(e)g (b)q(oth)e(comm)o(utativ)o(e)h(and)g(asso)q(ciativ)o(e.)37 b(If)21 b Fp(commute)f Ft(=)h Fg(false)p Ft(,)75 2107 y(then)d(the)f(order)h(of)f(op)q(erands)h(is)g(\014xed)g(and)f(is)i (de\014ned)f(to)f(b)q(e)h(in)h(ascending,)g(pro)q(cess)e(rank)h(order,) -2036 b Fo(>)75 2164 y Ft(b)q(eginning)17 b(with)e(pro)q(cess)g(zero.) 20 b(The)15 b(order)g(of)f(ev)m(aluation)i(can)f(b)q(e)h(c)o(hanged,)f (talking)g(adv)m(an)o(tage)f(of)75 2220 y(the)i(asso)q(ciativit)o(y)g (of)g(the)g(op)q(eration.)22 b(If)16 b Fp(commute)f Ft(=)h Fg(true)g Ft(then)g(the)g(order)g(of)f(ev)m(aluation)i(can)f(b)q(e)75 2277 y(c)o(hanged,)f(taking)g(adv)m(an)o(tage)g(of)g(comm)o(utativit)o (y)f(and)i(asso)q(ciativit)o(y)l(.)-1478 b Fo(?)166 2333 y Fp(function)20 b Ft(is)g(the)e(user-de\014ned)j(function,)f(whic)o(h) g(m)o(ust)e(ha)o(v)o(e)g(the)h(follo)o(wing)h(four)e(argumen)o(ts:)75 2390 y Fp(invec,)d(inoutvec,)i(len)e Ft(and)h Fp(datat)o(yp)q(e)p Ft(.)166 2446 y(The)f(ANSI-C)h(protot)o(yp)q(e)f(for)f(the)i(function)g (is)f(the)h(follo)o(wing.)75 2540 y Fg(typedef)23 b(void)g (MPI_User_function\()f(void)h(*invec,)g(void)g(*inoutvec,)g(int)g (*len,)1220 2596 y(MPI_Datatype)g(*datatype\);)166 2690 y Ft(The)15 b(F)l(ortran)f(declaration)i(of)f(the)g(user-de\014ned)i (function)f(app)q(ears)g(b)q(elo)o(w.)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 31 33 31 32 bop 75 -100 a Fk(1.9.)29 b(GLOBAL)16 b(REDUCTION)g(OPERA)l(TIONS) 842 b Ft(31)75 45 y Fg(FUNCTION)23 b(USER_FUNCTION\()f(INVEC\(*\),)g (INOUTVEC\(*\),)h(LEN,)g(TYPE\))75 102 y()g(INVEC\(LEN\),)f (INOUTVEC\(LEN\))99 158 y(INTEGER)h(LEN,)g(TYPE)166 252 y Ft(The)18 b Fp(datat)o(yp)q(e)h Ft(argumen)o(t)e(is)h(a)f(handle)i (to)e(the)h(data)e(t)o(yp)q(e)i(that)f(w)o(as)g(passed)h(in)o(to)f(the) h(call)h(to)75 308 y Fp(MPI)p 160 308 14 2 v 16 w(REDUCE)p Ft(.)f(The)g(user)g(reduce)h(function)g(should)g(b)q(e)f(written)g(suc) o(h)h(that)e(the)h(follo)o(wing)h(holds:)75 365 y(Let)h Fp(u[0],)g(...)32 b(,)20 b(u[len-1])g Ft(b)q(e)g(the)g Fp(len)g Ft(elemen)o(ts)g(in)g(the)g(comm)o(unication)g(bu\013er)f (describ)q(ed)j(b)o(y)d(the)75 421 y(argumen)o(ts)14 b Fp(invec,)i(len)f Ft(and)g Fp(datat)o(yp)q(e)i Ft(when)f(the)f (function)h(is)f(in)o(v)o(ok)o(ed;)g(let)h Fp(v[0],)e(...)19 b(,)c(v[len-1])h Ft(b)q(e)f Fp(len)75 478 y Ft(elemen)o(ts)e(in)g(the)f (comm)o(unication)h(bu\013er)f(describ)q(ed)i(b)o(y)e(the)g(argumen)o (ts)f Fp(inoutvec,)j(len)e Ft(and)g Fp(datat)o(yp)q(e)75 534 y Ft(when)17 b(the)g(function)g(is)h(in)o(v)o(ok)o(ed;)f(let)g Fp(w[0],)g(...)23 b(,)17 b(w[len-1])g Ft(b)q(e)g Fp(len)h Ft(elemen)o(ts)f(in)g(the)g(comm)o(unication)75 591 y(bu\013er)i (describ)q(ed)j(b)o(y)d(the)h(argumen)o(ts)e Fp(inoutvec,)k(len)e Ft(and)g Fp(datat)o(yp)q(e)h Ft(when)f(the)f(function)h(returns;)75 647 y(then)d Fp(w[i])f(=)h(u[i])p Fo(\016)p Fp(v[i])p Ft(,)g(for)f Fp(i=0)g(,)g(...)23 b(,)16 b(len-1)p Ft(,)h(where)g Fo(\016)f Ft(is)h(the)f(reduce)h(op)q(eration)g(that)f(the)g(function) 75 703 y(computes.)166 760 y(Informally)l(,)j(w)o(e)e(can)g(think)h(of) f Fp(invec)h Ft(and)g Fp(inoutvec)h Ft(as)e(arra)o(ys)f(of)h Fp(len)h Ft(elemen)o(ts)g(that)e Fp(function)75 816 y Ft(is)i(com)o(bining.)28 b(The)18 b(result)g(of)f(the)h(reduction)h(o)o (v)o(er-writes)e(v)m(alues)h(in)h Fp(inoutvec)p Ft(,)g(hence)g(the)e (name.)75 873 y(Eac)o(h)g(in)o(v)o(o)q(cation)g(of)f(the)h(function)h (results)f(in)h(the)f(p)q(oin)o(t)o(wise)h(ev)m(aluation)g(of)e(the)h (reduce)h(op)q(erator)75 929 y(on)h Fp(len)h Ft(elemen)o(ts:)29 b(I.e,)21 b(the)e(function)h(returns)g(in)g Fp(inoutvec[i])h Ft(the)e(v)m(alue)i Fp(invec)p Ft([)p Fp(i)p Ft(])12 b Fo(\016)h Fp(inoutvec)p Ft([)p Fp(i)p Ft(],)21 b(for)75 986 y Fp(i)12 b Ft(=)h Fp(0)p Fj(;)8 b(:)g(:)g(:)d(;)j Fp(count)j Fo(\000)f Fp(1)p Ft(,)15 b(where)g Fo(\016)g Ft(is)h(the)f(com)o(bining)h(op)q(eration)g(computed)f(b)o(y)g(the)h (function.)189 1092 y Fm(R)n(ationale.)47 b Ft(The)16 b(len)i(argumen)o(t)d(allo)o(ws)i Fp(MPI)p 1033 1092 V 16 w(REDUCE)g Ft(to)f(a)o(v)o(oid)g(calling)i(the)e(function)i(for) 189 1148 y(eac)o(h)13 b(elemen)o(t)h(in)g(the)f(input)h(bu\013er.)19 b(Rather,)13 b(the)h(system)e(can)h(c)o(ho)q(ose)h(to)e(apply)i(the)f (function)189 1205 y(to)k(c)o(h)o(unks)i(of)f(input.)30 b(In)19 b(C,)e(it)i(is)g(passed)f(in)h(as)f(a)g(reference)h(for)f (reasons)g(of)f(compatibilit)o(y)189 1261 y(with)e(F)l(ortran.)189 1337 y(By)e(in)o(ternally)h(comparing)f(the)g(v)m(alue)h(of)e(the)h Fp(datat)o(yp)q(e)h Ft(argumen)o(t)e(to)g(kno)o(wn,)h(global)g (handles,)189 1393 y(it)h(is)h(p)q(ossible)h(to)e(o)o(v)o(erload)g(the) g(use)h(of)f(a)g(single)h(user-de\014ned)i(function)e(for)e(sev)o (eral,)i(di\013eren)o(t)189 1449 y(data)f(t)o(yp)q(es.)20 b(\()p Fm(End)c(of)g(r)n(ationale.)p Ft(\))166 1556 y(General)e(datat)o (yp)q(es)f(ma)o(y)g(b)q(e)i(passed)f(to)f(the)g(user)h(function.)21 b(Ho)o(w)o(ev)o(er,)12 b(use)i(of)g(datat)o(yp)q(es)f(that)75 1612 y(are)i(not)g(con)o(tiguous)g(is)h(lik)o(ely)h(to)d(lead)i(to)f (ine\016ciencies.)166 1669 y(No)g Fp(MPI)g Ft(comm)o(unication)i (function)f(ma)o(y)f(b)q(e)h(called)h(inside)h(the)d(user)h(function.) 22 b Fp(MPI)p 1708 1669 V 16 w(ABORT)75 1725 y Ft(ma)o(y)14 b(b)q(e)i(called)h(inside)g(the)e(function)h(in)h(case)e(of)g(an)g (error.)189 1831 y Fm(A)n(dvic)n(e)g(to)i(users.)42 b Ft(Supp)q(ose)17 b(one)e(de\014nes)i(a)e(library)i(of)e(user-de\014ned) i(reduce)f(functions)h(that)189 1888 y(are)10 b(o)o(v)o(erloaded:)17 b(the)11 b Fp(datat)o(yp)q(e)h Ft(argumen)o(t)d(is)i(used)g(to)f (select)h(the)g(righ)o(t)f(execution)i(path)e(at)g(eac)o(h)189 1944 y(in)o(v)o(o)q(cation,)15 b(according)h(to)e(the)i(t)o(yp)q(es)f (of)g(the)g(op)q(erands.)21 b(The)15 b(user-de\014ned)i(reduce)f (function)189 2001 y(cannot)11 b(\\deco)q(de")h(the)f Fp(datat)o(yp)q(e)i Ft(argumen)o(t)d(that)h(it)h(is)f(passed,)h(and)g (cannot)f(iden)o(tify)l(,)i(b)o(y)e(itself,)189 2057 y(the)18 b(corresp)q(ondence)h(b)q(et)o(w)o(een)f(the)h(datat)o(yp)q(e) e(handles)i(and)g(the)f(datat)o(yp)q(e)f(they)h(represen)o(t.)189 2114 y(This)i(corresp)q(ondence)h(w)o(as)d(established)j(when)g(the)e (datat)o(yp)q(es)g(w)o(ere)g(created.)33 b(Before)20 b(the)189 2170 y(library)g(is)h(used,)h(a)d(library)i(initialization)i (pream)o(ble)e(m)o(ust)e(b)q(e)i(executed.)36 b(This)20 b(pream)o(ble)189 2227 y(co)q(de)14 b(will)i(de\014ne)f(the)e(datat)o (yp)q(es)h(that)f(are)g(used)i(b)o(y)f(the)g(library)l(,)g(and)g(store) f(handles)j(to)d(these)189 2283 y(datat)o(yp)q(es)i(in)h(global,)g (static)f(v)m(ariables)i(that)e(are)g(shared)h(b)o(y)f(the)h(user)g(co) q(de)g(and)g(the)f(library)189 2339 y(co)q(de.)189 2415 y(The)f(F)l(ortran)f(v)o(ersion)i(of)e Fp(MPI)p 734 2415 V 16 w(REDUCE)i Ft(will)h(in)o(v)o(ok)o(e)e(a)g(user-de\014ned)i (reduce)f(function)g(using)189 2471 y(the)g(F)l(ortran)g(calling)i(con) o(v)o(en)o(tions)f(and)g(will)h(pass)f(a)f(F)l(ortran-t)o(yp)q(e)g (datat)o(yp)q(e)g(argumen)o(t;)g(the)189 2527 y(C)d(v)o(ersion)h(will)h (use)f(C)g(calling)h(con)o(v)o(en)o(tion)f(and)g(the)f(C)h(represen)o (tation)f(of)h(a)f(datat)o(yp)q(e)g(handle.)189 2584 y(Users)i(who)h(plan)h(to)e(mix)h(languages)g(should)h(de\014ne)g (their)g(reduction)f(functions)h(accordingly)l(.)189 2640 y(\()p Fm(End)f(of)i(advic)n(e)f(to)g(users.)p Ft(\))-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 32 34 32 33 bop 75 -100 a Ft(32)708 b Fk(CHAPTER)15 b(1.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)189 45 y Fm(A)n(dvic)n(e)f(to)h (implementors.)39 b Ft(W)l(e)15 b(outline)h(b)q(elo)o(w)f(a)g(naiv)o(e) g(and)g(ine\016cien)o(t)h(implemen)o(tation)g(of)189 102 y Fp(MPI)p 274 102 14 2 v 15 w(REDUCE)p Ft(.)189 227 y Fg(if)23 b(\(rank)g(>)h(0\))g({)284 283 y(RECV\(tempbuf,)e (count,)h(datatype,)g(rank-1,...\))284 339 y(User_reduce\()f(tempbuf,)h (sendbuf,)g(count,)g(datatype\))189 396 y(})189 452 y(if)g(\(rank)g(<)h (groupsize-1\))e({)284 509 y(SEND\()h(sendbuf,)g(count,)g(datatype,)g (rank+1,)g(...\))189 565 y(})189 622 y(/*)g(answer)g(now)h(resides)f (in)g(process)g(groupsize-1)f(...)i(now)f(send)h(to)f(root)212 678 y(*/)189 735 y(if)g(\(rank)g(==)h(groupsize-1\))e({)284 791 y(SEND\()h(sendbuf,)g(count,)g(datatype,)g(root,)g(...\))189 848 y(})189 904 y(if)g(\(rank)g(==)h(root\))f({)284 960 y(RECV\(recvbuf,)f(count,)h(datatype,)g(groupsize-1,...\))189 1017 y(})189 1142 y Ft(The)16 b(reduction)h(computation)e(pro)q(ceeds,) i(sequen)o(tially)l(,)h(from)d(pro)q(cess)h Fg(0)f Ft(to)h(pro)q(cess)g Fg(group-)189 1198 y(size-1)p Ft(.)i(This)13 b(order)g(is)g(c)o(hosen)g (so)f(as)h(to)f(resp)q(ect)h(the)g(order)f(of)h(a)f(p)q(ossibly)i (non-comm)o(utativ)o(e)189 1255 y(op)q(erator)g(de\014ned)j(b)o(y)f (the)g(function)g Fg(User)p 949 1255 15 2 v 17 w(reduce\(\))p Ft(.)j(A)d(more)f(e\016cien)o(t)i(implemen)o(tation)f(is)189 1311 y(ac)o(hiev)o(ed)f(b)o(y)g(taking)g(adv)m(an)o(tage)g(of)f(asso)q (ciativit)o(y)h(and)h(using)f(a)g(logarithmic)h(tree)e(reduction.)189 1368 y(Comm)o(utativit)o(y)e(can)i(b)q(e)g(used)g(to)f(adv)m(an)o (tage,)g(for)g(those)g(cases)g(in)i(whic)o(h)f(the)g Fp(commute)e Ft(argu-)189 1424 y(men)o(t)h(to)g Fp(MPI)p 441 1424 14 2 v 16 w(OP)p 519 1424 V 17 w(CREA)l(TE)i Ft(is)f(true.)20 b(Also,)14 b(the)g(amoun)o(t)f(of)g(temp)q(orary)g (bu\013er)h(required)h(can)189 1481 y(b)q(e)i(reduced,)i(and)e(comm)o (unication)h(can)f(b)q(e)h(pip)q(elined)j(with)c(computation,)g(b)o(y)g (transferring)189 1537 y(and)e(reducing)h(the)g(elemen)o(ts)g(in)g(c)o (h)o(unks)f(of)g(size)h Fg(len)f Fj(<)p Fg(count)p Ft(.)189 1612 y(The)21 b(prede\014ned)h(reduce)g(op)q(erations)f(can)g(b)q(e)h (implemen)o(ted)g(as)f(a)f(library)i(of)e(user-de\014ned)189 1669 y(op)q(erations.)g(Ho)o(w)o(ev)o(er,)14 b(b)q(etter)h(p)q (erformance)g(migh)o(t)g(b)q(e)h(ac)o(hiev)o(ed)g(if)g Fp(MPI)p 1521 1669 V 16 w(REDUCE)g Ft(handles)189 1725 y(these)f(functions)h(as)f(a)g(sp)q(ecial)i(case.)j(\()p Fm(End)15 b(of)i(advic)n(e)f(to)g(implementors.)p Ft(\))75 1926 y Fp(MPI)p 160 1926 V 16 w(OP)p 238 1926 V 17 w(FREE\()f(op\))117 2003 y Fh(IN)155 b Fp(op)541 b Fh(op)q(eration)14 b(\(handle\))75 2127 y Fg(int)23 b(MPI)p 245 2127 15 2 v 17 w(op)p 310 2127 V 17 w(free\()g(MPI)p 542 2127 V 17 w(Op)h(*op\))75 2214 y(MPI)p 150 2214 V 17 w(OP)p 215 2214 V 17 w(FREE\()f(OP,)g (IERROR\))170 2270 y(INTEGER)g(OP,)h(IERROR)166 2357 y Ft(Marks)12 b(a)h(user-de\014ned)i(reduction)f(op)q(eration)g(for)f (deallo)q(cation)i(and)e(sets)g Fp(op)g Ft(to)g Fd(MPI)p 1675 2357 13 2 v 14 w(OP)p 1747 2357 V 14 w(NULL)p Ft(.)75 2477 y Fp(Example)i(of)g(User-de\014ned)i(Reduce)75 2563 y Ft(It)e(is)h(time)f(for)g(an)g(example)h(of)f(user-de\014ned)i (reduction.)75 2669 y Fn(Example)h(1.20)23 b Ft(Compute)14 b(the)i(pro)q(duct)f(of)g(an)g(arra)o(y)f(of)h(complex)h(n)o(um)o(b)q (ers,)f(in)h(C.)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 33 35 33 34 bop 75 -100 a Fk(1.9.)34 b(GLOBAL)16 b(REDUCTION)g(OPERA)l(TIONS) 837 b Ft(33)75 45 y Fg(typedef)23 b(struct)g({)170 102 y(double)g(real,imag;)75 158 y(})h(Complex;)75 271 y(/*)g(the)f (user-defined)f(function)99 327 y(*/)75 384 y(void)h(myProd\()g (Complex)g(*in,)g(Complex)g(*inout,)g(int)h(*len,)f(MPI_Datatype)f (*dptr)h(\))75 440 y({)170 497 y(int)h(i;)170 553 y(Complex)f(c;)170 666 y(for)h(\(i=0;)f(i<)h(*len;)f(++i\))g({)266 723 y(c.real)g(=)h (inout->real*in->real)d(-)528 779 y(inout->imag*in->imag;)266 835 y(c.imag)i(=)h(inout->real*in->imag)d(+)528 892 y (inout->imag*in->real;)266 948 y(*inout)i(=)h(c;)266 1005 y(in++;)f(inout++;)170 1061 y(})75 1118 y(})75 1231 y(/*)h(and,)f(to)g(call)h(it...)99 1287 y(*/)75 1344 y(...)170 1456 y(/*)g(each)f(process)g(has)h(an)f(array)g(of)h(100)g (Complexes)194 1513 y(*/)170 1569 y(Complex)f(a[100],)g(answer[100];) 170 1626 y(MPI_Op)g(myOp;)170 1682 y(MPI_Datatype)g(ctype;)170 1795 y(/*)h(explain)f(to)h(MPI)f(how)g(type)h(Complex)f(is)g(defined) 194 1852 y(*/)170 1908 y(MPI_Type_contiguous\()f(2,)h(MPI_DOUBLE,)g (&ctype)g(\);)170 1965 y(MPI_Type_commit\()f(&ctype)h(\);)170 2021 y(/*)h(create)f(the)h(complex-product)d(user-op)194 2077 y(*/)170 2134 y(MPI_Op_create\()h(myProd,)h(True,)g(&myOp)h(\);) 170 2247 y(MPI_Reduce\()f(a,)g(answer,)g(100,)h(ctype,)f(myOp,)g(root,) g(comm)g(\);)170 2360 y(/*)h(At)g(this)f(point,)g(the)g(answer,)g (which)h(consists)e(of)i(100)f(Complexes,)194 2416 y(*)h(resides)f(on)h (process)e(root)194 2473 y(*/)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 34 36 34 35 bop 75 -100 a Ft(34)708 b Fk(CHAPTER)15 b(1.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Ff(1.9.5)49 b(All-Reduce)75 131 y Fp(MPI)17 b Ft(includes)j(v)m(arian)o(ts)e(of)f (eac)o(h)h(of)f(the)h(reduce)g(op)q(erations)g(where)g(the)g(result)g (is)g(returned)g(to)f(all)75 187 y(pro)q(cesses)i(in)h(the)f(group.)32 b Fp(MPI)18 b Ft(requires)i(that)e(all)i(pro)q(cesses)g(participating)g (in)g(these)f(op)q(erations)75 244 y(receiv)o(e)d(iden)o(tical)h (results.)75 395 y Fp(MPI)p 160 395 14 2 v 16 w(ALLREDUCE\()e(sendbuf,) i(recvbuf,)f(count,)g(datat)o(yp)q(e,)g(op,)f(comm\))117 472 y Fh(IN)155 b Fp(sendbuf)444 b Fh(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 547 y(OUT)108 b Fp(recvbuf)450 b Fh(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er) g(\(c)o(hoice\))117 622 y(IN)155 b Fp(count)482 b Fh(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(send)h(bu\013er)f(\(in)o(teger\))117 697 y(IN)155 b Fp(datat)o(yp)q(e)424 b Fh(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(send)i(bu\013er)g(\(handle\))117 772 y(IN)155 b Fp(op)541 b Fh(op)q(eration)14 b(\(handle\))117 847 y(IN)155 b Fp(comm)470 b Fh(comm)o(unicator)11 b(\(handle\))75 972 y Fg(int)23 b(MPI)p 245 972 15 2 v 17 w(Allreduce\(void*)f (sendbuf,)h(void*)g(recvbuf,)g(int)g(count,)393 1028 y(MPI)p 468 1028 V 17 w(Datatype)g(datatype,)f(MPI)p 1010 1028 V 17 w(Op)i(op,)f(MPI)p 1266 1028 V 17 w(Comm)g(comm\))75 1115 y(MPI)p 150 1115 V 17 w(ALLREDUCE\(SENDBUF,)e(RECVBUF,)i(COUNT,)g (DATATYPE,)g(OP,)g(COMM,)g(IERROR\))170 1171 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1227 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 1314 y Ft(Same)18 b(as)g Fp(MPI)p 434 1314 14 2 v 15 w(REDUCE)h Ft(except)g(that)e(the)h(result)g(app)q(ears)h(in) f(the)g(receiv)o(e)h(bu\013er)f(of)g(all)h(the)75 1370 y(group)c(mem)o(b)q(ers.)189 1476 y Fm(A)n(dvic)n(e)20 b(to)h(implementors.)71 b Ft(The)21 b(all-reduce)h(op)q(erations)f(can) f(b)q(e)i(implemen)o(ted)g(as)e(a)g(re-)189 1533 y(duce,)d(follo)o(w)o (ed)h(b)o(y)e(a)h(broadcast.)24 b(Ho)o(w)o(ev)o(er,)16 b(a)g(direct)i(implemen)o(tation)g(can)f(lead)h(to)e(b)q(etter)189 1589 y(p)q(erformance.)k(\()p Fm(End)15 b(of)i(advic)n(e)f(to)g (implementors.)p Ft(\))75 1695 y Fn(Example)i(1.21)23 b Ft(A)c(routine)h(that)f(computes)h(the)f(pro)q(duct)h(of)g(a)f(v)o (ector)g(and)h(an)f(arra)o(y)f(that)h(are)75 1751 y(distributed)13 b(across)d(a)h(group)g(of)g(pro)q(cesses)h(and)f(returns)g(the)h(answ)o (er)e(at)h(all)h(no)q(des)g(\(see)f(also)g(Example)75 1808 y(1.16\).)75 1913 y Fg(SUBROUTINE)22 b(PAR_BLAS2\(m,)h(n,)g(a,)h (b,)f(c,)h(comm\))75 1970 y(REAL)f(a\(m\),)g(b\(m,n\))95 b(!)24 b(local)f(slice)g(of)h(array)75 2026 y(REAL)f(c\(n\))286 b(!)24 b(result)75 2083 y(REAL)f(sum\(n\))75 2139 y(INTEGER)g(n,)g (comm,)h(i,)f(j,)h(ierr)75 2252 y(!)g(local)f(sum)75 2308 y(DO)h(j=)f(1,)h(n)123 2365 y(sum\(j\))f(=)g(0.0)123 2421 y(DO)g(i)h(=)g(1,)f(m)170 2478 y(sum\(j\))g(=)h(sum\(j\))f(+)h (a\(i\)*b\(i,j\))123 2534 y(END)f(DO)75 2591 y(END)g(DO)75 2704 y(!)h(global)f(sum)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 35 37 35 36 bop 75 -100 a Fk(1.10.)34 b(REDUCE-SCA)l(TTER)1196 b Ft(35)75 45 y Fg(CALL)23 b(MPI_ALLREDUCE\(sum,)f(c,)h(n,)h(MPI_REAL,) f(MPI_SUM,)f(0,)i(comm,)f(ierr\))75 158 y(!)h(return)f(result)g(at)g (all)h(nodes)75 214 y(RETURN)75 358 y Fq(1.10)59 b(Reduce-Scatte)o(r)75 459 y Fp(MPI)16 b Ft(includes)k(v)m(arian)o(ts)d(of)f(eac)o(h)h(of)g (the)g(reduce)h(op)q(erations)f(where)g(the)g(result)h(is)f(scattered)g (to)f(all)75 516 y(pro)q(cesses)g(in)g(the)f(group)g(on)g(return.)75 667 y Fp(MPI)p 160 667 14 2 v 16 w(REDUCE)p 352 667 V 17 w(SCA)l(TTER\()h(sendbuf,)h(recvbuf,)f(recvcounts,)g(datat)o(yp)q (e,)h(op,)e(comm\))117 744 y Fh(IN)155 b Fp(sendbuf)444 b Fh(starting)14 b(address)h(of)f(send)g(bu\013er)h(\(c)o(hoice\))117 819 y(OUT)108 b Fp(recvbuf)450 b Fh(starting)14 b(address)h(of)f (receiv)o(e)h(bu\013er)g(\(c)o(hoice\))117 894 y(IN)155 b Fp(recvcounts)388 b Fh(in)o(teger)14 b(arra)o(y)f(sp)q(ecifying)g (the)h(n)o(um)o(b)q(er)e(of)h(elemen)o(ts)g(in)f(re-)905 950 y(sult)i(distributed)h(to)f(eac)o(h)h(pro)q(cess.)21 b(Arra)o(y)14 b(m)o(ust)f(b)q(e)i(iden-)905 1007 y(tical)e(on)h(all)f (calling)f(pro)q(cesses.)117 1082 y(IN)155 b Fp(datat)o(yp)q(e)424 b Fh(data)14 b(t)o(yp)q(e)g(of)f(elemen)o(ts)h(of)f(input)h(bu\013er)h (\(handle\))117 1157 y(IN)155 b Fp(op)541 b Fh(op)q(eration)14 b(\(handle\))117 1232 y(IN)155 b Fp(comm)470 b Fh(comm)o(unicator)11 b(\(handle\))75 1357 y Fg(int)23 b(MPI)p 245 1357 15 2 v 17 w(Reduce)p 406 1357 V 17 w(scatter\(void*)f(sendbuf,)g(void*)i (recvbuf,)e(int)i(*recvcounts,)393 1413 y(MPI)p 468 1413 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 1413 V 17 w(Op)i(op,)f(MPI)p 1266 1413 V 17 w(Comm)g(comm\))75 1500 y(MPI)p 150 1500 V 17 w(REDUCE)p 311 1500 V 16 w(SCATTER\(SENDBUF,)f(RECVBUF,)h (RECVCOUNTS,)f(DATATYPE,)h(OP,)g(COMM,)393 1556 y(IERROR\))170 1613 y()g(SENDBUF\(*\),)g(RECVBUF\(*\))170 1669 y(INTEGER)g(RECVCOUNTS\(*\),)f(DATATYPE,)h(OP,)g(COMM,)h(IERROR)166 1756 y Fp(MPI)p 251 1756 14 2 v 16 w(REDUCE)p 443 1756 V 17 w(SCA)l(TTER)f Ft(\014rst)e(do)q(es)h(an)g(elemen)o(t-wise)h (reduction)g(on)e(v)o(ector)g(of)h Fp(count)j Ft(=)75 1780 y Fc(P)119 1825 y Fd(i)138 1812 y Fp(recvcounts)p Ft([)p Fp(i)p Ft(])18 b(elemen)o(ts)g(in)f(the)g(send)h(bu\013er)f (de\014ned)h(b)o(y)f Fp(sendbuf,)i(count)g Ft(and)e Fp(datat)o(yp)q(e)p Ft(.)26 b(Next,)75 1868 y(the)19 b(resulting)h(v)o(ector)e(of)g (results)h(is)h(split)g(in)o(to)e Fg(n)h Ft(disjoin)o(t)h(segmen)o(ts,) e(where)h Fg(n)g Ft(is)g(the)g(n)o(um)o(b)q(er)g(of)75 1925 y(mem)o(b)q(ers)c(in)h(the)f(group.)k(Segmen)o(t)c Fg(i)g Ft(con)o(tains)g Fp(recvcounts[i])h Ft(elemen)o(ts.)21 b(The)15 b Fg(i)p Ft(th)f(segmen)o(t)h(is)g(sen)o(t)75 1981 y(to)g(pro)q(cess)g Fg(i)g Ft(and)g(stored)g(in)h(the)f(receiv)o (e)i(bu\013er)e(de\014ned)h(b)o(y)f Fp(recvbuf,)h(recvcounts[i])h Ft(and)e Fp(datat)o(yp)q(e)p Ft(.)189 2088 y Fm(A)n(dvic)n(e)d(to)h (implementors.)37 b Ft(The)12 b Fp(MPI)p 875 2088 V 16 w(REDUCE)p 1067 2088 V 17 w(SCA)l(TTER)h Ft(routine)f(is)g (functionally)i(equiv)m(a-)189 2144 y(len)o(t)c(to:)17 b(A)10 b Fp(MPI)p 473 2144 V 16 w(REDUCE)h Ft(op)q(eration)g(function)g (with)g Fp(count)g Ft(equal)g(to)f(the)g(sum)g(of)g Fp(recvcounts[i]) 189 2201 y Ft(follo)o(w)o(ed)17 b(b)o(y)g Fp(MPI)p 516 2201 V 16 w(SCA)l(TTERV)h Ft(with)f Fp(sendcounts)k Ft(equal)c(to)g Fp(recvcounts)p Ft(.)27 b(Ho)o(w)o(ev)o(er,)16 b(a)g(direct)189 2257 y(implemen)o(tation)g(ma)o(y)f(run)g(faster.)k(\()p Fm(End)d(of)g(advic)n(e)g(to)h(implementors.)p Ft(\))-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 36 38 36 37 bop 75 -100 a Ft(36)703 b Fk(CHAPTER)15 b(1.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)75 45 y Fq(1.11)59 b(Scan)75 241 y Fp(MPI)p 160 241 14 2 v 16 w(SCAN\()15 b(sendbuf,)j(recvbuf,)d(count,)h(datat)o(yp)q(e,)h(op,)e(comm)e(\))117 318 y Fh(IN)155 b Fp(sendbuf)444 b Fh(starting)14 b(address)h(of)f (send)g(bu\013er)h(\(c)o(hoice\))117 393 y(OUT)108 b Fp(recvbuf)450 b Fh(starting)14 b(address)h(of)f(receiv)o(e)h(bu\013er) g(\(c)o(hoice\))117 469 y(IN)155 b Fp(count)482 b Fh(n)o(um)o(b)q(er)13 b(of)h(elemen)o(ts)f(in)h(input)g(bu\013er)g(\(in)o(teger\))117 544 y(IN)155 b Fp(datat)o(yp)q(e)424 b Fh(data)14 b(t)o(yp)q(e)g(of)f (elemen)o(ts)h(of)f(input)h(bu\013er)h(\(handle\))117 619 y(IN)155 b Fp(op)541 b Fh(op)q(eration)14 b(\(handle\))117 694 y(IN)155 b Fp(comm)470 b Fh(comm)o(unicator)11 b(\(handle\))75 818 y Fg(int)23 b(MPI)p 245 818 15 2 v 17 w(Scan\(void*)g(sendbuf,)f (void*)i(recvbuf,)e(int)i(count,)393 875 y(MPI)p 468 875 V 17 w(Datatype)f(datatype,)f(MPI)p 1010 875 V 17 w(Op)i(op,)f(MPI)p 1266 875 V 17 w(Comm)g(comm)h(\))75 961 y(MPI)p 150 961 V 17 w(SCAN\(SENDBUF,)e(RECVBUF,)h(COUNT,)g (DATATYPE,)f(OP,)i(COMM,)f(IERROR\))170 1018 y()g(SENDBUF\(*\),)g (RECVBUF\(*\))170 1074 y(INTEGER)g(COUNT,)g(DATATYPE,)g(OP,)g(COMM,)h (IERROR)166 1161 y Fp(MPI)p 251 1161 14 2 v 16 w(SCAN)15 b Ft(is)h(used)f(to)g(p)q(erform)f(a)h(pre\014x)g(reduction)h(on)f (data)f(distributed)i(across)f(the)g(group.)75 1217 y(The)k(op)q (eration)g(returns,)h(in)f(the)g(receiv)o(e)h(bu\013er)f(of)f(the)h (pro)q(cess)g(with)h(rank)e Fg(i)p Ft(,)i(the)f(reduction)g(of)75 1274 y(the)h(v)m(alues)h(in)g(the)f(send)h(bu\013ers)e(of)h(pro)q (cesses)g(with)h(ranks)e Fg(0,...,i)g Ft(\(inclusiv)o(e\).)36 b(The)20 b(t)o(yp)q(e)g(of)75 1330 y(op)q(erations)c(supp)q(orted,)h (their)g(seman)o(tics,)f(and)g(the)h(constrain)o(ts)f(on)g(send)h(and)f (receiv)o(e)h(bu\013ers)f(are)75 1386 y(as)f(for)f Fp(MPI)p 285 1386 V 16 w(REDUCE)p Ft(.)189 1493 y Fm(R)n(ationale.)38 b Ft(W)l(e)13 b(ha)o(v)o(e)g(de\014ned)h(an)f(inclusiv)o(e)i(scan,)e (that)f(is,)i(the)f(pre\014x)g(reduction)h(on)f(pro)q(cess)189 1549 y Fg(i)k Ft(includes)j(the)e(data)f(from)f(pro)q(cess)i Fg(i)p Ft(.)27 b(An)18 b(alternativ)o(e)g(is)g(to)f(de\014ne)i(scan)e (in)i(an)e(exclusiv)o(e)189 1606 y(manner,)11 b(where)f(the)h(result)f (on)h Fg(i)f Ft(only)h(includes)h(data)e(up)h(to)e Fg(i-1)p Ft(.)18 b(Both)10 b(de\014nitions)i(are)e(useful.)189 1662 y(The)k(latter)g(has)g(some)g(adv)m(an)o(tages:)19 b(the)c(inclusiv)o(e)h(scan)f(can)f(alw)o(a)o(ys)g(b)q(e)h(computed)f (from)g(the)189 1719 y(exclusiv)o(e)k(scan)f(with)g(no)g(additional)h (comm)o(unication;)g(for)f(non-in)o(v)o(ertible)i(op)q(erations)e(suc)o (h)189 1775 y(as)f(max)g(and)g(min,)i(comm)o(unication)f(is)g(required) g(to)f(compute)h(the)f(exclusiv)o(e)j(scan)d(from)g(the)189 1831 y(inclusiv)o(e)i(scan.)23 b(There)16 b(is,)g(ho)o(w)o(ev)o(er,)f (a)h(complication)i(with)e(exclusiv)o(e)i(scan)e(since)h(one)g(m)o(ust) 189 1888 y(de\014ne)g(the)f(\\unit")g(elemen)o(t)h(for)e(the)h (reduction)h(in)g(this)f(case.)22 b(That)15 b(is,)i(one)f(m)o(ust)f (explicitly)189 1944 y(sa)o(y)21 b(what)h(o)q(ccurs)g(for)g(pro)q(cess) g Fg(0)p Ft(.)41 b(This)23 b(w)o(as)e(though)o(t)g(to)h(b)q(e)h (complex)g(for)e(user-de\014ned)189 2001 y(op)q(erations)15 b(and)g(hence,)h(the)g(exclusiv)o(e)g(scan)g(w)o(as)e(dropp)q(ed.)21 b(\()p Fm(End)15 b(of)i(r)n(ationale.)p Ft(\))75 2123 y Ff(1.11.1)49 b(Example)17 b(using)g(MPI)p 646 2123 15 2 v 18 w(SCAN)75 2208 y Fn(Example)h(1.22)23 b Ft(This)17 b(example)h(uses)f(a)f(user-de\014ned)j(op)q(eration)e(to)f(pro)q(duce) i(a)e Fm(se)n(gmente)n(d)g(sc)n(an)p Ft(.)75 2265 y(A)k(segmen)o(ted)g (scan)g(tak)o(es,)f(as)h(input,)h(a)f(set)f(of)h(v)m(alues)h(and)f(a)f (set)h(of)f(logicals,)j(and)e(the)g(logicals)75 2321 y(delineate)d(the)e(v)m(arious)h(segmen)o(ts)f(of)f(the)i(scan.)k(F)l (or)14 b(example:)321 2419 y Fj(v)r(al)q(ues)66 b(v)540 2426 y Fi(1)650 2419 y Fj(v)672 2426 y Fi(2)781 2419 y Fj(v)803 2426 y Fi(3)913 2419 y Fj(v)935 2426 y Fi(4)1143 2419 y Fj(v)1165 2426 y Fi(5)1323 2419 y Fj(v)1345 2426 y Fi(6)1455 2419 y Fj(v)1477 2426 y Fi(7)1587 2419 y Fj(v)1609 2426 y Fi(8)321 2476 y Fj(l)q(og)r(ical)q(s)49 b Ft(0)109 b(0)g(1)g(1)206 b(1)158 b(0)109 b(0)g(1)321 2532 y Fj(r)q(esul)q(t)76 b(v)540 2539 y Fi(1)601 2532 y Fj(v)623 2539 y Fi(1)653 2532 y Ft(+)10 b Fj(v)720 2539 y Fi(2)781 2532 y Fj(v)803 2539 y Fi(3)865 2532 y Fj(v)887 2539 y Fi(3)917 2532 y Ft(+)g Fj(v)984 2539 y Fi(4)1045 2532 y Fj(v)1067 2539 y Fi(3)1097 2532 y Ft(+)h Fj(v)1165 2539 y Fi(4)1194 2532 y Ft(+)g Fj(v)1262 2539 y Fi(5)1323 2532 y Fj(v)1345 2539 y Fi(6)1406 2532 y Fj(v)1428 2539 y Fi(6)1458 2532 y Ft(+)g Fj(v)1526 2539 y Fi(7)1587 2532 y Fj(v)1609 2539 y Fi(8)166 2631 y Ft(The)k(op)q(erator)g(that)f(pro)q(duces)i(this)g(e\013ect)f(is,) 1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 37 39 37 38 bop 75 -100 a Fk(1.11.)29 b(SCAN)1505 b Ft(37)711 51 y Fc( )764 95 y Fj(u)769 151 y(i)811 51 y Fc(!)854 123 y Fo(\016)887 51 y Fc( )940 95 y Fj(v)942 151 y(j)985 51 y Fc(!)1030 123 y Ft(=)1078 51 y Fc( )1132 95 y Fj(w)1138 151 y(j)1186 51 y Fc(!)1227 123 y Fj(;)166 234 y Ft(where,)735 369 y Fj(w)13 b Ft(=)829 297 y Fc(\()883 340 y Fj(u)d Ft(+)h Fj(v)43 b Ft(if)16 b Fj(i)c Ft(=)h Fj(j)883 397 y(v)125 b Ft(if)16 b Fj(i)c Fo(6)p Ft(=)h Fj(j)1203 369 y(:)166 480 y Ft(Note)20 b(that)g(this)h(is)g(a)f(non-comm)o(utativ)o (e)g(op)q(erator.)36 b(C)20 b(co)q(de)h(that)f(implemen)o(ts)i(it)e(is) h(giv)o(en)75 537 y(b)q(elo)o(w.)75 638 y Fg(typedef)i(struct)g({)170 694 y(double)g(val;)170 751 y(int)h(log;)75 807 y(})g(SegScanPair;)75 920 y(/*)g(the)f(user-defined)f(function)99 977 y(*/)75 1033 y(void)h(segScan\()g(SegScanPair)f(*in,)i(SegScanPair)e(*inout,)h (int)g(*len,)75 1090 y(MPI_Datatype)f(*dptr)h(\))75 1146 y({)170 1202 y(int)h(i;)170 1259 y(SegScanPair)f(c;)170 1372 y(for)h(\(i=0;)f(i<)h(*len;)f(++i\))g({)266 1428 y(if)g(\()h(in->log)f(==)h(inout->log)e(\))361 1485 y(c.val)i(=)f (in->val)g(+)h(inout->val;)266 1541 y(else)361 1598 y(c.val)g(=)f (inout->val;)266 1654 y(c.log)g(=)h(inout->log;)266 1711 y(*inout)f(=)h(c;)266 1767 y(in++;)f(inout++;)170 1823 y(})75 1880 y(})166 1981 y Ft(Note)c(that)f(the)i Fp(inout)g Ft(argumen)o(t)e(to)h(the)g(user-de\014ned)i(function)f(corresp)q(onds) g(to)f(the)g(righ)o(t-)75 2038 y(hand)13 b(op)q(erand)g(of)g(the)f(op)q (erator.)19 b(When)13 b(using)g(this)g(op)q(erator,)f(w)o(e)h(m)o(ust)f (b)q(e)h(careful)h(to)e(sp)q(ecify)i(that)75 2094 y(it)h(is)h(non-comm) o(utativ)o(e,)f(as)f(in)i(the)g(follo)o(wing.)170 2195 y Fg(int)24 b(i,base;)170 2252 y(SeqScanPair)47 b(a,)23 b(answer;)170 2308 y(MPI_Op)167 b(myOp;)170 2365 y(MPI_Datatype)23 b(type[2])g(=)g({MPI_DOUBLE,)g(MPI_INT};)170 2421 y(MPI_Aint)119 b(disp[2];)170 2478 y(int)239 b(blocklen[2])22 b(=)i({)f(1,)h(1};)170 2534 y(MPI_Datatype)f(sspair;)170 2647 y(/*)h(explain)f(to)h(MPI)f(how) g(type)h(SegScanPair)e(is)i(defined)194 2704 y(*/)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 38 40 38 39 bop 75 -100 a Ft(38)708 b Fk(CHAPTER)15 b(1.)30 b(COLLECTIVE)16 b(COMMUNICA)l(TION)170 45 y Fg(MPI_Address\()23 b(a,)g(disp\);)170 102 y(MPI_Address\()g(a.log,)g(disp+1\);)170 158 y(base)h(=)f(disp[0];)170 214 y(for)h(\(i=0;)f(i<2;)g(++i\))h (disp[i])f(-=)g(base;)170 271 y(MPI_Type_struct\()f(2,)i(blocklen,)e (disp,)i(type,)f(&sspair)g(\);)170 327 y(MPI_Type_commit\()f(&sspair)h (\);)170 384 y(/*)h(create)f(the)h(segmented-scan)d(user-op)194 440 y(*/)170 497 y(MPI_Op_create\()h(segScan,)h(False,)g(&myOp)g(\);) 170 553 y(...)170 610 y(MPI_Scan\()g(a,)h(answer,)f(1,)g(sspair,)g (myOp,)g(root,)g(comm)h(\);)75 753 y Fq(1.12)59 b(Co)n(rrectness)75 854 y Ft(A)11 b(correct,)f(p)q(ortable)h(program)f(m)o(ust)g(in)o(v)o (ok)o(e)g(collectiv)o(e)i(comm)o(unications)g(so)e(that)g(deadlo)q(c)o (k)h(will)h(not)75 911 y(o)q(ccur,)h(whether)g(collectiv)o(e)h(comm)o (unications)g(are)e(sync)o(hronizing)i(or)e(not.)19 b(The)13 b(follo)o(wing)g(examples)75 967 y(illustrate)j(dangerous)f(use)h(of)f (collectiv)o(e)i(routines.)75 1074 y Fn(Example)h(1.23)23 b Ft(The)15 b(follo)o(wing)h(is)g(erroneous.)75 1180 y Fg(switch\(rank\))22 b({)170 1236 y(case)i(0:)266 1293 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h(comm\);)266 1349 y(MPI_Bcast\(buf2,)e(count,)h(type,)g(1,)h(comm\);)266 1406 y(break;)170 1462 y(case)g(1:)266 1519 y(MPI_Bcast\(buf2,)e (count,)h(type,)g(1,)h(comm\);)266 1575 y(MPI_Bcast\(buf1,)e(count,)h (type,)g(0,)h(comm\);)266 1631 y(break;)75 1688 y(})166 1794 y Ft(W)l(e)19 b(assume)h(that)f(the)g(group)g(of)g Fp(comm)f Ft(is)i Fo(f)p Ft(0,1)p Fo(g)p Ft(.)31 b(Tw)o(o)19 b(pro)q(cesses)g(execute)i(t)o(w)o(o)d(broadcast)75 1851 y(op)q(erations)d(in)h(rev)o(erse)f(order.)20 b(If)15 b(the)h(op)q(eration)f(is)h(sync)o(hronizing)h(then)e(a)g(deadlo)q(c)o (k)h(will)h(o)q(ccur.)166 1907 y(Collectiv)o(e)h(op)q(erations)e(m)o (ust)g(b)q(e)g(executed)i(in)f(the)f(same)g(order)g(at)f(all)j(mem)o(b) q(ers)e(of)g(the)g(com-)75 1964 y(m)o(unication)g(group.)75 2070 y Fn(Example)i(1.24)23 b Ft(The)15 b(follo)o(wing)h(is)g (erroneous.)75 2176 y Fg(switch\(rank\))22 b({)170 2233 y(case)i(0:)266 2289 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h (comm0\);)266 2345 y(MPI_Bcast\(buf2,)e(count,)h(type,)g(2,)h(comm2\);) 266 2402 y(break;)170 2458 y(case)g(1:)266 2515 y(MPI_Bcast\(buf1,)e (count,)h(type,)g(1,)h(comm1\);)266 2571 y(MPI_Bcast\(buf2,)e(count,)h (type,)g(0,)h(comm0\);)266 2628 y(break;)170 2684 y(case)g(2:)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 39 41 39 40 bop 75 -100 a Fk(1.12.)29 b(CORRECTNESS)1285 b Ft(39)266 45 y Fg(MPI_Bcast\(buf1,)22 b(count,)h(type,)g(2,)h(comm2\);) 266 102 y(MPI_Bcast\(buf2,)e(count,)h(type,)g(1,)h(comm1\);)266 158 y(break;)75 214 y(})166 319 y Ft(Assume)13 b(that)f(the)i(group)e (of)h Fp(comm0)e Ft(is)i Fo(f)p Ft(0,1)p Fo(g)p Ft(,)f(of)h Fp(comm1)d Ft(is)k Fo(f)p Ft(1,)e(2)p Fo(g)h Ft(and)g(of)g Fp(comm2)e Ft(is)i Fo(f)p Ft(2,0)p Fo(g)p Ft(.)18 b(If)75 375 y(the)12 b(broadcast)g(is)h(a)f(sync)o(hronizing)i(op)q(eration,)e (then)h(there)g(is)f(a)g(cyclic)j(dep)q(endency:)20 b(the)13 b(broadcast)75 432 y(in)i Fp(comm2)d Ft(completes)j(only)f(after)g(the) g(broadcast)f(in)i Fp(comm0)p Ft(;)d(the)i(broadcast)f(in)i Fp(comm0)d Ft(completes)75 488 y(only)18 b(after)e(the)h(broadcast)g (in)h Fp(comm1)p Ft(;)d(and)i(the)h(broadcast)e(in)i Fp(comm1)d Ft(completes)j(only)f(after)g(the)75 545 y(broadcast)d(in)i Fp(comm2)p Ft(.)i(Th)o(us,)d(the)g(co)q(de)h(will)h(deadlo)q(c)o(k.)166 601 y(Collectiv)o(e)c(op)q(erations)f(m)o(ust)f(b)q(e)i(executed)g(in)f (an)g(order)g(so)f(that)g(no)h(cyclic)i(dep)q(endences)g(o)q(ccur.)75 705 y Fn(Example)k(1.25)23 b Ft(The)15 b(follo)o(wing)h(is)g (erroneous.)75 810 y Fg(switch\(rank\))22 b({)170 866 y(case)i(0:)266 923 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h(comm\);) 266 979 y(MPI_Send\(buf2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)266 1036 y(break;)170 1092 y(case)h(1:)266 1149 y(MPI_Recv\(buf2,)e(count,) h(type,)g(0,)h(tag,)f(comm\);)266 1205 y(MPI_Bcast\(buf1,)f(count,)h (type,)g(0,)h(comm\);)266 1261 y(break;)75 1318 y(})166 1422 y Ft(Pro)q(cess)15 b(zero)f(executes)i(a)e(broadcast,)g(follo)o(w) o(ed)h(b)o(y)g(a)f(blo)q(c)o(king)j(send)e(op)q(eration.)20 b(Pro)q(cess)15 b(one)75 1479 y(\014rst)21 b(executes)g(a)g(blo)q(c)o (king)h(receiv)o(e)g(that)e(matc)o(hes)h(the)g(send,)h(follo)o(w)o(ed)g (b)o(y)e(broadcast)h(call)h(that)75 1535 y(matc)o(hes)14 b(the)h(broadcast)e(of)i(pro)q(cess)f(zero.)20 b(This)15 b(program)e(ma)o(y)h(deadlo)q(c)o(k.)21 b(The)15 b(broadcast)e(call)j (on)75 1592 y(pro)q(cess)g(zero)f Fm(may)g Ft(blo)q(c)o(k)h(un)o(til)h (pro)q(cess)e(one)h(executes)f(the)h(matc)o(hing)f(broadcast)f(call,)i (so)f(that)g(the)75 1648 y(send)i(is)h(not)e(executed.)26 b(Pro)q(cess)17 b(one)g(will)h(de\014nitely)h(blo)q(c)o(k)f(on)e(the)h (receiv)o(e)h(and)f(so,)f(in)i(this)f(case,)75 1705 y(nev)o(er)e (executes)h(the)f(broadcast.)166 1761 y(The)j(relativ)o(e)g(order)f(of) g(execution)h(of)f(collectiv)o(e)i(op)q(erations)f(and)g(p)q(oin)o (t-to-p)q(oin)o(t)g(op)q(erations)75 1817 y(should)f(b)q(e)h(suc)o(h,)e (so)g(that)g(ev)o(en)h(if)g(the)g(collectiv)o(e)h(op)q(erations)e(and)h (the)g(p)q(oin)o(t-to-p)q(oin)o(t)g(op)q(erations)75 1874 y(are)e(sync)o(hronizing,)h(no)f(deadlo)q(c)o(k)i(will)f(o)q (ccur.)75 1978 y Fn(Example)i(1.26)23 b Ft(A)15 b(correct,)f(but)h (non-deterministic)j(program.)75 2083 y Fg(switch\(rank\))k({)170 2139 y(case)i(0:)266 2195 y(MPI_Bcast\(buf1,)e(count,)h(type,)g(0,)h (comm\);)266 2252 y(MPI_Send\(buf2,)e(count,)h(type,)g(1,)h(tag,)f (comm\);)266 2308 y(break;)170 2365 y(case)h(1:)266 2421 y(MPI_Recv\(buf2,)e(count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h(comm\);) 266 2478 y(MPI_Bcast\(buf1,)f(count,)h(type,)g(0,)h(comm\);)266 2534 y(MPI_Recv\(buf2,)e(count,)h(type,)g(MPI_ANY_SOURCE,)f(tag,)h (comm\);)266 2591 y(break;)170 2647 y(case)h(2:)266 2704 y(MPI_Send\(buf2,)e(count,)h(type,)g(1,)h(tag,)f(comm\);)-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Page: 40 42 40 41 bop 75 -100 a Ft(40)703 b Fk(CHAPTER)15 b(1.)35 b(COLLECTIVE)16 b(COMMUNICA)l(TION)383 4 y 18945146 16482269 0 0 29733355 25918013 startTexFig 383 4 a %%BeginDocument: coll-matchings.ps /$F2psDict 200 dict def $F2psDict begin $F2psDict /mtrx matrix put /l {lineto} bind def /m {moveto} bind def /s {stroke} bind def /n {newpath} bind def /gs {gsave} bind def /gr {grestore} bind def /clp {closepath} bind def /graycol {dup dup currentrgbcolor 4 -2 roll mul 4 -2 roll mul 4 -2 roll mul setrgbcolor} bind def /col-1 {} def /col0 {0 0 0 setrgbcolor} bind def /col1 {0 0 1 setrgbcolor} bind def /col2 {0 1 0 setrgbcolor} bind def /col3 {0 1 1 setrgbcolor} bind def /col4 {1 0 0 setrgbcolor} bind def /col5 {1 0 1 setrgbcolor} bind def /col6 {1 1 0 setrgbcolor} bind def /col7 {1 1 1 setrgbcolor} bind def end /$F2psBegin {$F2psDict begin /$F2psEnteredState save def} def /$F2psEnd {$F2psEnteredState restore end} def $F2psBegin 0 setlinecap 0 setlinejoin -50 0 translate -40.0 436.0 translate 0.900 -0.900 scale 0.500 setlinewidth % Polyline n 469 134 m 334 134 l gs col-1 s gr n 342.000 136.000 m 334.000 134.000 l 342.000 132.000 l gs 2 setlinejoin col-1 s gr % Polyline n 154 399 m 284 399 l gs col-1 s gr n 276.000 397.000 m 284.000 399.000 l 276.000 401.000 l gs 2 setlinejoin col-1 s gr % Polyline n 469 454 m 329 454 l gs col-1 s gr n 337.000 456.000 m 329.000 454.000 l 337.000 452.000 l gs 2 setlinejoin col-1 s gr /Times-Italic findfont 18.00 scalefont setfont 279 59 m gs 1 -1 scale (First Execution) col-1 show gr /Times-Italic findfont 18.00 scalefont setfont 274 299 m gs 1 -1 scale (Second Execution) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 139 94 m gs 1 -1 scale (0) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 319 94 m gs 1 -1 scale (1) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 509 94 m gs 1 -1 scale (2) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 194 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 484 164 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 194 m gs 1 -1 scale (send) col-1 show gr % Polyline n 159 189 m 279 189 l gs col-1 s gr n 271.000 187.000 m 279.000 189.000 l 271.000 191.000 l gs 2 setlinejoin col-1 s gr /Times-Roman findfont 16.00 scalefont setfont 294 139 m gs 1 -1 scale (recv) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 44 94 m gs 1 -1 scale (process:) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 484 139 m gs 1 -1 scale (send) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 199 184 m gs 1 -1 scale (match) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 384 129 m gs 1 -1 scale (match) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 429 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 404 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 294 459 m gs 1 -1 scale (recv) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 479 459 m gs 1 -1 scale (send) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 479 484 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 404 m gs 1 -1 scale (send) col-1 show gr /Times-Roman findfont 16.00 scalefont setfont 119 379 m gs 1 -1 scale (broadcast) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 194 394 m gs 1 -1 scale (match) col-1 show gr /Times-Italic findfont 16.00 scalefont setfont 379 449 m gs 1 -1 scale (match) col-1 show gr $F2psEnd %%EndDocument endTexFig 75 1146 a Ft(Figure)d(1.10:)k(A)c(race)f(condition)i(causes)f (non-deterministic)i(matc)o(hing)d(of)g(sends)i(and)e(receiv)o(es.)20 b(One)75 1202 y(cannot)15 b(rely)h(on)f(sync)o(hronization)h(from)e(a)h (broadcast)g(to)f(mak)o(e)h(the)g(program)f(deterministic.)266 1335 y Fg(MPI_Bcast\(buf1,)22 b(count,)h(type,)g(0,)h(comm\);)266 1392 y(break;)75 1448 y(})166 1555 y Ft(All)19 b(three)f(pro)q(cesses)g (participate)h(in)g(a)e(broadcast.)28 b(Pro)q(cess)17 b(0)h(sends)g(a)g(message)f(to)h(pro)q(cess)75 1611 y(1)h(after)g(the)h (broadcast,)f(and)h(pro)q(cess)f(2)h(sends)g(a)f(message)g(to)g(pro)q (cess)h(1)f(b)q(efore)g(the)h(broadcast.)75 1667 y(Pro)q(cess)15 b(1)g(receiv)o(es)h(b)q(efore)g(and)f(after)f(the)i(broadcast,)e(with)h (a)g(wildcard)i(source)e(argumen)o(t.)166 1724 y(Tw)o(o)10 b(p)q(ossible)j(executions)g(of)e(this)g(program,)g(with)h(di\013eren)o (t)f(matc)o(hings)g(of)g(sends)h(and)f(receiv)o(es,)75 1780 y(are)k(illustrated)i(in)f(\014gure)g(1.10.)j(Note)c(that)g(the)g (second)h(execution)g(has)g(the)f(p)q(eculiar)i(e\013ect)f(that)e(a)75 1837 y(send)k(executed)g(after)f(the)h(broadcast)e(is)i(receiv)o(ed)h (at)e(another)g(no)q(de)h(b)q(efore)g(the)f(broadcast.)26 b(This)75 1893 y(example)15 b(illustrates)g(the)g(fact)f(that)f(one)i (should)g(not)f(rely)h(on)f(collectiv)o(e)i(comm)o(unication)f (functions)75 1950 y(to)h(ha)o(v)o(e)g(particular)h(sync)o(hronization) h(e\013ects.)23 b(A)17 b(program)f(that)f(w)o(orks)h(correctly)h(only)g (when)g(the)75 2006 y(\014rst)e(execution)h(o)q(ccurs)g(\(only)f(when)h (broadcast)e(is)i(sync)o(hronizing\))g(is)g(erroneous.)166 2112 y(Finally)l(,)h(in)g(m)o(ultithreaded)g(implemen)o(tations,)g(one) f(can)g(ha)o(v)o(e)g(more)g(than)f(one,)h(concurren)o(tly)75 2169 y(executing,)e(collectiv)o(e)h(comm)o(unication)f(call)h(at)d(a)h (pro)q(cess.)20 b(In)13 b(these)h(situations,)f(it)h(is)g(the)f(user's) g(re-)75 2225 y(sp)q(onsibilit)o(y)i(to)d(ensure)h(that)f(the)h(same)f (comm)o(unicator)h(is)g(not)f(used)h(concurren)o(tly)h(b)o(y)e(t)o(w)o (o)g(di\013eren)o(t)75 2282 y(collectiv)o(e)17 b(comm)o(unication)f (calls)g(at)f(the)g(same)g(pro)q(cess.)189 2388 y Fm(A)n(dvic)n(e)e(to) h(implementors.)39 b Ft(Assume)13 b(that)f(broadcast)g(is)h(implemen)o (ted)i(using)e(p)q(oin)o(t-to-p)q(oin)o(t)189 2445 y Fp(MPI)h Ft(comm)o(unication.)21 b(Supp)q(ose)16 b(the)f(follo)o(wing)h (t)o(w)o(o)e(rules)i(are)f(follo)o(w)o(ed.)231 2538 y(1.)22 b(All)16 b(receiv)o(es)g(sp)q(ecify)h(their)e(source)h(explicitly)i (\(no)c(wildcards\).)231 2611 y(2.)22 b(Eac)o(h)13 b(pro)q(cess)g (sends)h(all)g(messages)f(that)f(p)q(ertain)i(to)f(one)g(collectiv)o(e) i(call)f(b)q(efore)g(sending)289 2668 y(an)o(y)g(message)h(that)g(p)q (ertain)h(to)e(a)h(subsequen)o(t)h(collectiv)o(e)h(call.)1967 46 y Fl(1)1967 103 y(2)1967 159 y(3)1967 215 y(4)1967 272 y(5)1967 328 y(6)1967 385 y(7)1967 441 y(8)1967 498 y(9)1959 554 y(10)1959 611 y(11)1959 667 y(12)1959 724 y(13)1959 780 y(14)1959 836 y(15)1959 893 y(16)1959 949 y(17)1959 1006 y(18)1959 1062 y(19)1959 1119 y(20)1959 1175 y(21)1959 1232 y(22)1959 1288 y(23)1959 1345 y(24)1959 1401 y(25)1959 1457 y(26)1959 1514 y(27)1959 1570 y(28)1959 1627 y(29)1959 1683 y(30)1959 1740 y(31)1959 1796 y(32)1959 1853 y(33)1959 1909 y(34)1959 1966 y(35)1959 2022 y(36)1959 2078 y(37)1959 2135 y(38)1959 2191 y(39)1959 2248 y(40)1959 2304 y(41)1959 2361 y(42)1959 2417 y(43)1959 2474 y(44)1959 2530 y(45)1959 2587 y(46)1959 2643 y(47)1959 2699 y(48)p eop %%Page: 41 43 41 42 bop 75 -100 a Fk(1.12.)34 b(CORRECTNESS)1280 b Ft(41)189 45 y(Then,)17 b(messages)f(b)q(elonging)j(to)d(successiv)o(e) i(broadcasts)e(cannot)g(b)q(e)i(confused,)f(as)f(the)h(order)189 102 y(of)d(p)q(oin)o(t-to-p)q(oin)o(t)i(messages)f(is)h(preserv)o(ed.) 189 177 y(It)g(is)h(the)g(implemen)o(tor's)g(resp)q(onsibilit)o(y)i(to) d(ensure)i(that)e(p)q(oin)o(t-to-p)q(oin)o(t)h(messages)f(are)g(not)189 233 y(confused)d(with)g(collectiv)o(e)h(messages.)19 b(One)13 b(w)o(a)o(y)f(to)g(accomplish)i(this)f(is,)g(whenev)o(er)g(a)f (comm)o(u-)189 290 y(nicator)j(is)g(created,)g(to)g(also)g(create)g(a)g (\\hidden)h(comm)o(unicator")f(for)f(collectiv)o(e)j(comm)o(unica-)189 346 y(tion.)j(One)15 b(could)h(ac)o(hiev)o(e)f(a)f(similar)i(e\013ect)e (more)g(c)o(heaply)l(,)i(for)e(example,)h(b)o(y)g(using)g(a)f(hidden) 189 403 y(tag)f(or)h(con)o(text)g(bit)h(to)f(indicate)i(whether)f(the)f (comm)o(unicator)g(is)h(used)g(for)f(p)q(oin)o(t-to-p)q(oin)o(t)h(or) 189 459 y(collectiv)o(e)i(comm)o(unication.)j(\()p Fm(End)c(of)g(advic) n(e)g(to)h(implementors.)p Ft(\))-32 46 y Fl(1)-32 103 y(2)-32 159 y(3)-32 215 y(4)-32 272 y(5)-32 328 y(6)-32 385 y(7)-32 441 y(8)-32 498 y(9)-40 554 y(10)-40 611 y(11)-40 667 y(12)-40 724 y(13)-40 780 y(14)-40 836 y(15)-40 893 y(16)-40 949 y(17)-40 1006 y(18)-40 1062 y(19)-40 1119 y(20)-40 1175 y(21)-40 1232 y(22)-40 1288 y(23)-40 1345 y(24)-40 1401 y(25)-40 1457 y(26)-40 1514 y(27)-40 1570 y(28)-40 1627 y(29)-40 1683 y(30)-40 1740 y(31)-40 1796 y(32)-40 1853 y(33)-40 1909 y(34)-40 1966 y(35)-40 2022 y(36)-40 2078 y(37)-40 2135 y(38)-40 2191 y(39)-40 2248 y(40)-40 2304 y(41)-40 2361 y(42)-40 2417 y(43)-40 2474 y(44)-40 2530 y(45)-40 2587 y(46)-40 2643 y(47)-40 2699 y(48)p eop %%Trailer end userdict /end-hook known{end-hook}if %%EOF .