Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
T
TAPIOCA
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
Francois Tessier
TAPIOCA
Commits
1c90dfb0
Commit
1c90dfb0
authored
Aug 15, 2017
by
Francois Tessier
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
In HACC-IO, add MPI file information to set custom values for MPI I/O hints + update run scripts
parent
ba106125
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
44 additions
and
25 deletions
+44
-25
examples/HACC-IO/miniHACC-AoS-MPIIO.cpp
examples/HACC-IO/miniHACC-AoS-MPIIO.cpp
+7
-2
examples/HACC-IO/miniHACC-AoS.cpp
examples/HACC-IO/miniHACC-AoS.cpp
+4
-3
examples/HACC-IO/miniHACC-SoA-MPIIO.cpp
examples/HACC-IO/miniHACC-SoA-MPIIO.cpp
+7
-2
examples/HACC-IO/run_check_bgq.sh
examples/HACC-IO/run_check_bgq.sh
+6
-5
examples/HACC-IO/run_check_xc40.sh
examples/HACC-IO/run_check_xc40.sh
+20
-13
No files found.
examples/HACC-IO/miniHACC-AoS-MPIIO.cpp
View file @
1c90dfb0
...
...
@@ -21,6 +21,7 @@ int main (int argc, char * argv[])
double
io_bw
;
MPI_Comm
sub_comm
;
MPI_File
file_handle
;
MPI_Info
info
;
MPI_Status
status
;
char
output
[
100
];
Tapioca
tp
;
...
...
@@ -105,10 +106,14 @@ int main (int argc, char * argv[])
mycolor
,
(
double
)
file_size
/
(
1024
*
1024
),
sub_particles
);
}
MPI_Info_create
(
&
info
);
MPI_Info_set
(
info
,
"cb_nodes"
,
getenv
(
"TAPIOCA_NBAGGR"
)
);
MPI_Info_set
(
info
,
"bg_nodes_pset"
,
getenv
(
"TAPIOCA_NBAGGR"
)
);
start_time
=
MPI_Wtime
();
MPI_File_open
(
sub_comm
,
output
,
MPI_MODE_WRONLY
,
MPI_INFO_NULL
,
&
file_handle
);
MPI_MODE_WRONLY
,
info
,
&
file_handle
);
offset
=
scan_size
*
particle_size
;
...
...
@@ -172,7 +177,7 @@ int main (int argc, char * argv[])
start_time
=
MPI_Wtime
();
MPI_File_open
(
sub_comm
,
output
,
MPI_MODE_RDONLY
,
MPI_INFO_NULL
,
&
file_handle
);
MPI_MODE_RDONLY
,
info
,
&
file_handle
);
if
(
0
==
sub_myrank
)
fprintf
(
stdout
,
GREEN
"[INFO]"
RESET
" [%08d] Read output file
\n
"
,
mycolor
);
...
...
examples/HACC-IO/miniHACC-AoS.cpp
View file @
1c90dfb0
...
...
@@ -30,9 +30,10 @@ int main (int argc, char * argv[])
MPI_Init
(
&
argc
,
&
argv
);
MPI_Comm_size
(
MPI_COMM_WORLD
,
&
world_numtasks
);
MPI_Comm_rank
(
MPI_COMM_WORLD
,
&
world_myrank
);
mycolor
=
tp
.
topology
.
BridgeNodeId
();
mykey
=
world_myrank
;
//mycolor = tp.topology.BridgeNodeId ();
mycolor
=
42
;
mykey
=
world_myrank
;
MPI_Comm_split
(
MPI_COMM_WORLD
,
mycolor
,
mykey
,
&
sub_comm
);
MPI_Comm_size
(
sub_comm
,
&
sub_numtasks
);
...
...
examples/HACC-IO/miniHACC-SoA-MPIIO.cpp
View file @
1c90dfb0
...
...
@@ -21,6 +21,7 @@ int main (int argc, char * argv[])
double
io_bw
;
MPI_Comm
sub_comm
;
MPI_File
file_handle
;
MPI_Info
info
;
MPI_Status
status
;
char
output
[
100
];
Tapioca
tp
;
...
...
@@ -105,10 +106,14 @@ int main (int argc, char * argv[])
mycolor
,
(
double
)
file_size
/
(
1024
*
1024
),
sub_particles
);
}
MPI_Info_create
(
&
info
);
MPI_Info_set
(
info
,
"cb_nodes"
,
getenv
(
"TAPIOCA_NBAGGR"
)
);
MPI_Info_set
(
info
,
"bg_nodes_pset"
,
getenv
(
"TAPIOCA_NBAGGR"
)
);
start_time
=
MPI_Wtime
();
MPI_File_open
(
sub_comm
,
output
,
MPI_MODE_WRONLY
,
MPI_INFO_NULL
,
&
file_handle
);
MPI_MODE_WRONLY
,
info
,
&
file_handle
);
offset
=
scan_size
*
sizeof
(
float
);
...
...
@@ -172,7 +177,7 @@ int main (int argc, char * argv[])
start_time
=
MPI_Wtime
();
MPI_File_open
(
sub_comm
,
output
,
MPI_MODE_RDONLY
,
MPI_INFO_NULL
,
&
file_handle
);
MPI_MODE_RDONLY
,
info
,
&
file_handle
);
if
(
0
==
sub_myrank
)
fprintf
(
stdout
,
GREEN
"[INFO]"
RESET
" [%08d] Read output file
\n
"
,
mycolor
);
...
...
examples/HACC-IO/run_check_bgq.sh
View file @
1c90dfb0
#!/bin/bash
NODES
=
1
024
NODES
=
1
28
PPN
=
16
NPROCS
=
$((
NODES
*
PPN
))
TARGET
=
"/projects/visualization/ftessier/debug"
...
...
@@ -9,7 +9,7 @@ cd $HOME/install/$ARCHI/bin/
export
TAPIOCA_DEVNULL
=
false
export
TAPIOCA_COMMSPLIT
=
true
export
TAPIOCA_STRATEGY
=
TOPOLOGY_AWARE
export
TAPIOCA_STRATEGY
=
UNIFORM
export
TAPIOCA_NBAGGR
=
16
export
TAPIOCA_BUFFERSIZE
=
16777216
export
TAPIOCA_PIPELINING
=
true
...
...
@@ -34,7 +34,7 @@ function updateSettings()
function
outputFile
()
{
OUTPUT
=
"
$HOME
/
PUB-IPDPS18/dat/hacc_rw
"
OUTPUT
=
"
$HOME
/
xp_tapioca/contention
"
OUTPUT
=
"
${
OUTPUT
}
/HACC_BGQ_MIRA_
${
BIN
}
_
${
NODES
}
_
${
PPN
}
"
OUTPUT
=
"
${
OUTPUT
}
_
${
TAPIOCA_STRATEGY
}
_
${
TAPIOCA_NBAGGR
}
_
${
TAPIOCA_BUFFERSIZE
}
_
${
PARTICLES
}
"
}
...
...
@@ -43,13 +43,14 @@ updateSettings
for
run
in
{
0..9
}
do
for
PARTICLES
in
5000 15000 25000 35000 50000 10
0000
for
PARTICLES
in
15000 25000 35000 5
0000
do
for
BIN
in
miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
rm
$TARGET
/
*
outputFile
runjob
--block
$COBALT_PARTNAME
--envs
$SETTINGS
-p
$PPN
--np
$NPROCS
: ./
$BIN
$PARTICLES
>>
$OUTPUT
runjob
--block
$COBALT_PARTNAME
--envs
$SETTINGS
-p
$PPN
--np
$NPROCS
: ./
$BIN
$PARTICLES
#>> $OUTPUT
sleep
3
done
done
...
...
examples/HACC-IO/run_check_xc40.sh
View file @
1c90dfb0
#!/bin/bash
NODES
=
256
NODES
=
1024
PPN
=
16
NPROCS
=
$((
NODES
*
PPN
))
TARGET
=
"/lus/theta-fs0/projects/Performance/ftessier/HACC"
DDT
=
"/soft/debuggers/forge/bin/ddt --connect"
STRIPE_COUNT
=
48
STRIPE_SIZE
=
8388608
AGGR_MULT
=
1
cd
$HOME
/install/
$ARCHI
/bin/
export
TAPIOCA_DEVNULL
=
false
export
TAPIOCA_COMMSPLIT
=
true
export
TAPIOCA_STRATEGY
=
TOPOLOGY_AWARE
export
TAPIOCA_NBAGGR
=
48
export
TAPIOCA_NBAGGR
=
192
export
TAPIOCA_BUFFERSIZE
=
16777216
function
setLustreFS
()
...
...
@@ -35,28 +36,34 @@ function updateSettings()
SETTINGS
=
"
$SETTINGS
-e MPICH_RMA_OVER_DMAPP=1"
SETTINGS
=
"
$SETTINGS
-e MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1"
SETTINGS
=
"
$SETTINGS
-e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=
1
"
SETTINGS
=
"
$SETTINGS
-e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=
$AGGR_MULT
"
}
function
outputFile
()
{
OUTPUT
=
"
$HOME
/PUB-IPDPS18/dat/
hacc_rw
"
OUTPUT
=
"
$HOME
/PUB-IPDPS18/dat/
aggr_count
"
OUTPUT
=
"
${
OUTPUT
}
/HACC_XC40_
${
BIN
}
_
${
NODES
}
_
${
PPN
}
"
OUTPUT
=
"
${
OUTPUT
}
_
${
TAPIOCA_STRATEGY
}
_
${
TAPIOCA_NBAGGR
}
_
${
TAPIOCA_BUFFERSIZE
}
_
${
PARTICLES
}
"
}
updateSettings
for
run
in
{
0..9
}
do
for
PARTICLES
in
5000 15000 25000 35000 50000 100
000
for
PARTICLES
in
25
000
do
for
BIN
in
miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
outputFile
setLustreFS
>>
$OUTPUT
aprun
$SETTINGS
-n
$NPROCS
-N
$PPN
./
$BIN
$PARTICLES
>>
$OUTPUT
sleep
5
for
AGGR
in
48 96 144 192
do
AGGR_MULT
=
$((
AGGR/STRIPE_COUNT
))
echo
"cray_cb_nodes_multiplier=
$AGGR_MULT
"
export
TAPIOCA_NBAGGR
=
$AGGR
updateSettings
for
BIN
in
miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
outputFile
setLustreFS
>>
$OUTPUT
aprun
$SETTINGS
-n
$NPROCS
-N
$PPN
./
$BIN
$PARTICLES
>>
$OUTPUT
sleep
5
done
done
done
done
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment