Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Francois Tessier
TAPIOCA
Commits
dbe6daa4
Commit
dbe6daa4
authored
Aug 15, 2017
by
Francois Tessier
Browse files
In HACC-IO, add MPI file information to set custom values for MPI I/O hints + update run scripts
parent
b2e0c318
Changes
5
Hide whitespace changes
Inline
Side-by-side
examples/HACC-IO/miniHACC-AoS-MPIIO.cpp
View file @
dbe6daa4
...
...
@@ -21,6 +21,7 @@ int main (int argc, char * argv[])
double
io_bw
;
MPI_Comm
sub_comm
;
MPI_File
file_handle
;
MPI_Info
info
;
MPI_Status
status
;
char
output
[
100
];
Tapioca
tp
;
...
...
@@ -105,10 +106,14 @@ int main (int argc, char * argv[])
mycolor
,
(
double
)
file_size
/
(
1024
*
1024
),
sub_particles
);
}
MPI_Info_create
(
&
info
);
MPI_Info_set
(
info
,
"cb_nodes"
,
getenv
(
"TAPIOCA_NBAGGR"
)
);
MPI_Info_set
(
info
,
"bg_nodes_pset"
,
getenv
(
"TAPIOCA_NBAGGR"
)
);
start_time
=
MPI_Wtime
();
MPI_File_open
(
sub_comm
,
output
,
MPI_MODE_WRONLY
,
MPI_INFO_NULL
,
&
file_handle
);
MPI_MODE_WRONLY
,
info
,
&
file_handle
);
offset
=
scan_size
*
particle_size
;
...
...
@@ -172,7 +177,7 @@ int main (int argc, char * argv[])
start_time
=
MPI_Wtime
();
MPI_File_open
(
sub_comm
,
output
,
MPI_MODE_RDONLY
,
MPI_INFO_NULL
,
&
file_handle
);
MPI_MODE_RDONLY
,
info
,
&
file_handle
);
if
(
0
==
sub_myrank
)
fprintf
(
stdout
,
GREEN
"[INFO]"
RESET
" [%08d] Read output file
\n
"
,
mycolor
);
...
...
examples/HACC-IO/miniHACC-AoS.cpp
View file @
dbe6daa4
...
...
@@ -30,9 +30,10 @@ int main (int argc, char * argv[])
MPI_Init
(
&
argc
,
&
argv
);
MPI_Comm_size
(
MPI_COMM_WORLD
,
&
world_numtasks
);
MPI_Comm_rank
(
MPI_COMM_WORLD
,
&
world_myrank
);
mycolor
=
tp
.
topology
.
BridgeNodeId
();
mykey
=
world_myrank
;
//mycolor = tp.topology.BridgeNodeId ();
mycolor
=
42
;
mykey
=
world_myrank
;
MPI_Comm_split
(
MPI_COMM_WORLD
,
mycolor
,
mykey
,
&
sub_comm
);
MPI_Comm_size
(
sub_comm
,
&
sub_numtasks
);
...
...
examples/HACC-IO/miniHACC-SoA-MPIIO.cpp
View file @
dbe6daa4
...
...
@@ -21,6 +21,7 @@ int main (int argc, char * argv[])
double
io_bw
;
MPI_Comm
sub_comm
;
MPI_File
file_handle
;
MPI_Info
info
;
MPI_Status
status
;
char
output
[
100
];
Tapioca
tp
;
...
...
@@ -105,10 +106,14 @@ int main (int argc, char * argv[])
mycolor
,
(
double
)
file_size
/
(
1024
*
1024
),
sub_particles
);
}
MPI_Info_create
(
&
info
);
MPI_Info_set
(
info
,
"cb_nodes"
,
getenv
(
"TAPIOCA_NBAGGR"
)
);
MPI_Info_set
(
info
,
"bg_nodes_pset"
,
getenv
(
"TAPIOCA_NBAGGR"
)
);
start_time
=
MPI_Wtime
();
MPI_File_open
(
sub_comm
,
output
,
MPI_MODE_WRONLY
,
MPI_INFO_NULL
,
&
file_handle
);
MPI_MODE_WRONLY
,
info
,
&
file_handle
);
offset
=
scan_size
*
sizeof
(
float
);
...
...
@@ -172,7 +177,7 @@ int main (int argc, char * argv[])
start_time
=
MPI_Wtime
();
MPI_File_open
(
sub_comm
,
output
,
MPI_MODE_RDONLY
,
MPI_INFO_NULL
,
&
file_handle
);
MPI_MODE_RDONLY
,
info
,
&
file_handle
);
if
(
0
==
sub_myrank
)
fprintf
(
stdout
,
GREEN
"[INFO]"
RESET
" [%08d] Read output file
\n
"
,
mycolor
);
...
...
examples/HACC-IO/run_check_bgq.sh
View file @
dbe6daa4
#!/bin/bash
NODES
=
1
024
NODES
=
1
28
PPN
=
16
NPROCS
=
$((
NODES
*
PPN
))
TARGET
=
"/projects/visualization/ftessier/debug"
...
...
@@ -9,7 +9,7 @@ cd $HOME/install/$ARCHI/bin/
export
TAPIOCA_DEVNULL
=
false
export
TAPIOCA_COMMSPLIT
=
true
export
TAPIOCA_STRATEGY
=
TOPOLOGY_AWARE
export
TAPIOCA_STRATEGY
=
UNIFORM
export
TAPIOCA_NBAGGR
=
16
export
TAPIOCA_BUFFERSIZE
=
16777216
export
TAPIOCA_PIPELINING
=
true
...
...
@@ -34,7 +34,7 @@ function updateSettings()
function
outputFile
()
{
OUTPUT
=
"
$HOME
/
PUB-IPDPS18/dat/hacc_rw
"
OUTPUT
=
"
$HOME
/
xp_tapioca/contention
"
OUTPUT
=
"
${
OUTPUT
}
/HACC_BGQ_MIRA_
${
BIN
}
_
${
NODES
}
_
${
PPN
}
"
OUTPUT
=
"
${
OUTPUT
}
_
${
TAPIOCA_STRATEGY
}
_
${
TAPIOCA_NBAGGR
}
_
${
TAPIOCA_BUFFERSIZE
}
_
${
PARTICLES
}
"
}
...
...
@@ -43,13 +43,14 @@ updateSettings
for
run
in
{
0..9
}
do
for
PARTICLES
in
5000
15000 25000 35000 50000
100000
for
PARTICLES
in
15000 25000 35000 50000
do
for
BIN
in
miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
rm
$TARGET
/
*
outputFile
runjob
--block
$COBALT_PARTNAME
--envs
$SETTINGS
-p
$PPN
--np
$NPROCS
: ./
$BIN
$PARTICLES
>>
$OUTPUT
runjob
--block
$COBALT_PARTNAME
--envs
$SETTINGS
-p
$PPN
--np
$NPROCS
: ./
$BIN
$PARTICLES
#>> $OUTPUT
sleep
3
done
done
...
...
examples/HACC-IO/run_check_xc40.sh
View file @
dbe6daa4
...
...
@@ -7,13 +7,14 @@ TARGET="/lus/theta-fs0/projects/Performance/ftessier/HACC"
DDT
=
"/soft/debuggers/forge/bin/ddt --connect"
STRIPE_COUNT
=
48
STRIPE_SIZE
=
8388608
AGGR_MULT
=
1
cd
$HOME
/install/
$ARCHI
/bin/
export
TAPIOCA_DEVNULL
=
false
export
TAPIOCA_COMMSPLIT
=
true
export
TAPIOCA_STRATEGY
=
TOPOLOGY_AWARE
export
TAPIOCA_NBAGGR
=
1
export
TAPIOCA_NBAGGR
=
1
92
export
TAPIOCA_BUFFERSIZE
=
16777216
function
setLustreFS
()
...
...
@@ -35,28 +36,34 @@ function updateSettings()
SETTINGS
=
"
$SETTINGS
-e MPICH_RMA_OVER_DMAPP=1"
SETTINGS
=
"
$SETTINGS
-e MPICH_MPIIO_AGGREGATOR_PLACEMENT_DISPLAY=1"
SETTINGS
=
"
$SETTINGS
-e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=
3
"
SETTINGS
=
"
$SETTINGS
-e MPICH_MPIIO_HINTS=*:cray_cb_nodes_multiplier=
$AGGR_MULT
"
}
function
outputFile
()
{
OUTPUT
=
"
$HOME
/PUB-IPDPS18/dat/
hacc_rw
"
OUTPUT
=
"
${
OUTPUT
}
/HACC_XC40_
FP8N_
${
BIN
}
_
${
NODES
}
_
${
PPN
}
"
OUTPUT
=
"
$HOME
/PUB-IPDPS18/dat/
aggr_count
"
OUTPUT
=
"
${
OUTPUT
}
/HACC_XC40_
${
BIN
}
_
${
NODES
}
_
${
PPN
}
"
OUTPUT
=
"
${
OUTPUT
}
_
${
TAPIOCA_STRATEGY
}
_
${
TAPIOCA_NBAGGR
}
_
${
TAPIOCA_BUFFERSIZE
}
_
${
PARTICLES
}
"
}
updateSettings
for
run
in
{
0..9
}
do
for
PARTICLES
in
5
000 15000 25000 35000 50000 100
000
for
PARTICLES
in
2
5000
do
for
BIN
in
miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
outputFile
setLustreFS
>>
$OUTPUT
aprun
$SETTINGS
-n
$NPROCS
-N
$PPN
./
$BIN
$PARTICLES
>>
$OUTPUT
sleep
5
for
AGGR
in
48 96 144 192
do
AGGR_MULT
=
$((
AGGR/STRIPE_COUNT
))
echo
"cray_cb_nodes_multiplier=
$AGGR_MULT
"
export
TAPIOCA_NBAGGR
=
$AGGR
updateSettings
for
BIN
in
miniHACC-AoS miniHACC-SoA miniHACC-AoS-MPIIO miniHACC-SoA-MPIIO
do
outputFile
setLustreFS
>>
$OUTPUT
aprun
$SETTINGS
-n
$NPROCS
-N
$PPN
./
$BIN
$PARTICLES
>>
$OUTPUT
sleep
5
done
done
done
done
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment