Skip to content
GitLab
Menu
Projects
Groups
Snippets
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
ExaHDF5
node_local_storage
Commits
52044695
Commit
52044695
authored
Apr 30, 2020
by
Huihuo Zheng
Browse files
fixed buff
parent
f820bb4a
Changes
1
Show whitespace changes
Inline
Side-by-side
hdf5/test_read_cache.cpp
View file @
52044695
...
...
@@ -124,7 +124,6 @@ int main(int argc, char **argv) {
fd
=
H5Fopen_cache
(
fname
,
H5F_ACC_RDONLY
,
plist_id
);
else
fd
=
H5Fopen
(
fname
,
H5F_ACC_RDONLY
,
plist_id
);
cout
<<
H5DRMM
.
mpi
.
rank
<<
" "
<<
H5DRMM
.
mpi
.
nproc
<<
endl
;
hid_t
dset
;
tt
.
start_clock
(
"H5Dopen"
);
if
(
cache
)
{
...
...
@@ -200,42 +199,7 @@ int main(int argc, char **argv) {
// First epoch -- reading the data from the file system and cache it to local storage
if
(
shuffle
)
::
shuffle
(
id
.
begin
(),
id
.
end
(),
g
);
for
(
int
nb
=
0
;
nb
<
num_batches
;
nb
++
)
{
//// reading from file system to memory using H5Dread.
tt
.
start_clock
(
"Select"
);
set_hyperslab_from_samples
(
&
id
[
fs_loc
+
nb
*
batch_size
],
batch_size
,
&
fspace
);
tt
.
stop_clock
(
"Select"
);
tt
.
start_clock
(
"H5Dread"
);
if
(
cache
)
{
tt
.
start_clock
(
"H5Dread_to_cache"
);
H5Dread_cache
(
dset
,
H5T_NATIVE_FLOAT
,
mspace
,
fspace
,
dxf_id
,
dat
);
tt
.
stop_clock
(
"H5Dread_to_cache"
);
}
else
{
H5Dread
(
dset
,
H5T_NATIVE_FLOAT
,
mspace
,
fspace
,
dxf_id
,
dat
);
}
tt
.
stop_clock
(
"H5Dread"
);
double
vm
,
rss
;
process_mem_usage
(
vm
,
rss
);
if
(
io_node
()
==
rank
and
debug_level
()
>
1
)
cout
<<
"VM: "
<<
vm
<<
"; RSS:"
<<
rss
<<
endl
;
msleep
(
int
(
compute
*
1000
));
}
if
(
io_node
()
==
rank
)
printf
(
"Epoch: %d --- time: %6.2f (sec) --- throughput: %6.2f (imgs/sec) --- rate: %6.2f (MB/sec)
\n
"
,
0
,
tt
[
"H5Dread"
].
t
,
nproc
*
num_batches
*
batch_size
/
tt
[
"H5Dread"
].
t
,
num_batches
*
batch_size
*
dim
*
sizeof
(
float
)
/
tt
[
"H5Dread"
].
t
/
1024
/
1024
*
nproc
);
// Epochs 1 - ... reading data directly from local storage
if
(
barrier
)
MPI_Barrier
(
MPI_COMM_WORLD
);
for
(
int
e
=
1
;
e
<
epochs
;
e
++
)
{
if
(
remap
)
{
tt
.
start_clock
(
"REMAP"
);
H5DRMMF_remap
();
tt
.
stop_clock
(
"REMAP"
);
}
for
(
int
e
=
0
;
e
<
epochs
;
e
++
)
{
double
vm
,
rss
;
process_mem_usage
(
vm
,
rss
);
if
(
io_node
()
==
rank
and
debug_level
()
>
1
)
...
...
@@ -254,9 +218,9 @@ int main(int argc, char **argv) {
tt
.
stop_clock
(
"Select"
);
tt
.
start_clock
(
"H5Dread"
);
if
(
cache
)
{
tt
.
start_clock
(
"H5Dread_
from_
cache"
);
tt
.
start_clock
(
"H5Dread_cache"
);
H5Dread_cache
(
dset
,
H5T_NATIVE_FLOAT
,
mspace
,
fspace
,
dxf_id
,
dat
);
tt
.
stop_clock
(
"H5Dread_
from_
cache"
);
tt
.
stop_clock
(
"H5Dread_cache"
);
}
else
{
H5Dread
(
dset
,
H5T_NATIVE_FLOAT
,
mspace
,
fspace
,
dxf_id
,
dat
);
}
...
...
@@ -280,6 +244,12 @@ int main(int argc, char **argv) {
printf
(
"Epoch: %d --- time: %6.2f (sec) --- throughput: %6.2f (imgs/sec) --- rate: %6.2f (MB/sec)
\n
"
,
e
,
t1
,
nproc
*
num_batches
*
batch_size
/
t1
,
num_batches
*
batch_size
*
dim
*
sizeof
(
float
)
/
t1
/
1024
/
1024
*
nproc
);
if
(
remap
)
{
tt
.
start_clock
(
"REMAP"
);
H5DRMMF_remap
();
tt
.
stop_clock
(
"REMAP"
);
}
}
H5Pclose
(
plist_id
);
H5Sclose
(
mspace
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment