Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
D
darshan
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
72
Issues
72
List
Boards
Labels
Milestones
Merge Requests
5
Merge Requests
5
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
darshan
darshan
Commits
4addc19b
Commit
4addc19b
authored
Sep 12, 2016
by
Philip Carns
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'dev-stdio-utils'
parents
c877a1c1
e95a46ed
Changes
3
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
3 changed files
with
269 additions
and
61 deletions
+269
-61
darshan-util/darshan-job-summary/bin/darshan-job-summary.pl.in
...an-util/darshan-job-summary/bin/darshan-job-summary.pl.in
+165
-54
darshan-util/darshan-job-summary/share/summary.tex
darshan-util/darshan-job-summary/share/summary.tex
+8
-1
darshan-util/darshan-parser.c
darshan-util/darshan-parser.c
+96
-6
No files found.
darshan-util/darshan-job-summary/bin/darshan-job-summary.pl.in
View file @
4addc19b
This diff is collapsed.
Click to expand it.
darshan-util/darshan-job-summary/share/summary.tex
View file @
4addc19b
...
...
@@ -51,8 +51,15 @@ which may skew results in this document.
\input
{
job-table.tex
}
}
\\
\vspace
{
1em
}
\ifdefined\inclperf
I/O performance
\emph
{
estimate
}
(at the
\perflayer
\
layer):
transferred
\textcolor
{
red
}{
\perfbytes
\
MiB
}
at
\textcolor
{
red
}{
\perfest
\
MiB/s
}
transferred
\textcolor
{
red
}{
\perfbytes
\
MiB
}
at
\textcolor
{
red
}{
\perfest
\
MiB/s
\\
}
\fi
\ifdefined\inclstdio
I/O performance
\emph
{
estimate
}
(at the STDIO layer):
transferred
\textcolor
{
red
}{
\stdioperfbytes
\
MiB
}
at
\textcolor
{
red
}{
\stdioperfest
\
MiB/s
}
\fi
\vspace
{
3em
}
\subfigure
{
...
...
darshan-util/darshan-parser.c
View file @
4addc19b
...
...
@@ -113,6 +113,8 @@ void mpiio_calc_file(hash_entry_t *file_hash, file_data_t *fdata);
void
mpiio_print_total_file
(
struct
darshan_mpiio_file
*
mfile
);
void
mpiio_file_list
(
hash_entry_t
*
file_hash
,
struct
darshan_name_record_ref
*
name_hash
,
int
detail_flag
);
void
stdio_accum_perf
(
struct
darshan_stdio_file
*
pfile
,
perf_data_t
*
pdata
);
void
calc_perf
(
perf_data_t
*
pdata
,
int64_t
nprocs
);
int
usage
(
char
*
exename
)
...
...
@@ -389,9 +391,10 @@ int main(int argc, char **argv)
"for module %s, SKIPPING.
\n
"
,
darshan_module_names
[
i
]);
continue
;
}
/* currently we only do base parsing for non MPI & POSIX modules */
else
if
((
i
!=
DARSHAN_POSIX_MOD
)
&&
(
i
!=
DARSHAN_MPIIO_MOD
)
&&
!
(
mask
&
OPTION_BASE
))
/* currently only POSIX, MPIIO, and STDIO modules support non-base
* parsing
*/
else
if
((
i
!=
DARSHAN_POSIX_MOD
)
&&
(
i
!=
DARSHAN_MPIIO_MOD
)
&&
(
i
!=
DARSHAN_STDIO_MOD
)
&&
!
(
mask
&
OPTION_BASE
))
continue
;
/* this module has data to be parsed and printed */
...
...
@@ -470,10 +473,10 @@ int main(int argc, char **argv)
mnt_pt
,
fs_type
,
fd
->
mod_ver
[
i
]);
}
/* we calculate more detailed stats for POSIX
and MPI-
IO modules,
/* we calculate more detailed stats for POSIX
, MPI-IO, and STD
IO modules,
* if the parser is executed with more than the base option
*/
if
(
i
!=
DARSHAN_POSIX_MOD
&&
i
!=
DARSHAN_MPIIO_MOD
)
if
(
i
!=
DARSHAN_POSIX_MOD
&&
i
!=
DARSHAN_MPIIO_MOD
&&
i
!=
DARSHAN_STDIO_MOD
)
continue
;
HASH_FIND
(
hlink
,
file_hash
,
&
(
base_rec
->
id
),
sizeof
(
darshan_record_id
),
hfile
);
...
...
@@ -510,6 +513,10 @@ int main(int argc, char **argv)
mpiio_accum_file
((
struct
darshan_mpiio_file
*
)
mod_buf
,
hfile
,
job
.
nprocs
);
mpiio_accum_perf
((
struct
darshan_mpiio_file
*
)
mod_buf
,
&
pdata
);
}
else
if
(
i
==
DARSHAN_STDIO_MOD
)
{
stdio_accum_perf
((
struct
darshan_stdio_file
*
)
mod_buf
,
&
pdata
);
}
memset
(
mod_buf
,
0
,
DEF_MOD_BUF_SIZE
);
...
...
@@ -523,7 +530,7 @@ int main(int argc, char **argv)
/* we calculate more detailed stats for POSIX and MPI-IO modules,
* if the parser is executed with more than the base option
*/
if
(
i
!=
DARSHAN_POSIX_MOD
&&
i
!=
DARSHAN_MPIIO_MOD
)
if
(
i
!=
DARSHAN_POSIX_MOD
&&
i
!=
DARSHAN_MPIIO_MOD
&&
i
!=
DARSHAN_STDIO_MOD
)
continue
;
/* Total Calc */
...
...
@@ -1042,6 +1049,89 @@ void mpiio_accum_file(struct darshan_mpiio_file *mfile,
return
;
}
void
stdio_accum_perf
(
struct
darshan_stdio_file
*
pfile
,
perf_data_t
*
pdata
)
{
pdata
->
total_bytes
+=
pfile
->
counters
[
STDIO_BYTES_READ
]
+
pfile
->
counters
[
STDIO_BYTES_WRITTEN
];
/*
* Calculation of Shared File Time
* Four Methods!!!!
* by_cumul: sum time counters and divide by nprocs
* (inaccurate if lots of variance between procs)
* by_open: difference between timestamp of open and close
* (inaccurate if file is left open without i/o happening)
* by_open_lastio: difference between timestamp of open and the
* timestamp of last i/o
* (similar to above but fixes case where file is left
* open after io is complete)
* by_slowest: use slowest rank time from log data
* (most accurate but requires newer log version)
*/
if
(
pfile
->
base_rec
.
rank
==
-
1
)
{
/* by_open */
if
(
pfile
->
fcounters
[
STDIO_F_CLOSE_END_TIMESTAMP
]
>
pfile
->
fcounters
[
STDIO_F_OPEN_START_TIMESTAMP
])
{
pdata
->
shared_time_by_open
+=
pfile
->
fcounters
[
STDIO_F_CLOSE_END_TIMESTAMP
]
-
pfile
->
fcounters
[
STDIO_F_OPEN_START_TIMESTAMP
];
}
/* by_open_lastio */
if
(
pfile
->
fcounters
[
STDIO_F_READ_END_TIMESTAMP
]
>
pfile
->
fcounters
[
STDIO_F_WRITE_END_TIMESTAMP
])
{
/* be careful: file may have been opened but not read or written */
if
(
pfile
->
fcounters
[
STDIO_F_READ_END_TIMESTAMP
]
>
pfile
->
fcounters
[
STDIO_F_OPEN_START_TIMESTAMP
])
{
pdata
->
shared_time_by_open_lastio
+=
pfile
->
fcounters
[
STDIO_F_READ_END_TIMESTAMP
]
-
pfile
->
fcounters
[
STDIO_F_OPEN_START_TIMESTAMP
];
}
}
else
{
/* be careful: file may have been opened but not read or written */
if
(
pfile
->
fcounters
[
STDIO_F_WRITE_END_TIMESTAMP
]
>
pfile
->
fcounters
[
STDIO_F_OPEN_START_TIMESTAMP
])
{
pdata
->
shared_time_by_open_lastio
+=
pfile
->
fcounters
[
STDIO_F_WRITE_END_TIMESTAMP
]
-
pfile
->
fcounters
[
STDIO_F_OPEN_START_TIMESTAMP
];
}
}
pdata
->
shared_time_by_cumul
+=
pfile
->
fcounters
[
STDIO_F_META_TIME
]
+
pfile
->
fcounters
[
STDIO_F_READ_TIME
]
+
pfile
->
fcounters
[
STDIO_F_WRITE_TIME
];
pdata
->
shared_meta_time
+=
pfile
->
fcounters
[
STDIO_F_META_TIME
];
/* by_slowest */
pdata
->
shared_time_by_slowest
+=
pfile
->
fcounters
[
STDIO_F_SLOWEST_RANK_TIME
];
}
/*
* Calculation of Unique File Time
* record the data for each file and sum it
*/
else
{
pdata
->
rank_cumul_io_time
[
pfile
->
base_rec
.
rank
]
+=
(
pfile
->
fcounters
[
STDIO_F_META_TIME
]
+
pfile
->
fcounters
[
STDIO_F_READ_TIME
]
+
pfile
->
fcounters
[
STDIO_F_WRITE_TIME
]);
pdata
->
rank_cumul_md_time
[
pfile
->
base_rec
.
rank
]
+=
pfile
->
fcounters
[
STDIO_F_META_TIME
];
}
return
;
}
void
posix_accum_perf
(
struct
darshan_posix_file
*
pfile
,
perf_data_t
*
pdata
)
{
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment