Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Jakob Luettgau
darshan
Commits
721e5e5c
Commit
721e5e5c
authored
Feb 09, 2015
by
Shane Snyder
Browse files
some code/header reorg + minor edits
parent
7fdf26f0
Changes
8
Hide whitespace changes
Inline
Side-by-side
darshan-log-format.h
View file @
721e5e5c
...
...
@@ -42,8 +42,8 @@ struct darshan_header
{
char
version_string
[
8
];
int64_t
magic_nr
;
uint8_t
comp_type
;
uint8_t
mod_count
;
uint8_t
comp_type
;
/* TODO */
uint8_t
mod_count
;
/* TODO: */
};
struct
darshan_record
...
...
@@ -64,126 +64,4 @@ struct darshan_job
char
metadata
[
DARSHAN_JOB_METADATA_LEN
];
/* TODO: what is this? */
};
/*** POSIX MODULE DATA STRUCTURES START */
/* TODO: where do these file record structs go? */
/* TODO: DARSHAN_* OR CP_* */
enum
darshan_posix_indices
{
CP_POSIX_READS
,
/* count of posix reads */
CP_POSIX_WRITES
,
/* count of posix writes */
CP_POSIX_OPENS
,
/* count of posix opens */
CP_POSIX_SEEKS
,
/* count of posix seeks */
CP_POSIX_STATS
,
/* count of posix stat/lstat/fstats */
CP_POSIX_MMAPS
,
/* count of posix mmaps */
CP_POSIX_FREADS
,
CP_POSIX_FWRITES
,
CP_POSIX_FOPENS
,
CP_POSIX_FSEEKS
,
CP_POSIX_FSYNCS
,
CP_POSIX_FDSYNCS
,
CP_MODE
,
/* mode of file */
CP_BYTES_READ
,
/* total bytes read */
CP_BYTES_WRITTEN
,
/* total bytes written */
CP_MAX_BYTE_READ
,
/* highest offset byte read */
CP_MAX_BYTE_WRITTEN
,
/* highest offset byte written */
CP_CONSEC_READS
,
/* count of consecutive reads */
CP_CONSEC_WRITES
,
/* count of consecutive writes */
CP_SEQ_READS
,
/* count of sequential reads */
CP_SEQ_WRITES
,
/* count of sequential writes */
CP_RW_SWITCHES
,
/* number of times switched between read and write */
CP_MEM_NOT_ALIGNED
,
/* count of accesses not mem aligned */
CP_MEM_ALIGNMENT
,
/* mem alignment in bytes */
CP_FILE_NOT_ALIGNED
,
/* count of accesses not file aligned */
CP_FILE_ALIGNMENT
,
/* file alignment in bytes */
CP_MAX_READ_TIME_SIZE
,
CP_MAX_WRITE_TIME_SIZE
,
/* buckets */
CP_SIZE_READ_0_100
,
/* count of posix read size ranges */
CP_SIZE_READ_100_1K
,
CP_SIZE_READ_1K_10K
,
CP_SIZE_READ_10K_100K
,
CP_SIZE_READ_100K_1M
,
CP_SIZE_READ_1M_4M
,
CP_SIZE_READ_4M_10M
,
CP_SIZE_READ_10M_100M
,
CP_SIZE_READ_100M_1G
,
CP_SIZE_READ_1G_PLUS
,
/* buckets */
CP_SIZE_WRITE_0_100
,
/* count of posix write size ranges */
CP_SIZE_WRITE_100_1K
,
CP_SIZE_WRITE_1K_10K
,
CP_SIZE_WRITE_10K_100K
,
CP_SIZE_WRITE_100K_1M
,
CP_SIZE_WRITE_1M_4M
,
CP_SIZE_WRITE_4M_10M
,
CP_SIZE_WRITE_10M_100M
,
CP_SIZE_WRITE_100M_1G
,
CP_SIZE_WRITE_1G_PLUS
,
/* counters */
CP_STRIDE1_STRIDE
,
/* the four most frequently appearing strides */
CP_STRIDE2_STRIDE
,
CP_STRIDE3_STRIDE
,
CP_STRIDE4_STRIDE
,
CP_STRIDE1_COUNT
,
/* count of each of the most frequent strides */
CP_STRIDE2_COUNT
,
CP_STRIDE3_COUNT
,
CP_STRIDE4_COUNT
,
CP_ACCESS1_ACCESS
,
/* the four most frequently appearing access sizes */
CP_ACCESS2_ACCESS
,
CP_ACCESS3_ACCESS
,
CP_ACCESS4_ACCESS
,
CP_ACCESS1_COUNT
,
/* count of each of the most frequent access sizes */
CP_ACCESS2_COUNT
,
CP_ACCESS3_COUNT
,
CP_ACCESS4_COUNT
,
CP_DEVICE
,
/* device id reported by stat */
CP_SIZE_AT_OPEN
,
CP_FASTEST_RANK
,
CP_FASTEST_RANK_BYTES
,
CP_SLOWEST_RANK
,
CP_SLOWEST_RANK_BYTES
,
CP_NUM_INDICES
,
};
/* floating point statistics */
enum
darshan_f_posix_indices
{
/* NOTE: adjust cp_normalize_timestamps() function if any TIMESTAMPS are
* added or modified in this list
*/
CP_F_OPEN_TIMESTAMP
=
0
,
/* timestamp of first open */
CP_F_READ_START_TIMESTAMP
,
/* timestamp of first read */
CP_F_WRITE_START_TIMESTAMP
,
/* timestamp of first write */
CP_F_CLOSE_TIMESTAMP
,
/* timestamp of last close */
CP_F_READ_END_TIMESTAMP
,
/* timestamp of last read */
CP_F_WRITE_END_TIMESTAMP
,
/* timestamp of last write */
CP_F_POSIX_READ_TIME
,
/* cumulative posix read time */
CP_F_POSIX_WRITE_TIME
,
/* cumulative posix write time */
CP_F_POSIX_META_TIME
,
/* cumulative posix meta time */
CP_F_MAX_READ_TIME
,
CP_F_MAX_WRITE_TIME
,
/* Total I/O and meta time consumed by fastest and slowest ranks,
* reported in either MPI or POSIX time depending on how the file
* was accessed.
*/
CP_F_FASTEST_RANK_TIME
,
CP_F_SLOWEST_RANK_TIME
,
CP_F_VARIANCE_RANK_TIME
,
CP_F_VARIANCE_RANK_BYTES
,
CP_F_NUM_INDICES
,
};
struct
darshan_posix_file
{
darshan_record_id
f_id
;
int64_t
rank
;
int64_t
counters
[
CP_NUM_INDICES
];
double
fcounters
[
CP_F_NUM_INDICES
];
};
/*** POSIX MODULE DATA STRUCTURES END */
#endif
/* __DARSHAN_LOG_FORMAT_H */
darshan-posix-log-format.h
0 → 100644
View file @
721e5e5c
/*
* (C) 2009 by Argonne National Laboratory.
* See COPYRIGHT in top-level directory.
*/
#ifndef __DARSHAN_POSIX_LOG_FORMAT_H
#define __DARSHAN_POSIX_LOG_FORMAT_H
#include
"darshan-log-format.h"
/* TODO: DARSHAN_* OR CP_* */
enum
darshan_posix_indices
{
CP_POSIX_READS
,
/* count of posix reads */
CP_POSIX_WRITES
,
/* count of posix writes */
CP_POSIX_OPENS
,
/* count of posix opens */
CP_POSIX_SEEKS
,
/* count of posix seeks */
CP_POSIX_STATS
,
/* count of posix stat/lstat/fstats */
CP_POSIX_MMAPS
,
/* count of posix mmaps */
CP_POSIX_FREADS
,
CP_POSIX_FWRITES
,
CP_POSIX_FOPENS
,
CP_POSIX_FSEEKS
,
CP_POSIX_FSYNCS
,
CP_POSIX_FDSYNCS
,
CP_MODE
,
/* mode of file */
CP_BYTES_READ
,
/* total bytes read */
CP_BYTES_WRITTEN
,
/* total bytes written */
CP_MAX_BYTE_READ
,
/* highest offset byte read */
CP_MAX_BYTE_WRITTEN
,
/* highest offset byte written */
CP_CONSEC_READS
,
/* count of consecutive reads */
CP_CONSEC_WRITES
,
/* count of consecutive writes */
CP_SEQ_READS
,
/* count of sequential reads */
CP_SEQ_WRITES
,
/* count of sequential writes */
CP_RW_SWITCHES
,
/* number of times switched between read and write */
CP_MEM_NOT_ALIGNED
,
/* count of accesses not mem aligned */
CP_MEM_ALIGNMENT
,
/* mem alignment in bytes */
CP_FILE_NOT_ALIGNED
,
/* count of accesses not file aligned */
CP_FILE_ALIGNMENT
,
/* file alignment in bytes */
CP_MAX_READ_TIME_SIZE
,
CP_MAX_WRITE_TIME_SIZE
,
/* buckets */
CP_SIZE_READ_0_100
,
/* count of posix read size ranges */
CP_SIZE_READ_100_1K
,
CP_SIZE_READ_1K_10K
,
CP_SIZE_READ_10K_100K
,
CP_SIZE_READ_100K_1M
,
CP_SIZE_READ_1M_4M
,
CP_SIZE_READ_4M_10M
,
CP_SIZE_READ_10M_100M
,
CP_SIZE_READ_100M_1G
,
CP_SIZE_READ_1G_PLUS
,
/* buckets */
CP_SIZE_WRITE_0_100
,
/* count of posix write size ranges */
CP_SIZE_WRITE_100_1K
,
CP_SIZE_WRITE_1K_10K
,
CP_SIZE_WRITE_10K_100K
,
CP_SIZE_WRITE_100K_1M
,
CP_SIZE_WRITE_1M_4M
,
CP_SIZE_WRITE_4M_10M
,
CP_SIZE_WRITE_10M_100M
,
CP_SIZE_WRITE_100M_1G
,
CP_SIZE_WRITE_1G_PLUS
,
/* counters */
CP_STRIDE1_STRIDE
,
/* the four most frequently appearing strides */
CP_STRIDE2_STRIDE
,
CP_STRIDE3_STRIDE
,
CP_STRIDE4_STRIDE
,
CP_STRIDE1_COUNT
,
/* count of each of the most frequent strides */
CP_STRIDE2_COUNT
,
CP_STRIDE3_COUNT
,
CP_STRIDE4_COUNT
,
CP_ACCESS1_ACCESS
,
/* the four most frequently appearing access sizes */
CP_ACCESS2_ACCESS
,
CP_ACCESS3_ACCESS
,
CP_ACCESS4_ACCESS
,
CP_ACCESS1_COUNT
,
/* count of each of the most frequent access sizes */
CP_ACCESS2_COUNT
,
CP_ACCESS3_COUNT
,
CP_ACCESS4_COUNT
,
CP_DEVICE
,
/* device id reported by stat */
CP_SIZE_AT_OPEN
,
CP_FASTEST_RANK
,
CP_FASTEST_RANK_BYTES
,
CP_SLOWEST_RANK
,
CP_SLOWEST_RANK_BYTES
,
CP_NUM_INDICES
,
};
/* floating point statistics */
enum
darshan_f_posix_indices
{
/* NOTE: adjust cp_normalize_timestamps() function if any TIMESTAMPS are
* added or modified in this list
*/
CP_F_OPEN_TIMESTAMP
=
0
,
/* timestamp of first open */
CP_F_READ_START_TIMESTAMP
,
/* timestamp of first read */
CP_F_WRITE_START_TIMESTAMP
,
/* timestamp of first write */
CP_F_CLOSE_TIMESTAMP
,
/* timestamp of last close */
CP_F_READ_END_TIMESTAMP
,
/* timestamp of last read */
CP_F_WRITE_END_TIMESTAMP
,
/* timestamp of last write */
CP_F_POSIX_READ_TIME
,
/* cumulative posix read time */
CP_F_POSIX_WRITE_TIME
,
/* cumulative posix write time */
CP_F_POSIX_META_TIME
,
/* cumulative posix meta time */
CP_F_MAX_READ_TIME
,
CP_F_MAX_WRITE_TIME
,
/* Total I/O and meta time consumed by fastest and slowest ranks,
* reported in either MPI or POSIX time depending on how the file
* was accessed.
*/
CP_F_FASTEST_RANK_TIME
,
CP_F_SLOWEST_RANK_TIME
,
CP_F_VARIANCE_RANK_TIME
,
CP_F_VARIANCE_RANK_BYTES
,
CP_F_NUM_INDICES
,
};
struct
darshan_posix_file
{
darshan_record_id
f_id
;
int64_t
rank
;
int64_t
counters
[
CP_NUM_INDICES
];
double
fcounters
[
CP_F_NUM_INDICES
];
};
#endif
/* __DARSHAN_POSIX_LOG_FORMAT_H */
darshan-runtime/lib/darshan-posix.c
View file @
721e5e5c
...
...
@@ -27,6 +27,7 @@
#include
"uthash.h"
#include
"darshan.h"
#include
"darshan-posix-log-format.h"
#ifndef HAVE_OFF64_T
typedef
int64_t
off64_t
;
...
...
darshan-util/darshan-logutils.h
View file @
721e5e5c
...
...
@@ -6,7 +6,8 @@
#ifndef __DARSHAN_LOG_UTILS_H
#define __DARSHAN_LOG_UTILS_H
#include
<darshan-log-format.h>
#include
"darshan-log-format.h"
#include
"darshan-posix-log-format.h"
#include
"uthash-1.9.2/src/uthash.h"
typedef
struct
darshan_fd_s
*
darshan_fd
;
...
...
darshan-util/darshan-posix-parser.c
View file @
721e5e5c
...
...
@@ -22,12 +22,14 @@ int main(int argc, char **argv)
{
int
ret
;
char
*
filename
;
char
tmp_string
[
4096
];
darshan_fd
file
;
struct
darshan_header
header
;
struct
darshan_job
job
;
struct
darshan_record_ref
*
rec_map
=
NULL
;
struct
darshan_record_ref
*
ref
,
*
tmp
;
struct
darshan_posix_file
next_rec
;
time_t
tmp_time
=
0
;
assert
(
argc
==
2
);
filename
=
argv
[
1
];
...
...
@@ -35,11 +37,6 @@ int main(int argc, char **argv)
struct
stat
sbuf
;
stat
(
filename
,
&
sbuf
);
printf
(
"
\n
Darshan log file size: %"
PRId64
"
\n
"
,
sbuf
.
st_size
);
printf
(
"size of header: %"
PRId64
"
\n
"
,
sizeof
(
struct
darshan_header
));
printf
(
"size of index map: %"
PRId64
"
\n
"
,
2
*
sizeof
(
int64_t
));
printf
(
"size of job data: %"
PRId64
"
\n
"
,
sizeof
(
struct
darshan_job
));
file
=
darshan_log_open
(
filename
,
"r"
);
if
(
!
file
)
{
...
...
@@ -47,7 +44,7 @@ int main(int argc, char **argv)
return
(
-
1
);
}
/* read
the
log header */
/* read
darshan
log header */
ret
=
darshan_log_getheader
(
file
,
&
header
);
if
(
ret
<
0
)
{
...
...
@@ -56,11 +53,7 @@ int main(int argc, char **argv)
return
(
-
1
);
}
printf
(
"
\n
*** HEADER DATA ***
\n
"
);
printf
(
"
\t
ver=%s
\n\t
magic_nr=%"
PRId64
"
\n\t
comp_type=%d
\n\t
mod_count=%d
\n
"
,
header
.
version_string
,
header
.
magic_nr
,
header
.
comp_type
,
header
.
mod_count
);
/* read job info */
/* read darshan job info */
ret
=
darshan_log_getjob
(
file
,
&
job
);
if
(
ret
<
0
)
{
...
...
@@ -69,10 +62,22 @@ int main(int argc, char **argv)
return
(
-
1
);
}
printf
(
"
\n
*** JOB DATA ***
\n
"
);
printf
(
"
\t
uid=%"
PRId64
"
\n\t
start_time=%"
PRId64
"
\n\t
end_time=%"
PRId64
"
\n\t
nprocs=%"
PRId64
"
\n\t
jobid=%"
PRId64
"
\n
"
,
job
.
uid
,
job
.
start_time
,
job
.
end_time
,
job
.
nprocs
,
job
.
jobid
);
/* print job summary */
printf
(
"# darshan log version: %s
\n
"
,
header
.
version_string
);
printf
(
"# size of POSIX file statistics: %zu bytes
\n
"
,
sizeof
(
next_rec
));
printf
(
"# size of job statistics: %zu bytes
\n
"
,
sizeof
(
job
));
printf
(
"# exe: %s
\n
"
,
tmp_string
);
printf
(
"# uid: %"
PRId64
"
\n
"
,
job
.
uid
);
printf
(
"# jobid: %"
PRId64
"
\n
"
,
job
.
jobid
);
printf
(
"# start_time: %"
PRId64
"
\n
"
,
job
.
start_time
);
tmp_time
+=
job
.
start_time
;
printf
(
"# start_time_asci: %s"
,
ctime
(
&
tmp_time
));
printf
(
"# end_time: %"
PRId64
"
\n
"
,
job
.
end_time
);
tmp_time
=
0
;
tmp_time
+=
job
.
end_time
;
printf
(
"# end_time_asci: %s"
,
ctime
(
&
tmp_time
));
printf
(
"# nprocs: %"
PRId64
"
\n
"
,
job
.
nprocs
);
printf
(
"# run time: %"
PRId64
"
\n
"
,
job
.
end_time
-
job
.
start_time
+
1
);
/* read record map */
ret
=
darshan_log_getmap
(
file
,
&
rec_map
);
...
...
@@ -111,6 +116,8 @@ int main(int argc, char **argv)
printf
(
"
\t
Record %d: id=%"
PRIu64
" (path=%s, rank=%"
PRId64
")
\n
"
,
i
,
next_rec
.
f_id
,
ref
->
rec
.
name
,
next_rec
.
rank
);
printf
(
"
\t\t
POSIX_OPENS:
\t
%"
PRIu64
"
\n\t\t
F_OPEN_TIMESTAMP:
\t
%lf
\n
"
,
next_rec
.
counters
[
CP_POSIX_OPENS
],
next_rec
.
fcounters
[
CP_F_OPEN_TIMESTAMP
]);
i
++
;
}
while
((
ret
=
darshan_log_getfile
(
file
,
&
next_rec
))
==
1
);
...
...
doc/darshan-dev-status.txt
0 → 100644
View file @
721e5e5c
Darshan-modular branch development notes
========================================
== Introduction
Darshan-modular is the new implementation of Darshan (version >= 3.0.0), a lightweight
toolkit for characterizing the I/O performance of instrumented HPC applications.
Previously, Darshan had been designed to gather I/O data from a static set of sources.
Adding instrumentation for additional sources of I/O data was only possible through
manual modification of the Darshan log file format, which consequentially breaks
other utilities reliant on that format.
Darshan-modular is differentiated from previous versions in that it modularizes the Darshan
runtime environment and log file format such that new "instrumentation modules" can be
added without breaking existing tools. Essentially, developers are given a framework to
implement arbitrary modules, which are responsible for gathering I/O data from a specific
system component (which could be from an I/O library, platform-specific data, etc.). Darshan
can then manage these modules and create a valid Darshan log regardless of how many or what
types of modules are used.
== Status
=== Runtime status
Text.
=== Util status
Text.
=== Counters status
Text.
== Adding new modules
darshan-modularization-design-notes.txt
→
doc/
darshan-modularization-design-notes.txt
View file @
721e5e5c
...
...
@@ -115,11 +115,11 @@ TODO NOTES:
- why are darshan mutexes recursive?
- how do we allow modules to specify there necessary linker/wrapper flags
- configurable amount of max runtime memory
- how do we associate handles with file/object records?
- easily doable within module, but can we refactor and make this common code?
- fs mount information -- should we export this info from darshan-core?
- would the posix module be the only one to leverage this info?
- it is used to correlate filenames with mount points, but not sure what is needed where
- test fortran and c apps to make sure command line args are handled properly (see email with Phil 11/4)
- how do we abstract shared file reduction so each module doesn't completely reimplement?
- when is DARSHAN_MPI_CALL macro needed? All MPI functions, or just those intercepted by darshan?
- darshan link order causes issues when compiling mpi-io-test -- had to add extra -lmpi to resolve
- is there more coordination between core and modules for shared file reductions?
- do we change darshan file extension (.dz? .darshan?)
darshan-modularization-whiteboard.pdf
→
doc/
darshan-modularization-whiteboard.pdf
View file @
721e5e5c
File moved
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment