Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
darshan
darshan
Commits
093b9f22
Commit
093b9f22
authored
Aug 17, 2015
by
Shane Snyder
Browse files
update log put functions for new log format
parent
d8b6e0dc
Changes
3
Hide whitespace changes
Inline
Side-by-side
darshan-log-format.h
View file @
093b9f22
...
...
@@ -31,21 +31,30 @@
/* max length of exe string within job record (not counting '\0') */
#define DARSHAN_EXE_LEN (DARSHAN_JOB_RECORD_SIZE - sizeof(struct darshan_job) - 1)
typedef
uint64_t
darshan_record_id
;
#define DARSHAN_MAX_MODS 16
/* TODO: do we want the logutil defs here ? */
/* X-macro for keeping module ordering consistent */
/* NOTE: first val used to define module enum values,
* second val used to define module name strings, and
* third val is used to provide the name of a
* corresponding logutils structure for parsing module
* data out of the log file (only used in darshan-util,
* just pass NULL (no quotes) if no log parsing
* functions are required).
*/
#define DARSHAN_MODULE_IDS \
X(DARSHAN_NULL_MOD,
"NULL") \
X(DARSHAN_POSIX_MOD,
"POSIX") \
X(DARSHAN_MPIIO_MOD,
"MPI-IO") \
X(DARSHAN_HDF5_MOD,
"HDF5") \
X(DARSHAN_PNETCDF_MOD,
"PNETCDF")
X(DARSHAN_NULL_MOD, "NULL"
, NULL
) \
X(DARSHAN_POSIX_MOD, "POSIX"
, posix_logutils
) \
X(DARSHAN_MPIIO_MOD, "MPI-IO"
, mpiio_logutils
) \
X(DARSHAN_HDF5_MOD, "HDF5"
, hdf5_logutils
) \
X(DARSHAN_PNETCDF_MOD, "PNETCDF"
, pnetcdf_logutils
)
/* unique identifiers to distinguish between available darshan modules */
/* NOTES: - valid ids range from [0...DARSHAN_MAX_MODS-1]
* - order of ids control module shutdown order (and consequently, order in log file)
*/
#define X(a, b) a,
#define X(a, b
, c
) a,
typedef
enum
{
DARSHAN_MODULE_IDS
...
...
@@ -53,7 +62,7 @@ typedef enum
#undef X
/* module name strings */
#define X(a, b) b,
#define X(a, b
, c
) b,
static
char
*
const
darshan_module_names
[]
=
{
DARSHAN_MODULE_IDS
...
...
@@ -67,6 +76,8 @@ enum darshan_comp_type
DARSHAN_BZIP2_COMP
,
};
typedef
uint64_t
darshan_record_id
;
/* the darshan_log_map structure is used to indicate the location of
* specific module data in a Darshan log. Note that 'off' and 'len' are
* the respective offset and length of the data in the file, in *uncompressed*
...
...
darshan-util/darshan-logutils.c
View file @
093b9f22
...
...
@@ -26,6 +26,8 @@ static int darshan_log_read(darshan_fd fd, void *buf, int len);
static
int
darshan_log_write
(
darshan_fd
fd
,
void
*
buf
,
int
len
);
static
int
darshan_decompress_buf
(
char
*
comp_buf
,
int
comp_buf_sz
,
char
*
decomp_buf
,
int
*
inout_decomp_buf_sz
);
static
int
darshan_compress_buf
(
char
*
decomp_buf
,
int
decomp_buf_sz
,
char
*
comp_buf
,
int
*
inout_comp_buf_sz
);
/* TODO: can we make this s.t. we don't care about ordering (i.e., X macro it ) */
struct
darshan_mod_logutil_funcs
*
mod_logutils
[
DARSHAN_MAX_MODS
]
=
...
...
@@ -56,31 +58,30 @@ struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS] =
*/
darshan_fd
darshan_log_open
(
const
char
*
name
,
const
char
*
mode
)
{
int
o_flags
;
darshan_fd
tmp_fd
;
/* we only allow "w" or "r" modes, nothing fancy */
assert
(
strlen
(
mode
)
==
1
);
assert
(
mode
[
0
]
==
'r'
||
mode
[
0
]
==
'w'
);
tmp_fd
=
malloc
(
sizeof
(
*
tmp_fd
));
if
(
!
tmp_fd
)
return
(
NULL
);
memset
(
tmp_fd
,
0
,
sizeof
(
*
tmp_fd
));
if
(
mode
[
0
]
==
'r'
)
{
o_flags
=
O_RDONLY
;
tmp_fd
->
fildes
=
open
(
name
,
O_RDONLY
)
;
}
else
if
(
mode
[
0
]
==
'w'
)
{
/* TODO: permissions when creating? umask */
/* when writing, we create the log file making sure not to overwrite
* an existing log
*/
o_flags
=
O_WRONLY
|
O_CREAT
|
O_EXCL
;
tmp_fd
->
fildes
=
open
(
name
,
O_WRONLY
|
O_CREAT
|
O_EXCL
,
0400
)
;
}
tmp_fd
=
malloc
(
sizeof
(
*
tmp_fd
));
if
(
!
tmp_fd
)
return
(
NULL
);
memset
(
tmp_fd
,
0
,
sizeof
(
*
tmp_fd
));
tmp_fd
->
fildes
=
open
(
name
,
o_flags
);
if
(
tmp_fd
->
fildes
<
0
)
{
perror
(
"darshan_log_open: "
);
...
...
@@ -161,15 +162,17 @@ int darshan_log_getheader(darshan_fd fd, struct darshan_header *header)
return
(
0
);
}
#if 0
/* darshan_log_putheader()
*
* write a darshan header to log file
* NOTE: the header is not passed in, and is instead built using
* contents of the given file descriptor
*
* returns 0 on success, -1 on failure
*/
int darshan_log_putheader(darshan_fd fd
, struct darshan_header *header
)
int
darshan_log_putheader
(
darshan_fd
fd
)
{
struct
darshan_header
header
;
int
ret
;
ret
=
darshan_log_seek
(
fd
,
0
);
...
...
@@ -179,21 +182,24 @@ int darshan_log_putheader(darshan_fd fd, struct darshan_header *header)
return
(
-
1
);
}
memset
(
&
header
,
0
,
sizeof
(
header
));
strcpy
(
header
.
version_string
,
DARSHAN_LOG_VERSION
);
header
.
magic_nr
=
DARSHAN_MAGIC_NR
;
/* copy the mapping information to the header */
memcpy
(
&
header
.
rec_map
,
&
fd
->
rec_map
,
sizeof
(
struct
darshan_log_map
));
memcpy
(
&
header
.
mod_map
,
&
fd
->
mod_map
,
DARSHAN_MAX_MODS
*
sizeof
(
struct
darshan_log_map
));
/* write header to file */
ret = darshan_log_write(fd, header, sizeof(
*
header));
if(ret != sizeof(
*
header))
ret
=
darshan_log_write
(
fd
,
&
header
,
sizeof
(
header
));
if
(
ret
!=
sizeof
(
header
))
{
fprintf
(
stderr
,
"Error: failed to write Darshan log file header.
\n
"
);
return
(
-
1
);
}
/* copy the mapping information to the file descriptor */
memcpy(&fd->rec_map, &header->rec_map, sizeof(struct darshan_log_map));
memcpy(&fd->mod_map, &header->mod_map, DARSHAN_MAX_MODS * sizeof(struct darshan_log_map));
return
(
0
);
}
#endif
/* darshan_log_getjob()
*
...
...
@@ -238,7 +244,6 @@ int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
free
(
comp_buf
);
return
(
-
1
);
}
assert
(
job_buf_sz
==
DARSHAN_JOB_RECORD_SIZE
);
free
(
comp_buf
);
memcpy
(
job
,
job_buf
,
sizeof
(
*
job
));
...
...
@@ -263,7 +268,6 @@ int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
return
(
0
);
}
#if 0
/* darshan_log_putjob()
*
* write job level metadat to darshan log file
...
...
@@ -273,6 +277,8 @@ int darshan_log_getjob(darshan_fd fd, struct darshan_job *job)
int
darshan_log_putjob
(
darshan_fd
fd
,
struct
darshan_job
*
job
)
{
struct
darshan_job
job_copy
;
char
*
comp_buf
;
int
comp_buf_sz
;
int
len
;
int
ret
;
...
...
@@ -297,9 +303,23 @@ int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
}
}
comp_buf
=
malloc
(
sizeof
(
*
job
));
if
(
!
comp_buf
)
return
(
-
1
);
comp_buf_sz
=
sizeof
(
*
job
);
/* compress the job data */
ret
=
darshan_compress_buf
((
char
*
)
&
job_copy
,
sizeof
(
*
job
),
comp_buf
,
&
comp_buf_sz
);
if
(
ret
<
0
)
{
fprintf
(
stderr
,
"Error: failed to decompress darshan job data.
\n
"
);
free
(
comp_buf
);
return
(
-
1
);
}
/* write job data to file */
ret = darshan_log_write(fd,
job, sizeof(*job)
);
if(ret !=
sizeof(*job)
)
ret
=
darshan_log_write
(
fd
,
comp_buf
,
comp_buf_sz
);
if
(
ret
!=
comp_buf_sz
)
{
fprintf
(
stderr
,
"Error: failed to write darshan log file job data.
\n
"
);
return
(
-
1
);
...
...
@@ -307,7 +327,6 @@ int darshan_log_putjob(darshan_fd fd, struct darshan_job *job)
return
(
0
);
}
#endif
/* darshan_log_getexe()
*
...
...
@@ -340,30 +359,34 @@ int darshan_log_getexe(darshan_fd fd, char *buf)
return
(
0
);
}
#if 0
/* darshan_log_putexe()
*
* wrties the application exe name to darshan log file
* NOTE: this needs to be called immediately following put_job as it
* expects the final pointer to be positioned immediately following
* the darshan job information
*
* returns 0 on success, -1 on failure
*/
int
darshan_log_putexe
(
darshan_fd
fd
,
char
*
buf
)
{
int tmp_off = sizeof(struct darshan_header) + sizeof(struct darshan_job);
int
len
;
int
ret
;
char
comp_buf
[
DARSHAN_EXE_LEN
]
=
{
0
};
int
comp_buf_sz
=
DARSHAN_EXE_LEN
;
len
=
strlen
(
buf
);
ret = darshan_log_seek(fd, tmp_off);
/* compress the input exe string */
ret
=
darshan_compress_buf
(
buf
,
len
,
comp_buf
,
&
comp_buf_sz
);
if
(
ret
<
0
)
{
fprintf(stderr, "Error: unable to
seek in darshan log file
.\n");
fprintf
(
stderr
,
"Error: unable to
compress exe string
.
\n
"
);
return
(
-
1
);
}
len = strlen(buf);
ret = darshan_log_write(fd, buf, len);
if(ret != len)
ret
=
darshan_log_write
(
fd
,
comp_buf
,
comp_buf_sz
);
if
(
ret
!=
comp_buf_sz
)
{
fprintf
(
stderr
,
"Error: failed to write exe string to darshan log file.
\n
"
);
return
(
-
1
);
...
...
@@ -371,7 +394,6 @@ int darshan_log_putexe(darshan_fd fd, char *buf)
return
(
0
);
}
#endif
/* darshan_log_getmounts()
*
...
...
@@ -445,7 +467,6 @@ int darshan_log_getmounts(darshan_fd fd, char*** mnt_pts,
return
(
0
);
}
#if 0
/* darshan_log_putmounts()
*
* writes mount information to the darshan log file
...
...
@@ -459,31 +480,40 @@ int darshan_log_putmounts(darshan_fd fd, char** mnt_pts, char** fs_types, int co
{
int
i
;
char
line
[
1024
];
char
mnt_dat
[
DARSHAN_EXE_LEN
]
=
{
0
};
int
mnt_dat_sz
=
0
;
char
comp_buf
[
DARSHAN_EXE_LEN
]
=
{
0
};
int
comp_buf_sz
=
DARSHAN_EXE_LEN
;
char
*
tmp
;
int
ret
;
/* write each mount entry to file */
tmp
=
mnt_dat
;
for
(
i
=
count
-
1
;
i
>=
0
;
i
--
)
{
sprintf
(
line
,
"
\n
%s
\t
%s"
,
fs_types
[
i
],
mnt_pts
[
i
]);
ret = darshan_log_write(fd, line, strlen(line));
if (ret != strlen(line))
{
fprintf(stderr, "Error: failed to write darshan log mount entry.\n");
return(-1);
}
memcpy
(
tmp
,
line
,
strlen
(
line
));
tmp
+=
strlen
(
line
);
mnt_dat_sz
+=
strlen
(
line
);
}
ret
=
darshan_compress_buf
(
mnt_dat
,
mnt_dat_sz
,
comp_buf
,
&
comp_buf_sz
);
if
(
ret
<
0
)
{
fprintf
(
stderr
,
"Error: unable to compress mount data.
\n
"
);
return
(
-
1
);
}
/* seek ahead to end of exe region, will be zero filled */
ret = darshan_log_seek(fd, DARSHAN_JOB_RECORD_SIZE);
if (ret < 0)
ret
=
darshan_log_write
(
fd
,
comp_buf
,
comp_buf_sz
);
if
(
ret
!=
comp_buf_sz
)
{
fprintf(stderr, "Error:
unab
le to
seek forward in
darshan log
file
.\n");
fprintf
(
stderr
,
"Error:
fai
le
d
to
write
darshan log
mount data
.
\n
"
);
return
(
-
1
);
}
return
(
0
);
}
#endif
/* darshan_log_gethash()
*
...
...
@@ -591,10 +621,12 @@ int darshan_log_gethash(darshan_fd fd, struct darshan_record_ref **hash)
return
(
0
);
}
#if 0
/* darshan_log_puthash()
*
* writes the hash table of records to the darshan log file
* NOTE: this function call should follow immediately after the call
* to darshan_log_putmounts(), as it assumes the darshan log file pointer
* is pointing to the offset immediately following the mount information
*
* returns 0 on success, -1 on failure
*/
...
...
@@ -606,15 +638,10 @@ int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
struct
darshan_record_ref
*
ref
,
*
tmp
;
uint32_t
name_len
;
size_t
record_sz
;
char
*
comp_buf
;
char
comp_buf_sz
;
int
ret
;
ret = darshan_log_seek(fd, fd->rec_map.off);
if(ret < 0)
{
fprintf(stderr, "Error: unable to seek in darshan log file.\n");
return(-1);
}
/* allocate a buffer to store 2 MiB worth of record data */
/* NOTE: this buffer may be reallocated if estimate is too small */
hash_buf_sz
=
2
*
1024
*
1024
;
...
...
@@ -663,13 +690,28 @@ int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
memcpy
(
hash_buf_off
,
ref
->
rec
.
name
,
name_len
);
hash_buf_off
+=
name_len
;
}
/* collectively write out the record hash to the darshan log */
hash_buf_sz
=
hash_buf_off
-
hash_buf
;
comp_buf
=
malloc
(
hash_buf_sz
);
if
(
!
comp_buf
)
return
(
-
1
);
comp_buf_sz
=
hash_buf_sz
;
/* compress the record hash */
ret
=
darshan_compress_buf
(
hash_buf
,
hash_buf_sz
,
comp_buf
,
&
comp_buf_sz
);
if
(
ret
<
0
)
{
fprintf
(
stderr
,
"Error: unable to compress darshan record hash.
\n
"
);
return
(
-
1
);
}
/* set the appropriate mapping info for the record hash in the file descriptor */
fd
->
rec_map
.
off
=
fd
->
pos
;
fd
->
rec_map
.
len
=
comp_buf_sz
;
/* write the record hash to file */
ret = darshan_log_write(fd,
hash
_buf,
hash
_buf_sz);
if(ret !=
hash
_buf_sz)
ret
=
darshan_log_write
(
fd
,
comp
_buf
,
comp
_buf_sz
);
if
(
ret
!=
comp
_buf_sz
)
{
fprintf
(
stderr
,
"Error: failed to write record hash to darshan log file.
\n
"
);
return
(
-
1
);
...
...
@@ -679,7 +721,6 @@ int darshan_log_puthash(darshan_fd fd, struct darshan_record_ref *hash)
return
(
0
);
}
#endif
/* darshan_log_getmod()
*
...
...
@@ -738,7 +779,7 @@ int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
return
(
1
);
}
#if 0
/* XXX */
/* darshan_log_putmod()
*
* write a chunk of module data to the darshan log file
...
...
@@ -748,15 +789,8 @@ int darshan_log_getmod(darshan_fd fd, darshan_module_id mod_id,
int
darshan_log_putmod
(
darshan_fd
fd
,
darshan_module_id
mod_id
,
void
*
mod_buf
,
int
mod_buf_sz
)
{
int mod_buf_end = fd->mod_map[mod_id].off + fd->mod_map[mod_id].len;
int
ret
;
if(!fd->mod_map[mod_id].len)
{
fprintf(stderr, "Error: attempting to write data for empty module.\n");
return(-1);
}
/* only seek to start of module data if current log file position
* is not within the given mod_id's range. This allows one to
* repeatedly call this function and put chunks of a module's
...
...
@@ -791,7 +825,6 @@ int darshan_log_putmod(darshan_fd fd, darshan_module_id mod_id,
return
(
0
);
}
#endif
/* darshan_log_close()
*
...
...
@@ -892,7 +925,7 @@ static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
{
if
(
tmp_stream
.
avail_out
==
0
)
{
/* We ran out of buffer space for compression. In theory,
/* We ran out of buffer space for
de
compression. In theory,
* we could just alloc more space, but probably just easier
* to bump up the default size of the output buffer.
*/
...
...
@@ -921,7 +954,53 @@ static int darshan_decompress_buf(char* comp_buf, int comp_buf_sz,
static
int
darshan_compress_buf
(
char
*
decomp_buf
,
int
decomp_buf_sz
,
char
*
comp_buf
,
int
*
inout_comp_buf_sz
)
{
int
ret
;
int
total_out
=
0
;
z_stream
tmp_stream
;
memset
(
&
tmp_stream
,
0
,
sizeof
(
tmp_stream
));
tmp_stream
.
zalloc
=
Z_NULL
;
tmp_stream
.
zfree
=
Z_NULL
;
tmp_stream
.
opaque
=
Z_NULL
;
tmp_stream
.
next_in
=
(
unsigned
char
*
)
decomp_buf
;
tmp_stream
.
avail_in
=
decomp_buf_sz
;
tmp_stream
.
next_out
=
(
unsigned
char
*
)
comp_buf
;
tmp_stream
.
avail_out
=
*
inout_comp_buf_sz
;
ret
=
deflateInit
(
&
tmp_stream
,
Z_DEFAULT_COMPRESSION
);
if
(
ret
!=
Z_OK
)
{
return
(
-
1
);
}
/* while we have not finished consuming all of the uncompressed input data */
while
(
tmp_stream
.
avail_in
)
{
if
(
tmp_stream
.
avail_out
==
0
)
{
/* We ran out of buffer space for compression. In theory,
* we could just alloc more space, but probably just easier
* to bump up the default size of the output buffer.
*/
deflateEnd
(
&
tmp_stream
);
return
(
-
1
);
}
/* compress data */
ret
=
deflate
(
&
tmp_stream
,
Z_FINISH
);
if
(
ret
!=
Z_STREAM_END
)
{
deflateEnd
(
&
tmp_stream
);
return
(
-
1
);
}
total_out
+=
tmp_stream
.
total_out
;
if
(
tmp_stream
.
avail_in
)
deflateReset
(
&
tmp_stream
);
}
deflateEnd
(
&
tmp_stream
);
*
inout_comp_buf_sz
=
total_out
;
return
(
0
);
}
...
...
darshan-util/darshan-logutils.h
View file @
093b9f22
...
...
@@ -68,7 +68,7 @@ extern struct darshan_mod_logutil_funcs *mod_logutils[DARSHAN_MAX_MODS];
darshan_fd
darshan_log_open
(
const
char
*
name
,
const
char
*
mode
);
int
darshan_log_getheader
(
darshan_fd
fd
,
struct
darshan_header
*
header
);
int
darshan_log_putheader
(
darshan_fd
fd
,
struct
darshan_header
*
header
);
int
darshan_log_putheader
(
darshan_fd
fd
);
int
darshan_log_getjob
(
darshan_fd
fd
,
struct
darshan_job
*
job
);
int
darshan_log_putjob
(
darshan_fd
fd
,
struct
darshan_job
*
job
);
int
darshan_log_getexe
(
darshan_fd
fd
,
char
*
buf
);
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
.
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment