Commit 9aeaecab authored by Philip Carns's avatar Philip Carns

fixes #130

The uncompressed header size for darshan logs has been increased from 1 KiB
to 4 KiB, which allows room for more metadata annotations


git-svn-id: https://svn.mcs.anl.gov/repos/darshan/trunk@858 3b7491f3-a168-0410-bf4b-c445ed680a29
parent 7c338aeb
...@@ -25,6 +25,8 @@ darshan-2.2.9-pre1 ...@@ -25,6 +25,8 @@ darshan-2.2.9-pre1
- Ability to disable shared-file reduction by setting the - Ability to disable shared-file reduction by setting the
DARSHAN_DISABLE_SHARED_REDUCTION environment variable. DARSHAN_DISABLE_SHARED_REDUCTION environment variable.
- More thorough output from darshan-parser --perf, suggested by Huong Luu. - More thorough output from darshan-parser --perf, suggested by Huong Luu.
- Increased uncompressed log header size from 1 KiB to 4 KiB to support
additional annotations.
darshan-2.2.8 darshan-2.2.8
============= =============
......
...@@ -19,13 +19,13 @@ ...@@ -19,13 +19,13 @@
#endif #endif
/* update this on file format changes */ /* update this on file format changes */
#define CP_VERSION "2.03" #define CP_VERSION "2.04"
/* magic number for validating output files and checking byte order */ /* magic number for validating output files and checking byte order */
#define CP_MAGIC_NR 6567223 #define CP_MAGIC_NR 6567223
/* size (in bytes) of job record */ /* size (in bytes) of job record */
#define CP_JOB_RECORD_SIZE 1024 #define CP_JOB_RECORD_SIZE 4096
/* max length of exe string within job record (not counting '\0') */ /* max length of exe string within job record (not counting '\0') */
#define CP_EXE_LEN (CP_JOB_RECORD_SIZE - sizeof(struct darshan_job) - 1) #define CP_EXE_LEN (CP_JOB_RECORD_SIZE - sizeof(struct darshan_job) - 1)
......
...@@ -34,6 +34,7 @@ struct darshan_fd_s ...@@ -34,6 +34,7 @@ struct darshan_fd_s
char version[10]; char version[10];
int job_struct_size; int job_struct_size;
char* name; char* name;
int COMPAT_CP_EXE_LEN;
}; };
/* isn't there a clever c way to avoid this? */ /* isn't there a clever c way to avoid this? */
...@@ -219,10 +220,14 @@ int (*getfile_internal)(darshan_fd fd, ...@@ -219,10 +220,14 @@ int (*getfile_internal)(darshan_fd fd,
struct darshan_file *file); struct darshan_file *file);
#define JOB_SIZE_124 28 #define JOB_SIZE_124 28
#define JOB_SIZE_200 56 #define JOB_SIZE_200 56
#define CP_JOB_RECORD_SIZE_200 1024
#define CP_JOB_RECORD_SIZE_1x 1024
/* internal routines for parsing different file versions */ /* internal routines for parsing different file versions */
static int getjob_internal_201(darshan_fd file, struct darshan_job *job); static int getjob_internal_201(darshan_fd file, struct darshan_job *job);
static int getjob_internal_200(darshan_fd file, struct darshan_job *job); static int getjob_internal_200(darshan_fd file, struct darshan_job *job);
static int getfile_internal_204(darshan_fd fd, struct darshan_job *job,
struct darshan_file *file);
static int getfile_internal_200(darshan_fd fd, struct darshan_job *job, static int getfile_internal_200(darshan_fd fd, struct darshan_job *job,
struct darshan_file *file); struct darshan_file *file);
static int getjob_internal_124(darshan_fd file, struct darshan_job *job); static int getjob_internal_124(darshan_fd file, struct darshan_job *job);
...@@ -369,35 +374,47 @@ int darshan_log_getjob(darshan_fd file, struct darshan_job *job) ...@@ -369,35 +374,47 @@ int darshan_log_getjob(darshan_fd file, struct darshan_job *job)
return(-1); return(-1);
} }
if(strcmp(file->version, "2.03") == 0) if(strcmp(file->version, "2.04") == 0)
{
getjob_internal = getjob_internal_201;
getfile_internal = getfile_internal_204;
file->job_struct_size = sizeof(*job);
file->COMPAT_CP_EXE_LEN = CP_EXE_LEN;
}
else if(strcmp(file->version, "2.03") == 0)
{ {
getjob_internal = getjob_internal_201; getjob_internal = getjob_internal_201;
getfile_internal = getfile_internal_200; getfile_internal = getfile_internal_200;
file->job_struct_size = sizeof(*job); file->job_struct_size = sizeof(*job);
file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_200-file->job_struct_size-1;
} }
else if(strcmp(file->version, "2.02") == 0) else if(strcmp(file->version, "2.02") == 0)
{ {
getjob_internal = getjob_internal_201; getjob_internal = getjob_internal_201;
getfile_internal = getfile_internal_200; getfile_internal = getfile_internal_200;
file->job_struct_size = sizeof(*job); file->job_struct_size = sizeof(*job);
file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_200-file->job_struct_size-1;
} }
else if(strcmp(file->version, "2.01") == 0) else if(strcmp(file->version, "2.01") == 0)
{ {
getjob_internal = getjob_internal_201; getjob_internal = getjob_internal_201;
getfile_internal = getfile_internal_200; getfile_internal = getfile_internal_200;
file->job_struct_size = sizeof(*job); file->job_struct_size = sizeof(*job);
file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_200-file->job_struct_size-1;
} }
else if(strcmp(file->version, "2.00") == 0) else if(strcmp(file->version, "2.00") == 0)
{ {
getjob_internal = getjob_internal_200; getjob_internal = getjob_internal_200;
getfile_internal = getfile_internal_200; getfile_internal = getfile_internal_200;
file->job_struct_size = JOB_SIZE_200; file->job_struct_size = JOB_SIZE_200;
file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_200-file->job_struct_size-1;
} }
else if(strcmp(file->version, "1.24") == 0) else if(strcmp(file->version, "1.24") == 0)
{ {
getjob_internal = getjob_internal_124; getjob_internal = getjob_internal_124;
getfile_internal = getfile_internal_124; getfile_internal = getfile_internal_124;
file->job_struct_size = JOB_SIZE_124; file->job_struct_size = JOB_SIZE_124;
file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_1x-file->job_struct_size-1;
} }
else if(strcmp(file->version, "1.23") == 0) else if(strcmp(file->version, "1.23") == 0)
{ {
...@@ -405,18 +422,21 @@ int darshan_log_getjob(darshan_fd file, struct darshan_job *job) ...@@ -405,18 +422,21 @@ int darshan_log_getjob(darshan_fd file, struct darshan_job *job)
getjob_internal = getjob_internal_124; getjob_internal = getjob_internal_124;
getfile_internal = getfile_internal_124; getfile_internal = getfile_internal_124;
file->job_struct_size = JOB_SIZE_124; file->job_struct_size = JOB_SIZE_124;
file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_1x-file->job_struct_size-1;
} }
else if(strcmp(file->version, "1.22") == 0) else if(strcmp(file->version, "1.22") == 0)
{ {
getjob_internal = getjob_internal_124; getjob_internal = getjob_internal_124;
getfile_internal = getfile_internal_122; getfile_internal = getfile_internal_122;
file->job_struct_size = JOB_SIZE_124; file->job_struct_size = JOB_SIZE_124;
file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_1x-file->job_struct_size-1;
} }
else if(strcmp(file->version, "1.21") == 0) else if(strcmp(file->version, "1.21") == 0)
{ {
getjob_internal = getjob_internal_124; getjob_internal = getjob_internal_124;
getfile_internal = getfile_internal_121; getfile_internal = getfile_internal_121;
file->job_struct_size = JOB_SIZE_124; file->job_struct_size = JOB_SIZE_124;
file->COMPAT_CP_EXE_LEN = CP_JOB_RECORD_SIZE_1x-file->job_struct_size-1;
} }
else else
{ {
...@@ -564,14 +584,14 @@ int darshan_log_getmounts(darshan_fd fd, int64_t** devs, char*** mnt_pts, char** ...@@ -564,14 +584,14 @@ int darshan_log_getmounts(darshan_fd fd, int64_t** devs, char*** mnt_pts, char**
int ret; int ret;
char* pos; char* pos;
int array_index = 0; int array_index = 0;
char buf[CP_EXE_LEN+1]; char buf[fd->COMPAT_CP_EXE_LEN+1];
ret = darshan_log_seek(fd, fd->job_struct_size); ret = darshan_log_seek(fd, fd->job_struct_size);
if(ret < 0) if(ret < 0)
return(ret); return(ret);
ret = darshan_log_read(fd, buf, (CP_EXE_LEN + 1)); ret = darshan_log_read(fd, buf, (fd->COMPAT_CP_EXE_LEN + 1));
if (ret < (CP_EXE_LEN + 1)) if (ret < (fd->COMPAT_CP_EXE_LEN + 1))
{ {
perror("darshan_log_read"); perror("darshan_log_read");
return(-1); return(-1);
...@@ -606,9 +626,9 @@ int darshan_log_getmounts(darshan_fd fd, int64_t** devs, char*** mnt_pts, char** ...@@ -606,9 +626,9 @@ int darshan_log_getmounts(darshan_fd fd, int64_t** devs, char*** mnt_pts, char**
while((pos = strrchr(buf, '\n')) != NULL) while((pos = strrchr(buf, '\n')) != NULL)
{ {
/* overestimate string lengths */ /* overestimate string lengths */
(*mnt_pts)[array_index] = malloc(CP_EXE_LEN); (*mnt_pts)[array_index] = malloc(fd->COMPAT_CP_EXE_LEN);
assert((*mnt_pts)[array_index]); assert((*mnt_pts)[array_index]);
(*fs_types)[array_index] = malloc(CP_EXE_LEN); (*fs_types)[array_index] = malloc(fd->COMPAT_CP_EXE_LEN);
assert((*fs_types)[array_index]); assert((*fs_types)[array_index]);
ret = sscanf(++pos, "%" PRId64 "\t%s\t%s", &(*devs)[array_index], ret = sscanf(++pos, "%" PRId64 "\t%s\t%s", &(*devs)[array_index],
...@@ -670,8 +690,8 @@ int darshan_log_getexe(darshan_fd fd, char *buf) ...@@ -670,8 +690,8 @@ int darshan_log_getexe(darshan_fd fd, char *buf)
if(ret < 0) if(ret < 0)
return(ret); return(ret);
ret = darshan_log_read(fd, buf, (CP_EXE_LEN + 1)); ret = darshan_log_read(fd, buf, (fd->COMPAT_CP_EXE_LEN + 1));
if (ret < (CP_EXE_LEN + 1)) if (ret < (fd->COMPAT_CP_EXE_LEN + 1))
{ {
perror("darshan_log_read"); perror("darshan_log_read");
return(-1); return(-1);
...@@ -735,12 +755,20 @@ void darshan_log_close(darshan_fd file) ...@@ -735,12 +755,20 @@ void darshan_log_close(darshan_fd file)
*/ */
void darshan_log_print_version_warnings(struct darshan_job *job) void darshan_log_print_version_warnings(struct darshan_job *job)
{ {
if(strcmp(job->version_string, "2.03") == 0) if(strcmp(job->version_string, "2.04") == 0)
{ {
/* current version */ /* current version */
return; return;
} }
if(strcmp(job->version_string, "2.03") == 0)
{
/* no meaningful change to interpretation of log file, 2.04 just
* increased the header space available for annotations.
*/
return;
}
if(strcmp(job->version_string, "2.02") == 0) if(strcmp(job->version_string, "2.02") == 0)
{ {
printf("# WARNING: version 2.01 log format has the following limitations:\n"); printf("# WARNING: version 2.01 log format has the following limitations:\n");
...@@ -1199,7 +1227,7 @@ static int getjob_internal_200(darshan_fd file, struct darshan_job *job) ...@@ -1199,7 +1227,7 @@ static int getjob_internal_200(darshan_fd file, struct darshan_job *job)
return(-1); return(-1);
} }
static int getfile_internal_200(darshan_fd fd, struct darshan_job *job, static int getfile_internal_204(darshan_fd fd, struct darshan_job *job,
struct darshan_file *file) struct darshan_file *file)
{ {
int ret; int ret;
...@@ -1255,6 +1283,62 @@ static int getfile_internal_200(darshan_fd fd, struct darshan_job *job, ...@@ -1255,6 +1283,62 @@ static int getfile_internal_200(darshan_fd fd, struct darshan_job *job,
return(-1); return(-1);
} }
static int getfile_internal_200(darshan_fd fd, struct darshan_job *job,
struct darshan_file *file)
{
int ret;
const char* err_string;
int i;
if(fd->pos < CP_JOB_RECORD_SIZE_200)
{
ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE_200);
if(ret < 0)
return(ret);
}
/* reset file record, so that diff compares against a zero'd out record
* if file is missing
*/
memset(file, 0, sizeof(&file));
ret = darshan_log_read(fd, file, sizeof(*file));
if(ret == sizeof(*file))
{
/* got exactly one, correct size record */
if(fd->swap_flag)
{
/* swap bytes if necessary */
DARSHAN_BSWAP64(&file->hash);
DARSHAN_BSWAP64(&file->rank);
for(i=0; i<CP_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->counters[i]);
for(i=0; i<CP_F_NUM_INDICES; i++)
DARSHAN_BSWAP64(&file->fcounters[i]);
}
return(1);
}
if(ret > 0)
{
/* got a short read */
fprintf(stderr, "Error: invalid file record (too small)\n");
return(-1);
}
if(ret == 0)
{
/* hit end of file */
return(0);
}
/* all other errors */
err_string = darshan_log_error(fd, &ret);
fprintf(stderr, "Error: %s\n", err_string);
return(-1);
}
/* If we see version 1.24, assume that it is stored in big endian 32 bit /* If we see version 1.24, assume that it is stored in big endian 32 bit
* format. Convert up to current format. * format. Convert up to current format.
*/ */
...@@ -1386,9 +1470,9 @@ static int getfile_internal_1x(darshan_fd fd, struct darshan_job *job, ...@@ -1386,9 +1470,9 @@ static int getfile_internal_1x(darshan_fd fd, struct darshan_job *job,
/* set file pointer if this is the first file record; otherwise pick up /* set file pointer if this is the first file record; otherwise pick up
* where we left off last time * where we left off last time
*/ */
if(fd->pos < CP_JOB_RECORD_SIZE) if(fd->pos < CP_JOB_RECORD_SIZE_1x)
{ {
ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE); ret = darshan_log_seek(fd, CP_JOB_RECORD_SIZE_1x);
if(ret < 0) if(ret < 0)
return(ret); return(ret);
} }
......
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment