Commit 76b3ca1f authored by Philip Carns's avatar Philip Carns

- added CP_SIZE_AT_OPEN metric (records size of file when opened)

- moved sync cost to cumulative write time rather than metadata time
- fixed more bugs in access size table in summary
- added sync to counter graph in summary


git-svn-id: https://svn.mcs.anl.gov/repos/darshan/trunk@144 3b7491f3-a168-0410-bf4b-c445ed680a29
parent 56368591
...@@ -178,6 +178,7 @@ enum darshan_indices ...@@ -178,6 +178,7 @@ enum darshan_indices
CP_ACCESS3_COUNT, CP_ACCESS3_COUNT,
CP_ACCESS4_COUNT, CP_ACCESS4_COUNT,
CP_DEVICE, /* device id reported by stat */ CP_DEVICE, /* device id reported by stat */
CP_SIZE_AT_OPEN,
CP_NUM_INDICES, CP_NUM_INDICES,
}; };
......
...@@ -150,6 +150,7 @@ char *darshan_names[] = { ...@@ -150,6 +150,7 @@ char *darshan_names[] = {
"CP_ACCESS3_COUNT", "CP_ACCESS3_COUNT",
"CP_ACCESS4_COUNT", "CP_ACCESS4_COUNT",
"CP_DEVICE", "CP_DEVICE",
"CP_SIZE_AT_OPEN",
"CP_NUM_INDICES" "CP_NUM_INDICES"
}; };
...@@ -178,9 +179,10 @@ char *darshan_f_names[] = { ...@@ -178,9 +179,10 @@ char *darshan_f_names[] = {
* *
* - added: * - added:
* - CP_DEVICE * - CP_DEVICE
* - CP_SIZE_AT_OPEN
* - changed params: * - changed params:
* - CP_FILE_RECORD_SIZE: 1240 to 1244 * - CP_FILE_RECORD_SIZE: 1240 to 1248
* - CP_NUM_INDICES: 138 to 139 * - CP_NUM_INDICES: 138 to 140
*/ */
#define CP_NUM_INDICES_1_22 138 #define CP_NUM_INDICES_1_22 138
struct darshan_file_1_22 struct darshan_file_1_22
...@@ -204,11 +206,12 @@ static void shift_missing_1_22(struct darshan_file* file); ...@@ -204,11 +206,12 @@ static void shift_missing_1_22(struct darshan_file* file);
* - CP_MAX_READ_TIME_SIZE * - CP_MAX_READ_TIME_SIZE
* - CP_MAX_WRITE_TIME_SIZE * - CP_MAX_WRITE_TIME_SIZE
* - CP_DEVICE * - CP_DEVICE
* - CP_SIZE_AT_OPEN
* - CP_F_MAX_READ_TIME * - CP_F_MAX_READ_TIME
* - CP_F_MAX_WRITE_TIME * - CP_F_MAX_WRITE_TIME
* - changed params: * - changed params:
* - CP_FILE_RECORD_SIZE: 1184 to 1244 * - CP_FILE_RECORD_SIZE: 1184 to 1248
* - CP_NUM_INDICES: 133 to 139 * - CP_NUM_INDICES: 133 to 140
* - CP_F_NUM_INDICES: 12 to 14 * - CP_F_NUM_INDICES: 12 to 14
* - so 60 bytes worth of new indices are the only difference * - so 60 bytes worth of new indices are the only difference
*/ */
...@@ -483,7 +486,9 @@ void darshan_log_print_version_warnings(struct darshan_job *job) ...@@ -483,7 +486,9 @@ void darshan_log_print_version_warnings(struct darshan_job *job)
{ {
printf("# WARNING: version 1.22 log format does not support the following parameters:\n"); printf("# WARNING: version 1.22 log format does not support the following parameters:\n");
printf("# CP_DEVICE\n"); printf("# CP_DEVICE\n");
printf("# It also does not record mounted file systems, mount points, or fs types.\n"); printf("# CP_SIZE_AT_OPEN\n");
printf("# It does not record mounted file systems, mount points, or fs types.\n");
printf("# It also attributes syncs to cumulative metadata time, rather than cumulative write time.\n");
return; return;
} }
...@@ -496,9 +501,11 @@ void darshan_log_print_version_warnings(struct darshan_job *job) ...@@ -496,9 +501,11 @@ void darshan_log_print_version_warnings(struct darshan_job *job)
printf("# CP_MAX_READ_TIME_SIZE\n"); printf("# CP_MAX_READ_TIME_SIZE\n");
printf("# CP_MAX_WRITE_TIME_SIZE\n"); printf("# CP_MAX_WRITE_TIME_SIZE\n");
printf("# CP_DEVICE\n"); printf("# CP_DEVICE\n");
printf("# CP_SIZE_AT_OPEN\n");
printf("# CP_F_MAX_READ_TIME\n"); printf("# CP_F_MAX_READ_TIME\n");
printf("# CP_F_MAX_WRITE_TIME\n"); printf("# CP_F_MAX_WRITE_TIME\n");
printf("# It also does not record mounted file systems, mount points, or fs types.\n"); printf("# It also does not record mounted file systems, mount points, or fs types.\n");
printf("# It also attributes syncs to cumulative metadata time, rather than cumulative write time.\n");
printf("#\n"); printf("#\n");
return; return;
} }
...@@ -523,6 +530,7 @@ static void shift_missing_1_21(struct darshan_file* file) ...@@ -523,6 +530,7 @@ static void shift_missing_1_21(struct darshan_file* file)
CP_MAX_READ_TIME_SIZE, CP_MAX_READ_TIME_SIZE,
CP_MAX_WRITE_TIME_SIZE, CP_MAX_WRITE_TIME_SIZE,
CP_DEVICE, CP_DEVICE,
CP_SIZE_AT_OPEN,
-1}; -1};
int missing_f_counters[] = { int missing_f_counters[] = {
CP_F_MAX_READ_TIME, CP_F_MAX_READ_TIME,
...@@ -574,6 +582,7 @@ static void shift_missing_1_22(struct darshan_file* file) ...@@ -574,6 +582,7 @@ static void shift_missing_1_22(struct darshan_file* file)
int c_index = 0; int c_index = 0;
int missing_counters[] = { int missing_counters[] = {
CP_DEVICE, CP_DEVICE,
CP_SIZE_AT_OPEN,
-1}; -1};
c_index = 0; c_index = 0;
......
...@@ -81,7 +81,7 @@ int main(int argc, char **argv) ...@@ -81,7 +81,7 @@ int main(int argc, char **argv)
/* print table of mounted file systems */ /* print table of mounted file systems */
ret = darshan_log_getmounts(file, &devs, &mnt_pts, &fs_types, &mount_count, ret = darshan_log_getmounts(file, &devs, &mnt_pts, &fs_types, &mount_count,
&no_files_flag); &no_files_flag);
printf("\n# mounted file systems (device, fs type, and mount point)\n"); printf("\n# mounted file systems (device, mount point, and fs type)\n");
printf("# -------------------------------------------------------\n"); printf("# -------------------------------------------------------\n");
for(i=0; i<mount_count; i++) for(i=0; i<mount_count; i++)
{ {
...@@ -135,6 +135,7 @@ int main(int argc, char **argv) ...@@ -135,6 +135,7 @@ int main(int argc, char **argv)
printf("# CP_ACCESS*_ACCESS: the four most common access sizes.\n"); printf("# CP_ACCESS*_ACCESS: the four most common access sizes.\n");
printf("# CP_ACCESS*_COUNT: count of the four most common access sizes.\n"); printf("# CP_ACCESS*_COUNT: count of the four most common access sizes.\n");
printf("# CP_DEVICE: device id reported by stat().\n"); printf("# CP_DEVICE: device id reported by stat().\n");
printf("# CP_SIZE_AT_OPEN: size of file when first opened.\n");
printf("# CP_F_OPEN_TIMESTAMP: timestamp of first open (mpi or posix).\n"); printf("# CP_F_OPEN_TIMESTAMP: timestamp of first open (mpi or posix).\n");
printf("# CP_F_*_START_TIMESTAMP: timestamp of first read/write (mpi or posix).\n"); printf("# CP_F_*_START_TIMESTAMP: timestamp of first read/write (mpi or posix).\n");
printf("# CP_F_*_END_TIMESTAMP: timestamp of last read/write (mpi or posix).\n"); printf("# CP_F_*_END_TIMESTAMP: timestamp of last read/write (mpi or posix).\n");
......
...@@ -551,7 +551,7 @@ int MPI_File_sync(MPI_File fh) ...@@ -551,7 +551,7 @@ int MPI_File_sync(MPI_File fh)
file = darshan_file_by_fh(fh); file = darshan_file_by_fh(fh);
if(file) if(file)
{ {
CP_F_INC(file, CP_F_MPI_META_TIME, (tm2-tm1)); CP_F_INC(file, CP_F_MPI_WRITE_TIME, (tm2-tm1));
CP_INC(file, CP_SYNCS, 1); CP_INC(file, CP_SYNCS, 1);
} }
CP_UNLOCK(); CP_UNLOCK();
...@@ -1317,8 +1317,9 @@ static void darshan_file_reduce(void* infile_v, ...@@ -1317,8 +1317,9 @@ static void darshan_file_reduce(void* infile_v,
inoutfile->counters[CP_MAX_READ_TIME_SIZE]; inoutfile->counters[CP_MAX_READ_TIME_SIZE];
} }
/* pick one device id */ /* pick one device id and file size */
tmp_file.counters[CP_DEVICE] = infile->counters[CP_DEVICE]; tmp_file.counters[CP_DEVICE] = infile->counters[CP_DEVICE];
tmp_file.counters[CP_SIZE_AT_OPEN] = infile->counters[CP_SIZE_AT_OPEN];
/* pick one name suffix */ /* pick one name suffix */
strcpy(tmp_file.name_suffix, infile->name_suffix); strcpy(tmp_file.name_suffix, infile->name_suffix);
......
...@@ -195,6 +195,7 @@ static void cp_access_counter(struct darshan_file_runtime* file, ssize_t size, ...@@ -195,6 +195,7 @@ static void cp_access_counter(struct darshan_file_runtime* file, ssize_t size,
if(fstat(file->fd, &cp_stat_buf) == 0) {\ if(fstat(file->fd, &cp_stat_buf) == 0) {\
CP_SET(file, CP_DEVICE, cp_stat_buf.st_dev); \ CP_SET(file, CP_DEVICE, cp_stat_buf.st_dev); \
CP_SET(file, CP_FILE_ALIGNMENT, cp_stat_buf.st_blksize); \ CP_SET(file, CP_FILE_ALIGNMENT, cp_stat_buf.st_blksize); \
CP_SET(file, CP_SIZE_AT_OPEN, cp_stat_buf.st_size); \
}\ }\
}\ }\
file->log_file->rank = my_rank; \ file->log_file->rank = my_rank; \
...@@ -326,7 +327,7 @@ int __wrap_fsync(int fd) ...@@ -326,7 +327,7 @@ int __wrap_fsync(int fd)
file = darshan_file_by_fd(fd); file = darshan_file_by_fd(fd);
if(file) if(file)
{ {
CP_F_INC(file, CP_F_POSIX_META_TIME, (tm2-tm1)); CP_F_INC(file, CP_F_POSIX_WRITE_TIME, (tm2-tm1));
CP_INC(file, CP_POSIX_FSYNCS, 1); CP_INC(file, CP_POSIX_FSYNCS, 1);
} }
CP_UNLOCK(); CP_UNLOCK();
...@@ -350,7 +351,7 @@ int __wrap_fdatasync(int fd) ...@@ -350,7 +351,7 @@ int __wrap_fdatasync(int fd)
file = darshan_file_by_fd(fd); file = darshan_file_by_fd(fd);
if(file) if(file)
{ {
CP_F_INC(file, CP_F_POSIX_META_TIME, (tm2-tm1)); CP_F_INC(file, CP_F_POSIX_WRITE_TIME, (tm2-tm1));
CP_INC(file, CP_POSIX_FDSYNCS, 1); CP_INC(file, CP_POSIX_FDSYNCS, 1);
} }
CP_UNLOCK(); CP_UNLOCK();
...@@ -1078,6 +1079,7 @@ void darshan_condense(void) ...@@ -1078,6 +1079,7 @@ void darshan_condense(void)
/* pick one */ /* pick one */
case CP_DEVICE: case CP_DEVICE:
case CP_SIZE_AT_OPEN:
CP_SET(base_file, i, CP_VALUE(iter_file, i)); CP_SET(base_file, i, CP_VALUE(iter_file, i));
break; break;
......
...@@ -97,6 +97,7 @@ while ($line = <TRACE>) { ...@@ -97,6 +97,7 @@ while ($line = <TRACE>) {
open(COUNTS, ">$tmp_dir/counts.dat") || die("error opening output file: $!\n"); open(COUNTS, ">$tmp_dir/counts.dat") || die("error opening output file: $!\n");
print COUNTS "# P=POSIX, MI=MPI-IO indep., MC=MPI-IO coll., R=read, W=write\n"; print COUNTS "# P=POSIX, MI=MPI-IO indep., MC=MPI-IO coll., R=read, W=write\n";
print COUNTS "# PR, MIR, MCR, PW, MIW, MCW, Popen, Pseek, Pstat\n"; print COUNTS "# PR, MIR, MCR, PW, MIW, MCW, Popen, Pseek, Pstat\n";
my $total_syncs = $summary{CP_POSIX_FSYNCS} + $summary{CP_POSIX_FDSYNCS};
print COUNTS "Read, ", $summary{CP_POSIX_READS}, ", ", print COUNTS "Read, ", $summary{CP_POSIX_READS}, ", ",
$summary{CP_INDEP_READS}, ", ", $summary{CP_COLL_READS}, "\n", $summary{CP_INDEP_READS}, ", ", $summary{CP_COLL_READS}, "\n",
"Write, ", $summary{CP_POSIX_WRITES}, ", ", "Write, ", $summary{CP_POSIX_WRITES}, ", ",
...@@ -105,7 +106,8 @@ print COUNTS "Read, ", $summary{CP_POSIX_READS}, ", ", ...@@ -105,7 +106,8 @@ print COUNTS "Read, ", $summary{CP_POSIX_READS}, ", ",
$summary{CP_COLL_OPENS}, "\n", $summary{CP_COLL_OPENS}, "\n",
"Stat, ", $summary{CP_POSIX_STATS}, ", 0, 0\n", "Stat, ", $summary{CP_POSIX_STATS}, ", 0, 0\n",
"Seek, ", $summary{CP_POSIX_SEEKS}, ", 0, 0\n", "Seek, ", $summary{CP_POSIX_SEEKS}, ", 0, 0\n",
"Mmap, ", $summary{CP_POSIX_MMAPS}, ", 0, 0\n"; "Mmap, ", $summary{CP_POSIX_MMAPS}, ", 0, 0\n",
"Fsync, ", $total_syncs, ", 0, 0\n";
close COUNTS; close COUNTS;
# histograms of reads and writes # histograms of reads and writes
...@@ -259,8 +261,8 @@ access size \& count \\\\ ...@@ -259,8 +261,8 @@ access size \& count \\\\
# sort access sizes (descending) # sort access sizes (descending)
my $i = 0; my $i = 0;
foreach $value (sort {$access_hash{$b} cmp $access_hash{$a} } keys %access_hash) { foreach $value (sort {$access_hash{$b} <=> $access_hash{$a} } keys %access_hash) {
if($i == 3) { if($i == 4) {
last; last;
} }
if($access_hash{$value} == 0) { if($access_hash{$value} == 0) {
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment