Commit 1ecd4cee authored by Kevin Harms's avatar Kevin Harms

Darshan utilities for loading log files in to a mysql database.


git-svn-id: https://svn.mcs.anl.gov/repos/darshan/trunk@157 3b7491f3-a168-0410-bf4b-c445ed680a29
parent a79aaa6c
#include <stdlib.h>
#include <stdio.h>
#include <string.h>
#include <ftw.h>
#include <libgen.h>
#include <mysql.h>
#include <regex.h>
#include <sys/types.h>
#include "darshan-logutils.h"
#define MAXSQL (1024*1024)
const char *insert_job_fmt = "insert into %s values('%d','%s','%s','%s',\
'%d','%d','%d','%d')";
const char *insert_mnt_fmt = "insert into %s values('%d','%d','%d','%s','%s')";
const char *insert_file_fmt = "insert into %s values('%d','%d','%lld','%d',\
'%s',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%.16lf','%.16lf','%.16lf','%.16lf','%.16lf',\
'%.16lf','%.16lf','%.16lf','%.16lf','%.16lf',\
'%.16lf','%.16lf','%.16lf','%.16lf')";
MYSQL *mysql = NULL;
int debug = 0;
int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
{
struct darshan_file file;
struct darshan_job job;
darshan_fd dfile;
int ret;
int nofiles;
char exe[1024];
char *base;
char *dash;
char *username;
char *jobid;
char *sqlstmt;
int count;
int i;
int *devs;
char **mnts;
char **fstypes;
regex_t regex;
regmatch_t match[1];
/* Only Process Files */
if (typeflag != FTW_F) return 0;
sqlstmt = malloc(MAXSQL);
if (!sqlstmt)
{
return -1;
}
/* Process Log Files */
dfile = darshan_log_open(fpath);
if (dfile == NULL)
{
perror("darshan_log_open");
return -1;
}
ret = darshan_log_getjob(dfile, &job);
if (ret < 0)
{
perror("darshan_log_getjob");
fprintf(stderr, "%s\n", fpath);
darshan_log_close(dfile);
return -1;
}
memset(exe, 0, sizeof(exe));
ret = darshan_log_getexe(dfile, exe, &nofiles);
if (ret < 0)
{
perror("darshan_log_getexe");
fprintf(stderr, "%s\n", fpath);
darshan_log_close(dfile);
return -1;
}
base = basename(fpath);
username = base;
dash = index(base, '_');
*dash = '\0';
jobid = dash+1;
/*
* Find jobid for log file name
*/
ret = regcomp(&regex, "_id[[:digit:]]+_", REG_EXTENDED);
if (ret != 0)
{
char buf[256];
regerror(ret, &regex, buf, sizeof(buf));
fprintf(stderr, "regcomp: %s\n", buf);
return -1;
}
ret = regexec(&regex, jobid, 1, match, 0);
if (ret != 0)
{
char buf[256];
regerror(ret, &regex, buf, sizeof(buf));
fprintf(stderr, "regexec: %s\n", buf);
return -1;
}
regfree(&regex);
dash = jobid;
jobid += (match[0].rm_so + 3);
dash += (match[0].rm_eo - 1);
*dash = 0;
/*
* Insert Job Record
*/
snprintf(sqlstmt, MAXSQL, insert_job_fmt, "darshan_job_surveyor",
atoi(jobid), username, job.version_string, exe, job.uid,
job.start_time, job.end_time, job.nprocs);
if (debug) printf("sql: %s\n", sqlstmt);
ret = mysql_query(mysql, sqlstmt);
if (ret)
{
fprintf(stderr, "log not processed: %s [mysql: %d (%s)\n",
fpath, mysql_errno(mysql), mysql_error(mysql));
goto exit;
}
/*
* Insert MountPoint Record (if present)
*/
ret = darshan_log_getmounts(dfile,&devs,&mnts,&fstypes,&count,&nofiles);
if (ret < 0)
{
perror("darshan_log_getmounts");
fprintf(stderr, "%s\n", fpath);
darshan_log_close(dfile);
return -1;
}
printf ("mounts: %d files: %d\n", count, nofiles);
for (i=0; (i<count); i++)
{
snprintf(sqlstmt,MAXSQL,insert_mnt_fmt, "darshan_mountpoints_surveyor",
atoi(jobid), job.start_time, devs[i], mnts[i], fstypes[i]);
if (debug) printf("sql: %s\n", sqlstmt);
ret = mysql_query(mysql, sqlstmt);
if (ret)
{
fprintf(stderr, "mysql: %d (%s)\n", mysql_errno(mysql),
mysql_error(mysql));
exit(1);
}
}
/*
* Insert File Records (if present)
*/
if (!nofiles)
{
while ((ret = darshan_log_getfile(dfile, &job, &file)) == 1)
{
snprintf(sqlstmt, MAXSQL, insert_file_fmt, "darshan_file_surveyor",
atoi(jobid), job.start_time, file.hash, file.rank, file.name_suffix,
file.counters[CP_INDEP_OPENS],
file.counters[CP_COLL_OPENS],
file.counters[CP_INDEP_READS],
file.counters[CP_INDEP_WRITES],
file.counters[CP_COLL_READS],
file.counters[CP_COLL_WRITES],
file.counters[CP_SPLIT_READS],
file.counters[CP_SPLIT_WRITES],
file.counters[CP_NB_READS],
file.counters[CP_NB_WRITES],
file.counters[CP_SYNCS],
file.counters[CP_POSIX_READS],
file.counters[CP_POSIX_WRITES],
file.counters[CP_POSIX_OPENS],
file.counters[CP_POSIX_SEEKS],
file.counters[CP_POSIX_STATS],
file.counters[CP_POSIX_MMAPS],
file.counters[CP_POSIX_FREADS],
file.counters[CP_POSIX_FWRITES],
file.counters[CP_POSIX_FOPENS],
file.counters[CP_POSIX_FSEEKS],
file.counters[CP_POSIX_FSYNCS],
file.counters[CP_POSIX_FDSYNCS],
file.counters[CP_INDEP_NC_OPENS],
file.counters[CP_COLL_NC_OPENS],
file.counters[CP_HDF5_OPENS],
file.counters[CP_COMBINER_NAMED],
file.counters[CP_COMBINER_DUP],
file.counters[CP_COMBINER_CONTIGUOUS],
file.counters[CP_COMBINER_VECTOR],
file.counters[CP_COMBINER_HVECTOR_INTEGER],
file.counters[CP_COMBINER_HVECTOR],
file.counters[CP_COMBINER_INDEXED],
file.counters[CP_COMBINER_HINDEXED_INTEGER],
file.counters[CP_COMBINER_HINDEXED],
file.counters[CP_COMBINER_INDEXED_BLOCK],
file.counters[CP_COMBINER_STRUCT_INTEGER],
file.counters[CP_COMBINER_STRUCT],
file.counters[CP_COMBINER_SUBARRAY],
file.counters[CP_COMBINER_DARRAY],
file.counters[CP_COMBINER_F90_REAL],
file.counters[CP_COMBINER_F90_COMPLEX],
file.counters[CP_COMBINER_F90_INTEGER],
file.counters[CP_COMBINER_RESIZED],
file.counters[CP_HINTS],
file.counters[CP_VIEWS],
file.counters[CP_MODE],
file.counters[CP_BYTES_READ],
file.counters[CP_BYTES_WRITTEN],
file.counters[CP_MAX_BYTE_READ],
file.counters[CP_MAX_BYTE_WRITTEN],
file.counters[CP_CONSEC_READS],
file.counters[CP_CONSEC_WRITES],
file.counters[CP_SEQ_READS],
file.counters[CP_SEQ_WRITES],
file.counters[CP_RW_SWITCHES],
file.counters[CP_MEM_NOT_ALIGNED],
file.counters[CP_MEM_ALIGNMENT],
file.counters[CP_FILE_NOT_ALIGNED],
file.counters[CP_FILE_ALIGNMENT],
file.counters[CP_MAX_READ_TIME_SIZE],
file.counters[CP_MAX_WRITE_TIME_SIZE],
file.counters[CP_SIZE_READ_0_100],
file.counters[CP_SIZE_READ_100_1K],
file.counters[CP_SIZE_READ_1K_10K],
file.counters[CP_SIZE_READ_10K_100K],
file.counters[CP_SIZE_READ_100K_1M],
file.counters[CP_SIZE_READ_1M_4M],
file.counters[CP_SIZE_READ_4M_10M],
file.counters[CP_SIZE_READ_10M_100M],
file.counters[CP_SIZE_READ_100M_1G],
file.counters[CP_SIZE_READ_1G_PLUS],
file.counters[CP_SIZE_WRITE_0_100],
file.counters[CP_SIZE_WRITE_100_1K],
file.counters[CP_SIZE_WRITE_1K_10K],
file.counters[CP_SIZE_WRITE_10K_100K],
file.counters[CP_SIZE_WRITE_100K_1M],
file.counters[CP_SIZE_WRITE_1M_4M],
file.counters[CP_SIZE_WRITE_4M_10M],
file.counters[CP_SIZE_WRITE_10M_100M],
file.counters[CP_SIZE_WRITE_100M_1G],
file.counters[CP_SIZE_WRITE_1G_PLUS],
file.counters[CP_SIZE_READ_AGG_0_100],
file.counters[CP_SIZE_READ_AGG_100_1K],
file.counters[CP_SIZE_READ_AGG_1K_10K],
file.counters[CP_SIZE_READ_AGG_10K_100K],
file.counters[CP_SIZE_READ_AGG_100K_1M],
file.counters[CP_SIZE_READ_AGG_1M_4M],
file.counters[CP_SIZE_READ_AGG_4M_10M],
file.counters[CP_SIZE_READ_AGG_10M_100M],
file.counters[CP_SIZE_READ_AGG_100M_1G],
file.counters[CP_SIZE_READ_AGG_1G_PLUS],
file.counters[CP_SIZE_WRITE_AGG_0_100],
file.counters[CP_SIZE_WRITE_AGG_100_1K],
file.counters[CP_SIZE_WRITE_AGG_1K_10K],
file.counters[CP_SIZE_WRITE_AGG_10K_100K],
file.counters[CP_SIZE_WRITE_AGG_100K_1M],
file.counters[CP_SIZE_WRITE_AGG_1M_4M],
file.counters[CP_SIZE_WRITE_AGG_4M_10M],
file.counters[CP_SIZE_WRITE_AGG_10M_100M],
file.counters[CP_SIZE_WRITE_AGG_100M_1G],
file.counters[CP_SIZE_WRITE_AGG_1G_PLUS],
file.counters[CP_EXTENT_READ_0_100],
file.counters[CP_EXTENT_READ_100_1K],
file.counters[CP_EXTENT_READ_1K_10K],
file.counters[CP_EXTENT_READ_10K_100K],
file.counters[CP_EXTENT_READ_100K_1M],
file.counters[CP_EXTENT_READ_1M_4M],
file.counters[CP_EXTENT_READ_4M_10M],
file.counters[CP_EXTENT_READ_10M_100M],
file.counters[CP_EXTENT_READ_100M_1G],
file.counters[CP_EXTENT_READ_1G_PLUS],
file.counters[CP_EXTENT_WRITE_0_100],
file.counters[CP_EXTENT_WRITE_100_1K],
file.counters[CP_EXTENT_WRITE_1K_10K],
file.counters[CP_EXTENT_WRITE_10K_100K],
file.counters[CP_EXTENT_WRITE_100K_1M],
file.counters[CP_EXTENT_WRITE_1M_4M],
file.counters[CP_EXTENT_WRITE_4M_10M],
file.counters[CP_EXTENT_WRITE_10M_100M],
file.counters[CP_EXTENT_WRITE_100M_1G],
file.counters[CP_EXTENT_WRITE_1G_PLUS],
file.counters[CP_STRIDE1_STRIDE],
file.counters[CP_STRIDE2_STRIDE],
file.counters[CP_STRIDE3_STRIDE],
file.counters[CP_STRIDE4_STRIDE],
file.counters[CP_STRIDE1_COUNT],
file.counters[CP_STRIDE2_COUNT],
file.counters[CP_STRIDE3_COUNT],
file.counters[CP_STRIDE4_COUNT],
file.counters[CP_ACCESS1_ACCESS],
file.counters[CP_ACCESS2_ACCESS],
file.counters[CP_ACCESS3_ACCESS],
file.counters[CP_ACCESS4_ACCESS],
file.counters[CP_ACCESS1_COUNT],
file.counters[CP_ACCESS2_COUNT],
file.counters[CP_ACCESS3_COUNT],
file.counters[CP_ACCESS4_COUNT],
file.counters[CP_DEVICE],
file.counters[CP_SIZE_AT_OPEN],
file.fcounters[CP_F_OPEN_TIMESTAMP],
file.fcounters[CP_F_READ_START_TIMESTAMP],
file.fcounters[CP_F_WRITE_START_TIMESTAMP],
file.fcounters[CP_F_CLOSE_TIMESTAMP],
file.fcounters[CP_F_READ_END_TIMESTAMP],
file.fcounters[CP_F_WRITE_END_TIMESTAMP],
file.fcounters[CP_F_POSIX_READ_TIME],
file.fcounters[CP_F_POSIX_WRITE_TIME],
file.fcounters[CP_F_POSIX_META_TIME],
file.fcounters[CP_F_MPI_META_TIME],
file.fcounters[CP_F_MPI_READ_TIME],
file.fcounters[CP_F_MPI_WRITE_TIME],
file.fcounters[CP_F_MAX_READ_TIME],
file.fcounters[CP_F_MAX_WRITE_TIME]);
if (debug) printf("sql: %s\n", sqlstmt);
ret = mysql_query(mysql, sqlstmt);
if (ret)
{
fprintf(stderr, "mysql: %d (%s)\n", mysql_errno(mysql),
mysql_error(mysql));
exit(1);
}
}
}
exit:
if (dfile) darshan_log_close(dfile);
if (count > 0)
{
for(i=0; i<count; i++)
{
free(mnts[i]);
free(fstypes[i]);
}
free(devs);
free(mnts);
free(fstypes);
}
ret = mysql_commit(mysql);
if (ret)
{
fprintf(stderr, "mysql: %d (%s)\n", mysql_errno(mysql),
mysql_error(mysql));
exit(1);
}
if (sqlstmt)
{
free(sqlstmt);
}
return 0;
}
int main (int argc, char **argv)
{
const char *base;
const char *host = "mgt6-mgmt.ether.alcf.anl.gov";
const char *user = "harms";
const char *passwd = "borken";
const char *db = "darshan";
int ret = 0;
if(argc != 2)
{
fprintf(stderr, "Error: bad arguments.\n");
return(-1);
}
base = argv[1];
mysql = mysql_init(NULL);
if (mysql == NULL)
{
fprintf(stderr, "mysql_init failed");
exit(-1);
}
mysql = mysql_real_connect(mysql, host, user, passwd, db, 0, NULL, 0);
if (mysql == NULL)
{
fprintf(stderr, "mysql_real_connect");
exit(-1);
}
ret = ftw(base, tree_walk, 512);
if(ret != 0)
{
fprintf(stderr, "Error: failed to walk path: %s\n", base);
return(-1);
}
mysql_close(mysql);
return 0;
}
#
# Darshan Database Descriptoin
#
#
# Darshan Job Records
#
create table darshan_job_surveyor
(
# Column Type Constraint Comment
jobid int unsigned not null comment 'Cobalt job id',
user varchar (8) not null comment 'username of user running the job',
version_string varchar (10) not null comment 'Darshan log version',
cmdline text not null comment 'Command line given to mpirun',
uid int not null comment 'Numerical user id',
start_time int not null comment 'Unix time job started running',
end_time int not null comment 'Unix time job stopped running',
nprocs int not null comment 'Number of processors job used',
# Table Contraints
constraint primary key(jobid, start_time)
);
#
# Darshan File Records
#
create table darshan_file_surveyor
(
# Column Type Constraint Comment
jobid int unsigned not null comment 'Cobalt job id',
start_time int not null comment 'Unix time job started running',
hash bigint not null comment 'Hash of path and filename',
rank bigint not null comment 'Rank of process that accessed file or -1 for all procs',
name_suffix varchar (11) not null comment 'Last 11 charaters of filename',
mpi_indep_opens bigint not null comment 'Count of MPI independent opens',
mpi_coll_opens bigint not null comment 'Count of MPI collective opens',
mpi_indep_reads bigint not null comment 'Count of independent MPI reads',
mpi_indep_writes bigint not null comment 'Count of independent MPI writes',
mpi_coll_reads bigint not null comment 'Count of collective MPI reads',
mpi_coll_writes bigint not null comment 'Count of collective MPI writes',
mpi_split_reads bigint not null comment 'Count of split reads',
mpi_split_writes bigint not null comment 'Count of split writes',
mpi_nb_reads bigint not null comment 'Count of nonblocking MPI reads',
mpi_nb_writes bigint not null comment 'Count of nonblocking MPI writes',
syncs bigint not null comment 'Count of syncs',
posix_reads bigint not null comment 'Count of posix reads',
posix_writes bigint not null comment 'Count of posix writes',
posix_opens bigint not null comment 'Count of posix opens',
posix_seeks bigint not null comment 'Count of posix seeks',
posix_stats bigint not null comment 'Count of posix stat/istat/fstats',
posix_mmaps bigint not null comment 'Count of posix mmaps',
posix_freads bigint not null comment 'Count of posix freads',
posix_fwrites bigint not null comment 'Count of posix fwrites',
posix_fopens bigint not null comment 'Count of posix fopens',
posix_fseeks bigint not null comment 'Count of posix fseeks',
posix_fsyncs bigint not null comment 'Count of posix fsyncs',
posix_fdsyncs bigint not null comment 'Count of posix fdsyncs',
nc_indep_opens bigint not null comment 'Count of NetCDF independent opens',
nc_coll_opens bigint not null comment 'Count of NetCDF collective opens',
hdf5_opens bigint not null comment 'Count of HDF5 opens',
mpi_combiner_named bigint not null comment 'Count of each MPI datatype category',
mpi_combiner_dup bigint not null comment 'Count of duped MPI datatypes',
mpi_combiner_contiguous bigint not null comment 'Count of contiguous MPI datatypes',
mpi_combiner_vector bigint not null comment 'Count of vectored MPI datatypes',
mpi_combiner_hvector_integer bigint not null comment 'Count of hvector MPI datatypes',
mpi_combiner_hvector bigint not null comment 'Count of hvector MPI datatypes',
mpi_combiner_indexed bigint not null comment 'Count of indexed MPI datatypes',
mpi_combiner_hindexed_integer bigint not null comment 'Count of hindex MPI datatypes',
mpi_combiner_hindexed bigint not null comment 'Count of hindex MPI datatypes',
mpi_combiner_indexed_block bigint not null comment 'Count of indexed block MPI datatypes',
mpi_combiner_struct_integer bigint not null comment 'Count of struct PMI datatypes',
mpi_combiner_struct bigint not null comment 'Count of struct MPI datatypes',
mpi_combiner_subarray bigint not null comment 'Count of subarray MPI datatypes',
mpi_combiner_darray bigint not null comment 'Count of darray MPI datatypes',
mpi_combiner_f90_real bigint not null comment 'Count of F90 real MPI datatypes',
mpi_combiner_f90_complex bigint not null comment 'Count of F90 complex MPI datatypes',
mpi_combiner_f90_integer bigint not null comment 'Count of F90 integer MPI datatypes',
mpi_combiner_resized bigint not null comment 'Count of resized MPI datatypes',
mpi_hints bigint not null comment 'Count of MPI hints used',
mpi_views bigint not null comment 'Count of MPI set view calls',
mode bigint not null comment 'Mode of file',
bytes_read bigint not null comment 'Total bytes read',
bytes_written bigint not null comment 'Total byets written',
max_byte_read bigint not null comment 'Highest offset byte read',
max_byte_written bigint not null comment 'Highest offset byte written',
consec_reads bigint not null comment 'Count of consecutive reads',
consec_writes bigint not null comment 'Count of consecutive writes',
seq_reads bigint not null comment 'Count of sequential reads',
seq_writes bigint not null comment 'Count of sequential writes',
rw_switches bigint not null comment 'Number of times switched between read and write',
mem_not_aligned bigint not null comment 'Count of accesses not mem aligned',
mem_alignment bigint not null comment 'Memory alignment in bytes',
file_not_aligned bigint not null comment 'Count of accesses not file aligned',
file_alignment bigint not null comment 'File alignment in bytes',
max_read_time_size bigint not null comment 'Size of read that took the maximum amount of time to complete',
max_write_time_size bigint not null comment 'Size of write that took the maximum amount of time to complete',
size_read_0_100 bigint not null comment 'Count of posix read size in range 0 <= x <= 100',
size_read_100_1K bigint not null comment 'Count of posix read size in range 100 < x <= 1k',
size_read_1K_10K bigint not null comment 'Count of posix read size in range 1k < x <= 10k',
size_read_10K_100K bigint not null comment 'Count of posix read size in range 10k < x <= 100k',
size_read_100K_1M bigint not null comment 'Count of posix read size in range 100k < x <= 1M',
size_read_1M_4M bigint not null comment 'Count of posix read size in range 1M < x <= 4M',
size_read_4M_10M bigint not null comment 'Count of posix read size in reange 4M < x <= 10M',
size_read_10M_100M bigint not null comment 'Count of posix read size in range 10M < x <= 100M',
size_read_100M_1G bigint not null comment 'Count of posix read size in range 100M < x <= 1G',
size_read_1G_plus bigint not null comment 'Count of posix read size in range 1G < x',
size_write_0_100 bigint not null comment 'Count of posix write sizes in range 0 <= x < 100',
size_write_100_1K bigint not null comment 'Count of posix write sizes in range 100 < x < 1k',
size_write_1K_10K bigint not null comment 'Count of posix write sizes in range 1k < x < 10k',
size_write_10K_100K bigint not null comment 'Count of posix write sizes in range 10k < x < 100k',
size_write_100K_1M bigint not null comment 'Count of posix write sizes in range 100k < x < 1M',
size_write_1M_4M bigint not null comment 'Count of posix write sizes in range 1M < x < 4M',
size_write_4M_10M bigint not null comment 'Count of posix write sizes in reange 4M < x < 10M',
size_write_10M_100M bigint not null comment 'Count of posix write sizes in range 10M < x < 100M',
size_write_100M_1G bigint not null comment 'Count of posix write sizes in range 100M < x < 1G',
size_write_1G_plus bigint not null comment 'Count of posix write sizes in range 1G < x',
size_read_agg_0_100 bigint not null comment 'Count of mpi read size in range 0 <= x < 100',
size_read_agg_100_1K bigint not null comment 'Count of mpi read size in range 100 < x < 1k',
size_read_agg_1K_10K bigint not null comment 'Count of mpi read size in range 1k < x < 10k',
size_read_agg_10K_100K bigint not null comment 'Count of mpi read size in range 10k < x < 100k',
size_read_agg_100K_1M bigint not null comment 'Count of mpi read size in range 100k < x < 1M',
size_read_agg_1M_4M bigint not null comment 'Count of mpi read size in range 1M < x < 4M',
size_read_agg_4M_10M bigint not null comment 'Count of mpi read size in reange 4M < x < 10M',
size_read_agg_10M_100M bigint not null comment 'Count of mpi read size in range 10M < x < 100M',
size_read_agg_100M_1G bigint not null comment 'Count of mpi read size in range 100M < x < 1G',
size_read_agg_1G_plus bigint not null comment 'Count of mpi read size in range 1G < x',
size_write_agg_0_100 bigint not null comment 'Count of mpi write sizes in range 0 <= x < 100',
size_write_agg_100_1K bigint not null comment 'Count of mpi write sizes in range 100 < x < 1k',
size_write_agg_1K_10K bigint not null comment 'Count of mpi write sizes in range 1k < x < 10k',
size_write_agg_10K_100K bigint not null comment 'Count of mpi write sizes in range 10k < x < 100k',
size_write_agg_100K_1M bigint not null comment 'Count of mpi write sizes in range 100k < x < 1M',
size_write_agg_1M_4M bigint not null comment 'Count of mpi write sizes in range 1M < x < 4M',
size_write_agg_4M_10M bigint not null comment 'Count of mpi write sizes in reange 4M < x < 10M',
size_write_agg_10M_100M bigint not null comment 'Count of mpi write sizes in range 10M < x < 100M',
size_write_agg_100M_1G bigint not null comment 'Count of mpi write sizes in range 100M < x < 1G',
size_write_agg_1G_plus bigint not null comment 'Count of mpi write sizes in range 1G < x',
extent_read_0_100 bigint not null comment 'Count of mpi read extents in range 0 <= x < 100',
extent_read_100_1K bigint not null comment 'Count of mpi read extents in range 100 < x < 1k',
extent_read_1K_10K bigint not null comment 'Count of mpi read extents in range 1k < x < 10k',
extent_read_10K_100K bigint not null comment 'Count of mpi read extents in range 10k < x < 100k',
extent_read_100K_1M bigint not null comment 'Count of mpi read extents in range 100k < x < 1M',
extent_read_1M_4M bigint not null comment 'Count of mpi read extents in range 1M < x < 4M',
extent_read_4M_10M bigint not null comment 'Count of mpi read extents in reange 4M < x < 10M',
extent_read_10M_100M bigint not null comment 'Count of mpi read extents in range 10M < x < 100M',
extent_read_100M_1G bigint not null comment 'Count of mpi read extents in range 100M < x < 1G',
extent_read_1G_plus bigint not null comment 'Count of mpi read extents in range 1G < x',
extent_write_0_100 bigint not null comment 'Count of mpi write extents in range 0 <= x < 100',
extent_write_100_1K bigint not null comment 'Count of mpi write extents in range 100 < x < 1k',
extent_write_1K_10K bigint not null comment 'Count of mpi write extents in range 1k < x < 10k',
extent_write_10K_100K bigint not null comment 'Count of mpi write extents in range 10k < x < 100k',
extent_write_100K_1M bigint not null comment 'Count of mpi write extents in range 100k < x < 1M',
extent_write_1M_4M bigint not null comment 'Count of mpi write extents in range 1M < x < 4M',
extent_write_4M_10M bigint not null comment 'Count of mpi write extents in reange 4M < x < 10M',
extent_write_10M_100M bigint not null comment 'Count of mpi write extents in range 10M < x < 100M',
extent_write_100M_1G bigint not null comment 'Count of mpi write extents in range 100M < x < 1G',
extent_write_1G_plus bigint not null comment 'Count of mpi write extents in range 1G < x',
write_stride1_stride bigint not null comment 'The most frequent strided access size',
write_stride2_stride bigint not null comment 'The 2nd most frequent strided access size',
write_stride3_stride bigint not null comment 'The 3rd most frequent strided access size',
write_stride4_stride bigint not null comment 'The 4th most frequent strided access size',
write_stride1_count bigint not null comment 'The count of the stride1 accesses',
write_stride2_count bigint not null comment 'The count of the stride2 accesses',
write_stride3_count bigint not null comment 'The count of the stride3 accesses',
write_stride4_count bigint not null comment 'The count of the stride4 accesses',
access1_access bigint not null comment 'The most frequent access size',
access2_access bigint not null comment 'The 2nd most frequent access size',
access3_access bigint not null comment 'The 3rd most frequent access size',
access4_access bigint not null comment 'The 4th most frequent access size',
access1_count bigint not null comment 'The count of access1 size',
access2_count bigint not null comment 'The count of access2 size',
access3_count bigint not null comment 'The count of access3 size',
access4_count bigint not null comment 'The count of access4 size',
device bigint not null comment 'Device of filesystem the file resides on',
size_at_open bigint not null comment 'File size at open',
open_timestamp double not null comment 'Timestamp of first open',
read_start_timestamp double not null comment 'Timestamp of first read',
write_start_timestamp double not null comment 'Timestamp of first write',
close_timestamp double not null comment 'Timestamp of last close',
read_end_timestamp double not null comment 'Timestamp of last read',
write_end_timestamp double not null comment 'Timestamp of last write',
posix_read_time double not null comment 'Cumulative posix read time (seconds)',
posix_write_time double not null comment 'Cumulative posix write time (seconds)',
posix_meta_time double not null comment 'Cumulative posix meta time (seconds)',
mpi_meta_time double not null comment 'Cumulative MPI meta time (seconds)',
mpi_read_time double not null comment 'Cumulative MPI read time (seconds)',
mpi_write_time double not null comment 'Cumulative MPI write time (seconds)',
max_read_time double not null comment 'Time of the maximum read (seconds)',
max_write_time double not null comment 'Time of the maximum write (seconds)',
# Table Contraints
constraint primary key(jobid, start_time, hash, rank),
constraint foreign key(jobid, start_time) references darshan_job_surveyor(jobid, start_time) on delete cascade
);
#
# Darshan Mount Points
# for determining which file used which file systems
#
create table darshan_mountpoints_surveyor
(
# Column Type Constraint Comment
jobid int unsigned not null comment 'Cobalt job id',
start_time int not null comment 'Unix time job started running',
device bigint not null comment 'Device of filesystem',
mountpoint text not null comment 'Mount point (string)',
fstype text not null comment 'File system type (string)',
# Table Contraints
constraint primary key(jobid, start_time, device),
constraint foreign key(jobid, start_time) references darshan_job_surveyor(jobid, start_time) on delete cascade
);
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment