Skip to content

GitLab

Commit 7d61ab47 authored by Kevin Harms's avatar Kevin Harms
Browse files

Updates to the mysql loader. Commits transactions in largers sets and 

adds command line options for the needed items.


git-svn-id: https://svn.mcs.anl.gov/repos/darshan/trunk@331 3b7491f3-a168-0410-bf4b-c445ed680a29
parent 3986ae58
Hide whitespace changes
Inline Side-by-side
darshan-load-mysql.c
View file @ 7d61ab47
......@@ -6,6 +6,8 @@
#include <mysql.h>
#include <regex.h>
#include <sys/types.h>
#include <getopt.h>
#include <readline/readline.h>
#include "darshan-logutils.h"
......@@ -13,10 +15,16 @@
#define STOPWALK (1)
#define CONTWALK (0)
#define OPT_HOST (1)
#define OPT_USER (2)
#define OPT_PASS (3)
#define OPT_DB (4)
#define OPT_PATH (5)
const char *insert_job_fmt = "insert into %s values('%d','%s','%s','%s',\
'%d','%d','%d','%d')";
'%d','%ld','%ld','%d')";
const char *insert_mnt_fmt = "insert into %s values('%d','%d','%lld','%s','%s')";
const char *insert_file_fmt = "insert into %s values('%d','%d','%lld','%d',\
const char *insert_file_fmt = "insert into %s values('%d','%ld','%lld','%d',\
'%s',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
......@@ -43,7 +51,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
{
struct darshan_file file;
struct darshan_job job;
darshan_fd dfile;
darshan_fd dfile = NULL;
int ret;
int rc;
int nofiles;
......@@ -52,7 +60,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
char *dash;
char *username;
char *jobid;
char *sqlstmt;
char *sqlstmt = NULL;
int count;
int i;
int64_t *devs;
......@@ -60,6 +68,8 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
char **fstypes;
regex_t regex;
regmatch_t match[1];
char *filepath = NULL;
int prev_rank;
rc = CONTWALK;
count = 0;
......@@ -70,7 +80,17 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
sqlstmt = malloc(MAXSQL);
if (!sqlstmt)
{
return STOPWALK;
perror("malloc");
rc = STOPWALK;
goto exit;
}
filepath = strdup(fpath);
if (!filepath)
{
perror("strdup");
rc = STOPWALK;
goto exit;
}
/* Process Log Files */
......@@ -102,7 +122,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
goto exit;
}
base = basename(fpath);
base = basename(filepath);
username = base;
dash = index(base, '_');
*dash = '\0';
......@@ -141,7 +161,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
/*
* Insert Job Record
*/
snprintf(sqlstmt, MAXSQL, insert_job_fmt, "darshan_job_surveyor",
snprintf(sqlstmt, MAXSQL, insert_job_fmt, "darshan_job_intrepid",
atoi(jobid), username, job.version_string, exe, job.uid,
job.start_time, job.end_time, job.nprocs);
......@@ -149,8 +169,10 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
ret = mysql_query(mysql, sqlstmt);
if (ret)
{
fprintf(stderr, "log not processed: %s [mysql: %d (%s)\n",
fpath, mysql_errno(mysql), mysql_error(mysql));
fprintf(stderr, "log not processed: %s [mysql: %d (%s)] : \
jobid=%d start_time=%ld\n",
fpath, mysql_errno(mysql), mysql_error(mysql),
atoi(jobid), job.start_time);
rc = CONTWALK;
goto exit;
}
......@@ -169,7 +191,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
for (i=0; (i<count); i++)
{
snprintf(sqlstmt,MAXSQL,insert_mnt_fmt, "darshan_mountpoints_surveyor",
snprintf(sqlstmt,MAXSQL,insert_mnt_fmt, "darshan_mountpoints_intrepid",
atoi(jobid), job.start_time, lld(devs[i]), mnts[i], fstypes[i]);
if (debug) printf("sql: %s\n", sqlstmt);
......@@ -190,7 +212,18 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
{
while ((ret = darshan_log_getfile(dfile, &job, &file)) == 1)
{
snprintf(sqlstmt, MAXSQL, insert_file_fmt, "darshan_file_surveyor",
/*
* Work around issue where bogus file data was in older logs.
* Bogus data was files taht were 'stat'd but not opened.
*/
if (file.rank != -1 && file.rank < prev_rank)
{
continue;
}
if (file.rank != -1)
prev_rank = file.rank;
snprintf(sqlstmt, MAXSQL, insert_file_fmt, "darshan_file_intrepid",
atoi(jobid), job.start_time, file.hash, file.rank, file.name_suffix,
file.counters[CP_INDEP_OPENS],
file.counters[CP_COLL_OPENS],
......@@ -374,16 +407,14 @@ exit:
if (fstypes) free(fstypes);
}
ret = mysql_commit(mysql);
if (ret)
if (sqlstmt)
{
fprintf(stderr, "mysql: %d (%s)\n", mysql_errno(mysql),
mysql_error(mysql));
free(sqlstmt);
}
if (sqlstmt)
if (filepath)
{
free(sqlstmt);
free(filepath);
}
return rc;
......@@ -391,20 +422,55 @@ exit:
int main (int argc, char **argv)
{
const char *base;
const char *host = "db-host";
const char *user = "username";
const char *passwd = "password";
const char *db = "darshan";
int ret = 0;
if(argc != 2)
char base[256] = "";
char host[256] = "";
char user[256] = "";
char pass[256] = "";
char db[256] = "";
int ret = 0;
while(1)
{
fprintf(stderr, "Error: bad arguments.\n");
return(-1);
static struct option options[] = {
{"host", 1, NULL, OPT_HOST},
{"user", 1, NULL, OPT_USER},
{"pass", 1, NULL, OPT_PASS},
{"db", 1, NULL, OPT_DB},
{"path", 1, NULL, OPT_PATH},
{NULL, 0, NULL, 0}
};
int o = getopt_long(argc, argv, "", options, NULL);
if (o == -1) break;
switch(o)
{
case OPT_HOST:
strncpy(host, optarg, sizeof(host));
break;
case OPT_USER:
strncpy(user, optarg, sizeof(user));
break;
case OPT_PASS:
strncpy(pass, optarg, sizeof(pass));
break;
case OPT_DB:
strncpy(db, optarg, sizeof(db));
break;
case OPT_PATH:
strncpy(base, optarg, sizeof(base));
break;
}
}
base = argv[1];
if (strcmp(pass, "") == 0)
{
char *line = readline(NULL);
if (line)
{
strncpy(pass, line, sizeof(pass));
}
}
mysql = mysql_init(NULL);
if (mysql == NULL)
......@@ -413,13 +479,16 @@ int main (int argc, char **argv)
exit(-1);
}
mysql = mysql_real_connect(mysql, host, user, passwd, db, 0, NULL, 0);
mysql = mysql_real_connect(mysql, host, user, pass, db, 0, NULL, 0);
if (mysql == NULL)
{
fprintf(stderr, "mysql_real_connect");
exit(-1);
}
/* Turn off auto commits, hopefuly for performance sake */
(void) mysql_autocommit(mysql, 0);
ret = ftw(base, tree_walk, 512);
if(ret != 0)
{
......@@ -427,6 +496,13 @@ int main (int argc, char **argv)
return(-1);
}
ret = mysql_commit(mysql);
if (ret)
{
fprintf(stderr, "mysql: %d (%s)\n", mysql_errno(mysql),
mysql_error(mysql));
}
mysql_close(mysql);
return 0;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment