Commit 7d61ab47 authored by Kevin Harms's avatar Kevin Harms
Browse files

Updates to the mysql loader. Commits transactions in largers sets and

adds command line options for the needed items.


git-svn-id: https://svn.mcs.anl.gov/repos/darshan/trunk@331 3b7491f3-a168-0410-bf4b-c445ed680a29
parent 3986ae58
...@@ -6,6 +6,8 @@ ...@@ -6,6 +6,8 @@
#include <mysql.h> #include <mysql.h>
#include <regex.h> #include <regex.h>
#include <sys/types.h> #include <sys/types.h>
#include <getopt.h>
#include <readline/readline.h>
#include "darshan-logutils.h" #include "darshan-logutils.h"
...@@ -13,10 +15,16 @@ ...@@ -13,10 +15,16 @@
#define STOPWALK (1) #define STOPWALK (1)
#define CONTWALK (0) #define CONTWALK (0)
#define OPT_HOST (1)
#define OPT_USER (2)
#define OPT_PASS (3)
#define OPT_DB (4)
#define OPT_PATH (5)
const char *insert_job_fmt = "insert into %s values('%d','%s','%s','%s',\ const char *insert_job_fmt = "insert into %s values('%d','%s','%s','%s',\
'%d','%d','%d','%d')"; '%d','%ld','%ld','%d')";
const char *insert_mnt_fmt = "insert into %s values('%d','%d','%lld','%s','%s')"; const char *insert_mnt_fmt = "insert into %s values('%d','%d','%lld','%s','%s')";
const char *insert_file_fmt = "insert into %s values('%d','%d','%lld','%d',\ const char *insert_file_fmt = "insert into %s values('%d','%ld','%lld','%d',\
'%s',\ '%s',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\ '%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
'%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\ '%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld','%lld',\
...@@ -43,7 +51,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag) ...@@ -43,7 +51,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
{ {
struct darshan_file file; struct darshan_file file;
struct darshan_job job; struct darshan_job job;
darshan_fd dfile; darshan_fd dfile = NULL;
int ret; int ret;
int rc; int rc;
int nofiles; int nofiles;
...@@ -52,7 +60,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag) ...@@ -52,7 +60,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
char *dash; char *dash;
char *username; char *username;
char *jobid; char *jobid;
char *sqlstmt; char *sqlstmt = NULL;
int count; int count;
int i; int i;
int64_t *devs; int64_t *devs;
...@@ -60,6 +68,8 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag) ...@@ -60,6 +68,8 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
char **fstypes; char **fstypes;
regex_t regex; regex_t regex;
regmatch_t match[1]; regmatch_t match[1];
char *filepath = NULL;
int prev_rank;
rc = CONTWALK; rc = CONTWALK;
count = 0; count = 0;
...@@ -70,7 +80,17 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag) ...@@ -70,7 +80,17 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
sqlstmt = malloc(MAXSQL); sqlstmt = malloc(MAXSQL);
if (!sqlstmt) if (!sqlstmt)
{ {
return STOPWALK; perror("malloc");
rc = STOPWALK;
goto exit;
}
filepath = strdup(fpath);
if (!filepath)
{
perror("strdup");
rc = STOPWALK;
goto exit;
} }
/* Process Log Files */ /* Process Log Files */
...@@ -102,7 +122,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag) ...@@ -102,7 +122,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
goto exit; goto exit;
} }
base = basename(fpath); base = basename(filepath);
username = base; username = base;
dash = index(base, '_'); dash = index(base, '_');
*dash = '\0'; *dash = '\0';
...@@ -141,7 +161,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag) ...@@ -141,7 +161,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
/* /*
* Insert Job Record * Insert Job Record
*/ */
snprintf(sqlstmt, MAXSQL, insert_job_fmt, "darshan_job_surveyor", snprintf(sqlstmt, MAXSQL, insert_job_fmt, "darshan_job_intrepid",
atoi(jobid), username, job.version_string, exe, job.uid, atoi(jobid), username, job.version_string, exe, job.uid,
job.start_time, job.end_time, job.nprocs); job.start_time, job.end_time, job.nprocs);
...@@ -149,8 +169,10 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag) ...@@ -149,8 +169,10 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
ret = mysql_query(mysql, sqlstmt); ret = mysql_query(mysql, sqlstmt);
if (ret) if (ret)
{ {
fprintf(stderr, "log not processed: %s [mysql: %d (%s)\n", fprintf(stderr, "log not processed: %s [mysql: %d (%s)] : \
fpath, mysql_errno(mysql), mysql_error(mysql)); jobid=%d start_time=%ld\n",
fpath, mysql_errno(mysql), mysql_error(mysql),
atoi(jobid), job.start_time);
rc = CONTWALK; rc = CONTWALK;
goto exit; goto exit;
} }
...@@ -169,7 +191,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag) ...@@ -169,7 +191,7 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
for (i=0; (i<count); i++) for (i=0; (i<count); i++)
{ {
snprintf(sqlstmt,MAXSQL,insert_mnt_fmt, "darshan_mountpoints_surveyor", snprintf(sqlstmt,MAXSQL,insert_mnt_fmt, "darshan_mountpoints_intrepid",
atoi(jobid), job.start_time, lld(devs[i]), mnts[i], fstypes[i]); atoi(jobid), job.start_time, lld(devs[i]), mnts[i], fstypes[i]);
if (debug) printf("sql: %s\n", sqlstmt); if (debug) printf("sql: %s\n", sqlstmt);
...@@ -190,7 +212,18 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag) ...@@ -190,7 +212,18 @@ int tree_walk (const char *fpath, const struct stat *sb, int typeflag)
{ {
while ((ret = darshan_log_getfile(dfile, &job, &file)) == 1) while ((ret = darshan_log_getfile(dfile, &job, &file)) == 1)
{ {
snprintf(sqlstmt, MAXSQL, insert_file_fmt, "darshan_file_surveyor", /*
* Work around issue where bogus file data was in older logs.
* Bogus data was files taht were 'stat'd but not opened.
*/
if (file.rank != -1 && file.rank < prev_rank)
{
continue;
}
if (file.rank != -1)
prev_rank = file.rank;
snprintf(sqlstmt, MAXSQL, insert_file_fmt, "darshan_file_intrepid",
atoi(jobid), job.start_time, file.hash, file.rank, file.name_suffix, atoi(jobid), job.start_time, file.hash, file.rank, file.name_suffix,
file.counters[CP_INDEP_OPENS], file.counters[CP_INDEP_OPENS],
file.counters[CP_COLL_OPENS], file.counters[CP_COLL_OPENS],
...@@ -374,16 +407,14 @@ exit: ...@@ -374,16 +407,14 @@ exit:
if (fstypes) free(fstypes); if (fstypes) free(fstypes);
} }
ret = mysql_commit(mysql); if (sqlstmt)
if (ret)
{ {
fprintf(stderr, "mysql: %d (%s)\n", mysql_errno(mysql), free(sqlstmt);
mysql_error(mysql));
} }
if (sqlstmt) if (filepath)
{ {
free(sqlstmt); free(filepath);
} }
return rc; return rc;
...@@ -391,20 +422,55 @@ exit: ...@@ -391,20 +422,55 @@ exit:
int main (int argc, char **argv) int main (int argc, char **argv)
{ {
const char *base; char base[256] = "";
const char *host = "db-host"; char host[256] = "";
const char *user = "username"; char user[256] = "";
const char *passwd = "password"; char pass[256] = "";
const char *db = "darshan"; char db[256] = "";
int ret = 0; int ret = 0;
if(argc != 2) while(1)
{ {
fprintf(stderr, "Error: bad arguments.\n"); static struct option options[] = {
return(-1); {"host", 1, NULL, OPT_HOST},
{"user", 1, NULL, OPT_USER},
{"pass", 1, NULL, OPT_PASS},
{"db", 1, NULL, OPT_DB},
{"path", 1, NULL, OPT_PATH},
{NULL, 0, NULL, 0}
};
int o = getopt_long(argc, argv, "", options, NULL);
if (o == -1) break;
switch(o)
{
case OPT_HOST:
strncpy(host, optarg, sizeof(host));
break;
case OPT_USER:
strncpy(user, optarg, sizeof(user));
break;
case OPT_PASS:
strncpy(pass, optarg, sizeof(pass));
break;
case OPT_DB:
strncpy(db, optarg, sizeof(db));
break;
case OPT_PATH:
strncpy(base, optarg, sizeof(base));
break;
}
} }
base = argv[1]; if (strcmp(pass, "") == 0)
{
char *line = readline(NULL);
if (line)
{
strncpy(pass, line, sizeof(pass));
}
}
mysql = mysql_init(NULL); mysql = mysql_init(NULL);
if (mysql == NULL) if (mysql == NULL)
...@@ -413,13 +479,16 @@ int main (int argc, char **argv) ...@@ -413,13 +479,16 @@ int main (int argc, char **argv)
exit(-1); exit(-1);
} }
mysql = mysql_real_connect(mysql, host, user, passwd, db, 0, NULL, 0); mysql = mysql_real_connect(mysql, host, user, pass, db, 0, NULL, 0);
if (mysql == NULL) if (mysql == NULL)
{ {
fprintf(stderr, "mysql_real_connect"); fprintf(stderr, "mysql_real_connect");
exit(-1); exit(-1);
} }
/* Turn off auto commits, hopefuly for performance sake */
(void) mysql_autocommit(mysql, 0);
ret = ftw(base, tree_walk, 512); ret = ftw(base, tree_walk, 512);
if(ret != 0) if(ret != 0)
{ {
...@@ -427,6 +496,13 @@ int main (int argc, char **argv) ...@@ -427,6 +496,13 @@ int main (int argc, char **argv)
return(-1); return(-1);
} }
ret = mysql_commit(mysql);
if (ret)
{
fprintf(stderr, "mysql: %d (%s)\n", mysql_errno(mysql),
mysql_error(mysql));
}
mysql_close(mysql); mysql_close(mysql);
return 0; return 0;
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment