initial commit of base code from CODES repo

- simplified configure.ac
- moved code from src/common to src directory
- misc minor fixes

Make.rules

+# flex & bison deps
+#
+%.c %.h: %.l
+	$(AM_V_GEN)$(LEX) --header-file=$(@:.c=.h) -o $(@:.h=.c) $< \
+	   || ( $(RM) $(basename $@).h $(basename $@).c ; exit 1) 
+
+%.c %.h: %.y
+	@test "x$(bison_ok)" != "yes" || echo "*** WARNING *** Bison version might be too old"
+	$(AM_V_GEN)$(YACC) --defines=$(@:.c=.h) -o $(@:.h=.c)  $< \
+	   || ( $(RM) $(basename $@).h $(basename $@).c ; exit 1)
+
+
+
+# %.y: %.y.in Makefile
+#	$(AM_V_GEN)$(SED) -e 's,[@]CODES_PURE_PARSER_DEFINES[@],$(CODES_PURE_PARSER_DEFINES),g' \
+#	-e 's,[@]CODES_PUSH_PARSER_DEFINES[@],$(CODES_PUSH_PARSER_DEFINES),g' \
+#	< src/common/iokernellang/codesparser.y.in > src/common/iokernellang/codesparser.y
+
+#
+# Output dist version
+#
+.phony: distversion
+distversion:
+	@echo $(VERSION)
+
+#
+# Easy way to build unit tests without running them
+# 
+.phony: tests
+tests: $(check_PROGRAMS)

Makefile.am

+AUTOMAKE_OPTIONS = foreign
+ACLOCAL_AMFLAGS = -I m4
+
+bin_PROGRAMS =
+bin_SCRIPTS =
+noinst_LIBRARIES =
+noinst_PROGRAMS =
+lib_LIBRARIES =
+noinst_HEADERS =
+TESTS =
+check_PROGRAMS =
+EXTRA_PROGRAMS =
+CLEANFILES = $(bin_SCRIPTS)
+EXTRA_DIST =
+BUILT_SOURCES =
+src_models_CESOURCES =
+
+
+EXTRA_DIST += prepare.sh
+
+AM_CPPFLAGS = -I$(top_srcdir)/src ${ROSS_CPPFLAGS}
+
+AM_CFLAGS = ${ROSS_CFLAGS}
+
+AM_LIBS = ${ROSS_LIBS}
+
+AM_CXXFLAGS = $(AM_CFLAGS)
+
+include Make.rules
+
+include $(top_srcdir)/src/Makefile.subdir

README.txt

+NOTE: see bottom of this file for suggested configurations on particular ANL
+machines.
+
+0 - Checkout, build, and install the trunk version of ROSS
+
+    mkdir ross
+    cd ross
+    svn co https://subversion.cs.rpi.edu/svn/rossnet/trunk
+    mkdir build
+    cd build
+    ARCH=x86_64 CC=mpicc CXX=mpicxx cmake -DCMAKE_INSTALL_PREFIX=../install ../trunk
+    make -j 3
+    make install
+
+    <the result should be that the trunk version of ROSS is installed in the 
+    ross/install/ directory>
+
+1 - If this is the first time you are building codes-base, run
+
+    ./prepare.sh
+
+2- Configure codes-base. This can be done in the source directory or in a
+   dedicated build directory if you prefer out-of-tree builds.
+
+    ./configure --with-ross=/path/to/ross/install
+
+3 - Build codes-base
+
+    make clean && make
+
+4 - codes-base uses flex and bison (or lex and yacc) to generate several parsers.
+These tools auto-generate C source files. To remove all of the autogenerated
+files for these parsers, execute
+
+    make maintainer-clean-local
+
+
+Machine-specific configurations:
+----------------
+
+- Fusion (ANL): add the following keys to your ~/.soft file and run "resoft"
+  prior to following the steps described in this file:
+
+  +python-2.7.3
+  +autoconf-2.68
+  +git
+  +cmake

configure.ac

+#                                               -*- Autoconf -*-
+# Process this file with autoconf to produce a configure script.
+
+AC_PREREQ([2.67])
+AC_INIT([codes-base], [0.2], [http://trac.mcs.anl.gov/projects/codes/newticket],[],[http://trac.mcs.anl.gov/projects/codes/wiki])
+
+AC_CANONICAL_TARGET
+AC_CANONICAL_SYSTEM
+AC_CANONICAL_HOST
+
+AM_INIT_AUTOMAKE([foreign subdir-objects -Wall])
+
+m4_ifdef([AM_SILENT_RULES], [AM_SILENT_RULES([yes])])
+
+AC_CONFIG_SRCDIR([README.txt])
+AC_CONFIG_HEADERS([codes_config.h])
+
+AX_PROG_BISON([],[AC_MSG_WARN([Could not find bison])])
+AX_PROG_FLEX([],[AC_MSG_WARN([Could not find flex])])
+AC_SUBST([BISON])
+AC_SUBST([FLEX])
+
+# Checks for programs.
+AC_PROG_CC
+AM_PROG_CC_C_O
+AC_PROG_CXX
+AC_PROG_CXXCPP
+AC_PROG_RANLIB
+
+AC_REQUIRE_CPP
+
+# Checks for header files.
+AC_HEADER_STDC
+AC_CHECK_HEADERS([stdlib.h string.h unistd.h execinfo.h pthread.h malloc.h])
+
+# Checks for typedefs, structures, and compiler characteristics.
+AC_C_CONST
+AC_C_INLINE
+AC_TYPE_INT8_T
+AC_TYPE_INT16_T
+AC_TYPE_INT32_T
+AC_TYPE_INT64_T
+AC_TYPE_UINT8_T
+AC_TYPE_UINT16_T
+AC_TYPE_UINT32_T
+AC_TYPE_UINT64_T
+AC_TYPE_SIZE_T
+AC_TYPE_SSIZE_T
+
+# Checks for library functions.
+AC_CHECK_FUNCS([memset])
+AC_CHECK_LIB([pthread],[pthread_create],,[AC_MSG_ERROR([Could not find pthread_create!])])
+AC_CHECK_LIB([m],[sqrt],,[AC_MSG_ERROR([Could not find sqrt!])])
+
+
+AX_PROG_BISON_CLFEATURES([],[AC_MSG_WARN([Could not find bison])],
+[bison_ok="yes"], [bison_ok="no"])
+AC_SUBST([bison_ok])
+
+dnl Check to see if CC is an MPI compiler
+AC_MSG_CHECKING(whether the mpicc compiler works)
+AC_TRY_COMPILE([#include <mpi.h>], [int ret = MPI_Init(0, (void*)0)],
+    AC_MSG_RESULT(yes),
+    AC_MSG_RESULT(no)
+    AC_MSG_ERROR(CC doesn't appear to be a valid MPI compiler.  See INSTALL document or try adding CC=mpicc to your configure command line.)
+)
+
+#
+# clear ROSS options
+#
+ROSS_CFLAGS=""
+ROSS_CPPFLAGS=""
+ROSS_LDFLAGS=""
+ROSS_LIBS=""
+ENABLE_ROSS=0
+AC_ARG_WITH([ross],[AS_HELP_STRING([--with-ross@<:@=DIR@:>@],
+                        [location of ROSS installation])])
+#
+# if we have ROSS, add ROSS flags
+#
+if test "x${with_ross}" != "x" ; then
+    ORIG_LDFLAGS="${LDFLAGS}"
+    ORIG_CPPFLAGS="${CPPFLAGS}"
+    ORIG_LIBS="${LIBS}"
+    ORIG_CFLAGS="${CFLAGS}"
+
+    checkpath="${with_ross}/bin"
+    ROSS_CONFIG=""
+    AC_CHECK_PROGS(ROSS_CONFIG, [ross-config], [NOT_FOUND], [$checkpath])
+    if test "${ROSS_CONFIG}" = "NOT_FOUND" ; then 
+        AC_MSG_ERROR(ROSS not found at: $with_ross)
+    fi
+
+    if test "${ROSS_CONFIG}x" != "x" ; then
+    
+        LDFLAGS=`${with_ross}/bin/ross-config --ldflags`
+        CFLAGS=`${with_ross}/bin/ross-config --cflags`
+        CPPFLAGS=`${with_ross}/bin/ross-config --cflags`
+        LIBS=`${with_ross}/bin/ross-config --libs`
+
+        ENABLE_ROSS=1
+        AC_CHECK_LIB([ROSS],[tw_init],,[ENABLE_ROSS=;])
+        if test -z "$ENABLE_ROSS" ; then
+            ENABLE_ROSS=0
+            AC_MSG_ERROR([Could not find usable ROSS library])
+        fi
+
+        ENABLE_ROSS=1
+
+        #
+        # TODO this is weird... need to add -c back to CPPFLAGS or CHECK_HEADERS
+        # tries to compile an exe using a c file that includes ross.h and no
+        # main()
+        #
+        CPPFLAGS_2="${CPPFLAGS}"
+        CPPFLAGS="-c ${CPPFLAGS}"
+        AC_CHECK_HEADERS([ross.h], [], [ENABLE_ROSS=;])
+        if test -z "$ENABLE_ROSS" ; then
+            ENABLE_ROSS=0
+            AC_MSG_ERROR([Could not find usable ross.h])
+        else
+            CPPFLAGS="${CPPFLAGS_2}"
+            checkpath="$PATH"
+            AC_SUBST([ROSS_LDFLAGS],["${LDFLAGS}"])
+            AC_SUBST([ROSS_CFLAGS],["${CFLAGS}"])
+            AC_SUBST([ROSS_CPPFLAGS],["${CPPFLAGS}"])
+            AC_SUBST([ROSS_LIBS],["${LIBS}"])
+        fi
+        CFLAGS="${ORIG_CFLAGS}"
+        CPPFLAGS="${ORIG_CPPFLAGS}"
+        LDFLAGS="${ORIG_LDFLAGS}"
+        LIBS="${ORIG_LIBS}"
+
+        AC_DEFINE([HAVE_ROSS],[1],[If ROSS is supported on target build platform])
+    fi
+else
+    ENABLE_ROSS=0
+    AC_MSG_ERROR([Could not find usable ROSS installation, try --with-ross=DIR])
+fi
+
+#
+# To reactivate rewriting of the codesparser.y file,
+# uncomment next line.
+#
+# NOTE: this will force the latest bison for building,
+# even when building from a dist.
+#
+#
+dnl AC_CONFIG_FILES([src/iokernellang/codesparser.y])
+
+AC_CONFIG_FILES([Makefile])
+
+AC_OUTPUT
+

prepare.sh

+#!/bin/sh
+
+echo "Regenerating build files..."
+autoreconf -fi -Im4

src/Makefile.subdir

+lib_LIBRARIES += src/libcodesmodelconfig.a \
+                 src/libcodesutil.a \
+                 src/libcodeslogging.a
+BUILT_SOURCES += src/modelconfig/configparser.h \
+                 src/modelconfig/configlex.h
+
+EXTRA_DIST += src/iokernellang/codesparser.y.in \
+              src/iokernellang/codeslexer.l \
+              src/modelconfig/configparser.y \
+              src/modelconfig/configlex.l \
+              src/util/quicklist.h \
+              src/util/quickhash.h
+
+PHONY: maintainer-clean-local
+maintainer-clean-local::
+	-rm -f src/iokernellang/codesparser.c
+	-rm -f src/iokernellang/codesparser.h
+	-rm -f src/iokernellang/codesparser.y
+	-rm -f src/iokernellang/codeslexer.c
+	-rm -f src/iokernellang/codeslexer.h
+	-rm -f src/modelconfig/configlex.h
+	-rm -f src/modelconfig/configlex.c
+	-rm -f src/modelconfig/configparser.h
+	-rm -f src/modelconfig/configparser.c
+
+
+src/modelconfig/configparser.h: src/modelconfig/configparser.y
+src/modelconfig/configlex.h: src/modelconfig/configlex.l
+
+lib_LIBRARIES += src/libcodeskernellang.a
+
+BUILT_SOURCES += src/iokernellang/codesparser.h \
+		 src/iokernellang/codesparser.c \
+                 src/iokernellang/codeslexer.h
+
+
+#src/iokernellang/codesparser.y: src/iokernellang/codesparser.y.in Makefile
+src/iokernellang/codesparser.c src/iokernellang/codesparser.h: src/iokernellang/codesparser.y
+
+EXTRA_DIST += src/iokernellang/codesparser.y
+
+src_libcodeskernellang_a_SOURCES = src/iokernellang/codesparser.h \
+                                          src/iokernellang/codesparser.c \
+                                          src/iokernellang/codeslexer.h \
+                                          src/iokernellang/codeslexer.c \
+                                          src/iokernellang/codesImpl.c \
+                                          src/iokernellang/CodesIOKernelContext.h \
+                                          src/iokernellang/CodesIOKernelParser.h \
+                                          src/iokernellang/CodesIOKernelTypes.h \
+                                          src/iokernellang/CodesKernelHelpers.h \
+                                          src/iokernellang/CodesKernelHelpers.c
+
+src_libcodesmodelconfig_a_SOURCES = \
+                                           src/modelconfig/configlex.c \
+                                           src/modelconfig/configlex.h \
+                                           src/modelconfig/configparser.c \
+                                           src/modelconfig/configparser.h \
+                                           src/modelconfig/configfile.c \
+                                           src/modelconfig/configfile.h \
+                                           src/modelconfig/configglue.h \
+                                           src/modelconfig/configglue.c \
+                                           src/modelconfig/configfile.h \
+                                           src/modelconfig/configstore.h \
+                                           src/modelconfig/configstore.c \
+                                           src/modelconfig/configstoreadapter.h \
+                                           src/modelconfig/configstoreadapter.c \
+                                           src/modelconfig/txt_configfile.c \
+                                           src/modelconfig/txt_configfile.h
+
+src_libcodesutil_a_SOURCES = src/util/tools.h \
+                                    src/util/tools.c \
+                                    src/util/interval-merge-tree.h \
+                                    src/util/interval-merge-tree.c \
+                                    src/util/red-black-tree.h \
+                                    src/util/red-black-tree.c \
+                                    src/util/codes-callbacks.h \
+                                    src/util/codes-callbacks.c
+
+src_libcodeslogging_a_SOURCES = src/logging/codeslogging.h \
+                                       src/logging/codeslogging.c \
+                                       src/logging/timeline.h \
+                                       src/logging/timeline.c

src/iokernellang/CodesIOKernelContext.h

+#ifndef CODESIOKERNELCONTEXT_H
+#define CODESIOKERNELCONTEXT_H
+
+typedef struct CodesIOKernelContext
+{
+    void *scanner_;
+    char *text;
+    void *lval;
+    void *locval;
+
+    int64_t sym[26];
+    int64_t var[10];
+    int64_t next_var[10];
+
+    int inst_ready;
+
+    /* Ning's additions for the bgp storage model */
+    /* XXX Does this belong here? */
+    int GroupRank;
+    int GroupSize;
+
+} CodesIOKernelContext;
+
+void CodesIOKernelScannerInit(
+    CodesIOKernelContext * context);
+void CodesIOKernelScannerDestroy(
+    CodesIOKernelContext * context);
+
+void CodesIOKernelScannerSetSymTable(
+    CodesIOKernelContext * context);
+
+#endif

src/iokernellang/CodesIOKernelParser.h

+#ifndef CODESIOKERNELPARSER_H
+#define CODESIOKERNELPARSER_H
+
+#include "codesparser.h"
+#include "CodesIOKernelContext.h"
+
+YYLTYPE *CodesIOKernel_get_lloc  (yyscan_t yyscanner);
+int CodesIOKernel_lex_init (yyscan_t* scanner);
+int CodesIOKernel_lex(YYSTYPE * lvalp, YYLTYPE * llocp, void * scanner);
+//YY_BUFFER_STATE CodesIOKernel__scan_string (yyconst char *yy_str ,yyscan_t yyscanner);
+
+#endif

src/iokernellang/CodesIOKernelTypes.h

+#ifndef CODES_H
+#define CODES_H
+
+#include "CodesIOKernelContext.h"
+
+typedef enum { typeCon, typeId, typeOpr } nodeEnum;
+
+/* constants */
+typedef struct {
+    int64_t value;                  /* value of constant */
+} conNodeType;
+
+/* identifiers */
+typedef struct {
+    int i;                      /* subscript to sym array */
+} idNodeType;
+
+/* operators */
+typedef struct {
+    int oper;                   /* operator */
+    int nops;                   /* number of operands */
+    struct nodeTypeTag *op[1];  /* operands (expandable) */
+} oprNodeType;
+
+typedef struct nodeTypeTag {
+    nodeEnum type;              /* type of node */
+
+    /* union must be last entry in nodeType */
+    /* because operNodeType may dynamically increase */
+    union {
+        conNodeType con;        /* constants */
+        idNodeType id;          /* identifiers */
+        oprNodeType opr;        /* operators */
+    };
+} nodeType;
+
+#ifndef YY_TYPEDEF_YY_SCANNER_T
+#define YY_TYPEDEF_YY_SCANNER_T
+typedef void* yyscan_t;
+#endif
+
+typedef union codesYYType {
+    int64_t iValue;                 /* integer value */
+    int sIndex;              /* symbol table index */
+    nodeType *nPtr;             /* node pointer */
+} codesYYType;
+#define YYSTYPE codesYYType 
+
+typedef struct codesParserParam
+{
+        yyscan_t scanner;
+        void * nPtr;
+}codesParserParam;
+
+// the parameter name (of the reentrant 'yyparse' function)
+// data is a pointer to a 'codesParserParam' structure
+//#define YYPARSE_PARAM data
+
+// the argument for the 'yylex' function
+//#define YYLEX_PARAM   ((codesParserParam *)data)->scanner
+
+extern int64_t * sym;
+extern int64_t * var;
+extern int * inst_ready;
+extern int * group_rank;
+extern int * group_size;
+#endif

src/iokernellang/CodesKernelHelpers.c

+#include "CodesKernelHelpers.h"
+
+#include <stdlib.h>
+#include <stdio.h>
+#include <string.h>
+#include <fcntl.h>
+#include <errno.h>
+#include <assert.h>
+
+#define CK_LINE_LIMIT 8192
+#define CL_DEFAULT_GID 0
+
+char * code_kernel_helpers_kinstToStr(int inst)
+{
+    switch(inst)
+    {
+        case WRITEAT:
+            return "WRITEAT";
+        case GETGROUPRANK:
+            return "GETGROUPRANK";
+        case GETGROUPSIZE:
+            return "GETGROUPSIZE";
+        case CLOSE:
+            return "CLOSE";
+        case OPEN:
+            return "OPEN";
+        case SYNC:
+            return "SYNC";
+        case SLEEP:
+            return "SLEEP";
+        case EXIT:
+            return "EXIT";
+        case DELETE:
+            return "DELETE";
+        default:
+            return "UNKNOWN";
+    };
+    
+    return "UNKNOWN";
+}
+
+char * code_kernel_helpers_cleventToStr(int inst)
+{
+    switch(inst)
+    {
+        case CL_WRITEAT:
+            return "CL_WRITEAT";
+        case CL_GETRANK:
+            return "CL_GETRANK";
+        case CL_GETSIZE:
+            return "CL_GETSIZE";
+        case CL_CLOSE:
+            return "CL_CLOSE";
+        case CL_OPEN:
+            return "CL_OPEN";
+        case CL_SYNC:
+            return "CL_SYNC";
+        case CL_SLEEP:
+            return "CL_SLEEP";
+        case CL_EXIT:
+            return "CL_EXIT";
+        case CL_DELETE:
+            return "CL_DELETE";
+        default:
+            return "CL_UNKNOWN";
+    };
+    
+    return "CL_UNKNOWN";
+}
+
+static int convertKLInstToEvent(int inst)
+{
+    switch(inst)
+    {
+        case WRITEAT:
+            return CL_WRITEAT;
+        case GETGROUPRANK:
+            return CL_GETRANK;
+        case GETGROUPSIZE:
+            return CL_GETSIZE;
+        case CLOSE:
+            return CL_CLOSE;
+        case OPEN:
+            return CL_OPEN;
+        case SYNC:
+            return CL_SYNC;
+        case SLEEP:
+            return CL_SLEEP;
+        case EXIT:
+            return CL_EXIT;
+        case DELETE:
+            return CL_DELETE;
+        default:
+            return CL_UNKNOWN;
+    };
+
+    return CL_UNKNOWN;
+}
+
+static void codes_kernel_helper_parse_cf(char * io_kernel_path, char *
+        io_kernel_def_path, char * io_kernel_meta_path, int task_rank, app_cf_info_t * task_info)
+{
+       int foundit = 0;
+       char line[CK_LINE_LIMIT];
+       FILE * ikmp = NULL;
+
+       /* open the config file */
+       ikmp = fopen(io_kernel_meta_path, "r");
+       if(ikmp == NULL)
+       {
+           fprintf(stderr, "%s:%i could not open meta file (%s)... bail\n", __func__,
+                   __LINE__, io_kernel_meta_path);
+           exit(1);
+       }
+
+       /* for each line in the config file */
+       while(fgets(line, CK_LINE_LIMIT, ikmp) != NULL)
+       {
+               char * token = NULL;
+               int min = 0;
+               int max = 0;
+               int gid = 0;
+               char * ctx = NULL;
+
+               /* parse the first element... the gid */
+               token = strtok_r(line, " \n", &ctx);
+               if(token)
+                   gid = atoi(token);
+
+               if(gid == CL_DEFAULT_GID)
+               {
+                   fprintf(stderr, "%s:%i incorrect GID detected in kernel meta\
+                           file. Cannot use the reserved GID\
+                           CL_DEFAULT_GID(%i)\n", __func__, __LINE__,
+                           CL_DEFAULT_GID);
+               }
+
+               /* parse the second element... min rank */
+               token = strtok_r(NULL, " \n", &ctx);
+               if(token)
+                       min = atoi(token);
+
+               /* parse the third element... max rank */
+               token = strtok_r(NULL, " \n", &ctx);
+               if(token)
+                       max = atoi(token);
+
+               /* parse the last element... kernel path */
+               token = strtok_r(NULL, " \n", &ctx);
+               if(token)
+                       strcpy(io_kernel_path, token);
+
+               /* if our rank is on this range... end processing of the config
+                * file */
+               if(task_rank >= min && task_rank <= max)
+               {
+                       task_info->gid = gid;
+                       task_info->min = min;
+                       task_info->max = max;
+                       task_info->lrank = task_rank - min;
+                       task_info->num_lrank = max - min + 1;
+
+                       foundit = 1;
+
+                       break;
+               }