[refactor] remove C++, add autotools, pkg-config

This patch cleans up the code, use better names for the headers, remove
the last bits of C++ and add autotools to compile the library.

This patch also splits the PMPI library into a separate .so, to make it
easier to only deal with the core downstream API.

.gitignore

+Makefile.in
+Makefile
+*.o
+*.lo
+*.la
+build-aux
+aclocal.m4
+m4/libtool.m4
+m4/ltoptions.m4
+m4/ltsugar.m4
+m4/ltversion.m4
+m4/lt~obsolete.m4
+autom4te.cache
+configure
+version.m4
+*~
+*.pc
+m4/ar-lib
+m4/compile
+m4/config.guess
+m4/config.sub
+m4/depcomp
+m4/install-sh
+m4/ltmain.sh
+m4/missing
+src/config.h.in

Makefile

-CC = gcc
-CXX = g++
-FORT = gfortran
-AR = ar
-OPT = -O2 -g
-LIBS = -L. -lrt
-MPI_INCLUDE = ../include
-ARFLAGS = rcs
-SHARED_LIB_FLAGS = -shared
-ZMQ_PATH = ../.libs
-NRM_LIBS = -Wl,-R $(ZMQ_PATH) -lzmq
-
-# Input and output strings for MPI_NRM library
-INPUT = mpi_nrm
-OUTPUT = $(INPUT)
-
-# Source files for LIB
-SOURCEINPUT = $(INPUT).cpp 
-SOURCECPP   = basic_functions.cpp f2c_mpi.cpp
-SOURCEC     = downstream_api.c
-SOURCEF     = $(INPUT)_fort.f
-
-OBJS = ${SOURCECPP:.cpp=.o} ${SOURCEC:.c=.o} \
-	${SOURCEF:.f=.o} ${SOURCEINPUT:.cpp=.o}
-OBJSCLEAN = ${SOURCECPP:.cpp=.o} ${SOURCEC:.c=.o} \
-	${SOURCEF:.f=.o} ${SOURCEINPUT:.cpp=.o}
-
-# Build MPI_NRM 
-#all : static shared object_clean
-all : shared object_clean
-again : clean all
-
-.c.o:
-	$(CC) $(OPT) -fPIC -c $<
-.f.o:
-	$(FORT) $(OPT) -fPIC -I. -I$(MPI_INCLUDE) -c $<
-
-.cpp.o:
-	$(CXX) $(OPT) -fPIC -I. -I$(MPI_INCLUDE) -c $<
-
-static: $(OBJS)
-	$(AR) $(ARFLAGS) -o lib$(OUTPUT).a $(OBJS)
-
-shared: $(OBJS)
-	$(CXX) $(SHARED_LIB_FLAGS) -o lib$(OUTPUT).so $(NRM_LIBS) $(OBJS)
-
-clean :
-	rm -f lib*
-
-# Clean only object files in current folder
-object_clean :
-	rm *.o

Makefile.am

+ACLOCAL_AMFLAGS = -I m4
+SUBDIRS = src
+
+pkgconfigdir = $(libdir)/pkgconfig
+pkgconfig_DATA = libnrm.pc
+
+EXTRA_DIST = autogen.sh libnrm.pc README.md

autogen.sh

+#!/bin/sh
+set -x
+autoreconf -vif -I m4

basic_functions.cpp

-/* Filename: basic_functions.ccpp 
- *
- * Basic Description: Contains commonly used functions
- *
- */
-
-#include "basic_functions.h"
-
-/************************
- * Returns the current time
- ***********************/
-extern "C" uint64_t return_current_time (void)
-{
-  struct timespec currentTime;
-  clock_gettime (CLOCK_MONOTONIC, &currentTime);
-  return ((currentTime.tv_sec * 1e9) + currentTime.tv_nsec);
-}

basic_functions.h

-/* Filename: basic_functions.h
- *
- * Includes required headers, functions and parameters used by basic_functions 
- *
- */
-
-#include <time.h>     // for clock_gettime
-#include <inttypes.h> // for uint64_t
-
-#ifdef __cplusplus
-extern "C"{
-#endif
-
-uint64_t return_current_time (void);
-
-#ifdef __cplusplus
-}
-#endif

configure.ac

+# see semver.org for version info
+AC_INIT([libnrm],[0.0.1],[sriduttb@anl.gov])
+
+# are we in the right source dir ?
+AC_CONFIG_SRCDIR([src/nrm.h])
+
+# build artefacts in separate dir
+AC_CONFIG_AUX_DIR([m4])
+AC_CONFIG_MACRO_DIR([m4])
+
+# automake should fail on any error
+AM_INIT_AUTOMAKE([-Wall -Werror foreign 1.12])
+AM_PROG_AR
+
+# check for libtool
+LT_INIT
+
+# check for programs
+AC_LANG([C])
+AC_PROG_CC
+AC_PROG_CC_STDC
+AC_PROG_CPP
+AC_TYPE_SIZE_T
+AC_TYPE_INTPTR_T
+
+# dependencies
+PKG_CHECK_MODULES([LIBZMQ],[libzmq])
+
+# PMPI lib
+AC_ARG_ENABLE(pmpi,
+	      [AS_HELP_STRING([--enable-pmpi],[Build PMPI library.])],
+	      [pmpi=true],[pmpi=false])
+if [[ "x$pmpi" = xtrue ]]; then
+	AX_MPI([],[AC_MSG_ERROR(could not find mpi library for --enable-pmpi)])
+fi
+AM_CONDITIONAL([PMPI_API],[test "x$pmpi" = xtrue])
+
+AC_CONFIG_HEADERS([src/config.h])
+
+AC_CONFIG_FILES([Makefile src/Makefile libnrm.pc])
+AC_OUTPUT

f2c_mpi.cpp

-/* Filename: f2c_mpi.ccp 
- *
- * Basic Description: Implements C wrapper for MPI functions to be called from
- * Fortran
- *
- */
-
-#include "f2c_mpi.h"
-
-/************************
- * mpi_init fortran intercept calling MPI_Init()
- ***********************/
-extern "C" void f2c_mpi_init_ (MPI_Fint *ierr)
-{
-  int argc;
-  char **argv;
-  MPI_Init (&argc, &argv);
-}
-
-/************************
- * mpi_finalize fortran intercept calling MPI_Finalize()
- ***********************/
-extern "C" void f2c_mpi_finalize_ ()
-{
-  MPI_Finalize ();
-}
-
-/************************
- * mpi_barrier fortran intercept calling MPI_Barrier()
- ***********************/
-extern "C" void f2c_mpi_barrier_ (MPI_Fint *f_handle, MPI_Fint *ierr)
-{
-  MPI_Comm commC = MPI_Comm_f2c (*f_handle); 
- 
-  MPI_Barrier (commC);
-  
-  return;
-}
-
-/************************
- * mpi_allreduce fortran intercept calling MPI_Allreduce)
- ***********************/
-extern "C" void f2c_mpi_allreduce_ (void *sendbuf, void *recvbuf, MPI_Fint *f_count, MPI_Fint *f_datatype, MPI_Fint *f_op,  MPI_Fint *f_handle, MPI_Fint *ierr)
-{
-  MPI_Datatype datatypeC = MPI_Type_f2c (*f_datatype);
-  MPI_Op opC = MPI_Op_f2c (*f_op);
-  MPI_Comm commC = MPI_Comm_f2c (*f_handle);
-  
-  MPI_Allreduce (sendbuf, recvbuf, *f_count, datatypeC, opC, commC);
-  
-  return;
-}

f2c_mpi.h

-/* Filename: f2c_mpi.h
- *
- * Includes required headers, functions and parameters used by f2c_mpi
- *
- */
-
-#include "mpi.h"
-
-#ifdef __cplusplus
-extern "C"{
-#endif
-
-void f2c_mpi_init_ (MPI_Fint *ierr);
-void f2c_mpi_finalize_ ();
-void f2c_mpi_barrier_ (MPI_Fint *f_handle, MPI_Fint *ierr);
-void f2c_mpi_allreduce_ (void *sendbuf, void *recvbuf, MPI_Fint *f_count, 
-    MPI_Fint *f_datatype, MPI_Fint *f_op,  MPI_Fint *f_handle, MPI_Fint *ierr);
-void f2c_mpi_reduce_ (void *sendbuf, void *recvbuf, MPI_Fint *f_count, 
-    MPI_Fint *f_datatype, MPI_Fint *f_op, MPI_Fint *f_root, MPI_Fint *f_handle,
-    MPI_Fint *ierr);
-
-#ifdef __cplusplus
-}
-#endif

libnrm.pc.in

+prefix=@prefix@
+exec_prefix=@prefix@
+libdir=@libdir@
+includedir=@includedir@
+
+Name: libnrm
+Description: Argo Node Resource Manager Client API
+Version: 0.0.1
+Requires: libzmq
+Libs: -L${libdir} -lnrm
+Cflags: -I${includedir}

m4/ax_mpi.m4

+# ===========================================================================
+#          https://www.gnu.org/software/autoconf-archive/ax_mpi.html
+# ===========================================================================
+#
+# SYNOPSIS
+#
+#   AX_MPI([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]])
+#
+# DESCRIPTION
+#
+#   This macro tries to find out how to compile programs that use MPI
+#   (Message Passing Interface), a standard API for parallel process
+#   communication (see http://www-unix.mcs.anl.gov/mpi/)
+#
+#   On success, it sets the MPICC, MPICXX, MPIF77, or MPIFC output variable
+#   to the name of the MPI compiler, depending upon the current language.
+#   (This may just be $CC/$CXX/$F77/$FC, but is more often something like
+#   mpicc/mpiCC/mpif77/mpif90.) It also sets MPILIBS to any libraries that
+#   are needed for linking MPI (e.g. -lmpi or -lfmpi, if a special
+#   MPICC/MPICXX/MPIF77/MPIFC was not found).
+#
+#   Note that this macro should be used only if you just have a few source
+#   files that need to be compiled using MPI. In particular, you should
+#   neither overwrite CC/CXX/F77/FC with the values of
+#   MPICC/MPICXX/MPIF77/MPIFC, nor assume that you can use the same flags
+#   etc. as the standard compilers. If you want to compile a whole program
+#   using the MPI compiler commands, use one of the macros
+#   AX_PROG_{CC,CXX,FC}_MPI.
+#
+#   ACTION-IF-FOUND is a list of shell commands to run if an MPI library is
+#   found, and ACTION-IF-NOT-FOUND is a list of commands to run if it is not
+#   found. If ACTION-IF-FOUND is not specified, the default action will
+#   define HAVE_MPI.
+#
+# LICENSE
+#
+#   Copyright (c) 2008 Steven G. Johnson <stevenj@alum.mit.edu>
+#   Copyright (c) 2008 Julian C. Cummings <cummings@cacr.caltech.edu>
+#
+#   This program is free software: you can redistribute it and/or modify it
+#   under the terms of the GNU General Public License as published by the
+#   Free Software Foundation, either version 3 of the License, or (at your
+#   option) any later version.
+#
+#   This program is distributed in the hope that it will be useful, but
+#   WITHOUT ANY WARRANTY; without even the implied warranty of
+#   MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General
+#   Public License for more details.
+#
+#   You should have received a copy of the GNU General Public License along
+#   with this program. If not, see <https://www.gnu.org/licenses/>.
+#
+#   As a special exception, the respective Autoconf Macro's copyright owner
+#   gives unlimited permission to copy, distribute and modify the configure
+#   scripts that are the output of Autoconf when processing the Macro. You
+#   need not follow the terms of the GNU General Public License when using
+#   or distributing such scripts, even though portions of the text of the
+#   Macro appear in them. The GNU General Public License (GPL) does govern
+#   all other use of the material that constitutes the Autoconf Macro.
+#
+#   This special exception to the GPL applies to versions of the Autoconf
+#   Macro released by the Autoconf Archive. When you make and distribute a
+#   modified version of the Autoconf Macro, you may extend this special
+#   exception to the GPL to apply to your modified version as well.
+
+#serial 9
+
+AU_ALIAS([ACX_MPI], [AX_MPI])
+AC_DEFUN([AX_MPI], [
+AC_PREREQ(2.50) dnl for AC_LANG_CASE
+
+AC_LANG_CASE([C], [
+	AC_REQUIRE([AC_PROG_CC])
+	AC_ARG_VAR(MPICC,[MPI C compiler command])
+	AC_CHECK_PROGS(MPICC, mpicc hcc mpxlc_r mpxlc mpcc cmpicc, $CC)
+	ax_mpi_save_CC="$CC"
+	CC="$MPICC"
+	AC_SUBST(MPICC)
+],
+[C++], [
+	AC_REQUIRE([AC_PROG_CXX])
+	AC_ARG_VAR(MPICXX,[MPI C++ compiler command])
+	AC_CHECK_PROGS(MPICXX, mpic++ mpicxx mpiCC hcp mpxlC_r mpxlC mpCC cmpic++, $CXX)
+	ax_mpi_save_CXX="$CXX"
+	CXX="$MPICXX"
+	AC_SUBST(MPICXX)
+],
+[Fortran 77], [
+	AC_REQUIRE([AC_PROG_F77])
+	AC_ARG_VAR(MPIF77,[MPI Fortran 77 compiler command])
+	AC_CHECK_PROGS(MPIF77, mpif77 hf77 mpxlf_r mpxlf mpf77 cmpifc, $F77)
+	ax_mpi_save_F77="$F77"
+	F77="$MPIF77"
+	AC_SUBST(MPIF77)
+],
+[Fortran], [
+	AC_REQUIRE([AC_PROG_FC])
+	AC_ARG_VAR(MPIFC,[MPI Fortran compiler command])
+	AC_CHECK_PROGS(MPIFC, mpif90 mpxlf95_r mpxlf90_r mpxlf95 mpxlf90 mpf90 cmpif90c, $FC)
+	ax_mpi_save_FC="$FC"
+	FC="$MPIFC"
+	AC_SUBST(MPIFC)
+])
+
+if test x = x"$MPILIBS"; then
+	AC_LANG_CASE([C], [AC_CHECK_FUNC(MPI_Init, [MPILIBS=" "])],
+		[C++], [AC_CHECK_FUNC(MPI_Init, [MPILIBS=" "])],
+		[Fortran 77], [AC_MSG_CHECKING([for MPI_Init])
+			AC_LINK_IFELSE([AC_LANG_PROGRAM([],[      call MPI_Init])],[MPILIBS=" "
+				AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)])],
+		[Fortran], [AC_MSG_CHECKING([for MPI_Init])
+			AC_LINK_IFELSE([AC_LANG_PROGRAM([],[      call MPI_Init])],[MPILIBS=" "
+				AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)])])
+fi
+AC_LANG_CASE([Fortran 77], [
+	if test x = x"$MPILIBS"; then
+		AC_CHECK_LIB(fmpi, MPI_Init, [MPILIBS="-lfmpi"])
+	fi
+	if test x = x"$MPILIBS"; then
+		AC_CHECK_LIB(fmpich, MPI_Init, [MPILIBS="-lfmpich"])
+	fi
+],
+[Fortran], [
+	if test x = x"$MPILIBS"; then
+		AC_CHECK_LIB(fmpi, MPI_Init, [MPILIBS="-lfmpi"])
+	fi
+	if test x = x"$MPILIBS"; then
+		AC_CHECK_LIB(mpichf90, MPI_Init, [MPILIBS="-lmpichf90"])
+	fi
+])
+if test x = x"$MPILIBS"; then
+	AC_CHECK_LIB(mpi, MPI_Init, [MPILIBS="-lmpi"])
+fi
+if test x = x"$MPILIBS"; then
+	AC_CHECK_LIB(mpich, MPI_Init, [MPILIBS="-lmpich"])
+fi
+
+dnl We have to use AC_TRY_COMPILE and not AC_CHECK_HEADER because the
+dnl latter uses $CPP, not $CC (which may be mpicc).
+AC_LANG_CASE([C], [if test x != x"$MPILIBS"; then
+	AC_MSG_CHECKING([for mpi.h])
+	AC_TRY_COMPILE([#include <mpi.h>],[],[AC_MSG_RESULT(yes)], [MPILIBS=""
+		AC_MSG_RESULT(no)])
+fi],
+[C++], [if test x != x"$MPILIBS"; then
+	AC_MSG_CHECKING([for mpi.h])
+	AC_TRY_COMPILE([#include <mpi.h>],[],[AC_MSG_RESULT(yes)], [MPILIBS=""
+		AC_MSG_RESULT(no)])
+fi],
+[Fortran 77], [if test x != x"$MPILIBS"; then
+	AC_MSG_CHECKING([for mpif.h])
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[      include 'mpif.h'])],[AC_MSG_RESULT(yes)], [MPILIBS=""
+		AC_MSG_RESULT(no)])
+fi],
+[Fortran], [if test x != x"$MPILIBS"; then
+	AC_MSG_CHECKING([for mpif.h])
+	AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[      include 'mpif.h'])],[AC_MSG_RESULT(yes)], [MPILIBS=""
+		AC_MSG_RESULT(no)])
+fi])
+
+AC_LANG_CASE([C], [CC="$ax_mpi_save_CC"],
+	[C++], [CXX="$ax_mpi_save_CXX"],
+	[Fortran 77], [F77="$ax_mpi_save_F77"],
+	[Fortran], [FC="$ax_mpi_save_FC"])
+
+AC_SUBST(MPILIBS)
+
+# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND:
+if test x = x"$MPILIBS"; then
+        $2
+        :
+else
+        ifelse([$1],,[AC_DEFINE(HAVE_MPI,1,[Define if you have the MPI library.])],[$1])
+        :
+fi
+])dnl AX_MPI

mpi_nrm.h

-/* Filename: mpi_nrm.h
- *
- * Includes required headers, functions and parameters used by the Message 
- * Passing Interface (MPI) Node Resource Manager (NRM) library
- *
- */
-
-#include <inttypes.h>   // for uint64_t
-#include <stdio.h>      // for printf
-#include <stdlib.h>     // for exit, atoi
-#include <signal.h>     // for signalHandling
-#include <sched.h>      // for sched_getcpu
-#include <unistd.h>     // for gethostname
-#include <ctype.h>
-
-#include "mpi.h"
-#include "basic_functions.h"
-#include "downstream_api.h"
-
-// Set to non-zero value to transmit to NRM using NRM_TRANSMIT environment variable
-static unsigned int _transmit = 0;   
-// Phase shorter than this will be aggregated - Set using NRM_DAMPER environ (in
-// seconds)
-static uint64_t _damper = 10000000;    
-
-// Book-keeping and Statistics
-static unsigned int _aggregation = 0;
-static unsigned int _damperAggregationCount = 0;
-static unsigned int _phaseSkipCount = 0;
-
-unsigned int cpu;
-int rank;
-// used to measure the computation time during a phase
-uint64_t startCompute, endCompute;
-
-struct nrm_context ctxt;

mpi_nrm_fort.f

-C Filename: mpi_nrm_fort.f
-C       
-C Description: This file implements the Fortran interface for the
-C MPI-NRM.
-C
-       
-       subroutine mpi_init(IERROR)
-       integer IERROR
-       call f2c_mpi_init(IERROR)
-
-       return
-       end
-
-       subroutine mpi_finalize(IERROR)
-       integer IERROR
-       call f2c_mpi_finalize(IERROR)
-       
-       return
-       end
-
-       subroutine mpi_barrier(COMM,IERROR)
-       integer IERROR
-       INTEGER COMM
-       call f2c_mpi_barrier(COMM,IERROR)
-
-       return
-       end
-       
-       subroutine mpi_allreduce(SBUF,RBUF,CNT,DTYP,OP,COMM,IERR)
-       use, intrinsic :: iso_c_binding
-       integer CNT,DTYP,OP,COMM,IERR
-       type(c_ptr), intent(out) :: SBUF
-       type(c_ptr), intent(in) :: RBUF
-       call f2c_mpi_allreduce(SBUF,RBUF,CNT,DTYP,OP,COMM,IERR)
-
-       return
-       end
-

src/Makefile.am

+AM_CPPFLAGS = @LIBZMQ_CFLAGS@
+
+lib_LTLIBRARIES = libnrm.la
+
+if PMPI_API
+lib_LTLIBRARIES += libnrm-pmpi.la
+endif
+
+libnrm_la_SOURCES = downstream_api.c nrm.h
+libnrm_la_LIBADD = @LIBZMQ_LIBS@
+libnrm_pmpi_la_SOURCES = downstream_api.c mpi_api.c nrm.h
+libnrm_pmpi_la_LIBADD = @LIBZMQ_LIBS@
+
+include_HEADERS = nrm.h

downstream_api.c → src/downstream_api.c

@@ -8,13 +8,14 @@
  * efficiency at the node level.
  */
 
-#include "downstream_api.h"
-#include<zmq.h>
-#include<stdio.h>
-#include<unistd.h>
-#include<string.h>
-#include<stdlib.h>
-#include<assert.h>
+#include <zmq.h>
+#include <stdio.h>
+#include <unistd.h>
+#include <string.h>
+#include <stdlib.h>
+#include <assert.h>
+
+#include "nrm.h"
 
 int nrm_init(struct nrm_context *ctxt, const char *uuid)
 {

mpi_nrm.cpp → src/mpi_api.c

@@ -15,27 +15,34 @@
  *
  * Written by Sridutt Bhalachandra, sriduttb@anl.gov
  */
-
-#include "mpi_nrm.h"
-
-/************************
- * Signal Handler to handle SIGTERM, SIGINT and other calls if needed
- ***********************/
-void handle_signal(int sig_num)
-{
-  printf("\nExiting MPI NRM Interface...\n");
-
-  exit(sig_num);
-}
-
+#include "nrm.h"
+#include <ctype.h>
+#include <sched.h>      // for sched_getcpu
+#include <stdio.h>      // for printf
+#include <stdlib.h>     // for exit, atoi
+#include <mpi.h>
+
+// Set to non-zero value to transmit to NRM using NRM_TRANSMIT environment variable
+static unsigned int _transmit = 0;   
+// Phase shorter than this will be aggregated - Set using NRM_DAMPER environ (in
+// seconds)
+static uint64_t _damper = 10000000;    
+
+// Book-keeping and Statistics
+static unsigned int _aggregation = 0;
+static unsigned int _damperAggregationCount = 0;
+static unsigned int _phaseSkipCount = 0;
+
+static unsigned int cpu;
+static int rank;
+// used to measure the computation time during a phase
+static uint64_t startCompute, endCompute;
+static struct nrm_context ctxt;
 /************************
  * Setup up the MPI NRM Interface
  ***********************/
-extern "C" void MPI_nrm_init()
+void MPI_nrm_init()
 {
-  signal(SIGTERM, handle_signal);
-  signal(SIGINT, handle_signal);
-
   cpu = sched_getcpu();
 
   MPI_Comm_rank(MPI_COMM_WORLD, &rank);
@@ -57,7 +64,7 @@ extern "C" void MPI_nrm_init()
   return;
 }
 
-extern "C" void MPI_nrm_fini()
+void MPI_nrm_fini()
 {
   // Cleanup NRM context
   nrm_fini(&ctxt);
@@ -66,7 +73,7 @@ extern "C" void MPI_nrm_fini()
 /************************
  * Prints the transmission statistics for an application
  ***********************/
-extern "C" void MPI_nrm_print_stats(void)
+void MPI_nrm_print_stats(void)
 {
   printf("Stats: CPU %u Damper %lf DamperAggreations %u PhaseSkips %u\n", cpu,
       _damper,_damperAggregationCount, _phaseSkipCount);
@@ -76,7 +83,7 @@ extern "C" void MPI_nrm_print_stats(void)
 /************************
  * Send appropriate phase context to NRM 
  ***********************/
-extern "C" void transmit_to_nrm(int cpu, uint64_t *startCompute, uint64_t
+void transmit_to_nrm(int cpu, uint64_t *startCompute, uint64_t
     endCompute, uint64_t startBarrier, uint64_t endBarrier)
 {
   uint64_t computeTime, barrierTime, totalPhaseTime;
@@ -100,7 +107,7 @@ extern "C" void transmit_to_nrm(int cpu, uint64_t *startCompute, uint64_t
   if(getenv("NRM_SKIP"))
   {
     // Reset phase start time
-    *startCompute = return_current_time();
+    *startCompute = nrm_gettime();
     _phaseSkipCount++;
 
     // Reset environment variable
@@ -113,14 +120,14 @@ extern "C" void transmit_to_nrm(int cpu, uint64_t *startCompute, uint64_t
 
   // Reset
   _aggregation = 0;
-  *startCompute = return_current_time();
+  *startCompute = nrm_gettime();
   
   return;
 }
 
 int MPI_Init(int *argc, char ***argv)
 {
-  startCompute = return_current_time();
+  startCompute = nrm_gettime();
   int ret_value = PMPI_Init(argc, argv);
   MPI_nrm_init();
 
@@ -168,12 +175,12 @@ int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source, int tag,
 
 int MPI_Barrier(MPI_Comm comm)
 {
-  endCompute = return_current_time();
+  endCompute = nrm_gettime();
   uint64_t startBarrier = endCompute;
 
   int ret_value = PMPI_Barrier(comm);
 
-  uint64_t endBarrier = return_current_time();
+  uint64_t endBarrier = nrm_gettime();
 
   if(_transmit)
   {
@@ -233,12 +240,12 @@ int MPI_Reduce(const void *sendbuf, void *recvbuf, int count,
 int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count,
          MPI_Datatype datatype, MPI_Op op, MPI_Comm comm)
 {
-  endCompute = return_current_time();
+  endCompute = nrm_gettime();
   uint64_t startBarrier = endCompute;
 
   int ret_value = PMPI_Allreduce(sendbuf, recvbuf, count, datatype, op, comm);
 
-  uint64_t endBarrier = return_current_time();
+  uint64_t endBarrier = nrm_gettime();
 
   if(_transmit)
   {

downstream_api.h → src/nrm.h

@@ -8,7 +8,12 @@
 #ifndef NRM_H
 #define NRM_H 1
 
-#include<time.h>
+#include <inttypes.h>
+#include <time.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
 
 /* min time in nsec between messages: necessary for rate-limiting progress
  * report. For now, 10ms is the threashold. */
@@ -30,17 +35,24 @@ struct nrm_context {
 #define NRM_PROGRESS_FORMAT "{\"type\":\"application\", \"event\":\"progress\", \"payload\": \"%lu\", \"uuid\": \"%s\"}"
 #define NRM_PHASE_CONTEXT_FORMAT "{\"type\":\"application\", \"event\":\"phase_context\", \"cpu\": \"%u\", \"aggregation\": \"%u\", \"computetime\": \"%llu\", \"totaltime\": \"%llu\", \"uuid\": \"%s\"}"
 #define NRM_EXIT_FORMAT "{\"type\":\"application\", \"event\":\"exit\", \"uuid\": \"%s\"}"
-#ifdef __cplusplus
-extern "C"{
-#endif
 
 int nrm_init(struct nrm_context *, const char *);
 int nrm_fini(struct nrm_context *);
 
-int nrm_send_progress(struct nrm_context *, unsigned long);
-int nrm_send_phase_context(struct nrm_context *ctxt, unsigned int cpu, unsigned
-    int aggregation, unsigned long long int computeTime, unsigned long long int
-    totalTime);
+int nrm_send_progress(struct nrm_context *, unsigned long progress);
+int nrm_send_phase_context(struct nrm_context *ctxt,
+			   unsigned int cpu,
+			   unsigned int aggregation,
+			   unsigned long long int computeTime,
+			   unsigned long long int totalTime);
+
+/* util function */
+inline long long int nrm_gettime(void)
+{
+	struct timespec now;
+	clock_gettime(CLOCK_REALTIME, &now);
+	return ((now.tv_sec * 1e9) + now.tv_nsec);
+}
 
 #ifdef __cplusplus
 }