From b46302e20edc4fa55613652a23d6f4f4a318748f Mon Sep 17 00:00:00 2001 From: Swann Perarnau Date: Thu, 1 Nov 2018 15:39:32 -0500 Subject: [PATCH] [refactor] remove C++, add autotools, pkg-config This patch cleans up the code, use better names for the headers, remove the last bits of C++ and add autotools to compile the library. This patch also splits the PMPI library into a separate .so, to make it easier to only deal with the core downstream API. --- .gitignore | 26 ++++ Makefile | 52 ------- Makefile.am | 7 + autogen.sh | 3 + basic_functions.cpp | 17 --- basic_functions.h | 18 --- configure.ac | 41 ++++++ f2c_mpi.cpp | 52 ------- f2c_mpi.h | 24 ---- libnrm.pc.in | 11 ++ m4/ax_mpi.m4 | 176 +++++++++++++++++++++++ mpi_nrm.h | 36 ----- mpi_nrm_fort.f | 38 ----- src/Makefile.am | 14 ++ downstream_api.c => src/downstream_api.c | 15 +- mpi_nrm.cpp => src/mpi_api.c | 61 ++++---- downstream_api.h => src/nrm.h | 28 ++-- 17 files changed, 340 insertions(+), 279 deletions(-) create mode 100644 .gitignore delete mode 100644 Makefile create mode 100644 Makefile.am create mode 100644 autogen.sh delete mode 100644 basic_functions.cpp delete mode 100644 basic_functions.h create mode 100644 configure.ac delete mode 100644 f2c_mpi.cpp delete mode 100644 f2c_mpi.h create mode 100644 libnrm.pc.in create mode 100644 m4/ax_mpi.m4 delete mode 100644 mpi_nrm.h delete mode 100644 mpi_nrm_fort.f create mode 100644 src/Makefile.am rename downstream_api.c => src/downstream_api.c (95%) rename mpi_nrm.cpp => src/mpi_api.c (83%) rename downstream_api.h => src/nrm.h (73%) diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..fdda218 --- /dev/null +++ b/.gitignore @@ -0,0 +1,26 @@ +Makefile.in +Makefile +*.o +*.lo +*.la +build-aux +aclocal.m4 +m4/libtool.m4 +m4/ltoptions.m4 +m4/ltsugar.m4 +m4/ltversion.m4 +m4/lt~obsolete.m4 +autom4te.cache +configure +version.m4 +*~ +*.pc +m4/ar-lib +m4/compile +m4/config.guess +m4/config.sub +m4/depcomp +m4/install-sh +m4/ltmain.sh +m4/missing +src/config.h.in diff --git a/Makefile b/Makefile deleted file mode 100644 index 445bd46..0000000 --- a/Makefile +++ /dev/null @@ -1,52 +0,0 @@ -CC = gcc -CXX = g++ -FORT = gfortran -AR = ar -OPT = -O2 -g -LIBS = -L. -lrt -MPI_INCLUDE = ../include -ARFLAGS = rcs -SHARED_LIB_FLAGS = -shared -ZMQ_PATH = ../.libs -NRM_LIBS = -Wl,-R $(ZMQ_PATH) -lzmq - -# Input and output strings for MPI_NRM library -INPUT = mpi_nrm -OUTPUT = $(INPUT) - -# Source files for LIB -SOURCEINPUT = $(INPUT).cpp -SOURCECPP = basic_functions.cpp f2c_mpi.cpp -SOURCEC = downstream_api.c -SOURCEF = $(INPUT)_fort.f - -OBJS = ${SOURCECPP:.cpp=.o} ${SOURCEC:.c=.o} \ - ${SOURCEF:.f=.o} ${SOURCEINPUT:.cpp=.o} -OBJSCLEAN = ${SOURCECPP:.cpp=.o} ${SOURCEC:.c=.o} \ - ${SOURCEF:.f=.o} ${SOURCEINPUT:.cpp=.o} - -# Build MPI_NRM -#all : static shared object_clean -all : shared object_clean -again : clean all - -.c.o: - $(CC) $(OPT) -fPIC -c $< -.f.o: - $(FORT) $(OPT) -fPIC -I. -I$(MPI_INCLUDE) -c $< - -.cpp.o: - $(CXX) $(OPT) -fPIC -I. -I$(MPI_INCLUDE) -c $< - -static: $(OBJS) - $(AR) $(ARFLAGS) -o lib$(OUTPUT).a $(OBJS) - -shared: $(OBJS) - $(CXX) $(SHARED_LIB_FLAGS) -o lib$(OUTPUT).so $(NRM_LIBS) $(OBJS) - -clean : - rm -f lib* - -# Clean only object files in current folder -object_clean : - rm *.o diff --git a/Makefile.am b/Makefile.am new file mode 100644 index 0000000..6832b99 --- /dev/null +++ b/Makefile.am @@ -0,0 +1,7 @@ +ACLOCAL_AMFLAGS = -I m4 +SUBDIRS = src + +pkgconfigdir = $(libdir)/pkgconfig +pkgconfig_DATA = libnrm.pc + +EXTRA_DIST = autogen.sh libnrm.pc README.md diff --git a/autogen.sh b/autogen.sh new file mode 100644 index 0000000..2fa9b02 --- /dev/null +++ b/autogen.sh @@ -0,0 +1,3 @@ +#!/bin/sh +set -x +autoreconf -vif -I m4 diff --git a/basic_functions.cpp b/basic_functions.cpp deleted file mode 100644 index 1c13004..0000000 --- a/basic_functions.cpp +++ /dev/null @@ -1,17 +0,0 @@ -/* Filename: basic_functions.ccpp - * - * Basic Description: Contains commonly used functions - * - */ - -#include "basic_functions.h" - -/************************ - * Returns the current time - ***********************/ -extern "C" uint64_t return_current_time (void) -{ - struct timespec currentTime; - clock_gettime (CLOCK_MONOTONIC, ¤tTime); - return ((currentTime.tv_sec * 1e9) + currentTime.tv_nsec); -} diff --git a/basic_functions.h b/basic_functions.h deleted file mode 100644 index 30bc674..0000000 --- a/basic_functions.h +++ /dev/null @@ -1,18 +0,0 @@ -/* Filename: basic_functions.h - * - * Includes required headers, functions and parameters used by basic_functions - * - */ - -#include // for clock_gettime -#include // for uint64_t - -#ifdef __cplusplus -extern "C"{ -#endif - -uint64_t return_current_time (void); - -#ifdef __cplusplus -} -#endif diff --git a/configure.ac b/configure.ac new file mode 100644 index 0000000..affead1 --- /dev/null +++ b/configure.ac @@ -0,0 +1,41 @@ +# see semver.org for version info +AC_INIT([libnrm],[0.0.1],[sriduttb@anl.gov]) + +# are we in the right source dir ? +AC_CONFIG_SRCDIR([src/nrm.h]) + +# build artefacts in separate dir +AC_CONFIG_AUX_DIR([m4]) +AC_CONFIG_MACRO_DIR([m4]) + +# automake should fail on any error +AM_INIT_AUTOMAKE([-Wall -Werror foreign 1.12]) +AM_PROG_AR + +# check for libtool +LT_INIT + +# check for programs +AC_LANG([C]) +AC_PROG_CC +AC_PROG_CC_STDC +AC_PROG_CPP +AC_TYPE_SIZE_T +AC_TYPE_INTPTR_T + +# dependencies +PKG_CHECK_MODULES([LIBZMQ],[libzmq]) + +# PMPI lib +AC_ARG_ENABLE(pmpi, + [AS_HELP_STRING([--enable-pmpi],[Build PMPI library.])], + [pmpi=true],[pmpi=false]) +if [[ "x$pmpi" = xtrue ]]; then + AX_MPI([],[AC_MSG_ERROR(could not find mpi library for --enable-pmpi)]) +fi +AM_CONDITIONAL([PMPI_API],[test "x$pmpi" = xtrue]) + +AC_CONFIG_HEADERS([src/config.h]) + +AC_CONFIG_FILES([Makefile src/Makefile libnrm.pc]) +AC_OUTPUT diff --git a/f2c_mpi.cpp b/f2c_mpi.cpp deleted file mode 100644 index f25cfa4..0000000 --- a/f2c_mpi.cpp +++ /dev/null @@ -1,52 +0,0 @@ -/* Filename: f2c_mpi.ccp - * - * Basic Description: Implements C wrapper for MPI functions to be called from - * Fortran - * - */ - -#include "f2c_mpi.h" - -/************************ - * mpi_init fortran intercept calling MPI_Init() - ***********************/ -extern "C" void f2c_mpi_init_ (MPI_Fint *ierr) -{ - int argc; - char **argv; - MPI_Init (&argc, &argv); -} - -/************************ - * mpi_finalize fortran intercept calling MPI_Finalize() - ***********************/ -extern "C" void f2c_mpi_finalize_ () -{ - MPI_Finalize (); -} - -/************************ - * mpi_barrier fortran intercept calling MPI_Barrier() - ***********************/ -extern "C" void f2c_mpi_barrier_ (MPI_Fint *f_handle, MPI_Fint *ierr) -{ - MPI_Comm commC = MPI_Comm_f2c (*f_handle); - - MPI_Barrier (commC); - - return; -} - -/************************ - * mpi_allreduce fortran intercept calling MPI_Allreduce) - ***********************/ -extern "C" void f2c_mpi_allreduce_ (void *sendbuf, void *recvbuf, MPI_Fint *f_count, MPI_Fint *f_datatype, MPI_Fint *f_op, MPI_Fint *f_handle, MPI_Fint *ierr) -{ - MPI_Datatype datatypeC = MPI_Type_f2c (*f_datatype); - MPI_Op opC = MPI_Op_f2c (*f_op); - MPI_Comm commC = MPI_Comm_f2c (*f_handle); - - MPI_Allreduce (sendbuf, recvbuf, *f_count, datatypeC, opC, commC); - - return; -} diff --git a/f2c_mpi.h b/f2c_mpi.h deleted file mode 100644 index 56c28b2..0000000 --- a/f2c_mpi.h +++ /dev/null @@ -1,24 +0,0 @@ -/* Filename: f2c_mpi.h - * - * Includes required headers, functions and parameters used by f2c_mpi - * - */ - -#include "mpi.h" - -#ifdef __cplusplus -extern "C"{ -#endif - -void f2c_mpi_init_ (MPI_Fint *ierr); -void f2c_mpi_finalize_ (); -void f2c_mpi_barrier_ (MPI_Fint *f_handle, MPI_Fint *ierr); -void f2c_mpi_allreduce_ (void *sendbuf, void *recvbuf, MPI_Fint *f_count, - MPI_Fint *f_datatype, MPI_Fint *f_op, MPI_Fint *f_handle, MPI_Fint *ierr); -void f2c_mpi_reduce_ (void *sendbuf, void *recvbuf, MPI_Fint *f_count, - MPI_Fint *f_datatype, MPI_Fint *f_op, MPI_Fint *f_root, MPI_Fint *f_handle, - MPI_Fint *ierr); - -#ifdef __cplusplus -} -#endif diff --git a/libnrm.pc.in b/libnrm.pc.in new file mode 100644 index 0000000..0596f03 --- /dev/null +++ b/libnrm.pc.in @@ -0,0 +1,11 @@ +prefix=@prefix@ +exec_prefix=@prefix@ +libdir=@libdir@ +includedir=@includedir@ + +Name: libnrm +Description: Argo Node Resource Manager Client API +Version: 0.0.1 +Requires: libzmq +Libs: -L${libdir} -lnrm +Cflags: -I${includedir} diff --git a/m4/ax_mpi.m4 b/m4/ax_mpi.m4 new file mode 100644 index 0000000..f6db3d0 --- /dev/null +++ b/m4/ax_mpi.m4 @@ -0,0 +1,176 @@ +# =========================================================================== +# https://www.gnu.org/software/autoconf-archive/ax_mpi.html +# =========================================================================== +# +# SYNOPSIS +# +# AX_MPI([ACTION-IF-FOUND[, ACTION-IF-NOT-FOUND]]) +# +# DESCRIPTION +# +# This macro tries to find out how to compile programs that use MPI +# (Message Passing Interface), a standard API for parallel process +# communication (see http://www-unix.mcs.anl.gov/mpi/) +# +# On success, it sets the MPICC, MPICXX, MPIF77, or MPIFC output variable +# to the name of the MPI compiler, depending upon the current language. +# (This may just be $CC/$CXX/$F77/$FC, but is more often something like +# mpicc/mpiCC/mpif77/mpif90.) It also sets MPILIBS to any libraries that +# are needed for linking MPI (e.g. -lmpi or -lfmpi, if a special +# MPICC/MPICXX/MPIF77/MPIFC was not found). +# +# Note that this macro should be used only if you just have a few source +# files that need to be compiled using MPI. In particular, you should +# neither overwrite CC/CXX/F77/FC with the values of +# MPICC/MPICXX/MPIF77/MPIFC, nor assume that you can use the same flags +# etc. as the standard compilers. If you want to compile a whole program +# using the MPI compiler commands, use one of the macros +# AX_PROG_{CC,CXX,FC}_MPI. +# +# ACTION-IF-FOUND is a list of shell commands to run if an MPI library is +# found, and ACTION-IF-NOT-FOUND is a list of commands to run if it is not +# found. If ACTION-IF-FOUND is not specified, the default action will +# define HAVE_MPI. +# +# LICENSE +# +# Copyright (c) 2008 Steven G. Johnson +# Copyright (c) 2008 Julian C. Cummings +# +# This program is free software: you can redistribute it and/or modify it +# under the terms of the GNU General Public License as published by the +# Free Software Foundation, either version 3 of the License, or (at your +# option) any later version. +# +# This program is distributed in the hope that it will be useful, but +# WITHOUT ANY WARRANTY; without even the implied warranty of +# MERCHANTABILITY or FITNESS FOR A PARTICULAR PURPOSE. See the GNU General +# Public License for more details. +# +# You should have received a copy of the GNU General Public License along +# with this program. If not, see . +# +# As a special exception, the respective Autoconf Macro's copyright owner +# gives unlimited permission to copy, distribute and modify the configure +# scripts that are the output of Autoconf when processing the Macro. You +# need not follow the terms of the GNU General Public License when using +# or distributing such scripts, even though portions of the text of the +# Macro appear in them. The GNU General Public License (GPL) does govern +# all other use of the material that constitutes the Autoconf Macro. +# +# This special exception to the GPL applies to versions of the Autoconf +# Macro released by the Autoconf Archive. When you make and distribute a +# modified version of the Autoconf Macro, you may extend this special +# exception to the GPL to apply to your modified version as well. + +#serial 9 + +AU_ALIAS([ACX_MPI], [AX_MPI]) +AC_DEFUN([AX_MPI], [ +AC_PREREQ(2.50) dnl for AC_LANG_CASE + +AC_LANG_CASE([C], [ + AC_REQUIRE([AC_PROG_CC]) + AC_ARG_VAR(MPICC,[MPI C compiler command]) + AC_CHECK_PROGS(MPICC, mpicc hcc mpxlc_r mpxlc mpcc cmpicc, $CC) + ax_mpi_save_CC="$CC" + CC="$MPICC" + AC_SUBST(MPICC) +], +[C++], [ + AC_REQUIRE([AC_PROG_CXX]) + AC_ARG_VAR(MPICXX,[MPI C++ compiler command]) + AC_CHECK_PROGS(MPICXX, mpic++ mpicxx mpiCC hcp mpxlC_r mpxlC mpCC cmpic++, $CXX) + ax_mpi_save_CXX="$CXX" + CXX="$MPICXX" + AC_SUBST(MPICXX) +], +[Fortran 77], [ + AC_REQUIRE([AC_PROG_F77]) + AC_ARG_VAR(MPIF77,[MPI Fortran 77 compiler command]) + AC_CHECK_PROGS(MPIF77, mpif77 hf77 mpxlf_r mpxlf mpf77 cmpifc, $F77) + ax_mpi_save_F77="$F77" + F77="$MPIF77" + AC_SUBST(MPIF77) +], +[Fortran], [ + AC_REQUIRE([AC_PROG_FC]) + AC_ARG_VAR(MPIFC,[MPI Fortran compiler command]) + AC_CHECK_PROGS(MPIFC, mpif90 mpxlf95_r mpxlf90_r mpxlf95 mpxlf90 mpf90 cmpif90c, $FC) + ax_mpi_save_FC="$FC" + FC="$MPIFC" + AC_SUBST(MPIFC) +]) + +if test x = x"$MPILIBS"; then + AC_LANG_CASE([C], [AC_CHECK_FUNC(MPI_Init, [MPILIBS=" "])], + [C++], [AC_CHECK_FUNC(MPI_Init, [MPILIBS=" "])], + [Fortran 77], [AC_MSG_CHECKING([for MPI_Init]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([],[ call MPI_Init])],[MPILIBS=" " + AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)])], + [Fortran], [AC_MSG_CHECKING([for MPI_Init]) + AC_LINK_IFELSE([AC_LANG_PROGRAM([],[ call MPI_Init])],[MPILIBS=" " + AC_MSG_RESULT(yes)], [AC_MSG_RESULT(no)])]) +fi +AC_LANG_CASE([Fortran 77], [ + if test x = x"$MPILIBS"; then + AC_CHECK_LIB(fmpi, MPI_Init, [MPILIBS="-lfmpi"]) + fi + if test x = x"$MPILIBS"; then + AC_CHECK_LIB(fmpich, MPI_Init, [MPILIBS="-lfmpich"]) + fi +], +[Fortran], [ + if test x = x"$MPILIBS"; then + AC_CHECK_LIB(fmpi, MPI_Init, [MPILIBS="-lfmpi"]) + fi + if test x = x"$MPILIBS"; then + AC_CHECK_LIB(mpichf90, MPI_Init, [MPILIBS="-lmpichf90"]) + fi +]) +if test x = x"$MPILIBS"; then + AC_CHECK_LIB(mpi, MPI_Init, [MPILIBS="-lmpi"]) +fi +if test x = x"$MPILIBS"; then + AC_CHECK_LIB(mpich, MPI_Init, [MPILIBS="-lmpich"]) +fi + +dnl We have to use AC_TRY_COMPILE and not AC_CHECK_HEADER because the +dnl latter uses $CPP, not $CC (which may be mpicc). +AC_LANG_CASE([C], [if test x != x"$MPILIBS"; then + AC_MSG_CHECKING([for mpi.h]) + AC_TRY_COMPILE([#include ],[],[AC_MSG_RESULT(yes)], [MPILIBS="" + AC_MSG_RESULT(no)]) +fi], +[C++], [if test x != x"$MPILIBS"; then + AC_MSG_CHECKING([for mpi.h]) + AC_TRY_COMPILE([#include ],[],[AC_MSG_RESULT(yes)], [MPILIBS="" + AC_MSG_RESULT(no)]) +fi], +[Fortran 77], [if test x != x"$MPILIBS"; then + AC_MSG_CHECKING([for mpif.h]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[ include 'mpif.h'])],[AC_MSG_RESULT(yes)], [MPILIBS="" + AC_MSG_RESULT(no)]) +fi], +[Fortran], [if test x != x"$MPILIBS"; then + AC_MSG_CHECKING([for mpif.h]) + AC_COMPILE_IFELSE([AC_LANG_PROGRAM([],[ include 'mpif.h'])],[AC_MSG_RESULT(yes)], [MPILIBS="" + AC_MSG_RESULT(no)]) +fi]) + +AC_LANG_CASE([C], [CC="$ax_mpi_save_CC"], + [C++], [CXX="$ax_mpi_save_CXX"], + [Fortran 77], [F77="$ax_mpi_save_F77"], + [Fortran], [FC="$ax_mpi_save_FC"]) + +AC_SUBST(MPILIBS) + +# Finally, execute ACTION-IF-FOUND/ACTION-IF-NOT-FOUND: +if test x = x"$MPILIBS"; then + $2 + : +else + ifelse([$1],,[AC_DEFINE(HAVE_MPI,1,[Define if you have the MPI library.])],[$1]) + : +fi +])dnl AX_MPI diff --git a/mpi_nrm.h b/mpi_nrm.h deleted file mode 100644 index 1af4fa5..0000000 --- a/mpi_nrm.h +++ /dev/null @@ -1,36 +0,0 @@ -/* Filename: mpi_nrm.h - * - * Includes required headers, functions and parameters used by the Message - * Passing Interface (MPI) Node Resource Manager (NRM) library - * - */ - -#include // for uint64_t -#include // for printf -#include // for exit, atoi -#include // for signalHandling -#include // for sched_getcpu -#include // for gethostname -#include - -#include "mpi.h" -#include "basic_functions.h" -#include "downstream_api.h" - -// Set to non-zero value to transmit to NRM using NRM_TRANSMIT environment variable -static unsigned int _transmit = 0; -// Phase shorter than this will be aggregated - Set using NRM_DAMPER environ (in -// seconds) -static uint64_t _damper = 10000000; - -// Book-keeping and Statistics -static unsigned int _aggregation = 0; -static unsigned int _damperAggregationCount = 0; -static unsigned int _phaseSkipCount = 0; - -unsigned int cpu; -int rank; -// used to measure the computation time during a phase -uint64_t startCompute, endCompute; - -struct nrm_context ctxt; diff --git a/mpi_nrm_fort.f b/mpi_nrm_fort.f deleted file mode 100644 index 3140316..0000000 --- a/mpi_nrm_fort.f +++ /dev/null @@ -1,38 +0,0 @@ -C Filename: mpi_nrm_fort.f -C -C Description: This file implements the Fortran interface for the -C MPI-NRM. -C - - subroutine mpi_init(IERROR) - integer IERROR - call f2c_mpi_init(IERROR) - - return - end - - subroutine mpi_finalize(IERROR) - integer IERROR - call f2c_mpi_finalize(IERROR) - - return - end - - subroutine mpi_barrier(COMM,IERROR) - integer IERROR - INTEGER COMM - call f2c_mpi_barrier(COMM,IERROR) - - return - end - - subroutine mpi_allreduce(SBUF,RBUF,CNT,DTYP,OP,COMM,IERR) - use, intrinsic :: iso_c_binding - integer CNT,DTYP,OP,COMM,IERR - type(c_ptr), intent(out) :: SBUF - type(c_ptr), intent(in) :: RBUF - call f2c_mpi_allreduce(SBUF,RBUF,CNT,DTYP,OP,COMM,IERR) - - return - end - diff --git a/src/Makefile.am b/src/Makefile.am new file mode 100644 index 0000000..b6c2f22 --- /dev/null +++ b/src/Makefile.am @@ -0,0 +1,14 @@ +AM_CPPFLAGS = @LIBZMQ_CFLAGS@ + +lib_LTLIBRARIES = libnrm.la + +if PMPI_API +lib_LTLIBRARIES += libnrm-pmpi.la +endif + +libnrm_la_SOURCES = downstream_api.c nrm.h +libnrm_la_LIBADD = @LIBZMQ_LIBS@ +libnrm_pmpi_la_SOURCES = downstream_api.c mpi_api.c nrm.h +libnrm_pmpi_la_LIBADD = @LIBZMQ_LIBS@ + +include_HEADERS = nrm.h diff --git a/downstream_api.c b/src/downstream_api.c similarity index 95% rename from downstream_api.c rename to src/downstream_api.c index 3d91031..7ebb80a 100644 --- a/downstream_api.c +++ b/src/downstream_api.c @@ -8,13 +8,14 @@ * efficiency at the node level. */ -#include "downstream_api.h" -#include -#include -#include -#include -#include -#include +#include +#include +#include +#include +#include +#include + +#include "nrm.h" int nrm_init(struct nrm_context *ctxt, const char *uuid) { diff --git a/mpi_nrm.cpp b/src/mpi_api.c similarity index 83% rename from mpi_nrm.cpp rename to src/mpi_api.c index 1ccb263..aff51b4 100644 --- a/mpi_nrm.cpp +++ b/src/mpi_api.c @@ -15,27 +15,34 @@ * * Written by Sridutt Bhalachandra, sriduttb@anl.gov */ - -#include "mpi_nrm.h" - -/************************ - * Signal Handler to handle SIGTERM, SIGINT and other calls if needed - ***********************/ -void handle_signal(int sig_num) -{ - printf("\nExiting MPI NRM Interface...\n"); - - exit(sig_num); -} - +#include "nrm.h" +#include +#include // for sched_getcpu +#include // for printf +#include // for exit, atoi +#include + +// Set to non-zero value to transmit to NRM using NRM_TRANSMIT environment variable +static unsigned int _transmit = 0; +// Phase shorter than this will be aggregated - Set using NRM_DAMPER environ (in +// seconds) +static uint64_t _damper = 10000000; + +// Book-keeping and Statistics +static unsigned int _aggregation = 0; +static unsigned int _damperAggregationCount = 0; +static unsigned int _phaseSkipCount = 0; + +static unsigned int cpu; +static int rank; +// used to measure the computation time during a phase +static uint64_t startCompute, endCompute; +static struct nrm_context ctxt; /************************ * Setup up the MPI NRM Interface ***********************/ -extern "C" void MPI_nrm_init() +void MPI_nrm_init() { - signal(SIGTERM, handle_signal); - signal(SIGINT, handle_signal); - cpu = sched_getcpu(); MPI_Comm_rank(MPI_COMM_WORLD, &rank); @@ -57,7 +64,7 @@ extern "C" void MPI_nrm_init() return; } -extern "C" void MPI_nrm_fini() +void MPI_nrm_fini() { // Cleanup NRM context nrm_fini(&ctxt); @@ -66,7 +73,7 @@ extern "C" void MPI_nrm_fini() /************************ * Prints the transmission statistics for an application ***********************/ -extern "C" void MPI_nrm_print_stats(void) +void MPI_nrm_print_stats(void) { printf("Stats: CPU %u Damper %lf DamperAggreations %u PhaseSkips %u\n", cpu, _damper,_damperAggregationCount, _phaseSkipCount); @@ -76,7 +83,7 @@ extern "C" void MPI_nrm_print_stats(void) /************************ * Send appropriate phase context to NRM ***********************/ -extern "C" void transmit_to_nrm(int cpu, uint64_t *startCompute, uint64_t +void transmit_to_nrm(int cpu, uint64_t *startCompute, uint64_t endCompute, uint64_t startBarrier, uint64_t endBarrier) { uint64_t computeTime, barrierTime, totalPhaseTime; @@ -100,7 +107,7 @@ extern "C" void transmit_to_nrm(int cpu, uint64_t *startCompute, uint64_t if(getenv("NRM_SKIP")) { // Reset phase start time - *startCompute = return_current_time(); + *startCompute = nrm_gettime(); _phaseSkipCount++; // Reset environment variable @@ -113,14 +120,14 @@ extern "C" void transmit_to_nrm(int cpu, uint64_t *startCompute, uint64_t // Reset _aggregation = 0; - *startCompute = return_current_time(); + *startCompute = nrm_gettime(); return; } int MPI_Init(int *argc, char ***argv) { - startCompute = return_current_time(); + startCompute = nrm_gettime(); int ret_value = PMPI_Init(argc, argv); MPI_nrm_init(); @@ -168,12 +175,12 @@ int MPI_Irecv(void *buf, int count, MPI_Datatype datatype, int source, int tag, int MPI_Barrier(MPI_Comm comm) { - endCompute = return_current_time(); + endCompute = nrm_gettime(); uint64_t startBarrier = endCompute; int ret_value = PMPI_Barrier(comm); - uint64_t endBarrier = return_current_time(); + uint64_t endBarrier = nrm_gettime(); if(_transmit) { @@ -233,12 +240,12 @@ int MPI_Reduce(const void *sendbuf, void *recvbuf, int count, int MPI_Allreduce(const void *sendbuf, void *recvbuf, int count, MPI_Datatype datatype, MPI_Op op, MPI_Comm comm) { - endCompute = return_current_time(); + endCompute = nrm_gettime(); uint64_t startBarrier = endCompute; int ret_value = PMPI_Allreduce(sendbuf, recvbuf, count, datatype, op, comm); - uint64_t endBarrier = return_current_time(); + uint64_t endBarrier = nrm_gettime(); if(_transmit) { diff --git a/downstream_api.h b/src/nrm.h similarity index 73% rename from downstream_api.h rename to src/nrm.h index 5a0c4c3..227579e 100644 --- a/downstream_api.h +++ b/src/nrm.h @@ -8,7 +8,12 @@ #ifndef NRM_H #define NRM_H 1 -#include +#include +#include + +#ifdef __cplusplus +extern "C" { +#endif /* min time in nsec between messages: necessary for rate-limiting progress * report. For now, 10ms is the threashold. */ @@ -30,17 +35,24 @@ struct nrm_context { #define NRM_PROGRESS_FORMAT "{\"type\":\"application\", \"event\":\"progress\", \"payload\": \"%lu\", \"uuid\": \"%s\"}" #define NRM_PHASE_CONTEXT_FORMAT "{\"type\":\"application\", \"event\":\"phase_context\", \"cpu\": \"%u\", \"aggregation\": \"%u\", \"computetime\": \"%llu\", \"totaltime\": \"%llu\", \"uuid\": \"%s\"}" #define NRM_EXIT_FORMAT "{\"type\":\"application\", \"event\":\"exit\", \"uuid\": \"%s\"}" -#ifdef __cplusplus -extern "C"{ -#endif int nrm_init(struct nrm_context *, const char *); int nrm_fini(struct nrm_context *); -int nrm_send_progress(struct nrm_context *, unsigned long); -int nrm_send_phase_context(struct nrm_context *ctxt, unsigned int cpu, unsigned - int aggregation, unsigned long long int computeTime, unsigned long long int - totalTime); +int nrm_send_progress(struct nrm_context *, unsigned long progress); +int nrm_send_phase_context(struct nrm_context *ctxt, + unsigned int cpu, + unsigned int aggregation, + unsigned long long int computeTime, + unsigned long long int totalTime); + +/* util function */ +inline long long int nrm_gettime(void) +{ + struct timespec now; + clock_gettime(CLOCK_REALTIME, &now); + return ((now.tv_sec * 1e9) + now.tv_nsec); +} #ifdef __cplusplus } -- 2.26.2