starting implementation (nearly done, actually)

.gitignore

+build/

CMakeLists.txt

+#
+#  general cmake flags:
+#    -DCMAKE_INSTALL_PREFIX=/usr/local     -- the prefix for installing
+#    -DCMAKE_BUILD_TYPE=type               -- type can be Debug, Release, ...
+#    -DCMAKE_PREFIX_PATH=/dir              -- external packages
+#
+#     note that CMAKE_PREFIX_PATH can be a list of directories:
+#      -DCMAKE_PREFIX_PATH='/dir1;/dir2;/dir3'
+#
+
+cmake_minimum_required (VERSION 3.14)
+project (mona C)
+enable_testing ()
+
+option(ENABLE_TESTS    "Build tests. May require CppUnit_ROOT" OFF)
+option(ENABLE_EXAMPLES "Build examples" OFF)
+
+add_definitions (-Wall -Wextra)
+
+# add our cmake module directory to the path
+set (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
+     "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
+
+# link shared lib with full rpath
+set (CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
+set (CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
+
+# setup cache variables for ccmake
+if (NOT CMAKE_BUILD_TYPE)
+    set (CMAKE_BUILD_TYPE Release
+         CACHE STRING "Choose the type of build." FORCE)
+    set_property (CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS 
+                  "Debug" "Release" "RelWithDebInfo" "MinSizeRel")
+endif ()
+set (CMAKE_PREFIX_PATH "" CACHE STRING "External dependencies path")
+set (BUILD_SHARED_LIBS "OFF" CACHE BOOL "Build a shared library")
+
+find_package (PkgConfig REQUIRED)
+find_package (mercury REQUIRED)
+pkg_check_modules (ARGOBOTS REQUIRED IMPORTED_TARGET argobots)
+
+add_subdirectory (src)
+
+if (ENABLE_TESTS)
+    find_package (CppUnit REQUIRED)
+    find_package (MPI REQUIRED)
+    include_directories(${CPPUNIT_INCLUDE_DIR})
+    enable_testing()
+    add_subdirectory (tests)
+endif (ENABLE_TESTS)
+
+if(${ENABLE_EXAMPLES})
+  add_subdirectory (examples)
+endif(${ENABLE_EXAMPLES})

COPYRIGHT

+Copyright (c) 2020, UChicago Argonne, LLC
+All Rights Reserved
+SDS TOOLS (ANL-SF-16-009)
+
+OPEN SOURCE LICENSE
+
+Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
+LLC, the U.S. Government retains certain rights in this software.
+
+Redistribution and use in source and binary forms, with or without
+modification, are permitted provided that the following conditions
+are met:
+
+1. Redistributions of source code must retain the above copyright notice,
+this list of conditions and the following disclaimer.
+
+2. Redistributions in binary form must reproduce the above copyright
+notice, this list of conditions and the following disclaimer in the
+documentation and/or other materials provided with the distribution.
+
+3. Neither the names of UChicago Argonne, LLC or the Department of
+Energy nor the names of its contributors may be used to endorse or
+promote products derived from this software without specific prior
+written permission.
+
+******************************************************************************
+DISCLAIMER
+
+THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND.
+
+NEITHER THE UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT
+OF ENERGY, NOR UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES,
+MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY
+OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY
+INFORMATION, DATA, APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS
+THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS.
+
+******************************************************************************

cmake/FindCppUnit.cmake

+#
+# Find the CppUnit includes and library
+#
+# This module defines
+# CPPUNIT_INCLUDE_DIR, where to find tiff.h, etc.
+# CPPUNIT_LIBRARIES, the libraries to link against to use CppUnit.
+# CPPUNIT_FOUND, If false, do not try to use CppUnit.
+
+# also defined, but not for general use are
+# CPPUNIT_LIBRARY, where to find the CppUnit library.
+# CPPUNIT_DEBUG_LIBRARY, where to find the CppUnit library in debug mode.
+
+FIND_PATH(CPPUNIT_INCLUDE_DIR cppunit/TestCase.h HINTS
+		/usr/local/include
+		/usr/include
+		ENV CppUnit_ROOT
+	 )
+
+# With Win32, important to have both
+IF(WIN32)
+	FIND_LIBRARY(CPPUNIT_LIBRARY cppunit
+			${CPPUNIT_INCLUDE_DIR}/../lib
+			/usr/local/lib
+			/usr/lib)
+	FIND_LIBRARY(CPPUNIT_DEBUG_LIBRARY cppunitd
+			${CPPUNIT_INCLUDE_DIR}/../lib
+			/usr/local/lib
+			/usr/lib)
+ELSE(WIN32)
+# On unix system, debug and release have the same name
+	FIND_LIBRARY(CPPUNIT_LIBRARY cppunit
+			${CPPUNIT_INCLUDE_DIR}/../lib
+			/usr/local/lib
+			/usr/lib)
+	FIND_LIBRARY(CPPUNIT_DEBUG_LIBRARY cppunit
+			${CPPUNIT_INCLUDE_DIR}/../lib
+			/usr/local/lib
+			/usr/lib)
+ENDIF(WIN32)
+
+	IF(CPPUNIT_INCLUDE_DIR)
+IF(CPPUNIT_LIBRARY)
+	SET(CPPUNIT_FOUND "YES")
+SET(CPPUNIT_LIBRARIES ${CPPUNIT_LIBRARY} ${CMAKE_DL_LIBS})
+	SET(CPPUNIT_DEBUG_LIBRARIES ${CPPUNIT_DEBUG_LIBRARY}
+			${CMAKE_DL_LIBS})
+	ENDIF(CPPUNIT_LIBRARY)
+ENDIF(CPPUNIT_INCLUDE_DIR)

examples/CMakeLists.txt

+add_executable (example ${CMAKE_CURRENT_SOURCE_DIR}/example.c)
+target_link_libraries (example mona)

examples/example.c

+#include <mpi.h>
+#include <mona.h>
+
+#define ASSERT_MESSAGE(__cond__, __msg__) \
+    if(!(__cond__)) { \
+        fprintf(stderr, "[%d] Assertion failed (%s): %s\n", rank, #__cond__, __msg__); \
+        exit(-1); \
+    }
+
+int main(int argc, char** argv) {
+
+    int rank, size, i;
+    na_return_t ret;
+
+    MPI_Init(&argc, &argv);
+    ABT_init(argc, argv);
+    MPI_Comm_rank(MPI_COMM_WORLD, &rank);
+    MPI_Comm_size(MPI_COMM_WORLD, &size);
+
+    mona_instance_t mona = mona_init("na+sm", NA_TRUE, NULL);
+    ASSERT_MESSAGE(mona != MONA_INSTANCE_NULL, "Could not initialize Mona instance"); 
+    printf("[%d] Correctly instanciated Mona\n", rank);
+
+    char addr_str[128];
+    na_size_t addr_size = 128;
+    na_addr_t addr = NA_ADDR_NULL;
+    ret = mona_addr_self(mona, &addr);
+    ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not get self address");
+
+    ret = mona_addr_to_string(mona, addr_str, &addr_size, addr);
+    ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not convert address to string");
+    printf("[%d] My address is %s\n", rank, addr_str);
+
+    ret = mona_addr_free(mona, addr);
+    ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not free address");
+
+    size_t msg_len = mona_msg_get_max_unexpected_size(mona);
+    printf("[%d] Message size: %ld\n", rank, msg_len);
+
+    void* plugin_data = NULL;
+    char* buf = (char*)mona_msg_buf_alloc(mona, msg_len, &plugin_data);
+    ASSERT_MESSAGE(buf != NULL, "Could not allocate message buffer");
+
+    if(rank == 0) {
+        MPI_Recv(addr_str, 128, MPI_CHAR, 1, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
+        
+        ret = mona_addr_lookup(mona, addr_str, &addr);
+        ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not lookup address");
+
+        printf("[0] Sending message to rank 1\n");
+
+        for(i = mona_msg_get_unexpected_header_size(mona); i < (int)msg_len; i++) {
+            buf[i] = i % 32;
+        }
+
+        ret = mona_msg_init_unexpected(mona, buf, msg_len);
+        ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not initialize message");
+
+        ret = mona_msg_send_unexpected(
+                mona, buf, msg_len, plugin_data, addr, 0, 0, NA_OP_ID_IGNORE);
+        ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not send message");
+
+        ret = mona_addr_free(mona, addr);
+        ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not free address");
+
+    } else {
+        MPI_Send(addr_str, 128, MPI_CHAR, 0, 0, MPI_COMM_WORLD);
+
+        ret = mona_msg_recv_unexpected(
+                mona, buf, msg_len, plugin_data, NA_OP_ID_IGNORE);
+        ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not receive message");
+
+        printf("[1] Receiving message from rank 0\n");
+
+        for(i = mona_msg_get_unexpected_header_size(mona); i < (int)msg_len; i++) {
+            ASSERT_MESSAGE(buf[i] == i % 32, "Incorrect byte received");
+        }
+    }
+
+    ret = mona_msg_buf_free(mona, buf, plugin_data);
+    ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not free buffer");
+
+    mona_finalize(mona);
+
+    ABT_finalize();
+    MPI_Finalize();
+
+    return 0;
+}

include/mona.h

+/*
+ * (C) 2020 The University of Chicago
+ * 
+ * See COPYRIGHT in top-level directory.
+ */
+#ifndef __MONA_H
+#define __MONA_H
+
+#include <mercury.h>
+#include <mercury_types.h>
+#include <mercury_bulk.h>
+#include <mercury_macros.h>
+#include <abt.h>
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+typedef struct mona_instance* mona_instance_t;
+typedef struct mona_request*  mona_request_t;
+
+#define MONA_INSTANCE_NULL ((mona_instance_t)NULL)
+#define MONA_REQUEST_NULL  ((mona_request_t)NULL)
+
+mona_instance_t mona_init(
+        const char *info_string,
+        na_bool_t listen,
+        const struct na_init_info *na_init_info);
+
+mona_instance_t mona_init_thread(
+        const char *info_string,
+        na_bool_t listen,
+        const struct na_init_info *na_init_info,
+        na_bool_t use_progress_es);
+
+mona_instance_t mona_init_pool(
+        const char *info_string,
+        na_bool_t listen,
+        const struct na_init_info *na_init_info,
+        ABT_pool progress_pool);
+
+mona_instance_t mona_init_na_pool(
+        na_class_t *na_class,
+        na_context_t *na_context,
+        ABT_pool progress_pool);
+
+na_return_t mona_finalize(mona_instance_t mona);
+
+const char* mona_get_class_name(mona_instance_t mona);
+
+const char* mona_get_class_protocol(mona_instance_t mona);
+
+na_bool_t mona_is_listening(mona_instance_t mona);
+
+na_return_t mona_addr_lookup(
+        mona_instance_t mona,
+        const char *name,
+        na_addr_t *addr);
+
+na_return_t mona_addr_free(
+        mona_instance_t mona,
+        na_addr_t addr);
+
+na_return_t mona_addr_set_remove(
+        mona_instance_t mona,
+        na_addr_t addr);
+
+na_return_t mona_addr_self(
+        mona_instance_t mona,
+        na_addr_t* addr);
+
+na_return_t mona_addr_dup(
+        mona_instance_t mona,
+        na_addr_t addr,
+        na_addr_t* dup_addr);
+
+na_bool_t mona_addr_cmp(
+        mona_instance_t mona,
+        na_addr_t addr1,
+        na_addr_t addr2);
+
+na_bool_t mona_addr_is_self(
+        mona_instance_t mona,
+        na_addr_t addr);
+
+na_return_t mona_addr_to_string(
+        mona_instance_t mona,
+        char *buf,
+        na_size_t *buf_size,
+        na_addr_t addr);
+
+na_size_t mona_addr_get_serialize_size(
+        mona_instance_t mona,
+        na_addr_t addr);
+
+na_return_t mona_addr_serialize(
+        mona_instance_t mona,
+        void *buf,
+        na_size_t buf_size,
+        na_addr_t addr);
+
+na_return_t mona_addr_deserialize(
+        mona_instance_t mona,
+        na_addr_t *addr,
+        const void *buf,
+        na_size_t buf_size);
+
+na_size_t mona_msg_get_max_unexpected_size(
+        mona_instance_t mona);
+
+na_size_t mona_msg_get_max_expected_size(
+        mona_instance_t mona);
+
+na_size_t mona_msg_get_unexpected_header_size(
+        mona_instance_t mona);
+
+na_size_t mona_msg_get_expected_header_size(
+        mona_instance_t mona);
+
+na_tag_t mona_msg_get_max_tag(mona_instance_t mona);
+
+na_op_id_t mona_op_create(mona_instance_t mona);
+
+na_return_t mona_op_destroy(
+        mona_instance_t mona,
+        na_op_id_t op_id);
+
+void* mona_msg_buf_alloc(
+        mona_instance_t mona,
+        na_size_t buf_size,
+        void **plugin_data);
+
+na_return_t mona_msg_buf_free(
+        mona_instance_t mona,
+        void *buf,
+        void *plugin_data);
+
+na_return_t mona_msg_init_unexpected(
+        mona_instance_t mona,
+        void *buf,
+        na_size_t buf_size);
+
+na_return_t mona_msg_send_unexpected(
+        mona_instance_t mona,
+        const void *buf,
+        na_size_t buf_size,
+        void *plugin_data,
+        na_addr_t dest_addr,
+        na_uint8_t dest_id,
+        na_tag_t tag,
+        na_op_id_t *op_id);
+
+na_return_t mona_msg_isend_unexpected(
+        mona_instance_t mona,
+        const void *buf,
+        na_size_t buf_size,
+        void *plugin_data,
+        na_addr_t dest_addr,
+        na_uint8_t dest_id,
+        na_tag_t tag,
+        na_op_id_t *op_id,
+        mona_request_t* req);
+
+na_return_t mona_msg_recv_unexpected(
+        mona_instance_t mona,
+        void *buf,
+        na_size_t buf_size,
+        void *plugin_data,
+        na_op_id_t *op_id);
+
+na_return_t mona_msg_irecv_unexpected(
+        mona_instance_t mona,
+        void *buf,
+        na_size_t buf_size,
+        void *plugin_data,
+        na_op_id_t *op_id,
+        mona_request_t* req);
+
+na_return_t mona_msg_init_expected(
+        mona_instance_t mona,
+        void *buf,
+        na_size_t buf_size);
+
+na_return_t mona_msg_send_expected(
+        mona_instance_t mona,
+        const void *buf,
+        na_size_t buf_size,
+        void *plugin_data,
+        na_addr_t dest_addr,
+        na_uint8_t dest_id,
+        na_tag_t tag,
+        na_op_id_t *op_id);
+
+na_return_t mona_msg_isend_expected(
+        mona_instance_t mona,
+        const void *buf,
+        na_size_t buf_size,
+        void *plugin_data,
+        na_addr_t dest_addr,
+        na_uint8_t dest_id,
+        na_tag_t tag,
+        na_op_id_t *op_id,
+        mona_request_t* req);
+
+na_return_t mona_msg_recv_expected(
+        mona_instance_t mona,
+        void *buf,
+        na_size_t buf_size,
+        void *plugin_data,
+        na_addr_t source_addr,
+        na_uint8_t source_id,
+        na_tag_t tag,
+        na_op_id_t *op_id);
+
+na_return_t mona_msg_irecv_expected(
+        mona_instance_t mona,
+        void *buf,
+        na_size_t buf_size,
+        void *plugin_data,
+        na_addr_t source_addr,
+        na_uint8_t source_id,
+        na_tag_t tag,
+        na_op_id_t *op_id,
+        mona_request_t* req);
+
+na_return_t mona_mem_handle_create(
+        mona_instance_t mona,
+        void *buf,
+        na_size_t buf_size,
+        unsigned long flags,
+        na_mem_handle_t *mem_handle);
+
+na_return_t mona_mem_handle_create_segments(
+        mona_instance_t mona,
+        struct na_segment *segments,
+        na_size_t segment_count,
+        unsigned long flags,
+        na_mem_handle_t *mem_handle);
+
+na_return_t mona_mem_handle_free(
+        mona_instance_t mona,
+        na_mem_handle_t mem_handle);
+
+na_return_t mona_mem_register(
+        mona_instance_t mona,
+        na_mem_handle_t mem_handle);
+
+na_return_t mona_mem_deregister(
+        mona_instance_t mona,
+        na_mem_handle_t mem_handle);
+
+na_return_t mona_mem_publish(
+        mona_instance_t mona,
+        na_mem_handle_t mem_handle);
+
+na_return_t mona_mem_unpublish(
+        mona_instance_t mona,
+        na_mem_handle_t mem_handle);
+
+na_size_t mona_mem_handle_get_serialize_size(
+        mona_instance_t mona,
+        na_mem_handle_t mem_handle);
+
+na_return_t mona_mem_handle_serialize(
+        mona_instance_t mona,
+        void *buf, na_size_t buf_size,
+        na_mem_handle_t mem_handle);
+
+na_return_t mona_mem_handle_deserialize(
+        mona_instance_t mona,
+        na_mem_handle_t *mem_handle,
+        const void *buf,
+        na_size_t buf_size);
+
+na_return_t mona_put(
+        mona_instance_t mona,
+        na_mem_handle_t local_mem_handle,
+        na_offset_t local_offset,
+        na_mem_handle_t remote_mem_handle,
+        na_offset_t remote_offset,
+        na_size_t data_size,
+        na_addr_t remote_addr,
+        na_uint8_t remote_id,
+        na_op_id_t *op_id);
+
+na_return_t mona_iput(
+        mona_instance_t mona,
+        na_mem_handle_t local_mem_handle,
+        na_offset_t local_offset,
+        na_mem_handle_t remote_mem_handle,
+        na_offset_t remote_offset,
+        na_size_t data_size,
+        na_addr_t remote_addr,
+        na_uint8_t remote_id,
+        na_op_id_t *op_id,
+        mona_request_t* req);
+
+na_return_t mona_get(
+        mona_instance_t mona,
+        na_mem_handle_t local_mem_handle,
+        na_offset_t local_offset,
+        na_mem_handle_t remote_mem_handle,
+        na_offset_t remote_offset,
+        na_size_t data_size,
+        na_addr_t remote_addr,
+        na_uint8_t remote_id,
+        na_op_id_t *op_id);
+
+na_return_t mona_iget(
+        mona_instance_t mona,
+        na_mem_handle_t local_mem_handle,
+        na_offset_t local_offset,
+        na_mem_handle_t remote_mem_handle,
+        na_offset_t remote_offset,
+        na_size_t data_size,
+        na_addr_t remote_addr,
+        na_uint8_t remote_id,
+        na_op_id_t *op_id,
+        mona_request_t* req);
+
+int mona_poll_get_fd(mona_instance_t mona);
+
+na_bool_t mona_poll_try_wait(mona_instance_t mona);
+
+na_return_t mona_cancel(
+        mona_instance_t mona,
+        na_op_id_t op_id);
+
+const char* mona_error_to_string(int errnum);
+
+na_return_t mona_wait(mona_request_t req);
+
+int mona_test(mona_request_t req, int* flag);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif

src/CMakeLists.txt

+# load package helper for generating cmake CONFIG packages
+include (CMakePackageConfigHelpers)
+
+# where to install files for "find_package"
+set (mona-pkg "share/cmake/mona")
+
+#
+# library version set here (e.g. for shared libs).
+#
+set (MONA_VERSION_MAJOR 0)
+set (MONA_VERSION_MINOR 1)
+set (MONA_VERSION_PATCH 0)
+set (mona-vers "${MONA_VERSION_MAJOR}.${MONA_VERSION_MINOR}")
+set (MONA_VERSION "${mona-vers}.${MONA_VERSION_PATCH}")
+
+add_library(mona ${CMAKE_CURRENT_SOURCE_DIR}/mona.c)
+target_link_libraries (mona mercury PkgConfig::ARGOBOTS)
+target_include_directories (mona PUBLIC $<INSTALL_INTERFACE:include>)
+
+# local include's BEFORE, in case old incompatable .h files in prefix/include
+target_include_directories (mona BEFORE PUBLIC
+    $<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include>)
+
+# for shared libs, establish the lib version
+set_target_properties (mona
+    PROPERTIES VERSION ${MONA_VERSION}
+    SOVERSION ${MONA_VERSION_MAJOR})
+
+#
+# installation stuff (packaging and install commands)
+#
+write_basic_package_version_file(
+    "mona-config-version.cmake"
+    VERSION ${MONA_VERSION}
+    COMPATIBILITY AnyNewerVersion)
+
+# generate our config file for find_package()
+configure_file (mona-config.cmake.in mona-config.cmake @ONLY)
+
+# some bits for the pkg-config file
+set (DEST_DIR "${CMAKE_INSTALL_PREFIX}")
+set (PRIVATE_LIBS "-lmona")
+configure_file ("mona.pc.in" "mona.pc" @ONLY)
+
+#
+# "make install" rules
+#
+install (TARGETS mona EXPORT mona-targets
+         ARCHIVE DESTINATION lib
+         LIBRARY DESTINATION lib)
+install (EXPORT mona-targets
+         DESTINATION ${mona-pkg}
+         FILE "mona-targets.cmake")
+install (FILES "${CMAKE_CURRENT_BINARY_DIR}/mona-config.cmake"
+