Commit 37703e17 authored by Matthieu Dorier's avatar Matthieu Dorier
Browse files

starting implementation (nearly done, actually)

parents
#
# general cmake flags:
# -DCMAKE_INSTALL_PREFIX=/usr/local -- the prefix for installing
# -DCMAKE_BUILD_TYPE=type -- type can be Debug, Release, ...
# -DCMAKE_PREFIX_PATH=/dir -- external packages
#
# note that CMAKE_PREFIX_PATH can be a list of directories:
# -DCMAKE_PREFIX_PATH='/dir1;/dir2;/dir3'
#
cmake_minimum_required (VERSION 3.14)
project (mona C)
enable_testing ()
option(ENABLE_TESTS "Build tests. May require CppUnit_ROOT" OFF)
option(ENABLE_EXAMPLES "Build examples" OFF)
add_definitions (-Wall -Wextra)
# add our cmake module directory to the path
set (CMAKE_MODULE_PATH ${CMAKE_MODULE_PATH}
"${CMAKE_CURRENT_SOURCE_DIR}/cmake")
# link shared lib with full rpath
set (CMAKE_INSTALL_RPATH "${CMAKE_INSTALL_PREFIX}/lib")
set (CMAKE_INSTALL_RPATH_USE_LINK_PATH TRUE)
# setup cache variables for ccmake
if (NOT CMAKE_BUILD_TYPE)
set (CMAKE_BUILD_TYPE Release
CACHE STRING "Choose the type of build." FORCE)
set_property (CACHE CMAKE_BUILD_TYPE PROPERTY STRINGS
"Debug" "Release" "RelWithDebInfo" "MinSizeRel")
endif ()
set (CMAKE_PREFIX_PATH "" CACHE STRING "External dependencies path")
set (BUILD_SHARED_LIBS "OFF" CACHE BOOL "Build a shared library")
find_package (PkgConfig REQUIRED)
find_package (mercury REQUIRED)
pkg_check_modules (ARGOBOTS REQUIRED IMPORTED_TARGET argobots)
add_subdirectory (src)
if (ENABLE_TESTS)
find_package (CppUnit REQUIRED)
find_package (MPI REQUIRED)
include_directories(${CPPUNIT_INCLUDE_DIR})
enable_testing()
add_subdirectory (tests)
endif (ENABLE_TESTS)
if(${ENABLE_EXAMPLES})
add_subdirectory (examples)
endif(${ENABLE_EXAMPLES})
Copyright (c) 2020, UChicago Argonne, LLC
All Rights Reserved
SDS TOOLS (ANL-SF-16-009)
OPEN SOURCE LICENSE
Under the terms of Contract No. DE-AC02-06CH11357 with UChicago Argonne,
LLC, the U.S. Government retains certain rights in this software.
Redistribution and use in source and binary forms, with or without
modification, are permitted provided that the following conditions
are met:
1. Redistributions of source code must retain the above copyright notice,
this list of conditions and the following disclaimer.
2. Redistributions in binary form must reproduce the above copyright
notice, this list of conditions and the following disclaimer in the
documentation and/or other materials provided with the distribution.
3. Neither the names of UChicago Argonne, LLC or the Department of
Energy nor the names of its contributors may be used to endorse or
promote products derived from this software without specific prior
written permission.
******************************************************************************
DISCLAIMER
THE SOFTWARE IS SUPPLIED "AS IS" WITHOUT WARRANTY OF ANY KIND.
NEITHER THE UNTED STATES GOVERNMENT, NOR THE UNITED STATES DEPARTMENT
OF ENERGY, NOR UCHICAGO ARGONNE, LLC, NOR ANY OF THEIR EMPLOYEES,
MAKES ANY WARRANTY, EXPRESS OR IMPLIED, OR ASSUMES ANY LEGAL LIABILITY
OR RESPONSIBILITY FOR THE ACCURACY, COMPLETENESS, OR USEFULNESS OF ANY
INFORMATION, DATA, APPARATUS, PRODUCT, OR PROCESS DISCLOSED, OR REPRESENTS
THAT ITS USE WOULD NOT INFRINGE PRIVATELY OWNED RIGHTS.
******************************************************************************
#
# Find the CppUnit includes and library
#
# This module defines
# CPPUNIT_INCLUDE_DIR, where to find tiff.h, etc.
# CPPUNIT_LIBRARIES, the libraries to link against to use CppUnit.
# CPPUNIT_FOUND, If false, do not try to use CppUnit.
# also defined, but not for general use are
# CPPUNIT_LIBRARY, where to find the CppUnit library.
# CPPUNIT_DEBUG_LIBRARY, where to find the CppUnit library in debug mode.
FIND_PATH(CPPUNIT_INCLUDE_DIR cppunit/TestCase.h HINTS
/usr/local/include
/usr/include
ENV CppUnit_ROOT
)
# With Win32, important to have both
IF(WIN32)
FIND_LIBRARY(CPPUNIT_LIBRARY cppunit
${CPPUNIT_INCLUDE_DIR}/../lib
/usr/local/lib
/usr/lib)
FIND_LIBRARY(CPPUNIT_DEBUG_LIBRARY cppunitd
${CPPUNIT_INCLUDE_DIR}/../lib
/usr/local/lib
/usr/lib)
ELSE(WIN32)
# On unix system, debug and release have the same name
FIND_LIBRARY(CPPUNIT_LIBRARY cppunit
${CPPUNIT_INCLUDE_DIR}/../lib
/usr/local/lib
/usr/lib)
FIND_LIBRARY(CPPUNIT_DEBUG_LIBRARY cppunit
${CPPUNIT_INCLUDE_DIR}/../lib
/usr/local/lib
/usr/lib)
ENDIF(WIN32)
IF(CPPUNIT_INCLUDE_DIR)
IF(CPPUNIT_LIBRARY)
SET(CPPUNIT_FOUND "YES")
SET(CPPUNIT_LIBRARIES ${CPPUNIT_LIBRARY} ${CMAKE_DL_LIBS})
SET(CPPUNIT_DEBUG_LIBRARIES ${CPPUNIT_DEBUG_LIBRARY}
${CMAKE_DL_LIBS})
ENDIF(CPPUNIT_LIBRARY)
ENDIF(CPPUNIT_INCLUDE_DIR)
add_executable (example ${CMAKE_CURRENT_SOURCE_DIR}/example.c)
target_link_libraries (example mona)
#include <mpi.h>
#include <mona.h>
#define ASSERT_MESSAGE(__cond__, __msg__) \
if(!(__cond__)) { \
fprintf(stderr, "[%d] Assertion failed (%s): %s\n", rank, #__cond__, __msg__); \
exit(-1); \
}
int main(int argc, char** argv) {
int rank, size, i;
na_return_t ret;
MPI_Init(&argc, &argv);
ABT_init(argc, argv);
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &size);
mona_instance_t mona = mona_init("na+sm", NA_TRUE, NULL);
ASSERT_MESSAGE(mona != MONA_INSTANCE_NULL, "Could not initialize Mona instance");
printf("[%d] Correctly instanciated Mona\n", rank);
char addr_str[128];
na_size_t addr_size = 128;
na_addr_t addr = NA_ADDR_NULL;
ret = mona_addr_self(mona, &addr);
ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not get self address");
ret = mona_addr_to_string(mona, addr_str, &addr_size, addr);
ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not convert address to string");
printf("[%d] My address is %s\n", rank, addr_str);
ret = mona_addr_free(mona, addr);
ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not free address");
size_t msg_len = mona_msg_get_max_unexpected_size(mona);
printf("[%d] Message size: %ld\n", rank, msg_len);
void* plugin_data = NULL;
char* buf = (char*)mona_msg_buf_alloc(mona, msg_len, &plugin_data);
ASSERT_MESSAGE(buf != NULL, "Could not allocate message buffer");
if(rank == 0) {
MPI_Recv(addr_str, 128, MPI_CHAR, 1, 0, MPI_COMM_WORLD, MPI_STATUS_IGNORE);
ret = mona_addr_lookup(mona, addr_str, &addr);
ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not lookup address");
printf("[0] Sending message to rank 1\n");
for(i = mona_msg_get_unexpected_header_size(mona); i < (int)msg_len; i++) {
buf[i] = i % 32;
}
ret = mona_msg_init_unexpected(mona, buf, msg_len);
ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not initialize message");
ret = mona_msg_send_unexpected(
mona, buf, msg_len, plugin_data, addr, 0, 0, NA_OP_ID_IGNORE);
ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not send message");
ret = mona_addr_free(mona, addr);
ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not free address");
} else {
MPI_Send(addr_str, 128, MPI_CHAR, 0, 0, MPI_COMM_WORLD);
ret = mona_msg_recv_unexpected(
mona, buf, msg_len, plugin_data, NA_OP_ID_IGNORE);
ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not receive message");
printf("[1] Receiving message from rank 0\n");
for(i = mona_msg_get_unexpected_header_size(mona); i < (int)msg_len; i++) {
ASSERT_MESSAGE(buf[i] == i % 32, "Incorrect byte received");
}
}
ret = mona_msg_buf_free(mona, buf, plugin_data);
ASSERT_MESSAGE(ret == NA_SUCCESS, "Could not free buffer");
mona_finalize(mona);
ABT_finalize();
MPI_Finalize();
return 0;
}
/*
* (C) 2020 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#ifndef __MONA_H
#define __MONA_H
#include <mercury.h>
#include <mercury_types.h>
#include <mercury_bulk.h>
#include <mercury_macros.h>
#include <abt.h>
#ifdef __cplusplus
extern "C" {
#endif
typedef struct mona_instance* mona_instance_t;
typedef struct mona_request* mona_request_t;
#define MONA_INSTANCE_NULL ((mona_instance_t)NULL)
#define MONA_REQUEST_NULL ((mona_request_t)NULL)
mona_instance_t mona_init(
const char *info_string,
na_bool_t listen,
const struct na_init_info *na_init_info);
mona_instance_t mona_init_thread(
const char *info_string,
na_bool_t listen,
const struct na_init_info *na_init_info,
na_bool_t use_progress_es);
mona_instance_t mona_init_pool(
const char *info_string,
na_bool_t listen,
const struct na_init_info *na_init_info,
ABT_pool progress_pool);
mona_instance_t mona_init_na_pool(
na_class_t *na_class,
na_context_t *na_context,
ABT_pool progress_pool);
na_return_t mona_finalize(mona_instance_t mona);
const char* mona_get_class_name(mona_instance_t mona);
const char* mona_get_class_protocol(mona_instance_t mona);
na_bool_t mona_is_listening(mona_instance_t mona);
na_return_t mona_addr_lookup(
mona_instance_t mona,
const char *name,
na_addr_t *addr);
na_return_t mona_addr_free(
mona_instance_t mona,
na_addr_t addr);
na_return_t mona_addr_set_remove(
mona_instance_t mona,
na_addr_t addr);
na_return_t mona_addr_self(
mona_instance_t mona,
na_addr_t* addr);
na_return_t mona_addr_dup(
mona_instance_t mona,
na_addr_t addr,
na_addr_t* dup_addr);
na_bool_t mona_addr_cmp(
mona_instance_t mona,
na_addr_t addr1,
na_addr_t addr2);
na_bool_t mona_addr_is_self(
mona_instance_t mona,
na_addr_t addr);
na_return_t mona_addr_to_string(
mona_instance_t mona,
char *buf,
na_size_t *buf_size,
na_addr_t addr);
na_size_t mona_addr_get_serialize_size(
mona_instance_t mona,
na_addr_t addr);
na_return_t mona_addr_serialize(
mona_instance_t mona,
void *buf,
na_size_t buf_size,
na_addr_t addr);
na_return_t mona_addr_deserialize(
mona_instance_t mona,
na_addr_t *addr,
const void *buf,
na_size_t buf_size);
na_size_t mona_msg_get_max_unexpected_size(
mona_instance_t mona);
na_size_t mona_msg_get_max_expected_size(
mona_instance_t mona);
na_size_t mona_msg_get_unexpected_header_size(
mona_instance_t mona);
na_size_t mona_msg_get_expected_header_size(
mona_instance_t mona);
na_tag_t mona_msg_get_max_tag(mona_instance_t mona);
na_op_id_t mona_op_create(mona_instance_t mona);
na_return_t mona_op_destroy(
mona_instance_t mona,
na_op_id_t op_id);
void* mona_msg_buf_alloc(
mona_instance_t mona,
na_size_t buf_size,
void **plugin_data);
na_return_t mona_msg_buf_free(
mona_instance_t mona,
void *buf,
void *plugin_data);
na_return_t mona_msg_init_unexpected(
mona_instance_t mona,
void *buf,
na_size_t buf_size);
na_return_t mona_msg_send_unexpected(
mona_instance_t mona,
const void *buf,
na_size_t buf_size,
void *plugin_data,
na_addr_t dest_addr,
na_uint8_t dest_id,
na_tag_t tag,
na_op_id_t *op_id);
na_return_t mona_msg_isend_unexpected(
mona_instance_t mona,
const void *buf,
na_size_t buf_size,
void *plugin_data,
na_addr_t dest_addr,
na_uint8_t dest_id,
na_tag_t tag,
na_op_id_t *op_id,
mona_request_t* req);
na_return_t mona_msg_recv_unexpected(
mona_instance_t mona,
void *buf,
na_size_t buf_size,
void *plugin_data,
na_op_id_t *op_id);
na_return_t mona_msg_irecv_unexpected(
mona_instance_t mona,
void *buf,
na_size_t buf_size,
void *plugin_data,
na_op_id_t *op_id,
mona_request_t* req);
na_return_t mona_msg_init_expected(
mona_instance_t mona,
void *buf,
na_size_t buf_size);
na_return_t mona_msg_send_expected(
mona_instance_t mona,
const void *buf,
na_size_t buf_size,
void *plugin_data,
na_addr_t dest_addr,
na_uint8_t dest_id,
na_tag_t tag,
na_op_id_t *op_id);
na_return_t mona_msg_isend_expected(
mona_instance_t mona,
const void *buf,
na_size_t buf_size,
void *plugin_data,
na_addr_t dest_addr,
na_uint8_t dest_id,
na_tag_t tag,
na_op_id_t *op_id,
mona_request_t* req);
na_return_t mona_msg_recv_expected(
mona_instance_t mona,
void *buf,
na_size_t buf_size,
void *plugin_data,
na_addr_t source_addr,
na_uint8_t source_id,
na_tag_t tag,
na_op_id_t *op_id);
na_return_t mona_msg_irecv_expected(
mona_instance_t mona,
void *buf,
na_size_t buf_size,
void *plugin_data,
na_addr_t source_addr,
na_uint8_t source_id,
na_tag_t tag,
na_op_id_t *op_id,
mona_request_t* req);
na_return_t mona_mem_handle_create(
mona_instance_t mona,
void *buf,
na_size_t buf_size,
unsigned long flags,
na_mem_handle_t *mem_handle);
na_return_t mona_mem_handle_create_segments(
mona_instance_t mona,
struct na_segment *segments,
na_size_t segment_count,
unsigned long flags,
na_mem_handle_t *mem_handle);
na_return_t mona_mem_handle_free(
mona_instance_t mona,
na_mem_handle_t mem_handle);
na_return_t mona_mem_register(
mona_instance_t mona,
na_mem_handle_t mem_handle);
na_return_t mona_mem_deregister(
mona_instance_t mona,
na_mem_handle_t mem_handle);
na_return_t mona_mem_publish(
mona_instance_t mona,
na_mem_handle_t mem_handle);
na_return_t mona_mem_unpublish(
mona_instance_t mona,
na_mem_handle_t mem_handle);
na_size_t mona_mem_handle_get_serialize_size(
mona_instance_t mona,
na_mem_handle_t mem_handle);
na_return_t mona_mem_handle_serialize(
mona_instance_t mona,
void *buf, na_size_t buf_size,
na_mem_handle_t mem_handle);
na_return_t mona_mem_handle_deserialize(
mona_instance_t mona,
na_mem_handle_t *mem_handle,
const void *buf,
na_size_t buf_size);
na_return_t mona_put(
mona_instance_t mona,
na_mem_handle_t local_mem_handle,
na_offset_t local_offset,
na_mem_handle_t remote_mem_handle,
na_offset_t remote_offset,
na_size_t data_size,
na_addr_t remote_addr,
na_uint8_t remote_id,
na_op_id_t *op_id);
na_return_t mona_iput(
mona_instance_t mona,
na_mem_handle_t local_mem_handle,
na_offset_t local_offset,
na_mem_handle_t remote_mem_handle,
na_offset_t remote_offset,
na_size_t data_size,
na_addr_t remote_addr,
na_uint8_t remote_id,
na_op_id_t *op_id,
mona_request_t* req);
na_return_t mona_get(
mona_instance_t mona,
na_mem_handle_t local_mem_handle,
na_offset_t local_offset,
na_mem_handle_t remote_mem_handle,
na_offset_t remote_offset,
na_size_t data_size,
na_addr_t remote_addr,
na_uint8_t remote_id,
na_op_id_t *op_id);
na_return_t mona_iget(
mona_instance_t mona,
na_mem_handle_t local_mem_handle,
na_offset_t local_offset,
na_mem_handle_t remote_mem_handle,
na_offset_t remote_offset,
na_size_t data_size,
na_addr_t remote_addr,
na_uint8_t remote_id,
na_op_id_t *op_id,
mona_request_t* req);
int mona_poll_get_fd(mona_instance_t mona);
na_bool_t mona_poll_try_wait(mona_instance_t mona);
na_return_t mona_cancel(
mona_instance_t mona,
na_op_id_t op_id);
const char* mona_error_to_string(int errnum);
na_return_t mona_wait(mona_request_t req);
int mona_test(mona_request_t req, int* flag);
#ifdef __cplusplus
}
#endif
#endif
# load package helper for generating cmake CONFIG packages
include (CMakePackageConfigHelpers)
# where to install files for "find_package"
set (mona-pkg "share/cmake/mona")
#
# library version set here (e.g. for shared libs).
#
set (MONA_VERSION_MAJOR 0)
set (MONA_VERSION_MINOR 1)
set (MONA_VERSION_PATCH 0)
set (mona-vers "${MONA_VERSION_MAJOR}.${MONA_VERSION_MINOR}")
set (MONA_VERSION "${mona-vers}.${MONA_VERSION_PATCH}")
add_library(mona ${CMAKE_CURRENT_SOURCE_DIR}/mona.c)
target_link_libraries (mona mercury PkgConfig::ARGOBOTS)
target_include_directories (mona PUBLIC $<INSTALL_INTERFACE:include>)
# local include's BEFORE, in case old incompatable .h files in prefix/include
target_include_directories (mona BEFORE PUBLIC
$<BUILD_INTERFACE:${CMAKE_CURRENT_SOURCE_DIR}/../include>)
# for shared libs, establish the lib version
set_target_properties (mona
PROPERTIES VERSION ${MONA_VERSION}
SOVERSION ${MONA_VERSION_MAJOR})
#
# installation stuff (packaging and install commands)
#
write_basic_package_version_file(
"mona-config-version.cmake"
VERSION ${MONA_VERSION}
COMPATIBILITY AnyNewerVersion)
# generate our config file for find_package()
configure_file (mona-config.cmake.in mona-config.cmake @ONLY)
# some bits for the pkg-config file
set (DEST_DIR "${CMAKE_INSTALL_PREFIX}")
set (PRIVATE_LIBS "-lmona")
configure_file ("mona.pc.in" "mona.pc" @ONLY)
#
# "make install" rules
#
install (TARGETS mona EXPORT mona-targets
ARCHIVE DESTINATION lib
LIBRARY DESTINATION lib)
install (EXPORT mona-targets
DESTINATION ${mona-pkg}
FILE "mona-targets.cmake")
install (FILES "${CMAKE_CURRENT_BINARY_DIR}/mona-config.cmake"