Add separate control library to communicate with plugin (for setting prefix)

CMakeLists.txt

 cmake_minimum_required(VERSION 3.13)
-project(tau-sdskeyval LANGUAGES CXX)
+project(
+    tau-sdskeyval
+    VERSION 0.1
+    LANGUAGES CXX)
 
 list(APPEND CMAKE_MODULE_PATH "${CMAKE_CURRENT_SOURCE_DIR}/cmake")
 
@@ -9,35 +12,6 @@ find_package(OpenMP REQUIRED)
 
 enable_testing()
 
-add_library(
-    tausdskvplugin
-    SHARED
-        src/lib.cpp
-        src/margo.cpp
-        src/mercury.cpp
-        src/plugin.cpp
-        src/sdskv_client.cpp
-)
-
-target_include_directories(
-    tausdskvplugin
-    PUBLIC
-        include
-    PRIVATE
-        internal
-)
-
-target_link_libraries(
-    tausdskvplugin
-    PRIVATE
-        tau::tau
-        sdskeyval::client
-)
-
-target_compile_features(
-    tausdskvplugin
-    PRIVATE
-        cxx_std_17
-)
-
+add_subdirectory(plugin)
+add_subdirectory(control)
 add_subdirectory(example)
\ No newline at end of file

control/CMakeLists.txt

+project(tau-sdskeyval-control LANGUAGES CXX)
+
+configure_file(cmake/version.hpp.in generated/version.hpp)
+
+add_library(
+    tau_sdskeyval_control
+        src/lib.cpp
+)
+add_library(tau_sdskeyval::control ALIAS tau_sdskeyval_control)
+
+target_include_directories(
+    tau_sdskeyval_control
+    PUBLIC
+        include
+    PRIVATE
+        ${CMAKE_CURRENT_BINARY_DIR}/generated
+)
+
+target_link_libraries(
+    tau_sdskeyval_control
+    PRIVATE
+        tau_sdskeyval::plugin_internal_control
+        dl
+)
+
+target_compile_features(
+    tau_sdskeyval_control
+    PRIVATE
+        cxx_std_17
+)

control/cmake/version.hpp.in

+#ifndef TAU_SDSKEYVAL_CONTROL_VERSION_HPP_
+#define TAU_SDSKEYVAL_CONTROL_VERSION_HPP_
+
+namespace tau_sdskeyval { namespace control {
+
+constexpr int VersionMajor = @CMAKE_PROJECT_VERSION_MAJOR@;
+constexpr int VersionMinor = @CMAKE_PROJECT_VERSION_MINOR@;
+
+} }
+
+#endif // TAU_SDSKEYVAL_CONTROL_VERSION_HPP_
\ No newline at end of file

control/include/tau-sdskeyval/plugin.h

+/**
+ * @file plugin.h
+ * @author Andrew Gaspar (agaspar@lanl.gov)
+ * @brief Routines for controlling plugin.
+ * @date 2019-05-29
+ *
+ * @copyright Copyright (c) 2019 Triad National Security, LLC
+ */
+
+#ifndef TAU_SDSKEYVAL_CONTROL_PLUGIN_H_
+#define TAU_SDSKEYVAL_CONTROL_PLUGIN_H_
+
+#ifdef __cplusplus
+extern "C" {
+#endif
+
+int tausdskeyval_set_dump_name(char const *dump_name);
+
+#ifdef __cplusplus
+}
+#endif
+
+#endif // TAU_SDSKEYVAL_CONTROL_PLUGIN_H_
\ No newline at end of file

control/src/lib.cpp

+/**
+ * @file lib.cpp
+ * @author Andrew Gaspar (agaspar@lanl.gov)
+ * @brief Implements the control library for tau_sdskeyval_plugin
+ * @date 2019-05-29
+ *
+ * @copyright Copyright (c) 2019 Triad National Security, LLC
+ */
+
+// STL Includes
+#include <iostream>
+#include <optional>
+#include <utility>
+
+// System Includes
+#include <dlfcn.h>
+
+// Internal Includes
+#include <tau-sdskeyval-plugin/control.h>
+
+// Local Includes
+#include <tau-sdskeyval/plugin.h>
+#include <version.hpp>
+
+#define LOAD_SYMBOL(so, symbol_name) (decltype(symbol_name) *)dlsym((so), #symbol_name)
+
+class TauSdskeyvalPlugin {
+  public:
+    static TauSdskeyvalPlugin *FindPlugin();
+
+    void SetDumpName(char const *dump_name) { set_dump_name_(dump_name); }
+
+  private:
+    explicit TauSdskeyvalPlugin(decltype(tausdskeyval_control_set_dump_name) set_dump_name)
+        : set_dump_name_(set_dump_name) {}
+
+    decltype(tausdskeyval_control_set_dump_name) *set_dump_name_;
+};
+
+TauSdskeyvalPlugin *TauSdskeyvalPlugin::FindPlugin() {
+    static std::optional<TauSdskeyvalPlugin> plugin = std::nullopt;
+    static void *so = nullptr;
+
+    if (plugin) return &plugin.value();
+
+    // Load library once and leak the handle
+    if (!so) so = dlopen("libtau_sdskeyval_plugin.so", RTLD_NOLOAD | RTLD_LAZY);
+    if (!so) {
+        std::cerr << "tau_sdskeyval: ERROR: Could not find libtau_sdskeyval_plugin.so" << std::endl;
+        return nullptr;
+    }
+
+    auto major = LOAD_SYMBOL(so, TAU_SDSKEYVAL_PLUGIN_VERSION_MAJOR);
+    auto minor = LOAD_SYMBOL(so, TAU_SDSKEYVAL_PLUGIN_VERSION_MINOR);
+    auto set_dump_name = LOAD_SYMBOL(so, tausdskeyval_control_set_dump_name);
+
+    if (!(major && minor && set_dump_name)) {
+        std::cerr
+            << "tau_sdskeyval: ERROR: Could not find required symbols in libtau_sdskeyval_plugin.so"
+            << std::endl;
+        return nullptr;
+    }
+
+    // Make sure the control and plugin are compatible versions
+    if (std::make_pair(tau_sdskeyval::control::VersionMajor,
+                       tau_sdskeyval::control::VersionMinor) != std::make_pair(*major, *minor)) {
+        std::cerr << "tau_sdskeyval: ERROR: The control library (v"
+                  << tau_sdskeyval::control::VersionMajor << "."
+                  << tau_sdskeyval::control::VersionMinor
+                  << ") is not compatible with the plugin library (v" << *major << "." << *minor
+                  << ")" << std::endl;
+        return nullptr;
+    }
+
+    plugin = TauSdskeyvalPlugin(set_dump_name);
+
+    return &plugin.value();
+}
+
+extern "C" int tausdskeyval_set_dump_name(char const *dump_name) {
+    if (auto plugin = TauSdskeyvalPlugin::FindPlugin()) {
+        plugin->SetDumpName(dump_name);
+        return 0;
+    } else {
+        return 1;
+    }
+}
\ No newline at end of file

example/CMakeLists.txt

@@ -10,6 +10,7 @@ add_executable(
 target_link_libraries(
     tautest
     PRIVATE
+        tau_sdskeyval::control
         tau::tau
         OpenMP::OpenMP_CXX
         # MPI::MPI_CXX
@@ -35,5 +36,5 @@ add_test(
 set_tests_properties(
     tausdskv-test
     PROPERTIES
-        ENVIRONMENT "TAU_PLUGINS_PATH=;TAU_PLUGINS=$<TARGET_FILE:tausdskvplugin>"
+        ENVIRONMENT "TAU_PLUGINS_PATH=;TAU_PLUGINS=$<TARGET_FILE:tau_sdskeyval_plugin>"
 )
\ No newline at end of file

example/src/main.cpp

@@ -2,11 +2,13 @@
 #include <atomic>
 #include <iostream>
 #include <numeric>
+#include <sstream>
 #include <vector>
 
 // Third Party Includes
 #include <TAU.h>
 #include <omp.h>
+#include <tau-sdskeyval/plugin.h>
 
 void recurse(int n) {
     TAU_PROFILE("recurse", "void(int)", TAU_DEFAULT);
@@ -22,31 +24,36 @@ int main(int argc, char **argv) {
 #pragma omp parallel
     { TAU_REGISTER_THREAD(); }
 
-    {
-        TAU_PROFILE("openmp_outer", "void()", TAU_DEFAULT);
-
-#pragma omp parallel
+    for (auto i = 0; i < 10; i++) {
         {
-            TAU_PROFILE("openmp_inner", "void()", TAU_DEFAULT);
-            int volatile N = 1000000;
-            std::vector<int> my_stuff(N);
-            std::iota(my_stuff.begin(), my_stuff.end(), 0);
+            std::stringstream ss;
+            ss << "ts" << i;
+            tausdskeyval_set_dump_name(ss.str().c_str());
         }
 
-#pragma omp parallel for
-        for (auto i = 0; i < 1000000; i++) {
-            TAU_PROFILE("opemp_loop", "void(int)", TAU_DEFAULT);
-        }
+        {
+            TAU_PROFILE("openmp_outer", "void()", TAU_DEFAULT);
 
 #pragma omp parallel
-        { recurse(1000); }
+            {
+                TAU_PROFILE("openmp_inner", "void()", TAU_DEFAULT);
+                int volatile N = 1000000;
+                std::vector<int> my_stuff(N);
+                std::iota(my_stuff.begin(), my_stuff.end(), 0);
+            }
+
+#pragma omp parallel for
+            for (auto i = 0; i < 1000000; i++) {
+                TAU_PROFILE("opemp_loop", "void(int)", TAU_DEFAULT);
+            }
 
 #pragma omp parallel
-        std::cout << "Hi from thread " << omp_get_thread_num() << "!" << std::endl;
-    }
+            { recurse(1000); }
+        }
 
 #pragma omp parallel
-    Tau_dump();
+        Tau_dump();
+    }
 
     return 0;
 }
\ No newline at end of file

include/tau-sdskv/control.h

-#ifndef TAUSDSKV_CONTROL_H_
-#define TAUSDSKV_CONTROL_H_
-
-#ifdef __cplusplus
-extern "C" {
-#endif // __cplusplus
-
-int tausdskv_set_dump_name(char const *dump_name);
-
-#ifdef __cplusplus
-}
-#endif // __cplusplus
-
-#endif // TAUSDSKV_CONTROL_H_
\ No newline at end of file

plugin/CMakeLists.txt

+project(tau-sdskeyval-plugin LANGUAGES CXX)
+
+# Header file for control of the plugin
+add_library(
+    plugin_internal_control
+    INTERFACE
+)
+add_library(tau_sdskeyval::plugin_internal_control ALIAS plugin_internal_control)
+
+target_include_directories(
+    plugin_internal_control
+    INTERFACE
+        include
+)
+
+# Implements plugin
+configure_file(cmake/version.cpp.in version.cpp)
+
+add_library(
+    tau_sdskeyval_plugin
+    SHARED
+        src/lib.cpp
+        src/margo.cpp
+        src/mercury.cpp
+        src/plugin.cpp
+        src/sdskv_client.cpp
+        ${CMAKE_CURRENT_BINARY_DIR}/version.cpp
+)
+add_library(tau_sdskeyval::plugin ALIAS tau_sdskeyval_plugin)
+
+target_include_directories(
+    tau_sdskeyval_plugin
+    PRIVATE
+        internal
+)
+
+target_link_libraries(
+    tau_sdskeyval_plugin
+    PRIVATE
+        tau_sdskeyval::plugin_internal_control
+        tau::tau
+        sdskeyval::client
+)
+
+target_compile_features(
+    tau_sdskeyval_plugin
+    PRIVATE
+        cxx_std_17
+)

plugin/cmake/version.cpp.in

+// Internal Includes
+#include <tau-sdskeyval-plugin/control.h>
+
+int TAU_SDSKEYVAL_PLUGIN_VERSION_MAJOR = @CMAKE_PROJECT_VERSION_MAJOR@;
+int TAU_SDSKEYVAL_PLUGIN_VERSION_MINOR = @CMAKE_PROJECT_VERSION_MINOR@;
\ No newline at end of file

plugin/include/tau-sdskeyval-plugin/control.h

+/**
+ * @file control.h
+ * @author Andrew Gaspar (agaspar@lanl.gov)
+ * @brief Declares internal plugin routines
+ * @date 2019-05-29
+ *
+ * @copyright Copyright (c) 2019 Triad National Security, LLC
+ */
+
+#ifndef TAU_SDSKEYVAL_CONTROL_H_
+#define TAU_SDSKEYVAL_CONTROL_H_
+
+extern int TAU_SDSKEYVAL_PLUGIN_VERSION_MAJOR;
+extern int TAU_SDSKEYVAL_PLUGIN_VERSION_MINOR;
+
+#ifdef __cplusplus
+extern "C" {
+#endif // __cplusplus
+
+int tausdskeyval_control_set_dump_name(char const *dump_name);
+
+#ifdef __cplusplus
+}
+#endif // __cplusplus
+
+#endif // TAU_SDSKEYVAL_CONTROL_H_
\ No newline at end of file

internal/margo.hpp → plugin/internal/margo.hpp

 /**
  * @file margo.hpp
- * @author Andrew Gaspar (you@domain.com)
+ * @author Andrew Gaspar (agaspar@lanl.gov)
  * @brief C++ interface to margo
  * @date 2019-05-24
  *
  * @copyright Copyright (c) 2019 Triad National Security, LLC
- *
  */
 
-#ifndef TAUSDSKV_MARGO_HPP_
-#define TAUSDSKV_MARGO_HPP_
+#ifndef TAU_SDSKEYVAL_MARGO_HPP_
+#define TAU_SDSKEYVAL_MARGO_HPP_
 
 // Third Party Includes
 #include <margo.h>
@@ -18,7 +17,7 @@
 #include <exception>
 #include <string_view>
 
-namespace tausdskv {
+namespace tau_sdskeyval {
 class MargoInstance;
 
 enum class MargoMode { Client = MARGO_CLIENT_MODE, Server = MARGO_SERVER_MODE };
@@ -75,6 +74,6 @@ class MargoInstance {
   private:
     margo_instance_id mid_ = nullptr;
 };
-} // namespace tausdskv
+} // namespace tau_sdskeyval
 
-#endif // TAUSDSKV_MARGO_HPP_
\ No newline at end of file
+#endif // TAU_SDSKEYVAL_MARGO_HPP_
\ No newline at end of file

internal/mercury.hpp → plugin/internal/mercury.hpp

 /**
  * @file margo.hpp
- * @author Andrew Gaspar (you@domain.com)
+ * @author Andrew Gaspar (agaspar@lanl.gov)
  * @brief C++ interface to mercury
  * @date 2019-05-24
  *
  * @copyright Copyright (c) 2019 Triad National Security, LLC
- *
  */
 
-#ifndef TAUSDSKV_MERCURY_HPP_
-#define TAUSDSKV_MERCURY_HPP_
+#ifndef TAU_SDSKEYVAL_MERCURY_HPP_
+#define TAU_SDSKEYVAL_MERCURY_HPP_
 
 // Third Party Includes
 #include <mercury.h>
@@ -18,7 +17,7 @@
 #include <exception>
 #include <string>
 
-namespace tausdskv {
+namespace tau_sdskeyval {
 class MercuryException {
   public:
     explicit MercuryException(hg_return_t status);
@@ -33,6 +32,6 @@ inline void MercuryCheck(hg_return_t status) {
         throw MercuryException(status);
     }
 }
-} // namespace tausdskv
+} // namespace tau_sdskeyval
 
-#endif // TAUSDSKV_MERCURY_HPP_
\ No newline at end of file
+#endif // TAU_SDSKEYVAL_MERCURY_HPP_
\ No newline at end of file

internal/plugin.hpp → plugin/internal/plugin.hpp

 /**
  * @file plugin.hpp
  * @author Andrew Gaspar (agaspar@lanl.gov)
- * @brief
+ * @brief TAU SDSKV Plugin
  * @date 2019-05-01
  *
  * @copyright Copyright (c) 2019 Triad National Security, LLC
- *
  */
 
-#ifndef TAUSDSKV_PLUGIN_HPP_
-#define TAUSDSKV_PLUGIN_HPP_
+#ifndef TAU_SDSKEYVAL_PLUGIN_HPP_
+#define TAU_SDSKEYVAL_PLUGIN_HPP_
 
 // STL Includes
 #include <map>
@@ -28,7 +27,7 @@
 
 enum class PluginStatus { Success = 0, Fail = 1 };
 
-namespace tausdskv {
+namespace tau_sdskeyval {
 class Plugin {
   public:
     static void InitializeInstance(int argc, char **argv);
@@ -55,6 +54,8 @@ class Plugin {
         };
     }
 
+    void SetPrefix(std::string_view prefix);
+
     PluginStatus Dump(Tau_plugin_event_dump_data_t const &data);
     PluginStatus Mpit(Tau_plugin_event_mpit_data_t const &data);
     PluginStatus
@@ -89,7 +90,7 @@ class Plugin {
     /**
      * @brief Guards thread_state_
      */
-    std::shared_mutex mutex_;
+    std::unique_ptr<std::shared_mutex> mutex_ = std::make_unique<std::shared_mutex>();
 
     /**
      * @brief Holds per-thread state.
@@ -109,6 +110,6 @@ class Plugin {
     sdskv_database_id_t db_id_;
 };
 
-} // namespace tausdskv
+} // namespace tau_sdskeyval
 
-#endif // TAUSDSKV_PLUGIN_HPP_
\ No newline at end of file
+#endif // TAU_SDSKEYVAL_PLUGIN_HPP_
\ No newline at end of file

internal/sdskv_client.hpp → plugin/internal/sdskv_client.hpp

 /**
  * @file sdskeyval_client.hpp
- * @author Andrew Gaspar (you@domain.com)
+ * @author Andrew Gaspar (agaspar@lanl.gov)
  * @brief C++ interface to sdskv
  * @date 2019-05-24
  *
  * @copyright Copyright (c) 2019 Triad National Security, LLC
  */
 
-#ifndef TAUSDSKV_SDSKV_CLIENT_HPP_
-#define TAUSDSKV_SDSKV_CLIENT_HPP_
+#ifndef TAU_SDSKEYVAL_SDSKV_CLIENT_HPP_
+#define TAU_SDSKEYVAL_SDSKV_CLIENT_HPP_
 
 // STL Includes
 #include <exception>
@@ -20,7 +20,7 @@
 // Internal Includes
 #include <margo.hpp>
 
-namespace tausdskv {
+namespace tau_sdskeyval {
 class SdskvException : public std::exception {
   public:
     explicit SdskvException(int status);
@@ -86,6 +86,6 @@ class SdskvClient {
   private:
     sdskv_client_t client_ = SDSKV_CLIENT_NULL;
 };
-} // namespace tausdskv
+} // namespace tau_sdskeyval
 
-#endif // TAUSDSKV_SDSKV_CLIENT_HPP_
\ No newline at end of file
+#endif // TAU_SDSKEYVAL_SDSKV_CLIENT_HPP_
\ No newline at end of file

src/lib.cpp → plugin/src/lib.cpp

+/**
+ * @file lib.cpp
+ * @author Andrew Gaspar (agaspar@lanl.gov)
+ * @brief Implements control routines for plugin.
+ * @date 2019-05-29
+ *
+ * @copyright Copyright (c) 2019 Triad National Security, LLC
+ */
+
 // STL Includes
 #include <cstring>
 #include <memory>
@@ -9,13 +18,11 @@
 
 // Local Includes
 #include <plugin.hpp>
-#include <tau-sdskv/control.h>
+#include <tau-sdskeyval-plugin/control.h>
 
-using namespace tausdskv;
+using namespace tau_sdskeyval;
 
 extern "C" int Tau_plugin_init_func(int argc, char **argv) {
-    // TAU_SoS
-
     Plugin::InitializeInstance(argc, argv);
 
     Tau_plugin_callbacks_t cb;
@@ -43,4 +50,10 @@ extern "C" int Tau_plugin_init_func(int argc, char **argv) {
     return 0;
 }
 
-extern "C" int tausdskv_set_dump_name(char const *dump_name) { return 0; }
\ No newline at end of file
+extern "C" int tausdskeyval_control_set_dump_name(char const *dump_name) {
+    auto instance = Plugin::GetInstance();
+    if (!instance) return 1;
+
+    instance->SetPrefix(dump_name);
+    return 0;
+}
\ No newline at end of file

src/margo.cpp → plugin/src/margo.cpp

 /**
  * @file margo.cpp
- * @author Andrew Gaspar (you@domain.com)
- * @brief
+ * @author Andrew Gaspar (agaspar@lanl.gov)
+ * @brief C++ Interface to Margo implementation
  * @date 2019-05-24
  *
  * @copyright Copyright (c) 2019 Triad National Security, LLC
@@ -19,7 +19,7 @@
 // Internal Includes
 #include <mercury.hpp>
 
-using namespace tausdskv;
+using namespace tau_sdskeyval;
 
 char const *MargoInitException::what() const noexcept { return "Margo failed to initialize!"; }
 
@@ -57,7 +57,9 @@ MargoAddress::~MargoAddress() { Reset(); }
 
 void MargoAddress::Reset() {
     if (addr_ != HG_ADDR_NULL) {
+        std::cout << "freeing MargoAddress " << addr_ << std::endl;
         MercuryCheck(margo_addr_free(mid_->InstanceId(), addr_));
+        std::cout << "freed MargoAddress" << std::endl;
         addr_ = HG_ADDR_NULL;
     }
 }

src/mercury.cpp → plugin/src/mercury.cpp

 /**
  * @file mercury.cpp
- * @author Andrew Gaspar (you@domain.com)
- * @brief
+ * @author Andrew Gaspar (agaspar@lanl.gov)
+ * @brief Mercury C++ Interface implementation
  * @date 2019-05-24
  *
  * @copyright Copyright (c) 2019 Triad National Security, LLC
- *
  */
 
 // Self Include
@@ -14,7 +13,7 @@
 // STL Includes
 #include <sstream>
 
-using namespace tausdskv;
+using namespace tau_sdskeyval;