Commit 898816e1 authored by Matthieu Dorier's avatar Matthieu Dorier

Merge branch 'dev-cereal' into 'master'

Support for cereal serialization library

See merge request !6
parents bb0a5840 245c1844
...@@ -48,8 +48,15 @@ if (NOT CMAKE_BUILD_TYPE) ...@@ -48,8 +48,15 @@ if (NOT CMAKE_BUILD_TYPE)
endif () endif ()
set (CMAKE_PREFIX_PATH "" CACHE STRING "External dependencies path") set (CMAKE_PREFIX_PATH "" CACHE STRING "External dependencies path")
set (BUILD_SHARED_LIBS "OFF" CACHE BOOL "Build a shared library") set (BUILD_SHARED_LIBS "OFF" CACHE BOOL "Build a shared library")
set (ENABLE_CEREAL "OFF" CACHE BOOL "Enable cereal serialization")
include_directories (${CMAKE_BINARY_DIR}/include)
# packages we depend on # packages we depend on
if (ENABLE_CEREAL)
find_package (cereal CONFIG REQUIRED)
endif (ENABLE_CEREAL)
include (xpkg-import) include (xpkg-import)
find_package (mercury CONFIG REQUIRED) find_package (mercury CONFIG REQUIRED)
xpkg_import_module (margo REQUIRED margo) xpkg_import_module (margo REQUIRED margo)
...@@ -62,4 +69,4 @@ if (ENABLE_EXAMPLES) ...@@ -62,4 +69,4 @@ if (ENABLE_EXAMPLES)
add_subdirectory (examples) add_subdirectory (examples)
endif (ENABLE_EXAMPLES) endif (ENABLE_EXAMPLES)
configure_file (include/thallium/config.hpp.in ${CMAKE_BINARY_DIR}/include/thallium/config.hpp)
...@@ -2,6 +2,7 @@ ...@@ -2,6 +2,7 @@
#define __THALLIUM_HPP #define __THALLIUM_HPP
#include <margo.h> #include <margo.h>
#include <thallium/config.hpp>
#include <thallium/abt.hpp> #include <thallium/abt.hpp>
#include <thallium/anonymous.hpp> #include <thallium/anonymous.hpp>
#include <thallium/bulk_mode.hpp> #include <thallium/bulk_mode.hpp>
......
...@@ -9,9 +9,6 @@ ...@@ -9,9 +9,6 @@
#include <thallium/margo_exception.hpp> #include <thallium/margo_exception.hpp>
#include <thallium/buffer.hpp> #include <thallium/buffer.hpp>
#include <thallium/packed_response.hpp> #include <thallium/packed_response.hpp>
#include <thallium/serialization/serialize.hpp>
#include <thallium/serialization/stl/vector.hpp>
#include <thallium/serialization/buffer_output_archive.hpp>
namespace thallium { namespace thallium {
......
...@@ -12,6 +12,7 @@ ...@@ -12,6 +12,7 @@
#include <margo.h> #include <margo.h>
#include <thallium/endpoint.hpp> #include <thallium/endpoint.hpp>
#include <thallium/margo_exception.hpp> #include <thallium/margo_exception.hpp>
#include <thallium/serialization/stl/vector.hpp>
namespace thallium { namespace thallium {
...@@ -335,10 +336,10 @@ public: ...@@ -335,10 +336,10 @@ public:
* @brief Function that serializes a bulk object into an archive. * @brief Function that serializes a bulk object into an archive.
* *
* @tparam A Archive type. * @tparam A Archive type.
* @param ar Input archive. * @param ar Output archive.
*/ */
template<typename A> template<typename A>
void save(A& ar) { void save(A& ar) const {
if(m_bulk == HG_BULK_NULL) { if(m_bulk == HG_BULK_NULL) {
std::vector<char> buf; std::vector<char> buf;
ar & buf; ar & buf;
...@@ -356,7 +357,7 @@ public: ...@@ -356,7 +357,7 @@ public:
* @brief Deserializes a bulk object from an output archive. * @brief Deserializes a bulk object from an output archive.
* *
* @tparam A Archive type. * @tparam A Archive type.
* @param ar Output archive. * @param ar Input archive.
*/ */
template<typename A> template<typename A>
void load(A& ar); void load(A& ar);
......
...@@ -11,14 +11,15 @@ ...@@ -11,14 +11,15 @@
#include <utility> #include <utility>
#include <chrono> #include <chrono>
#include <margo.h> #include <margo.h>
#include <thallium/config.hpp>
#include <thallium/buffer.hpp> #include <thallium/buffer.hpp>
#include <thallium/timeout.hpp> #include <thallium/timeout.hpp>
#include <thallium/packed_response.hpp> #include <thallium/packed_response.hpp>
#include <thallium/async_response.hpp> #include <thallium/async_response.hpp>
#include <thallium/margo_exception.hpp>
#include <thallium/serialization/serialize.hpp> #include <thallium/serialization/serialize.hpp>
#include <thallium/serialization/stl/vector.hpp>
#include <thallium/serialization/buffer_output_archive.hpp> #include <thallium/serialization/buffer_output_archive.hpp>
#include <thallium/margo_exception.hpp> #include <thallium/serialization/stl/vector.hpp>
namespace thallium { namespace thallium {
...@@ -199,7 +200,7 @@ public: ...@@ -199,7 +200,7 @@ public:
packed_response operator()(T&& ... args) const { packed_response operator()(T&& ... args) const {
buffer b; buffer b;
buffer_output_archive arch(b, *m_engine); buffer_output_archive arch(b, *m_engine);
serialize_many(arch, std::forward<T>(args)...); arch(std::forward<T>(args)...);
return forward(b); return forward(b);
} }
...@@ -221,7 +222,7 @@ public: ...@@ -221,7 +222,7 @@ public:
packed_response timed(const std::chrono::duration<R,P>& t, T&& ... args) const { packed_response timed(const std::chrono::duration<R,P>& t, T&& ... args) const {
buffer b; buffer b;
buffer_output_archive arch(b, *m_engine); buffer_output_archive arch(b, *m_engine);
serialize_many(arch, std::forward<T>(args)...); arch(std::forward<T>(args)...);
std::chrono::duration<double, std::milli> fp_ms = t; std::chrono::duration<double, std::milli> fp_ms = t;
double timeout_ms = fp_ms.count(); double timeout_ms = fp_ms.count();
return forward(b, timeout_ms); return forward(b, timeout_ms);
...@@ -267,7 +268,7 @@ public: ...@@ -267,7 +268,7 @@ public:
async_response async(T&& ... t) { async_response async(T&& ... t) {
buffer b; buffer b;
buffer_output_archive arch(b, *m_engine); buffer_output_archive arch(b, *m_engine);
serialize_many(arch, std::forward<T>(t)...); arch(std::forward<T>(t)...);
return iforward(b); return iforward(b);
} }
...@@ -288,7 +289,7 @@ public: ...@@ -288,7 +289,7 @@ public:
async_response timed_async(const std::chrono::duration<R,P>& t, T&& ... args) { async_response timed_async(const std::chrono::duration<R,P>& t, T&& ... args) {
buffer b; buffer b;
buffer_output_archive arch(b, *m_engine); buffer_output_archive arch(b, *m_engine);
serialize_many(arch, std::forward<T>(args)...); arch(std::forward<T>(t)...);
std::chrono::duration<double, std::milli> fp_ms = t; std::chrono::duration<double, std::milli> fp_ms = t;
double timeout_ms = fp_ms.count(); double timeout_ms = fp_ms.count();
return iforward(b, timeout_ms); return iforward(b, timeout_ms);
......
#undef THALLIUM_USE_CEREAL
#undef ENABLE_CEREAL
#cmakedefine ENABLE_CEREAL
#ifdef ENABLE_CEREAL
#define THALLIUM_USE_CEREAL
#undef ENABLE_CEREAL
#endif
...@@ -14,6 +14,7 @@ ...@@ -14,6 +14,7 @@
#include <vector> #include <vector>
#include <atomic> #include <atomic>
#include <margo.h> #include <margo.h>
#include <thallium/config.hpp>
#include <thallium/pool.hpp> #include <thallium/pool.hpp>
#include <thallium/tuple_util.hpp> #include <thallium/tuple_util.hpp>
#include <thallium/function_cast.hpp> #include <thallium/function_cast.hpp>
...@@ -320,9 +321,13 @@ public: ...@@ -320,9 +321,13 @@ public:
* *
* @return a remote_procedure object. * @return a remote_procedure object.
*/ */
template<typename ... Args> template<typename A1, typename ... Args>
remote_procedure define(const std::string& name,
const std::function<void(const request&, A1, Args...)>& fun,
uint16_t provider_id=0, const pool& p = pool());
remote_procedure define(const std::string& name, remote_procedure define(const std::string& name,
const std::function<void(const request&, Args...)>& fun, const std::function<void(const request&)>& fun,
uint16_t provider_id=0, const pool& p = pool()); uint16_t provider_id=0, const pool& p = pool());
/** /**
...@@ -391,15 +396,15 @@ public: ...@@ -391,15 +396,15 @@ public:
#include <thallium/remote_procedure.hpp> #include <thallium/remote_procedure.hpp>
#include <thallium/proc_buffer.hpp> #include <thallium/proc_buffer.hpp>
#include <thallium/serialization/stl/tuple.hpp>
#include <thallium/serialization/buffer_input_archive.hpp> #include <thallium/serialization/buffer_input_archive.hpp>
#include <thallium/serialization/buffer_output_archive.hpp> #include <thallium/serialization/buffer_output_archive.hpp>
#include <thallium/serialization/stl/tuple.hpp>
namespace thallium { namespace thallium {
template<typename ... Args> template<typename A1, typename ... Args>
remote_procedure engine::define(const std::string& name, remote_procedure engine::define(const std::string& name,
const std::function<void(const request&, Args...)>& fun, const std::function<void(const request&, A1, Args...)>& fun,
uint16_t provider_id, const pool& p) { uint16_t provider_id, const pool& p) {
hg_id_t id = margo_provider_register_name(m_mid, name.c_str(), hg_id_t id = margo_provider_register_name(m_mid, name.c_str(),
...@@ -410,15 +415,13 @@ remote_procedure engine::define(const std::string& name, ...@@ -410,15 +415,13 @@ remote_procedure engine::define(const std::string& name,
p.native_handle()); p.native_handle());
m_rpcs[id] = [fun,this](const request& r, const buffer& b) { m_rpcs[id] = [fun,this](const request& r, const buffer& b) {
std::function<void(Args...)> l = [&fun, &r](Args&&... args) { std::function<void(A1, Args...)> call_function = [&fun, &r](A1&& a1, Args&&... args) {
fun(r, std::forward<Args>(args)...); fun(r, std::forward<A1>(a1), std::forward<Args>(args)...);
}; };
std::tuple<typename std::decay<Args>::type...> iargs; std::tuple<typename std::decay<A1>::type, typename std::decay<Args>::type...> iargs;
if(sizeof...(Args) > 0) { buffer_input_archive iarch(b, *this);
buffer_input_archive iarch(b, *this); iarch(iargs);
iarch & iargs; apply_function_to_tuple(call_function, iargs);
}
apply_function_to_tuple(l,iargs);
}; };
rpc_callback_data* cb_data = new rpc_callback_data; rpc_callback_data* cb_data = new rpc_callback_data;
......
...@@ -53,7 +53,7 @@ public: ...@@ -53,7 +53,7 @@ public:
T as() const { T as() const {
T t; T t;
buffer_input_archive iarch(m_buffer, *m_engine); buffer_input_archive iarch(m_buffer, *m_engine);
iarch & t; iarch(t);
return t; return t;
} }
...@@ -77,8 +77,8 @@ public: ...@@ -77,8 +77,8 @@ public:
std::tuple<typename std::decay<T1>::type, std::tuple<typename std::decay<T1>::type,
typename std::decay<T2>::type, typename std::decay<T2>::type,
typename std::decay_t<Tn>::type...> t; typename std::decay_t<Tn>::type...> t;
buffer_input_archive iarch(m_buffer); buffer_input_archive iarch(m_buffer, *m_engine);
iarch & t; arch(t);
return t; return t;
} }
......
...@@ -114,7 +114,7 @@ public: ...@@ -114,7 +114,7 @@ public:
if(m_handle != HG_HANDLE_NULL) { if(m_handle != HG_HANDLE_NULL) {
buffer b; buffer b;
buffer_output_archive arch(b, *m_engine); buffer_output_archive arch(b, *m_engine);
serialize_many(arch, std::forward<T>(t)...); arch(std::forward<T>(t)...);
hg_return_t ret = margo_respond(m_handle, &b); hg_return_t ret = margo_respond(m_handle, &b);
MARGO_ASSERT(ret, margo_respond); MARGO_ASSERT(ret, margo_respond);
} }
......
...@@ -6,6 +6,21 @@ ...@@ -6,6 +6,21 @@
#ifndef __THALLIUM_BUFFER_INPUT_ARCHIVE_HPP #ifndef __THALLIUM_BUFFER_INPUT_ARCHIVE_HPP
#define __THALLIUM_BUFFER_INPUT_ARCHIVE_HPP #define __THALLIUM_BUFFER_INPUT_ARCHIVE_HPP
#include <thallium/config.hpp>
#ifdef THALLIUM_USE_CEREAL
#include <thallium/serialization/cereal/archives.hpp>
namespace thallium {
using buffer_input_archive = cereal_input_archive;
}
#else
#include <type_traits> #include <type_traits>
#include <stdexcept> #include <stdexcept>
#include <cstring> #include <cstring>
...@@ -133,3 +148,4 @@ public: ...@@ -133,3 +148,4 @@ public:
} }
#endif #endif
#endif
...@@ -6,6 +6,20 @@ ...@@ -6,6 +6,20 @@
#ifndef __THALLIUM_BUFFER_OUTPUT_ARCHIVE_HPP #ifndef __THALLIUM_BUFFER_OUTPUT_ARCHIVE_HPP
#define __THALLIUM_BUFFER_OUTPUT_ARCHIVE_HPP #define __THALLIUM_BUFFER_OUTPUT_ARCHIVE_HPP
#include <thallium/config.hpp>
#ifdef THALLIUM_USE_CEREAL
#include <thallium/serialization/cereal/archives.hpp>
namespace thallium {
using buffer_output_archive = cereal_output_archive;
}
#else
#include <type_traits> #include <type_traits>
#include <thallium/serialization/serialize.hpp> #include <thallium/serialization/serialize.hpp>
#include <thallium/buffer.hpp> #include <thallium/buffer.hpp>
...@@ -131,3 +145,4 @@ public: ...@@ -131,3 +145,4 @@ public:
} }
#endif #endif
#endif
/*
* Copyright (c) 2019 UChicago Argonne, LLC
*
* See COPYRIGHT in top-level directory.
*/
#ifndef THALLIUM_CEREAL_ARCHIVES_BINARY_HPP
#define THALLIUM_CEREAL_ARCHIVES_BINARY_HPP
#include <cstring>
#include <thallium/buffer.hpp>
#include <cereal/cereal.hpp>
namespace thallium {
class engine;
class cereal_output_archive : public cereal::OutputArchive<cereal_output_archive, cereal::AllowEmptyClassElision>
{
public:
cereal_output_archive(buffer& b, engine& e)
: cereal::OutputArchive<cereal_output_archive, cereal::AllowEmptyClassElision>(this)
, m_buffer(b)
, m_pos(0)
, m_engine(&e)
{
m_buffer.resize(0);
}
cereal_output_archive(buffer& b)
: cereal::OutputArchive<cereal_output_archive, cereal::AllowEmptyClassElision>(this)
, m_buffer(b)
, m_pos(0)
, m_engine(nullptr)
{
m_buffer.resize(0);
}
~cereal_output_archive() = default;
inline void write(const void* data, size_t size) {
if(m_pos+size > m_buffer.size()) {
if(m_pos+size > m_buffer.capacity()) {
m_buffer.reserve(m_buffer.capacity()*2);
}
m_buffer.resize(m_pos+size);
}
memcpy((void*)(m_buffer.data() + m_pos),(void*)data, size);
m_pos += size;
}
engine& get_engine() const {
return *m_engine;
}
private:
buffer& m_buffer;
std::size_t m_pos;
engine* m_engine;
};
class cereal_input_archive : public cereal::InputArchive<cereal_input_archive, cereal::AllowEmptyClassElision>
{
public:
cereal_input_archive(const buffer& b, engine& e)
: cereal::InputArchive<cereal_input_archive, cereal::AllowEmptyClassElision>(this)
, m_buffer(b)
, m_pos(0)
, m_engine(&e)
{}
cereal_input_archive(buffer& b)
: cereal::InputArchive<cereal_input_archive, cereal::AllowEmptyClassElision>(this)
, m_buffer(b)
, m_pos(0)
, m_engine(nullptr)
{}
~cereal_input_archive() = default;
inline void read(void* data, std::size_t size) {
if(m_pos + size > m_buffer.size()) {
throw std::runtime_error("Reading beyond buffer size");
}
std::memcpy((void*)data,(const void*)(m_buffer.data() + m_pos), size);
m_pos += size;
}
engine& get_engine() const {
return *m_engine;
}
private:
const buffer& m_buffer;
std::size_t m_pos;
engine* m_engine;
};
template<class T> inline
typename std::enable_if<std::is_arithmetic<T>::value, void>::type
CEREAL_SAVE_FUNCTION_NAME(cereal_output_archive & ar, T const & t)
{
ar.write(std::addressof(t), sizeof(t));
}
template<class T> inline
typename std::enable_if<std::is_arithmetic<T>::value, void>::type
CEREAL_LOAD_FUNCTION_NAME(cereal_input_archive & ar, T & t)
{
ar.read(std::addressof(t), sizeof(t));
}
template <class Archive, class T> inline
CEREAL_ARCHIVE_RESTRICT(cereal_input_archive, cereal_output_archive)
CEREAL_SERIALIZE_FUNCTION_NAME(Archive& ar, cereal::NameValuePair<T>& t)
{
ar(t.value);
}
template <class Archive, class T> inline
CEREAL_ARCHIVE_RESTRICT(cereal_input_archive, cereal_output_archive)
CEREAL_SERIALIZE_FUNCTION_NAME(Archive& ar, cereal::SizeTag<T>& t)
{
ar(t.size);
}
template <class T> inline
void CEREAL_SAVE_FUNCTION_NAME(cereal_output_archive& ar, cereal::BinaryData<T> const & bd)
{
ar.write(bd.data, static_cast<std::size_t>(bd.size));
}
template <class T> inline
void CEREAL_LOAD_FUNCTION_NAME(cereal_input_archive & ar, cereal::BinaryData<T> & bd)
{
ar.read(bd.data, static_cast<std::size_t>(bd.size));
}
}
// register archives for polymorphic support
CEREAL_REGISTER_ARCHIVE(thallium::cereal_output_archive)
CEREAL_REGISTER_ARCHIVE(thallium::cereal_input_archive)
// tie input and output archives together
CEREAL_SETUP_ARCHIVE_TRAITS(thallium::cereal_input_archive, thallium::cereal_output_archive)
#endif
...@@ -6,6 +6,10 @@ ...@@ -6,6 +6,10 @@
#ifndef SERIALIZE_H #ifndef SERIALIZE_H
#define SERIALIZE_H #define SERIALIZE_H
#include <thallium/config.hpp>
#ifndef THALLIUM_USE_CEREAL
#include <utility> #include <utility>
#include <type_traits> #include <type_traits>
...@@ -205,3 +209,4 @@ inline void serialize_many(A& ar, T&& t) { ...@@ -205,3 +209,4 @@ inline void serialize_many(A& ar, T&& t) {
} }
#endif #endif
#endif
...@@ -6,6 +6,12 @@ ...@@ -6,6 +6,12 @@
#ifndef __THALLIUM_ARRAY_SERIALIZATION_HPP #ifndef __THALLIUM_ARRAY_SERIALIZATION_HPP
#define __THALLIUM_ARRAY_SERIALIZATION_HPP #define __THALLIUM_ARRAY_SERIALIZATION_HPP
#include <thallium/config.hpp>
#ifdef THALLIUM_USE_CEREAL
#include <cereal/types/array.hpp>
#else
#include <type_traits> #include <type_traits>
#include <array> #include <array>
...@@ -52,3 +58,4 @@ inline void load(A& ar, std::array<T,N>& v) { ...@@ -52,3 +58,4 @@ inline void load(A& ar, std::array<T,N>& v) {
} // namespace thallium } // namespace thallium
#endif #endif
#endif
...@@ -6,6 +6,12 @@ ...@@ -6,6 +6,12 @@
#ifndef __THALLIUM_COMPLEX_SERIALIZATION_HPP #ifndef __THALLIUM_COMPLEX_SERIALIZATION_HPP
#define __THALLIUM_COMPLEX_SERIALIZATION_HPP #define __THALLIUM_COMPLEX_SERIALIZATION_HPP
#include <thallium/config.hpp>
#ifdef THALLIUM_USE_CEREAL
#include <cereal/types/complex.hpp>
#else
#include <complex> #include <complex>
namespace thallium { namespace thallium {
...@@ -25,3 +31,4 @@ inline void load(A& ar, std::complex<T>& t) { ...@@ -25,3 +31,4 @@ inline void load(A& ar, std::complex<T>& t) {
} }
#endif #endif
#endif
...@@ -6,6 +6,12 @@ ...@@ -6,6 +6,12 @@
#ifndef __THALLIUM_DEQUE_SERIALIZATION_HPP #ifndef __THALLIUM_DEQUE_SERIALIZATION_HPP
#define __THALLIUM_DEQUE_SERIALIZATION_HPP #define __THALLIUM_DEQUE_SERIALIZATION_HPP
#include <thallium/config.hpp>
#ifdef THALLIUM_USE_CEREAL
#include <cereal/types/deque.hpp>
#else
#include <deque> #include <deque>
namespace thallium { namespace thallium {
...@@ -33,3 +39,4 @@ inline void load(A& ar, std::deque<T,Alloc>& l) { ...@@ -33,3 +39,4 @@ inline void load(A& ar, std::deque<T,Alloc>& l) {
} }
#endif #endif
#endif
...@@ -6,6 +6,12 @@ ...@@ -6,6 +6,12 @@
#ifndef __THALLIUM_FORWARD_LIST_SERIALIZATION_HPP #ifndef __THALLIUM_FORWARD_LIST_SERIALIZATION_HPP
#define __THALLIUM_FORWARD_LIST_SERIALIZATION_HPP #define __THALLIUM_FORWARD_LIST_SERIALIZATION_HPP
#include <thallium/config.hpp>
#ifdef THALLIUM_USE_CEREAL
#include <cereal/types/forward_list.hpp>
#else
#include <forward_list> #include <forward_list>
namespace thallium { namespace thallium {
...@@ -33,3 +39,4 @@ inline void load(A& ar, std::forward_list<T,Alloc>& l) { ...@@ -33,3 +39,4 @@ inline void load(A& ar, std::forward_list<T,Alloc>& l) {
} }
#endif #endif
#endif
...@@ -6,6 +6,12 @@ ...@@ -6,6 +6,12 @@
#ifndef __THALLIUM_LIST_SERIALIZATION_HPP #ifndef __THALLIUM_LIST_SERIALIZATION_HPP
#define __THALLIUM_LIST_SERIALIZATION_HPP #define __THALLIUM_LIST_SERIALIZATION_HPP
#include <thallium/config.hpp>
#ifdef THALLIUM_USE_CEREAL
#include <cereal/types/list.hpp>
#else
#include <list> #include <list>
namespace thallium { namespace thallium {
...@@ -33,3 +39,4 @@ inline void load(A& ar, std::list<T,Alloc>& l) { ...@@ -33,3 +39,4 @@ inline void load(A& ar, std::list<T,Alloc>& l) {
} }
#endif #endif
#endif
/*
* (C) 2017 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#ifndef __THALLIUM_MAP_SERIALIZATION_HPP #ifndef __THALLIUM_MAP_SERIALIZATION_HPP
#define __THALLIUM_MAP_SERIALIZATION_HPP #define __THALLIUM_MAP_SERIALIZATION_HPP
#include <thallium/config.hpp>
#ifdef THALLIUM_USE_CEREAL
#include <cereal/types/map.hpp>
#else
#include <map> #include <map>
namespace thallium { namespace thallium {
...@@ -30,3 +41,4 @@ inline void load(A& ar, std::map<K,V,Compare,Alloc>& m) { ...@@ -30,3 +41,4 @@ inline void load(A& ar, std::map<K,V,Compare,Alloc>& m) {