Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
H
HEPnOS
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
sds
HEP
HEPnOS
Commits
8dcdec7b
Commit
8dcdec7b
authored
Sep 23, 2019
by
Matthieu Dorier
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'master' of xgitlab.cels.anl.gov:sds/HEPnOS
parents
bcf785f1
1a3f5f7f
Changes
32
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
32 changed files
with
769 additions
and
533 deletions
+769
-533
CMakeLists.txt
CMakeLists.txt
+3
-4
bin/CMakeLists.txt
bin/CMakeLists.txt
+1
-1
bin/hepnos-daemon.cpp
bin/hepnos-daemon.cpp
+0
-6
include/hepnos.hpp
include/hepnos.hpp
+1
-0
include/hepnos/DataSet.hpp
include/hepnos/DataSet.hpp
+6
-2
include/hepnos/DataStore.hpp
include/hepnos/DataStore.hpp
+16
-2
include/hepnos/Event.hpp
include/hepnos/Event.hpp
+7
-4
include/hepnos/Exception.hpp
include/hepnos/Exception.hpp
+3
-0
include/hepnos/KeyValueContainer.hpp
include/hepnos/KeyValueContainer.hpp
+40
-17
include/hepnos/ProductID.hpp
include/hepnos/ProductID.hpp
+4
-2
include/hepnos/Run.hpp
include/hepnos/Run.hpp
+6
-3
include/hepnos/SubRun.hpp
include/hepnos/SubRun.hpp
+6
-2
include/hepnos/WriteBatch.hpp
include/hepnos/WriteBatch.hpp
+54
-0
src/CMakeLists.txt
src/CMakeLists.txt
+4
-3
src/DataSet.cpp
src/DataSet.cpp
+36
-7
src/DataStore.cpp
src/DataStore.cpp
+18
-7
src/Event.cpp
src/Event.cpp
+23
-2
src/Run.cpp
src/Run.cpp
+35
-5
src/RunSet.cpp
src/RunSet.cpp
+1
-2
src/SubRun.cpp
src/SubRun.cpp
+35
-5
src/WriteBatch.cpp
src/WriteBatch.cpp
+16
-0
src/private/DataStoreImpl.hpp
src/private/DataStoreImpl.hpp
+126
-307
src/private/KeyTypes.hpp
src/private/KeyTypes.hpp
+0
-61
src/private/StringHash.hpp
src/private/StringHash.hpp
+18
-0
src/private/ValueTypes.hpp
src/private/ValueTypes.hpp
+0
-45
src/private/WriteBatchImpl.hpp
src/private/WriteBatchImpl.hpp
+101
-0
src/service/HEPnOSService.cpp
src/service/HEPnOSService.cpp
+5
-29
test/CMakeLists.txt
test/CMakeLists.txt
+17
-11
test/HEPnOSTestMain.cpp
test/HEPnOSTestMain.cpp
+1
-0
test/WriteBatchTest.cpp
test/WriteBatchTest.cpp
+154
-0
test/WriteBatchTest.hpp
test/WriteBatchTest.hpp
+31
-0
test/config.yaml
test/config.yaml
+1
-6
No files found.
CMakeLists.txt
View file @
8dcdec7b
...
...
@@ -7,10 +7,11 @@
# -DCMAKE_PREFIX_PATH='/dir1;/dir2;/dir3'
#
cmake_minimum_required
(
VERSION 3.
0
)
cmake_minimum_required
(
VERSION 3.
14
)
project
(
hepnos CXX
)
add_definitions
(
-g
)
set
(
CMAKE_CXX_STANDARD 14
)
# add our cmake module directory to the path
set
(
CMAKE_MODULE_PATH
${
CMAKE_MODULE_PATH
}
...
...
@@ -35,13 +36,11 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
# packages we depend on
include
(
xpkg-import
)
find_package
(
mercury CONFIG REQUIRED
)
find_package
(
Boost REQUIRED COMPONENTS serialization
system filesystem
)
find_package
(
Boost REQUIRED COMPONENTS serialization
)
include_directories
(
${
Boost_INCLUDE_DIRS
}
)
xpkg_import_module
(
margo REQUIRED margo
)
xpkg_import_module
(
sdskv-client REQUIRED sdskv-client
)
xpkg_import_module
(
bake-client REQUIRED bake-client
)
xpkg_import_module
(
sdskv-server REQUIRED sdskv-server
)
xpkg_import_module
(
bake-server REQUIRED bake-server
)
xpkg_import_module
(
ch-placement REQUIRED ch-placement
)
find_package
(
yaml-cpp REQUIRED
)
...
...
bin/CMakeLists.txt
View file @
8dcdec7b
add_executable
(
hepnos-daemon hepnos-daemon.cpp
)
target_link_libraries
(
hepnos-daemon hepnos-service yaml-cpp margo
bake-server
sdskv-server
)
target_link_libraries
(
hepnos-daemon hepnos-service yaml-cpp margo sdskv-server
)
add_executable
(
hepnos-shutdown hepnos-shutdown.cpp
)
target_link_libraries
(
hepnos-shutdown hepnos yaml-cpp margo
)
...
...
bin/hepnos-daemon.cpp
View file @
8dcdec7b
...
...
@@ -9,14 +9,8 @@
#include <vector>
#include <unistd.h>
#include <mpi.h>
#include <margo.h>
#include <bake-server.h>
#include <sdskv-server.h>
#include <yaml-cpp/yaml.h>
#include "hepnos-service.h"
#define ASSERT(__cond, __msg, ...) { if(!(__cond)) { fprintf(stderr, "[%s:%d] " __msg, __FILE__, __LINE__, __VA_ARGS__); exit(-1); } }
void
usage
(
void
)
{
fprintf
(
stderr
,
"Usage: hepnos-daemon <config-file> <connection-file>
\n
"
);
...
...
include/hepnos.hpp
View file @
8dcdec7b
...
...
@@ -13,5 +13,6 @@
#include <hepnos/RunSet.hpp>
#include <hepnos/SubRun.hpp>
#include <hepnos/SubRunNumber.hpp>
#include <hepnos/WriteBatch.hpp>
#endif
include/hepnos/DataSet.hpp
View file @
8dcdec7b
...
...
@@ -161,7 +161,10 @@ class DataSet : public KeyValueContainer {
* @return a valid ProductID if the key did not already exist and the write succeeded,
* an invalid one otherwise.
*/
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>&
buffer
)
override
;
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
/**
* @brief Loads binary data associated with a particular key from the DataSet.
...
...
@@ -174,7 +177,7 @@ class DataSet : public KeyValueContainer {
* @return true if the key exists and the read succeeded,
* false otherwise.
*/
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>&
buffer
)
const
override
;
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
value
)
const
override
;
/**
* @brief Comparison operator.
...
...
@@ -222,6 +225,7 @@ class DataSet : public KeyValueContainer {
* @return A Run instance pointing to the created run.
*/
Run
createRun
(
const
RunNumber
&
runNumber
);
Run
createRun
(
WriteBatch
&
batch
,
const
RunNumber
&
runNumber
);
typedef
DataStore
::
const_iterator
const_iterator
;
typedef
DataStore
::
iterator
iterator
;
...
...
include/hepnos/DataStore.hpp
View file @
8dcdec7b
...
...
@@ -19,6 +19,7 @@ class Run;
class
SubRun
;
class
Event
;
template
<
typename
T
,
typename
C
=
std
::
vector
<
T
>
>
class
Ptr
;
class
WriteBatch
;
/**
* The DataStore class is the main handle referencing an HEPnOS service.
...
...
@@ -32,6 +33,7 @@ class DataStore {
friend
class
Run
;
friend
class
SubRun
;
friend
class
Event
;
friend
class
WriteBatch
;
public:
...
...
@@ -249,6 +251,18 @@ class DataStore {
*/
DataSet
createDataSet
(
const
std
::
string
&
name
);
/**
* @brief Creates a dataset with a given name inside the data store.
* This function takes a WriteBatch instance, the dataset will be
* actually created when this batch is flushed or destroyed.
*
* @param batch WriteBatch in which to enqueue the creation.
* @param name Name of the dataset.
*
* @return A DataSet instance pointing to the created dataset.
*/
DataSet
createDataSet
(
WriteBatch
&
batch
,
const
std
::
string
&
name
);
/**
* @brief Create a pointer to a product. The type T used must
* match the type of the product corresponding to the provided
...
...
@@ -315,7 +329,7 @@ class DataStore {
*
* @return true if the data was loaded successfuly, false otherwise.
*/
bool
loadRawProduct
(
const
ProductID
&
productID
,
std
::
vector
<
char
>
&
buffer
);
bool
loadRawProduct
(
const
ProductID
&
productID
,
std
::
string
&
buffer
);
};
class
DataStore
::
const_iterator
{
...
...
@@ -560,7 +574,7 @@ Ptr<T,C> DataStore::makePtr(const ProductID& productID, std::size_t index) {
template
<
typename
T
>
bool
DataStore
::
loadProduct
(
const
ProductID
&
productID
,
T
&
t
)
{
std
::
vector
<
char
>
buffer
;
std
::
string
buffer
;
if
(
!
loadRawProduct
(
productID
,
buffer
))
{
return
false
;
}
...
...
include/hepnos/Event.hpp
View file @
8dcdec7b
...
...
@@ -105,21 +105,24 @@ class Event : public KeyValueContainer {
* @brief Stores raw key/value data in this Event.
*
* @param key Key
* @param
buffer
Value
* @param
value
Value
*
* @return a valid ProductID if the key did not already exist, an invalid one otherwise.
*/
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>&
buffer
)
override
;
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
/**
* @brief Loads raw key/value data from this Event.
*
* @param key Key
* @param
buffer
Buffer used to hold the value.
* @param
value
Buffer used to hold the value.
*
* @return true if the key exists, false otherwise.
*/
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>&
buffer
)
const
override
;
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
value
)
const
override
;
/**
* @brief Compares this Event with another Event. The Events must point to
...
...
include/hepnos/Exception.hpp
View file @
8dcdec7b
...
...
@@ -17,7 +17,10 @@ class Exception : public std::exception
public:
Exception
()
=
default
;
Exception
(
const
std
::
string
&
msg
)
:
m_msg
(
msg
){}
Exception
(
const
Exception
&
)
=
default
;
Exception
&
operator
=
(
const
Exception
&
)
=
default
;
virtual
const
char
*
what
()
const
noexcept
override
{
...
...
include/hepnos/KeyValueContainer.hpp
View file @
8dcdec7b
...
...
@@ -19,6 +19,8 @@
namespace
hepnos
{
class
WriteBatch
;
class
KeyValueContainer
{
public:
...
...
@@ -69,7 +71,13 @@ class KeyValueContainer {
*
* @return A valid ProductID if the key did not already exist, an invalid one otherwise.
*/
virtual
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>&
buffer
)
=
0
;
virtual
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
value
)
=
0
;
virtual
ProductID
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
value
)
=
0
;
virtual
ProductID
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
value
)
=
0
;
virtual
ProductID
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
value
)
=
0
;
/**
* @brief Loads raw key/value data from this KeyValueContainer.
...
...
@@ -80,7 +88,7 @@ class KeyValueContainer {
*
* @return true if the key exists, false otherwise.
*/
virtual
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>
&
buffer
)
const
=
0
;
virtual
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
=
0
;
/**
* @brief Stores a key/value pair into the KeyValueContainer.
...
...
@@ -99,18 +107,16 @@ class KeyValueContainer {
*/
template
<
typename
K
,
typename
V
>
ProductID
store
(
const
K
&
key
,
const
V
&
value
)
{
std
::
stringstream
ss_value
;
std
::
stringstream
ss_key
;
ss_key
<<
key
<<
"#"
<<
demangle
<
V
>
();
boost
::
archive
::
binary_oarchive
oa
(
ss_value
);
try
{
oa
<<
value
;
}
catch
(...)
{
throw
Exception
(
"Exception occured during serialization"
);
}
std
::
string
serialized
=
ss_value
.
str
();
std
::
vector
<
char
>
buffer
(
serialized
.
begin
(),
serialized
.
end
());
return
storeRawData
(
ss_key
.
str
(),
buffer
);
std
::
string
key_str
,
val_str
;
serializeKeyValue
(
key
,
value
,
key_str
,
val_str
);
return
storeRawData
(
std
::
move
(
key_str
),
std
::
move
(
val_str
));
}
template
<
typename
K
,
typename
V
>
ProductID
store
(
WriteBatch
&
batch
,
const
K
&
key
,
const
V
&
value
)
{
std
::
string
key_str
,
val_str
;
serializeKeyValue
(
key
,
value
,
key_str
,
val_str
);
return
storeRawData
(
batch
,
std
::
move
(
key_str
),
std
::
move
(
val_str
));
}
/**
...
...
@@ -130,15 +136,14 @@ class KeyValueContainer {
*/
template
<
typename
K
,
typename
V
>
bool
load
(
const
K
&
key
,
V
&
value
)
const
{
std
::
vector
<
char
>
buffer
;
std
::
string
buffer
;
std
::
stringstream
ss_key
;
ss_key
<<
key
<<
"#"
<<
demangle
<
V
>
();
if
(
!
loadRawData
(
ss_key
.
str
(),
buffer
))
{
return
false
;
}
try
{
std
::
string
serialized
(
buffer
.
begin
(),
buffer
.
end
());
std
::
stringstream
ss
(
serialized
);
std
::
stringstream
ss
(
buffer
);
//boost::archive::binary_iarchive ia(ss);
InputArchive
ia
(
getDataStore
(),
ss
);
ia
>>
value
;
...
...
@@ -147,6 +152,24 @@ class KeyValueContainer {
}
return
true
;
}
private:
template
<
typename
K
,
typename
V
>
static
void
serializeKeyValue
(
const
K
&
key
,
const
V
&
value
,
std
::
string
&
key_str
,
std
::
string
&
value_str
)
{
std
::
stringstream
ss_value
;
std
::
stringstream
ss_key
;
ss_key
<<
key
<<
"#"
<<
demangle
<
V
>
();
key_str
=
std
::
move
(
ss_key
.
str
());
boost
::
archive
::
binary_oarchive
oa
(
ss_value
);
try
{
oa
<<
value
;
}
catch
(...)
{
throw
Exception
(
"Exception occured during serialization"
);
}
value_str
=
ss_value
.
str
();
}
};
}
...
...
include/hepnos/ProductID.hpp
View file @
8dcdec7b
...
...
@@ -11,14 +11,16 @@
#include <boost/serialization/access.hpp>
#include <boost/serialization/string.hpp>
#include <hepnos/DataStore.hpp>
namespace
hepnos
{
class
WriteBatch
;
class
DataStore
;
class
ProductID
{
friend
class
DataStore
;
friend
class
DataStore
::
Impl
;
friend
class
WriteBatch
;
friend
class
boost
::
serialization
::
access
;
private:
...
...
include/hepnos/Run.hpp
View file @
8dcdec7b
...
...
@@ -115,8 +115,10 @@ class Run : public KeyValueContainer {
*
* @return a valid ProductID if the key did not already exist, an invalid one otherwise.
*/
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>&
buffer
)
override
;
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
/**
* @brief Loads raw key/value data from this Run.
*
...
...
@@ -125,7 +127,7 @@ class Run : public KeyValueContainer {
*
* @return true if the key exists, false otherwise.
*/
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>
&
buffer
)
const
override
;
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
override
;
/**
* @brief Compares this Run with another Run. The Runs must point to
...
...
@@ -311,6 +313,7 @@ class Run : public KeyValueContainer {
* @return a handle to the created or existing SubRun.
*/
SubRun
createSubRun
(
const
SubRunNumber
&
subRunNumber
);
SubRun
createSubRun
(
WriteBatch
&
batch
,
const
SubRunNumber
&
subRunNumber
);
};
class
Run
::
const_iterator
{
...
...
include/hepnos/SubRun.hpp
View file @
8dcdec7b
...
...
@@ -112,7 +112,10 @@ class SubRun : public KeyValueContainer {
*
* @return a valid ProductID if the key did not already exist, an invalid one otherwise.
*/
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>&
buffer
)
override
;
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
/**
* @brief Loads raw key/value data from this SubRun.
...
...
@@ -122,7 +125,7 @@ class SubRun : public KeyValueContainer {
*
* @return true if the key exists, false otherwise.
*/
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>
&
buffer
)
const
override
;
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
override
;
/**
* @brief Compares this SubRun with another SubRun. The SubRuns must point to
...
...
@@ -300,6 +303,7 @@ class SubRun : public KeyValueContainer {
* @return a handle to the created or existing Event.
*/
Event
createEvent
(
const
EventNumber
&
eventNumber
);
Event
createEvent
(
WriteBatch
&
batch
,
const
EventNumber
&
eventNumber
);
};
class
SubRun
::
const_iterator
{
...
...
include/hepnos/WriteBatch.hpp
0 → 100644
View file @
8dcdec7b
/*
* (C) 2019 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#ifndef __HEPNOS_WRITE_BATCH_H
#define __HEPNOS_WRITE_BATCH_H
#include <memory>
#include <string>
#include <vector>
#include <hepnos/KeyValueContainer.hpp>
#include <hepnos/ProductID.hpp>
#include <hepnos/DataStore.hpp>
#include <hepnos/Exception.hpp>
namespace
hepnos
{
class
DataStore
;
class
DataSet
;
class
Run
;
class
SubRun
;
class
Event
;
class
WriteBatch
{
friend
class
DataStore
;
friend
class
DataSet
;
friend
class
Run
;
friend
class
SubRun
;
friend
class
Event
;
friend
class
KeyValueContainer
;
private:
class
Impl
;
std
::
unique_ptr
<
Impl
>
m_impl
;
public:
WriteBatch
(
DataStore
&
ds
);
~
WriteBatch
();
WriteBatch
(
const
WriteBatch
&
)
=
delete
;
WriteBatch
&
operator
=
(
const
WriteBatch
&
)
=
delete
;
WriteBatch
(
WriteBatch
&&
)
=
delete
;
WriteBatch
&
operator
=
(
WriteBatch
&&
)
=
delete
;
};
}
#endif
src/CMakeLists.txt
View file @
8dcdec7b
...
...
@@ -4,7 +4,8 @@ set(hepnos-src DataStore.cpp
RunSet.cpp
Run.cpp
SubRun.cpp
Event.cpp
)
Event.cpp
WriteBatch.cpp
)
set
(
hepnos-service-src service/HEPnOSService.cpp
service/ServiceConfig.cpp
...
...
@@ -26,7 +27,7 @@ set (hepnos-vers "${HEPNOS_VERSION_MAJOR}.${HEPNOS_VERSION_MINOR}")
set
(
HEPNOS_VERSION
"
${
hepnos-vers
}
.
${
HEPNOS_VERSION_PATCH
}
"
)
add_library
(
hepnos
${
hepnos-src
}
)
target_link_libraries
(
hepnos mercury margo yaml-cpp sdskv-client
bake-client
ch-placement
)
target_link_libraries
(
hepnos mercury margo yaml-cpp sdskv-client ch-placement
)
target_include_directories
(
hepnos PUBLIC $<INSTALL_INTERFACE:include>
)
# local include's BEFORE, in case old incompatable .h files in prefix/include
...
...
@@ -40,7 +41,7 @@ set_target_properties (hepnos
SOVERSION
${
HEPNOS_VERSION_MAJOR
}
)
add_library
(
hepnos-service
${
hepnos-service-src
}
)
target_link_libraries
(
hepnos mercury margo yaml-cpp sdskv-client sdskv-server
bake-client bake-server
ch-placement
)
target_link_libraries
(
hepnos mercury margo yaml-cpp sdskv-client sdskv-server ch-placement
)
target_include_directories
(
hepnos-service PUBLIC $<INSTALL_INTERFACE:include>
)
# local include's BEFORE, in case old incompatable .h files in prefix/include
...
...
src/DataSet.cpp
View file @
8dcdec7b
...
...
@@ -10,6 +10,7 @@
#include "private/RunImpl.hpp"
#include "private/DataSetImpl.hpp"
#include "private/DataStoreImpl.hpp"
#include "private/WriteBatchImpl.hpp"
namespace
hepnos
{
...
...
@@ -93,7 +94,7 @@ bool DataSet::valid() const {
return
m_impl
&&
m_impl
->
m_datastore
;
}
ProductID
DataSet
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>
&
buffer
)
{
ProductID
DataSet
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
...
...
@@ -101,7 +102,27 @@ ProductID DataSet::storeRawData(const std::string& key, const std::vector<char>&
return
m_impl
->
m_datastore
->
m_impl
->
store
(
0
,
fullname
(),
key
,
buffer
);
}
bool
DataSet
::
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>&
buffer
)
const
{
ProductID
DataSet
::
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
return
storeRawData
(
key
,
buffer
);
// will call the function above
}
ProductID
DataSet
::
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
fullname
(),
key
,
buffer
);
}
ProductID
DataSet
::
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
fullname
(),
std
::
move
(
key
),
std
::
move
(
buffer
));
}
bool
DataSet
::
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
...
...
@@ -153,7 +174,7 @@ DataSet DataSet::createDataSet(const std::string& name) {
throw
Exception
(
"Invalid character '/' or '%' in dataset name"
);
}
std
::
string
parent
=
fullname
();
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
name
,
std
::
vector
<
char
>
());
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
name
,
std
::
string
());
return
DataSet
(
m_impl
->
m_datastore
,
m_impl
->
m_level
+
1
,
parent
,
name
);
}
...
...
@@ -163,7 +184,17 @@ Run DataSet::createRun(const RunNumber& runNumber) {
}
std
::
string
parent
=
fullname
();
std
::
string
runStr
=
Run
::
Impl
::
makeKeyStringFromRunNumber
(
runNumber
);
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
runStr
,
std
::
vector
<
char
>
());
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
runStr
,
std
::
string
());
return
Run
(
m_impl
->
m_datastore
,
m_impl
->
m_level
+
1
,
parent
,
runNumber
);
}
Run
DataSet
::
createRun
(
WriteBatch
&
batch
,
const
RunNumber
&
runNumber
)
{
if
(
InvalidRunNumber
==
runNumber
)
{
throw
Exception
(
"Trying to create a Run with InvalidRunNumber"
);
}
std
::
string
parent
=
fullname
();
std
::
string
runStr
=
Run
::
Impl
::
makeKeyStringFromRunNumber
(
runNumber
);
batch
.
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
runStr
,
std
::
string
());
return
Run
(
m_impl
->
m_datastore
,
m_impl
->
m_level
+
1
,
parent
,
runNumber
);
}
...
...
@@ -206,9 +237,7 @@ DataSet::iterator DataSet::find(const std::string& datasetPath) {
datasetName
=
datasetPath
.
substr
(
c
+
1
);
}
std
::
vector
<
char
>
data
;
bool
b
=
m_impl
->
m_datastore
->
m_impl
->
load
(
level
,
containerName
,
datasetName
,
data
);
bool
b
=
m_impl
->
m_datastore
->
m_impl
->
exists
(
level
,
containerName
,
datasetName
);
if
(
!
b
)
{
return
m_impl
->
m_datastore
->
end
();
}
...
...
src/DataStore.cpp
View file @
8dcdec7b
...
...
@@ -6,14 +6,12 @@
#include <vector>
#include <functional>
#include <iostream>
#include <yaml-cpp/yaml.h>
#include <sdskv-client.h>
#include <bake-client.h>
#include <ch-placement.h>
#include "hepnos/Exception.hpp"
#include "hepnos/DataStore.hpp"
#include "hepnos/DataSet.hpp"
#include "hepnos/WriteBatch.hpp"
#include "private/DataStoreImpl.hpp"
#include "private/WriteBatchImpl.hpp"
namespace
hepnos
{
...
...
@@ -44,6 +42,7 @@ DataStore& DataStore::operator=(DataStore&& other) {
m_impl
->
cleanup
();
}
m_impl
=
std
::
move
(
other
.
m_impl
);
return
*
this
;
}
DataStore
::~
DataStore
()
{
...
...
@@ -77,7 +76,7 @@ DataStore::iterator DataStore::find(const std::string& datasetPath) {
datasetName
=
datasetPath
.
substr
(
c
+
1
);
}
std
::
vector
<
char
>
data
;
std
::
string
data
;
bool
b
=
m_impl
->
load
(
level
,
containerName
,
datasetName
,
data
);
if
(
!
b
)
{
return
m_impl
->
m_end
;
...
...
@@ -178,7 +177,19 @@ DataSet DataStore::createDataSet(const std::string& name) {
||
name
.
find
(
'%'
)
!=
std
::
string
::
npos
)
{
throw
Exception
(
"Invalid character ('/' or '%') in dataset name"
);
}
m_impl
->
store
(
1
,
""
,
name
,
std
::
vector
<
char
>
());
m_impl
->
store
(
1
,
""
,
name
,
std
::
string
());
return
DataSet
(
this
,
1
,
""
,
name
);
}
DataSet
DataStore
::
createDataSet
(
WriteBatch
&
batch
,
const
std
::
string
&
name
)
{
if
(
!
m_impl
)
{
throw
Exception
(
"Calling DataStore member function on an invalid DataStore object"
);
}
if
(
name
.
find
(
'/'
)
!=
std
::
string
::
npos
||
name
.
find
(
'%'
)
!=
std
::
string
::
npos
)
{
throw
Exception
(
"Invalid character ('/' or '%') in dataset name"
);
}
batch
.
m_impl
->
store
(
1
,
""
,
name
,
std
::
string
());
return
DataSet
(
this
,
1
,
""
,
name
);
}
...
...
@@ -191,7 +202,7 @@ void DataStore::shutdown() {
}
}
bool
DataStore
::
loadRawProduct
(
const
ProductID
&
productID
,
std
::
vector
<
char
>
&
buffer
)
{
bool
DataStore
::
loadRawProduct
(
const
ProductID
&
productID
,
std
::
string
&
buffer
)
{
if
(
!
m_impl
)
{
throw
Exception
(
"Calling DataStore member function on an invalid DataStore object"
);
}
...
...
src/Event.cpp
View file @
8dcdec7b
...
...
@@ -6,6 +6,7 @@
#include "hepnos/Event.hpp"
#include "private/EventImpl.hpp"
#include "private/DataStoreImpl.hpp"
#include "private/WriteBatchImpl.hpp"
namespace
hepnos
{
...
...
@@ -64,7 +65,7 @@ bool Event::valid() const {
}
ProductID
Event
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>
&
buffer
)
{
ProductID
Event
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Event member function on an invalid Event object"
);
}
...
...
@@ -72,7 +73,27 @@ ProductID Event::storeRawData(const std::string& key, const std::vector<char>& b
return
m_impl
->
m_datastore
->
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
key
,
buffer
);
}
bool
Event
::
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>&
buffer
)
const
{
ProductID
Event
::
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
return
storeRawData
(
key
,
buffer
);
// call the above function
}
ProductID
Event
::
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Event member function on an invalid Event object"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
key
,
buffer
);
}
ProductID
Event
::
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Event member function on an invalid Event object"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
std
::
move
(
key
),
std
::
move
(
buffer
));
}
bool
Event
::
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Event member function on an invalid Event object"
);
}
...
...
src/Run.cpp
View file @
8dcdec7b
...
...
@@ -7,6 +7,7 @@
#include "private/RunImpl.hpp"
#include "private/SubRunImpl.hpp"
#include "private/DataStoreImpl.hpp"
#include "private/WriteBatchImpl.hpp"
namespace
hepnos
{
...
...
@@ -66,7 +67,7 @@ bool Run::valid() const {
}
ProductID
Run
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>
&
buffer
)
{
ProductID
Run
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Run member function on an invalid Run object"
);
}
...
...
@@ -74,7 +75,27 @@ ProductID Run::storeRawData(const std::string& key, const std::vector<char>& buf
return
m_impl
->
m_datastore
->
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
key
,
buffer
);
}
bool
Run
::
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>&
buffer
)
const
{
ProductID
Run
::
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
return
storeRawData
(
key
,
buffer
);
// calls the above function
}
ProductID
Run
::
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Run member function on an invalid Run object"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
key
,
buffer
);
}
ProductID
Run
::
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Run member function on an invalid Run object"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
std
::
move
(
key
),
std
::
move
(
buffer
));
}
bool
Run
::
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Run member function on an invalid Run object"
);
}
...
...
@@ -118,7 +139,17 @@ SubRun Run::createSubRun(const SubRunNumber& subRunNumber) {
}
std
::
string
parent
=
m_impl
->
fullpath
();
std
::
string
subRunStr
=
SubRun
::
Impl
::
makeKeyStringFromSubRunNumber
(
subRunNumber
);
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
subRunStr
,
std
::
vector
<
char
>
());
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
subRunStr
,
std
::
string
());
return
SubRun
(
m_impl
->
m_datastore
,
m_impl
->
m_level
+
1
,
parent
,
subRunNumber
);
}
SubRun
Run
::
createSubRun
(
WriteBatch
&
batch
,
const
SubRunNumber
&
subRunNumber
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Run member function on an invalid Run object"
);
}
std
::
string
parent
=
m_impl
->
fullpath
();
std
::
string
subRunStr
=
SubRun
::
Impl
::
makeKeyStringFromSubRunNumber
(
subRunNumber
);
batch
.
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
subRunStr
,
std
::
string
());
return
SubRun
(
m_impl
->
m_datastore
,
m_impl
->
m_level
+
1
,
parent
,
subRunNumber
);
}
...
...
@@ -134,10 +165,9 @@ Run::iterator Run::find(const SubRunNumber& subRunNumber) {
throw
Exception
(
"Calling Run member function on an invalid Run object"
);
}
int
ret
;
std
::
vector
<
char
>
data
;
std
::
string
parent
=
m_impl
->
fullpath
();
std
::
string
subRunStr
=
SubRun
::
Impl
::
makeKeyStringFromSubRunNumber
(
subRunNumber
);
bool
b
=
m_impl
->
m_datastore
->
m_impl
->
load
(
m_impl
->
m_level
+
1
,
parent
,
subRunStr
,
data
);