Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
H
HEPnOS
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
sds
HEP
HEPnOS
Commits
2f12aef2
Commit
2f12aef2
authored
Sep 03, 2019
by
Matthieu Dorier
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
WriteBatch working for datasets
parent
d637e054
Changes
23
Hide whitespace changes
Inline
Side-by-side
Showing
23 changed files
with
505 additions
and
147 deletions
+505
-147
include/hepnos.hpp
include/hepnos.hpp
+1
-0
include/hepnos/DataSet.hpp
include/hepnos/DataSet.hpp
+5
-2
include/hepnos/DataStore.hpp
include/hepnos/DataStore.hpp
+16
-2
include/hepnos/Event.hpp
include/hepnos/Event.hpp
+7
-4
include/hepnos/KeyValueContainer.hpp
include/hepnos/KeyValueContainer.hpp
+40
-17
include/hepnos/ProductID.hpp
include/hepnos/ProductID.hpp
+4
-2
include/hepnos/Run.hpp
include/hepnos/Run.hpp
+5
-3
include/hepnos/SubRun.hpp
include/hepnos/SubRun.hpp
+5
-2
include/hepnos/WriteBatch.hpp
include/hepnos/WriteBatch.hpp
+54
-0
src/CMakeLists.txt
src/CMakeLists.txt
+2
-1
src/DataSet.cpp
src/DataSet.cpp
+26
-7
src/DataStore.cpp
src/DataStore.cpp
+17
-3
src/Event.cpp
src/Event.cpp
+23
-2
src/Run.cpp
src/Run.cpp
+25
-5
src/RunSet.cpp
src/RunSet.cpp
+1
-2
src/SubRun.cpp
src/SubRun.cpp
+25
-5
src/WriteBatch.cpp
src/WriteBatch.cpp
+16
-0
src/private/DataStoreImpl.hpp
src/private/DataStoreImpl.hpp
+55
-29
src/private/KeyTypes.hpp
src/private/KeyTypes.hpp
+0
-61
src/private/WriteBatchImpl.hpp
src/private/WriteBatchImpl.hpp
+64
-0
test/CMakeLists.txt
test/CMakeLists.txt
+4
-0
test/WriteBatchTest.cpp
test/WriteBatchTest.cpp
+79
-0
test/WriteBatchTest.hpp
test/WriteBatchTest.hpp
+31
-0
No files found.
include/hepnos.hpp
View file @
2f12aef2
...
...
@@ -13,5 +13,6 @@
#include <hepnos/RunSet.hpp>
#include <hepnos/SubRun.hpp>
#include <hepnos/SubRunNumber.hpp>
#include <hepnos/WriteBatch.hpp>
#endif
include/hepnos/DataSet.hpp
View file @
2f12aef2
...
...
@@ -161,7 +161,10 @@ class DataSet : public KeyValueContainer {
* @return a valid ProductID if the key did not already exist and the write succeeded,
* an invalid one otherwise.
*/
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>&
buffer
)
override
;
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
/**
* @brief Loads binary data associated with a particular key from the DataSet.
...
...
@@ -174,7 +177,7 @@ class DataSet : public KeyValueContainer {
* @return true if the key exists and the read succeeded,
* false otherwise.
*/
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>&
buffer
)
const
override
;
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
value
)
const
override
;
/**
* @brief Comparison operator.
...
...
include/hepnos/DataStore.hpp
View file @
2f12aef2
...
...
@@ -19,6 +19,7 @@ class Run;
class
SubRun
;
class
Event
;
template
<
typename
T
,
typename
C
=
std
::
vector
<
T
>
>
class
Ptr
;
class
WriteBatch
;
/**
* The DataStore class is the main handle referencing an HEPnOS service.
...
...
@@ -32,6 +33,7 @@ class DataStore {
friend
class
Run
;
friend
class
SubRun
;
friend
class
Event
;
friend
class
WriteBatch
;
public:
...
...
@@ -249,6 +251,18 @@ class DataStore {
*/
DataSet
createDataSet
(
const
std
::
string
&
name
);
/**
* @brief Creates a dataset with a given name inside the data store.
* This function takes a WriteBatch instance, the dataset will be
* actually created when this batch is flushed or destroyed.
*
* @param batch WriteBatch in which to enqueue the creation.
* @param name Name of the dataset.
*
* @return A DataSet instance pointing to the created dataset.
*/
DataSet
createDataSet
(
WriteBatch
&
batch
,
const
std
::
string
&
name
);
/**
* @brief Create a pointer to a product. The type T used must
* match the type of the product corresponding to the provided
...
...
@@ -315,7 +329,7 @@ class DataStore {
*
* @return true if the data was loaded successfuly, false otherwise.
*/
bool
loadRawProduct
(
const
ProductID
&
productID
,
std
::
vector
<
char
>
&
buffer
);
bool
loadRawProduct
(
const
ProductID
&
productID
,
std
::
string
&
buffer
);
};
class
DataStore
::
const_iterator
{
...
...
@@ -560,7 +574,7 @@ Ptr<T,C> DataStore::makePtr(const ProductID& productID, std::size_t index) {
template
<
typename
T
>
bool
DataStore
::
loadProduct
(
const
ProductID
&
productID
,
T
&
t
)
{
std
::
vector
<
char
>
buffer
;
std
::
string
buffer
;
if
(
!
loadRawProduct
(
productID
,
buffer
))
{
return
false
;
}
...
...
include/hepnos/Event.hpp
View file @
2f12aef2
...
...
@@ -105,21 +105,24 @@ class Event : public KeyValueContainer {
* @brief Stores raw key/value data in this Event.
*
* @param key Key
* @param
buffer
Value
* @param
value
Value
*
* @return a valid ProductID if the key did not already exist, an invalid one otherwise.
*/
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>&
buffer
)
override
;
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
/**
* @brief Loads raw key/value data from this Event.
*
* @param key Key
* @param
buffer
Buffer used to hold the value.
* @param
value
Buffer used to hold the value.
*
* @return true if the key exists, false otherwise.
*/
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>&
buffer
)
const
override
;
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
value
)
const
override
;
/**
* @brief Compares this Event with another Event. The Events must point to
...
...
include/hepnos/KeyValueContainer.hpp
View file @
2f12aef2
...
...
@@ -19,6 +19,8 @@
namespace
hepnos
{
class
WriteBatch
;
class
KeyValueContainer
{
public:
...
...
@@ -69,7 +71,13 @@ class KeyValueContainer {
*
* @return A valid ProductID if the key did not already exist, an invalid one otherwise.
*/
virtual
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>&
buffer
)
=
0
;
virtual
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
value
)
=
0
;
virtual
ProductID
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
value
)
=
0
;
virtual
ProductID
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
value
)
=
0
;
virtual
ProductID
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
value
)
=
0
;
/**
* @brief Loads raw key/value data from this KeyValueContainer.
...
...
@@ -80,7 +88,7 @@ class KeyValueContainer {
*
* @return true if the key exists, false otherwise.
*/
virtual
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>
&
buffer
)
const
=
0
;
virtual
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
=
0
;
/**
* @brief Stores a key/value pair into the KeyValueContainer.
...
...
@@ -99,18 +107,16 @@ class KeyValueContainer {
*/
template
<
typename
K
,
typename
V
>
ProductID
store
(
const
K
&
key
,
const
V
&
value
)
{
std
::
stringstream
ss_value
;
std
::
stringstream
ss_key
;
ss_key
<<
key
<<
"#"
<<
demangle
<
V
>
();
boost
::
archive
::
binary_oarchive
oa
(
ss_value
);
try
{
oa
<<
value
;
}
catch
(...)
{
throw
Exception
(
"Exception occured during serialization"
);
}
std
::
string
serialized
=
ss_value
.
str
();
std
::
vector
<
char
>
buffer
(
serialized
.
begin
(),
serialized
.
end
());
return
storeRawData
(
ss_key
.
str
(),
buffer
);
std
::
string
key_str
,
val_str
;
serializeKeyValue
(
key
,
value
,
key_str
,
val_str
);
return
storeRawData
(
std
::
move
(
key_str
),
std
::
move
(
val_str
));
}
template
<
typename
K
,
typename
V
>
ProductID
store
(
WriteBatch
&
batch
,
const
K
&
key
,
const
V
&
value
)
{
std
::
string
key_str
,
val_str
;
serializeKeyValue
(
key
,
value
,
key_str
,
val_str
);
return
storeRawData
(
batch
,
std
::
move
(
key_str
),
std
::
move
(
val_str
));
}
/**
...
...
@@ -130,15 +136,14 @@ class KeyValueContainer {
*/
template
<
typename
K
,
typename
V
>
bool
load
(
const
K
&
key
,
V
&
value
)
const
{
std
::
vector
<
char
>
buffer
;
std
::
string
buffer
;
std
::
stringstream
ss_key
;
ss_key
<<
key
<<
"#"
<<
demangle
<
V
>
();
if
(
!
loadRawData
(
ss_key
.
str
(),
buffer
))
{
return
false
;
}
try
{
std
::
string
serialized
(
buffer
.
begin
(),
buffer
.
end
());
std
::
stringstream
ss
(
serialized
);
std
::
stringstream
ss
(
buffer
);
//boost::archive::binary_iarchive ia(ss);
InputArchive
ia
(
getDataStore
(),
ss
);
ia
>>
value
;
...
...
@@ -147,6 +152,24 @@ class KeyValueContainer {
}
return
true
;
}
private:
template
<
typename
K
,
typename
V
>
static
void
serializeKeyValue
(
const
K
&
key
,
const
V
&
value
,
std
::
string
&
key_str
,
std
::
string
&
value_str
)
{
std
::
stringstream
ss_value
;
std
::
stringstream
ss_key
;
ss_key
<<
key
<<
"#"
<<
demangle
<
V
>
();
key_str
=
std
::
move
(
ss_key
.
str
());
boost
::
archive
::
binary_oarchive
oa
(
ss_value
);
try
{
oa
<<
value
;
}
catch
(...)
{
throw
Exception
(
"Exception occured during serialization"
);
}
value_str
=
ss_value
.
str
();
}
};
}
...
...
include/hepnos/ProductID.hpp
View file @
2f12aef2
...
...
@@ -11,14 +11,16 @@
#include <boost/serialization/access.hpp>
#include <boost/serialization/string.hpp>
#include <hepnos/DataStore.hpp>
namespace
hepnos
{
class
WriteBatch
;
class
DataStore
;
class
ProductID
{
friend
class
DataStore
;
friend
class
DataStore
::
Impl
;
friend
class
WriteBatch
;
friend
class
boost
::
serialization
::
access
;
private:
...
...
include/hepnos/Run.hpp
View file @
2f12aef2
...
...
@@ -115,8 +115,10 @@ class Run : public KeyValueContainer {
*
* @return a valid ProductID if the key did not already exist, an invalid one otherwise.
*/
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>&
buffer
)
override
;
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
/**
* @brief Loads raw key/value data from this Run.
*
...
...
@@ -125,7 +127,7 @@ class Run : public KeyValueContainer {
*
* @return true if the key exists, false otherwise.
*/
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>
&
buffer
)
const
override
;
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
override
;
/**
* @brief Compares this Run with another Run. The Runs must point to
...
...
include/hepnos/SubRun.hpp
View file @
2f12aef2
...
...
@@ -112,7 +112,10 @@ class SubRun : public KeyValueContainer {
*
* @return a valid ProductID if the key did not already exist, an invalid one otherwise.
*/
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>&
buffer
)
override
;
ProductID
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
value
)
override
;
ProductID
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
value
)
override
;
/**
* @brief Loads raw key/value data from this SubRun.
...
...
@@ -122,7 +125,7 @@ class SubRun : public KeyValueContainer {
*
* @return true if the key exists, false otherwise.
*/
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>
&
buffer
)
const
override
;
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
override
;
/**
* @brief Compares this SubRun with another SubRun. The SubRuns must point to
...
...
include/hepnos/WriteBatch.hpp
0 → 100644
View file @
2f12aef2
/*
* (C) 2019 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#ifndef __HEPNOS_WRITE_BATCH_H
#define __HEPNOS_WRITE_BATCH_H
#include <memory>
#include <string>
#include <vector>
#include <hepnos/KeyValueContainer.hpp>
#include <hepnos/ProductID.hpp>
#include <hepnos/DataStore.hpp>
#include <hepnos/Exception.hpp>
namespace
hepnos
{
class
DataStore
;
class
DataSet
;
class
Run
;
class
SubRun
;
class
Event
;
class
WriteBatch
{
friend
class
DataStore
;
friend
class
DataSet
;
friend
class
Run
;
friend
class
SubRun
;
friend
class
Event
;
friend
class
KeyValueContainer
;
private:
class
Impl
;
std
::
unique_ptr
<
Impl
>
m_impl
;
public:
WriteBatch
(
DataStore
&
ds
);
~
WriteBatch
();
WriteBatch
(
const
WriteBatch
&
)
=
delete
;
WriteBatch
&
operator
=
(
const
WriteBatch
&
)
=
delete
;
WriteBatch
(
WriteBatch
&&
)
=
delete
;
WriteBatch
&
operator
=
(
WriteBatch
&&
)
=
delete
;
};
}
#endif
src/CMakeLists.txt
View file @
2f12aef2
...
...
@@ -4,7 +4,8 @@ set(hepnos-src DataStore.cpp
RunSet.cpp
Run.cpp
SubRun.cpp
Event.cpp
)
Event.cpp
WriteBatch.cpp
)
set
(
hepnos-service-src service/HEPnOSService.cpp
service/ServiceConfig.cpp
...
...
src/DataSet.cpp
View file @
2f12aef2
...
...
@@ -10,6 +10,7 @@
#include "private/RunImpl.hpp"
#include "private/DataSetImpl.hpp"
#include "private/DataStoreImpl.hpp"
#include "private/WriteBatchImpl.hpp"
namespace
hepnos
{
...
...
@@ -93,7 +94,7 @@ bool DataSet::valid() const {
return
m_impl
&&
m_impl
->
m_datastore
;
}
ProductID
DataSet
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>
&
buffer
)
{
ProductID
DataSet
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
...
...
@@ -101,7 +102,27 @@ ProductID DataSet::storeRawData(const std::string& key, const std::vector<char>&
return
m_impl
->
m_datastore
->
m_impl
->
store
(
0
,
fullname
(),
key
,
buffer
);
}
bool
DataSet
::
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>&
buffer
)
const
{
ProductID
DataSet
::
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
return
storeRawData
(
key
,
buffer
);
// will call the function above
}
ProductID
DataSet
::
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
fullname
(),
key
,
buffer
);
}
ProductID
DataSet
::
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
fullname
(),
std
::
move
(
key
),
std
::
move
(
buffer
));
}
bool
DataSet
::
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
...
...
@@ -153,7 +174,7 @@ DataSet DataSet::createDataSet(const std::string& name) {
throw
Exception
(
"Invalid character '/' or '%' in dataset name"
);
}
std
::
string
parent
=
fullname
();
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
name
,
std
::
vector
<
char
>
());
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
name
,
std
::
string
());
return
DataSet
(
m_impl
->
m_datastore
,
m_impl
->
m_level
+
1
,
parent
,
name
);
}
...
...
@@ -163,7 +184,7 @@ Run DataSet::createRun(const RunNumber& runNumber) {
}
std
::
string
parent
=
fullname
();
std
::
string
runStr
=
Run
::
Impl
::
makeKeyStringFromRunNumber
(
runNumber
);
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
runStr
,
std
::
vector
<
char
>
());
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
runStr
,
std
::
string
());
return
Run
(
m_impl
->
m_datastore
,
m_impl
->
m_level
+
1
,
parent
,
runNumber
);
}
...
...
@@ -206,9 +227,7 @@ DataSet::iterator DataSet::find(const std::string& datasetPath) {
datasetName
=
datasetPath
.
substr
(
c
+
1
);
}
std
::
vector
<
char
>
data
;
bool
b
=
m_impl
->
m_datastore
->
m_impl
->
load
(
level
,
containerName
,
datasetName
,
data
);
bool
b
=
m_impl
->
m_datastore
->
m_impl
->
exists
(
level
,
containerName
,
datasetName
);
if
(
!
b
)
{
return
m_impl
->
m_datastore
->
end
();
}
...
...
src/DataStore.cpp
View file @
2f12aef2
...
...
@@ -9,7 +9,9 @@
#include "hepnos/Exception.hpp"
#include "hepnos/DataStore.hpp"
#include "hepnos/DataSet.hpp"
#include "hepnos/WriteBatch.hpp"
#include "private/DataStoreImpl.hpp"
#include "private/WriteBatchImpl.hpp"
namespace
hepnos
{
...
...
@@ -74,7 +76,7 @@ DataStore::iterator DataStore::find(const std::string& datasetPath) {
datasetName
=
datasetPath
.
substr
(
c
+
1
);
}
std
::
vector
<
char
>
data
;
std
::
string
data
;
bool
b
=
m_impl
->
load
(
level
,
containerName
,
datasetName
,
data
);
if
(
!
b
)
{
return
m_impl
->
m_end
;
...
...
@@ -175,7 +177,19 @@ DataSet DataStore::createDataSet(const std::string& name) {
||
name
.
find
(
'%'
)
!=
std
::
string
::
npos
)
{
throw
Exception
(
"Invalid character ('/' or '%') in dataset name"
);
}
m_impl
->
store
(
1
,
""
,
name
,
std
::
vector
<
char
>
());
m_impl
->
store
(
1
,
""
,
name
,
std
::
string
());
return
DataSet
(
this
,
1
,
""
,
name
);
}
DataSet
DataStore
::
createDataSet
(
WriteBatch
&
batch
,
const
std
::
string
&
name
)
{
if
(
!
m_impl
)
{
throw
Exception
(
"Calling DataStore member function on an invalid DataStore object"
);
}
if
(
name
.
find
(
'/'
)
!=
std
::
string
::
npos
||
name
.
find
(
'%'
)
!=
std
::
string
::
npos
)
{
throw
Exception
(
"Invalid character ('/' or '%') in dataset name"
);
}
batch
.
m_impl
->
store
(
1
,
""
,
name
,
std
::
string
());
return
DataSet
(
this
,
1
,
""
,
name
);
}
...
...
@@ -188,7 +202,7 @@ void DataStore::shutdown() {
}
}
bool
DataStore
::
loadRawProduct
(
const
ProductID
&
productID
,
std
::
vector
<
char
>
&
buffer
)
{
bool
DataStore
::
loadRawProduct
(
const
ProductID
&
productID
,
std
::
string
&
buffer
)
{
if
(
!
m_impl
)
{
throw
Exception
(
"Calling DataStore member function on an invalid DataStore object"
);
}
...
...
src/Event.cpp
View file @
2f12aef2
...
...
@@ -6,6 +6,7 @@
#include "hepnos/Event.hpp"
#include "private/EventImpl.hpp"
#include "private/DataStoreImpl.hpp"
#include "private/WriteBatchImpl.hpp"
namespace
hepnos
{
...
...
@@ -64,7 +65,7 @@ bool Event::valid() const {
}
ProductID
Event
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>
&
buffer
)
{
ProductID
Event
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Event member function on an invalid Event object"
);
}
...
...
@@ -72,7 +73,27 @@ ProductID Event::storeRawData(const std::string& key, const std::vector<char>& b
return
m_impl
->
m_datastore
->
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
key
,
buffer
);
}
bool
Event
::
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>&
buffer
)
const
{
ProductID
Event
::
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
return
storeRawData
(
key
,
buffer
);
// call the above function
}
ProductID
Event
::
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Event member function on an invalid Event object"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
key
,
buffer
);
}
ProductID
Event
::
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Event member function on an invalid Event object"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
std
::
move
(
key
),
std
::
move
(
buffer
));
}
bool
Event
::
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Event member function on an invalid Event object"
);
}
...
...
src/Run.cpp
View file @
2f12aef2
...
...
@@ -7,6 +7,7 @@
#include "private/RunImpl.hpp"
#include "private/SubRunImpl.hpp"
#include "private/DataStoreImpl.hpp"
#include "private/WriteBatchImpl.hpp"
namespace
hepnos
{
...
...
@@ -66,7 +67,7 @@ bool Run::valid() const {
}
ProductID
Run
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>
&
buffer
)
{
ProductID
Run
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Run member function on an invalid Run object"
);
}
...
...
@@ -74,7 +75,27 @@ ProductID Run::storeRawData(const std::string& key, const std::vector<char>& buf
return
m_impl
->
m_datastore
->
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
key
,
buffer
);
}
bool
Run
::
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>&
buffer
)
const
{
ProductID
Run
::
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
return
storeRawData
(
key
,
buffer
);
// calls the above function
}
ProductID
Run
::
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Run member function on an invalid Run object"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
key
,
buffer
);
}
ProductID
Run
::
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Run member function on an invalid Run object"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
std
::
move
(
key
),
std
::
move
(
buffer
));
}
bool
Run
::
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Run member function on an invalid Run object"
);
}
...
...
@@ -118,7 +139,7 @@ SubRun Run::createSubRun(const SubRunNumber& subRunNumber) {
}
std
::
string
parent
=
m_impl
->
fullpath
();
std
::
string
subRunStr
=
SubRun
::
Impl
::
makeKeyStringFromSubRunNumber
(
subRunNumber
);
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
subRunStr
,
std
::
vector
<
char
>
());
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
subRunStr
,
std
::
string
());
return
SubRun
(
m_impl
->
m_datastore
,
m_impl
->
m_level
+
1
,
parent
,
subRunNumber
);
}
...
...
@@ -134,10 +155,9 @@ Run::iterator Run::find(const SubRunNumber& subRunNumber) {
throw
Exception
(
"Calling Run member function on an invalid Run object"
);
}
int
ret
;
std
::
vector
<
char
>
data
;
std
::
string
parent
=
m_impl
->
fullpath
();
std
::
string
subRunStr
=
SubRun
::
Impl
::
makeKeyStringFromSubRunNumber
(
subRunNumber
);
bool
b
=
m_impl
->
m_datastore
->
m_impl
->
load
(
m_impl
->
m_level
+
1
,
parent
,
subRunStr
,
data
);
bool
b
=
m_impl
->
m_datastore
->
m_impl
->
exists
(
m_impl
->
m_level
+
1
,
parent
,
subRunStr
);
if
(
!
b
)
{
return
m_impl
->
m_end
;
}
...
...
src/RunSet.cpp
View file @
2f12aef2
...
...
@@ -34,12 +34,11 @@ Run RunSet::operator[](const RunNumber& runNumber) {
RunSet
::
iterator
RunSet
::
find
(
const
RunNumber
&
runNumber
)
{
int
ret
;
std
::
vector
<
char
>
data
;
std
::
string
parent
=
m_impl
->
m_dataset
->
fullname
();
std
::
string
strNum
=
Run
::
Impl
::
makeKeyStringFromRunNumber
(
runNumber
);
auto
datastore
=
m_impl
->
m_dataset
->
m_impl
->
m_datastore
;
auto
level
=
m_impl
->
m_dataset
->
m_impl
->
m_level
;
bool
b
=
datastore
->
m_impl
->
load
(
level
+
1
,
parent
,
strNum
,
data
);
bool
b
=
datastore
->
m_impl
->
exists
(
level
+
1
,
parent
,
strNum
);
if
(
!
b
)
return
end
();
return
iterator
(
Run
(
datastore
,
level
+
1
,
parent
,
runNumber
));
}
...
...
src/SubRun.cpp
View file @
2f12aef2
...
...
@@ -8,6 +8,7 @@
#include "private/SubRunImpl.hpp"
#include "private/EventImpl.hpp"
#include "private/DataStoreImpl.hpp"
#include "private/WriteBatchImpl.hpp"
namespace
hepnos
{
...
...
@@ -67,7 +68,7 @@ bool SubRun::valid() const {
return
m_impl
&&
m_impl
->
m_datastore
;
}
ProductID
SubRun
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
vector
<
char
>
&
buffer
)
{
ProductID
SubRun
::
storeRawData
(
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling SubRun member function on invalid SubRun object"
);
}
...
...
@@ -75,7 +76,27 @@ ProductID SubRun::storeRawData(const std::string& key, const std::vector<char>&
return
m_impl
->
m_datastore
->
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
key
,
buffer
);
}
bool
SubRun
::
loadRawData
(
const
std
::
string
&
key
,
std
::
vector
<
char
>&
buffer
)
const
{
ProductID
SubRun
::
storeRawData
(
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
return
storeRawData
(
key
,
buffer
);
// call above function
}
ProductID
SubRun
::
storeRawData
(
WriteBatch
&
batch
,
const
std
::
string
&
key
,
const
std
::
string
&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling SubRun member function on invalid SubRun object"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
key
,
buffer
);
}
ProductID
SubRun
::
storeRawData
(
WriteBatch
&
batch
,
std
::
string
&&
key
,
std
::
string
&&
buffer
)
{
if
(
!
valid
())
{
throw
Exception
(
"Calling SubRun member function on invalid SubRun object"
);
}
// forward the call to the datastore's store function
return
batch
.
m_impl
->
store
(
0
,
m_impl
->
fullpath
(),
std
::
move
(
key
),
std
::
move
(
buffer
));
}
bool
SubRun
::
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling SubRun member function on invalid SubRun object"
);
}
...
...
@@ -109,7 +130,7 @@ Event SubRun::createEvent(const EventNumber& eventNumber) {
}
std
::
string
parent
=
m_impl
->
fullpath
();
std
::
string
eventStr
=
Event
::
Impl
::
makeKeyStringFromEventNumber
(
eventNumber
);
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
eventStr
,
std
::
vector
<
char
>
());
m_impl
->
m_datastore
->
m_impl
->
store
(
m_impl
->
m_level
+
1
,
parent
,
eventStr
,
std
::
string
());
return
Event
(
m_impl
->
m_datastore
,
m_impl
->
m_level
+
1
,
parent
,
eventNumber
);
}
...
...
@@ -125,10 +146,9 @@ SubRun::iterator SubRun::find(const EventNumber& eventNumber) {
throw
Exception
(
"Calling SubRun member function on invalid SubRun object"
);
}
int
ret
;
std
::
vector
<
char
>
data
;
std
::
string
parent
=
m_impl
->
fullpath
();
std
::
string
eventStr
=
Event
::
Impl
::
makeKeyStringFromEventNumber
(
eventNumber
);
bool
b
=
m_impl
->
m_datastore
->
m_impl
->
load
(
m_impl
->
m_level
+
1
,
parent
,
eventStr
,
data
);
bool
b
=
m_impl
->
m_datastore
->
m_impl
->
exists
(
m_impl
->
m_level
+
1
,
parent
,
eventStr
);
if
(
!
b
)
{
return
m_impl
->
m_end
;
}
...
...
src/WriteBatch.cpp
0 → 100644
View file @
2f12aef2
/*
* (C) 2019 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#include "hepnos.hpp"
#include "private/WriteBatchImpl.hpp"
namespace
hepnos
{
WriteBatch
::
WriteBatch
(
DataStore
&
datastore
)
:
m_impl
(
std
::
make_unique
<
Impl
>
(
&
datastore
))
{}
WriteBatch
::~
WriteBatch
()
{}
}
src/private/DataStoreImpl.hpp
View file @
2f12aef2
...
...
@@ -56,9 +56,9 @@ class DataStore::Impl {
// initialize SDSKV client
try
{
m_sdskv_client
=
sdskv
::
client
(
m_mid
);
}
catch
(
...
)
{
}
catch
(
sdskv
::
exception
&
ex
)
{
cleanup
();
throw
Exception
(
"Could not create SDSKV client"
);
throw
Exception
(
"Could not create SDSKV client
(SDSKV error="
+
std
::
to_string
(
ex
.
error
())
+
")
"
);
}
// create list of sdskv provider handles
YAML
::
Node
sdskv
=
config
[
"hepnos"
][
"providers"
][
"sdskv"
];
...
...
@@ -72,7 +72,7 @@ class DataStore::Impl {
if
(
hret
!=
HG_SUCCESS
)
{
margo_addr_free
(
m_mid
,
addr
);
cleanup
();
throw
Exception
(
"margo_addr_lookup failed"
);
throw
Exception
(
"margo_addr_lookup failed
(MARGO error="
+
std
::
to_string
(
hret
)
+
")
"
);
}
m_addrs
[
str_addr
]
=
addr
;
}
...
...
@@ -83,9 +83,9 @@ class DataStore::Impl {
try
{
sdskv
::
provider_handle
ph
(
m_sdskv_client
,
addr
,
provider_id
);
dbs
=
m_sdskv_client
.
open
(
ph
);
}
catch
(
...
)
{
}
catch
(
sdskv
::
exception
&
ex
)
{
cleanup
();
throw
Exception
(
"
sdskv_count_databases failed
"
);
throw
Exception
(
"
Could not open databases (SDSKV error="
+
std
::
to_string
(
ex
.
error
())
+
")
"
);