Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
sds
HEP
HEPnOS
Commits
8116e5be
Commit
8116e5be
authored
Mar 03, 2020
by
Matthieu Dorier
Browse files
Merge branch 'dev-event-set'
parents
4efe19bc
210b7dc8
Changes
27
Hide whitespace changes
Inline
Side-by-side
include/hepnos.hpp
View file @
8116e5be
...
...
@@ -7,6 +7,7 @@
#include
<hepnos/Demangle.hpp>
#include
<hepnos/Event.hpp>
#include
<hepnos/EventNumber.hpp>
#include
<hepnos/EventSet.hpp>
#include
<hepnos/Exception.hpp>
#include
<hepnos/KeyValueContainer.hpp>
#include
<hepnos/Run.hpp>
...
...
include/hepnos/DataSet.hpp
View file @
8116e5be
...
...
@@ -369,7 +369,7 @@ class DataSet : public KeyValueContainer {
/**
* @brief Returns the RunSet associated with this DataSet.
*
* @return
a reference to
the RunSet associated with this DataSet.
* @return the RunSet associated with this DataSet.
*/
RunSet
runs
()
const
;
...
...
@@ -384,6 +384,17 @@ class DataSet : public KeyValueContainer {
* @return a Run corresponding to the provided run number.
*/
Run
operator
[](
const
RunNumber
&
runNumber
)
const
;
/**
* @brief Returns an EventSet pointing to Events in
* the specified target. If target is -1, the EventSet
* will also iterate over targets.
*
* @param target Target index in which to find events.
*
* @return an EventSet associated with the DataSet.
*/
EventSet
events
(
int
target
=-
1
)
const
;
};
class
DataSet
::
const_iterator
{
...
...
include/hepnos/DataStore.hpp
View file @
8116e5be
...
...
@@ -9,6 +9,7 @@
#include
<vector>
#include
<string>
#include
<memory>
#include
<hepnos/ItemType.hpp>
namespace
hepnos
{
...
...
@@ -167,6 +168,16 @@ class DataStore {
*/
void
shutdown
();
/**
* @brief Returns the number of underlying targets for the
* specified item type.
*
* @param type Item type.
*
* @return The number of targets for the item type.
*/
size_t
numTargets
(
const
ItemType
&
type
)
const
;
private:
/**
...
...
include/hepnos/Event.hpp
View file @
8116e5be
...
...
@@ -15,11 +15,15 @@
namespace
hepnos
{
class
SubRun
;
class
EventSet
;
class
Event
:
public
KeyValueContainer
{
private:
friend
class
SubRun
;
friend
class
EventSet
;
std
::
shared_ptr
<
ItemImpl
>
m_impl
;
...
...
@@ -146,6 +150,12 @@ class Event : public KeyValueContainer {
*/
const
EventNumber
&
number
()
const
;
/**
* @brief Returns an intance of the enclosing SubRun.
*
* @return Parent SubRun.
*/
SubRun
subrun
()
const
;
};
}
...
...
include/hepnos/EventSet.hpp
0 → 100644
View file @
8116e5be
/*
* (C) 2018 The University of Chicago
*
* See COPYRIGHT in top-level directory.
*/
#ifndef __HEPNOS_EVENT_SET_H
#define __HEPNOS_EVENT_SET_H
#include
<memory>
#include
<hepnos/Exception.hpp>
#include
<hepnos/DataStore.hpp>
#include
<hepnos/EventNumber.hpp>
#include
<hepnos/RunNumber.hpp>
#include
<hepnos/SubRunNumber.hpp>
namespace
hepnos
{
class
EventSetImpl
;
/**
* @brief The EventSet class is a helper class to access Events
* stored down in a particular DataSet or Run (bypassing
* intermediate nesting levels).
*/
class
EventSet
{
friend
class
DataSet
;
private:
/**
* @brief Implementation class (used for the Pimpl idiom).
*/
std
::
shared_ptr
<
EventSetImpl
>
m_impl
;
/*!< Pointer to implementation. */
/**
* @brief Constructor.
*/
EventSet
(
const
std
::
shared_ptr
<
EventSetImpl
>&
impl
);
EventSet
(
std
::
shared_ptr
<
EventSetImpl
>&&
impl
);
public:
typedef
Event
value_type
;
/**
* @brief Copy-constructor.
*
* @param other EventSet to copy.
*/
EventSet
(
const
EventSet
&
other
)
=
default
;
/**
* @brief Move-constructor. Deleted.
*
* @param other EventSet to move.
*/
EventSet
(
EventSet
&&
other
)
=
default
;
/**
* @brief Copy-assignment operator.
*
* @param other EventSet to copy.
*
* @return this.
*/
EventSet
&
operator
=
(
const
EventSet
&
other
)
=
default
;
/**
* @brief Move-assignment operator. Deleted.
*
* @param other EventSet to move.
*
* @return this.
*/
EventSet
&
operator
=
(
EventSet
&&
other
)
=
default
;
/**
* @brief Destructor.
*/
~
EventSet
()
=
default
;
class
const_iterator
;
class
iterator
;
/**
* @brief Get the DataStore to which this EventSet belongs.
*/
DataStore
datastore
()
const
;
/**
* @brief Returns an iterator referring to the first Event
* in this EventSet.
*
* @return an iterator referring to the first Event in this EventSet.
*/
iterator
begin
();
/**
* @brief Returns an iterator referring to the end of the EventSet.
* The EventSet pointed to by this iterator is not valid (that is,
* `end()->valid()` returns `false`).
*
* @return an iterator referring to the end of the EventSet.
*/
iterator
end
();
/**
* @brief Returns a const_iterator referring to the first Event
* in this EventSet.
*
* @return an iterator referring to the first Event in this EventSet.
*/
const_iterator
begin
()
const
;
/**
* @brief Returns a const_iterator referring to the end of the EventSet.
* The EventSet pointed to by this iterator is not valid (that is,
* `end()->valid()` returns `false`).
*
* @return a const_iterator referring to the end of the EventSet.
*/
const_iterator
end
()
const
;
/**
* @brief Returns a const_iterator referring to the first Event
* in this EventSet.
*
* @return a const_iterator referring to the first Event in this EventSet.
*/
const_iterator
cbegin
()
const
;
/**
* @brief Returns a const_iterator referring to the end of the EventSet.
* The EventSet pointed to by this iterator is not valid (that is,
* `cend()->valid()` return `false`).
*
* @return a const_iterator referring to the end of the EventSet.
*/
const_iterator
cend
()
const
;
};
class
EventSet
::
const_iterator
{
friend
class
EventSet
;
protected:
/**
* @brief Implementation of the class (using Pimpl idiom)
*/
class
Impl
;
std
::
unique_ptr
<
Impl
>
m_impl
;
/*!< Pointer to implementation */
const_iterator
(
std
::
unique_ptr
<
Impl
>&&
impl
);
public:
/**
* @brief Constructor. Creates a const_iterator pointing
* to an invalid Event.
*/
const_iterator
();
typedef
const_iterator
self_type
;
typedef
Event
value_type
;
typedef
Event
&
reference
;
typedef
Event
*
pointer
;
typedef
int
difference_type
;
typedef
std
::
forward_iterator_tag
iterator_category
;
/**
* @brief Destructor. This destructor is virtual because
* the iterator class inherits from const_iterator.
*/
virtual
~
const_iterator
();
/**
* @brief Copy-constructor.
*
* @param other const_iterator to copy.
*/
const_iterator
(
const
const_iterator
&
other
);
/**
* @brief Move-constructor.
*
* @param other const_iterator to move.
*/
const_iterator
(
const_iterator
&&
other
);
/**
* @brief Copy-assignment operator.
*
* @param other const_iterator to copy.
*
* @return this.
*/
const_iterator
&
operator
=
(
const
const_iterator
&
);
/**
* @brief Move-assignment operator.
*
* @param other const_iterator to move.
*
* @return this.
*/
const_iterator
&
operator
=
(
const_iterator
&&
);
/**
* @brief Increments the const_iterator, returning
* a copy of the iterator after incrementation.
*
* @return a copy of the iterator after incrementation.
*/
self_type
operator
++
();
/**
* @brief Increments the const_iterator, returning
* a copy of the iterator before incrementation.
*
* @return a copy of the iterator after incrementation.
*/
self_type
operator
++
(
int
);
/**
* @brief Dereference operator. Returns a const reference
* to the DataSet this const_iterator points to.
*
* @return a const reference to the DataSet this
* const_iterator points to.
*/
const
reference
operator
*
();
/**
* @brief Returns a const pointer to the DataSet this
* const_iterator points to.
*
* @return a const pointer to the DataSet this
* const_iterator points to.
*/
const
pointer
operator
->
();
/**
* @brief Compares two const_iterators. The two const_iterators
* are equal if they point to the same DataSet or if both
* correspond to EventSet::cend().
*
* @param rhs const_iterator to compare with.
*
* @return true if the two const_iterators are equal, false otherwise.
*/
bool
operator
==
(
const
self_type
&
rhs
)
const
;
/**
* @brief Compares two const_iterators.
*
* @param rhs const_iterator to compare with.
*
* @return true if the two const_iterators are different, false atherwise.
*/
bool
operator
!=
(
const
self_type
&
rhs
)
const
;
};
class
EventSet
::
iterator
:
public
EventSet
::
const_iterator
{
friend
class
EventSet
;
private:
iterator
(
std
::
unique_ptr
<
EventSet
::
const_iterator
::
Impl
>&&
impl
);
public:
/**
* @brief Constructor. Builds an iterator pointing to an
* invalid Event.
*/
iterator
();
typedef
iterator
self_type
;
typedef
Event
value_type
;
typedef
Event
&
reference
;
typedef
Event
*
pointer
;
typedef
int
difference_type
;
typedef
std
::
forward_iterator_tag
iterator_category
;
/**
* @brief Destructor.
*/
~
iterator
();
/**
* @brief Copy constructor.
*
* @param other iterator to copy.
*/
iterator
(
const
iterator
&
other
);
/**
* @brief Move constructor.
*
* @param other iterator to move.
*/
iterator
(
iterator
&&
other
);
/**
* @brief Copy-assignment operator.
*
* @param other iterator to copy.
*
* @return this.
*/
iterator
&
operator
=
(
const
iterator
&
other
);
/**
* @brief Move-assignment operator.
*
* @param other iterator to move.
*
* @return this.
*/
iterator
&
operator
=
(
iterator
&&
other
);
/**
* @brief Dereference operator. Returns a reference
* to the DataSet this iterator points to.
*
* @return A reference to the DataSet this iterator
* points to.
*/
reference
operator
*
();
/**
* @brief Returns a pointer to the DataSet this iterator
* points to.
*
* @return A pointer to the DataSet this iterator points to.
*/
pointer
operator
->
();
};
}
#endif
include/hepnos/ItemType.hpp
0 → 100644
View file @
8116e5be
#ifndef __HEPNOS_ITEM_TYPE_HPP
#define __HEPNOS_ITEM_TYPE_HPP
namespace
hepnos
{
enum
class
ItemType
:
uint32_t
{
DATASET
,
RUN
,
SUBRUN
,
EVENT
,
PRODUCT
};
}
#endif
include/hepnos/Run.hpp
View file @
8116e5be
...
...
@@ -22,6 +22,7 @@ class Run : public KeyValueContainer {
private:
friend
class
SubRun
;
friend
class
RunSet
;
friend
class
DataSet
;
...
...
include/hepnos/SubRun.hpp
View file @
8116e5be
...
...
@@ -20,6 +20,7 @@ class SubRun : public KeyValueContainer {
private:
friend
class
Event
;
friend
class
Run
;
std
::
shared_ptr
<
ItemImpl
>
m_impl
;
...
...
@@ -149,6 +150,11 @@ class SubRun : public KeyValueContainer {
*/
const
SubRunNumber
&
number
()
const
;
/**
* @brief Returns an instance of the enclosing Run.
*/
Run
run
()
const
;
class
const_iterator
;
class
iterator
;
...
...
src/AsyncEngineImpl.hpp
View file @
8116e5be
...
...
@@ -96,10 +96,16 @@ class AsyncEngineImpl {
id
.
run
=
run_number
;
id
.
subrun
=
subrun_number
;
id
.
event
=
event_number
;
ItemType
type
=
ItemType
::
RUN
;
if
(
subrun_number
!=
InvalidSubRunNumber
)
{
type
=
ItemType
::
SUBRUN
;
if
(
event_number
!=
InvalidEventNumber
)
type
=
ItemType
::
EVENT
;
}
// make a thread that will store the data
m_pool
.
make_thread
([
this
,
id
,
ds
=
m_datastore
]()
{
m_pool
.
make_thread
([
this
,
id
,
type
,
ds
=
m_datastore
]()
{
// locate db
auto
&
db
=
ds
->
locateItemDb
(
id
);
auto
&
db
=
ds
->
locateItemDb
(
type
,
id
);
try
{
db
.
put
(
&
id
,
sizeof
(
id
),
nullptr
,
0
);
}
catch
(
sdskv
::
exception
&
ex
)
{
...
...
src/CMakeLists.txt
View file @
8116e5be
...
...
@@ -7,7 +7,8 @@ set(hepnos-src DataStore.cpp
Event.cpp
UUID.cpp
WriteBatch.cpp
AsyncEngine.cpp
)
AsyncEngine.cpp
EventSet.cpp
)
set
(
hepnos-service-src service/HEPnOSService.cpp
service/ServiceConfig.cpp
...
...
src/DataSet.cpp
View file @
8116e5be
...
...
@@ -9,6 +9,7 @@
#include
"hepnos/AsyncEngine.hpp"
#include
"ItemImpl.hpp"
#include
"DataSetImpl.hpp"
#include
"EventSetImpl.hpp"
#include
"DataStoreImpl.hpp"
#include
"AsyncEngineImpl.hpp"
#include
"WriteBatchImpl.hpp"
...
...
@@ -349,6 +350,24 @@ RunSet DataSet::runs() const {
return
RunSet
(
m_impl
);
}
EventSet
DataSet
::
events
(
int
target
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
auto
numTargets
=
m_impl
->
m_datastore
->
numTargets
(
ItemType
::
EVENT
);
if
(
target
>=
(
int
)
numTargets
)
{
throw
Exception
(
std
::
string
(
"Invalid target number "
)
+
std
::
to_string
(
target
)
+
" for EventSet (>= "
+
std
::
to_string
(
numTargets
)
+
")"
);
}
if
(
target
>=
0
)
return
EventSet
(
std
::
make_shared
<
EventSetImpl
>
(
*
m_impl
,
target
));
else
{
return
EventSet
(
std
::
make_shared
<
EventSetImpl
>
(
*
m_impl
,
0
,
numTargets
));
}
}
////////////////////////////////////////////////////////////////////////////////////////////
// DataSet::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////
...
...
src/DataStore.cpp
View file @
8116e5be
...
...
@@ -73,5 +73,12 @@ bool DataStore::loadRawProduct(const ProductID& productID, char* data, size_t* s
return
m_impl
->
loadRawProduct
(
productID
,
data
,
size
);
}
size_t
DataStore
::
numTargets
(
const
ItemType
&
type
)
const
{
if
(
!
m_impl
)
{
throw
Exception
(
"Calling DataStore member function on an invalid DataStore object"
);
}
return
m_impl
->
numTargets
(
type
);
}
}
src/DataStoreImpl.hpp
View file @
8116e5be
...
...
@@ -154,6 +154,22 @@ class DataStoreImpl {
if
(
m_mid
)
margo_finalize
(
m_mid
);
}
size_t
numTargets
(
const
ItemType
&
type
)
const
{
switch
(
type
)
{
case
ItemType
::
DATASET
:
return
m_dataset_dbs
.
dbs
.
size
();
case
ItemType
::
RUN
:
return
m_run_dbs
.
dbs
.
size
();
case
ItemType
::
SUBRUN
:
return
m_subrun_dbs
.
dbs
.
size
();
case
ItemType
::
EVENT
:
return
m_event_dbs
.
dbs
.
size
();
case
ItemType
::
PRODUCT
:
return
m_product_dbs
.
dbs
.
size
();
}
return
0
;
}
private:
static
void
checkConfig
(
YAML
::
Node
&
config
)
{
...
...
@@ -478,15 +494,20 @@ class DataStoreImpl {
// Access functions for numbered items (Runs, SubRuns, and Events)
///////////////////////////////////////////////////////////////////////////
const
sdskv
::
database
&
locateItemDb
(
const
Item
Descriptor
&
id
)
const
{
const
sdskv
::
database
&
locateItemDb
(
const
Item
Type
&
type
,
const
ItemDescriptor
&
id
,
int
target
=-
1
)
const
{
long
unsigned
db_idx
=
0
;
if
(
target
>=
0
)
{
if
(
type
==
ItemType
::
RUN
)
return
m_run_dbs
.
dbs
[
target
];
if
(
type
==
ItemType
::
SUBRUN
)
return
m_subrun_dbs
.
dbs
[
target
];
if
(
type
==
ItemType
::
EVENT
)
return
m_event_dbs
.
dbs
[
target
];
}
uint64_t
hash
;
size_t
prime
=
1099511628211ULL
;
hash
=
id
.
dataset
.
hash
();
if
(
id
.
subrun
==
InvalidSubRunNumber
)
{
// we are locating a Run
if
(
type
==
ItemType
::
RUN
)
{
// we are locating a Run
ch_placement_find_closest
(
m_run_dbs
.
chi
,
hash
,
1
,
&
db_idx
);
return
m_run_dbs
.
dbs
[
db_idx
];
}
else
if
(
id
.
event
==
InvalidEventNumber
)
{
// we are locating a SubRun
}
else
if
(
type
==
ItemType
::
SUBRUN
)
{
// we are locating a SubRun
hash
*=
prime
;
hash
=
hash
^
id
.
run
;
ch_placement_find_closest
(
m_subrun_dbs
.
chi
,
hash
,
1
,
&
db_idx
);
...
...
@@ -504,13 +525,17 @@ class DataStoreImpl {
* maxRuns shared_ptr to RunImpl coming after the
* current run. Returns the number of Runs read.
*/
size_t
nextItems
(
const
std
::
shared_ptr
<
ItemImpl
>&
current
,
size_t
nextItems
(
const
ItemType
&
item_type
,
const
ItemType
&
prefix_type
,
const
std
::
shared_ptr
<
ItemImpl
>&
current
,
std
::
vector
<
std
::
shared_ptr
<
ItemImpl
>>&
result
,
size_t
maxItems
)
const
{
size_t
maxItems
,
int
target
=-
1
)
const
{
int
ret
;
result
.
resize
(
0
);
const
ItemDescriptor
&
start_key
=
current
->
m_descriptor
;
auto
&
db
=
locateItemDb
(
start_key
);
auto
&
db
=
locateItemDb
(
item_type
,
start_key
,
target
);
// ignore keys that don't have the same uuid
// issue an sdskv_list_keys