Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
sds
HEP
HEPnOS
Commits
9d29aaa0
Commit
9d29aaa0
authored
Mar 11, 2020
by
Matthieu Dorier
Browse files
implemented a way to prefetch objects during iterations
parent
3cc9962a
Changes
16
Hide whitespace changes
Inline
Side-by-side
include/hepnos/DataSet.hpp
View file @
9d29aaa0
...
...
@@ -164,6 +164,8 @@ class DataSet : public KeyValueContainer {
*/
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
value
)
const
override
;
bool
loadRawData
(
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
override
;
bool
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
std
::
string
&
value
)
const
override
;
bool
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
override
;
/**
* @brief Comparison operator.
...
...
include/hepnos/Event.hpp
View file @
9d29aaa0
...
...
@@ -128,6 +128,8 @@ class Event : public KeyValueContainer {
*/
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
value
)
const
override
;
bool
loadRawData
(
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
override
;
bool
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
override
;
bool
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
override
;
/**
* @brief Compares this Event with another Event. The Events must point to
...
...
include/hepnos/KeyValueContainer.hpp
View file @
9d29aaa0
...
...
@@ -22,6 +22,7 @@ namespace hepnos {
class
WriteBatch
;
class
AsyncEngine
;
class
Prefetcher
;
class
KeyValueContainer
{
...
...
@@ -104,6 +105,10 @@ class KeyValueContainer {
virtual
bool
loadRawData
(
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
=
0
;
virtual
bool
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
=
0
;
virtual
bool
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
=
0
;
/**
* @brief Stores a key/value pair into the KeyValueContainer.
* The type of the key should have operator<< available
...
...
@@ -207,6 +212,16 @@ class KeyValueContainer {
return
loadVectorImpl
(
key
,
value
,
std
::
is_pod
<
V
>
());
}
template
<
typename
K
,
typename
V
>
bool
load
(
const
Prefetcher
&
prefetcher
,
const
K
&
key
,
V
&
value
)
const
{
return
loadImpl
(
prefetcher
,
key
,
value
,
std
::
is_pod
<
V
>
());
}
template
<
typename
K
,
typename
V
>
bool
load
(
const
Prefetcher
&
prefetcher
,
const
K
&
key
,
std
::
vector
<
V
>&
value
)
const
{
return
loadVectorImpl
(
prefetcher
,
key
,
value
,
std
::
is_pod
<
V
>
());
}
private:
/**
...
...
@@ -297,6 +312,19 @@ class KeyValueContainer {
return
vsize
==
sizeof
(
value
);
}
template
<
typename
K
,
typename
V
>
bool
loadImpl
(
const
Prefetcher
&
prefetcher
,
const
K
&
key
,
V
&
value
,
const
std
::
integral_constant
<
bool
,
true
>&
)
const
{
std
::
string
buffer
;
std
::
stringstream
ss_key
;
ss_key
<<
key
<<
"#"
<<
demangle
<
V
>
();
size_t
vsize
=
sizeof
(
value
);
if
(
!
loadRawData
(
prefetcher
,
ss_key
.
str
(),
reinterpret_cast
<
char
*>
(
&
value
),
&
vsize
))
{
return
false
;
}
return
vsize
==
sizeof
(
value
);
}
/**
* @brief Implementation of the load function when the value type is not a POD.
*/
...
...
@@ -319,6 +347,25 @@ class KeyValueContainer {
return
true
;
}
template
<
typename
K
,
typename
V
>
bool
loadImpl
(
const
Prefetcher
&
prefetcher
,
const
K
&
key
,
V
&
value
,
const
std
::
integral_constant
<
bool
,
false
>&
)
const
{
std
::
string
buffer
;
std
::
stringstream
ss_key
;
ss_key
<<
key
<<
"#"
<<
demangle
<
V
>
();
if
(
!
loadRawData
(
prefetcher
,
ss_key
.
str
(),
buffer
))
{
return
false
;
}
try
{
std
::
stringstream
ss
(
buffer
);
InputArchive
ia
(
datastore
(),
ss
);
ia
>>
value
;
}
catch
(...)
{
throw
Exception
(
"Exception occured during serialization"
);
}
return
true
;
}
/**
* @brief Implementation of the load function when the value type is a vector of POD.
*/
...
...
@@ -344,6 +391,28 @@ class KeyValueContainer {
return
true
;
}
template
<
typename
K
,
typename
V
>
bool
loadVectorImpl
(
const
Prefetcher
&
prefetcher
,
const
K
&
key
,
std
::
vector
<
V
>&
value
,
const
std
::
integral_constant
<
bool
,
true
>&
)
const
{
std
::
string
buffer
;
std
::
stringstream
ss_key
;
ss_key
<<
key
<<
"#"
<<
demangle
<
std
::
vector
<
V
>>
();
if
(
!
loadRawData
(
prefetcher
,
ss_key
.
str
(),
buffer
))
{
return
false
;
}
size_t
count
=
0
;
if
(
buffer
.
size
()
<
sizeof
(
count
))
{
return
false
;
}
std
::
memcpy
(
&
count
,
buffer
.
data
(),
sizeof
(
count
));
if
(
buffer
.
size
()
!=
sizeof
(
count
)
+
count
*
sizeof
(
V
))
{
return
false
;
}
value
.
resize
(
count
);
std
::
memcpy
(
value
.
data
(),
buffer
.
data
()
+
sizeof
(
count
),
count
*
sizeof
(
V
));
return
true
;
}
/**
* @brief Implementation of the load function when the value type is a vector of non-POD.
*/
...
...
@@ -370,6 +439,31 @@ class KeyValueContainer {
}
return
true
;
}
template
<
typename
K
,
typename
V
>
bool
loadVectorImpl
(
const
Prefetcher
&
prefetcher
,
const
K
&
key
,
std
::
vector
<
V
>&
value
,
const
std
::
integral_constant
<
bool
,
false
>&
)
const
{
std
::
string
buffer
;
std
::
stringstream
ss_key
;
ss_key
<<
key
<<
"#"
<<
demangle
<
std
::
vector
<
V
>>
();
if
(
!
loadRawData
(
prefetcher
,
ss_key
.
str
(),
buffer
))
{
return
false
;
}
try
{
std
::
stringstream
ss
(
buffer
);
InputArchive
ia
(
datastore
(),
ss
);
size_t
count
=
0
;
ia
>>
count
;
value
.
resize
(
count
);
for
(
unsigned
i
=
0
;
i
<
count
;
i
++
)
{
ia
>>
value
[
i
];
}
}
catch
(...)
{
throw
Exception
(
"Exception occured during serialization"
);
}
return
true
;
}
/**
* @brief Creates the string key based on the provided key
* and the type of the value. Serializes the value into a string.
...
...
include/hepnos/Prefetcher.hpp
View file @
9d29aaa0
...
...
@@ -2,6 +2,7 @@
#define __HEPNOS_PREFETCHER_HPP
#include
<memory>
#include
<hepnos/Demangle.hpp>
#include
<hepnos/Prefetchable.hpp>
namespace
hepnos
{
...
...
@@ -13,6 +14,7 @@ class RunSet;
class
EventSet
;
class
Run
;
class
SubRun
;
class
Event
;
class
Prefetcher
{
...
...
@@ -20,10 +22,7 @@ class Prefetcher {
friend
class
EventSet
;
friend
class
Run
;
friend
class
SubRun
;
private:
std
::
shared_ptr
<
PrefetcherImpl
>
m_impl
;
friend
class
Event
;
public:
...
...
@@ -48,6 +47,18 @@ class Prefetcher {
Prefetchable
<
Container
>
operator
()(
const
Container
&
c
)
const
{
return
Prefetchable
<
Container
>
(
c
,
*
this
);
}
template
<
typename
V
>
void
fetchProduct
(
const
std
::
string
&
label
,
bool
fetch
=
true
)
const
{
fetchProductImpl
(
label
+
"#"
+
demangle
<
V
>
(),
fetch
);
}
private:
std
::
shared_ptr
<
PrefetcherImpl
>
m_impl
;
void
fetchProductImpl
(
const
std
::
string
&
labelAndType
,
bool
fetch
)
const
;
};
}
...
...
include/hepnos/ProductID.hpp
View file @
9d29aaa0
...
...
@@ -20,6 +20,7 @@ class ProductID {
friend
class
DataStoreImpl
;
friend
class
AsyncEngineImpl
;
friend
class
WriteBatchImpl
;
friend
class
PrefetcherImpl
;
friend
class
boost
::
serialization
::
access
;
public:
...
...
include/hepnos/Run.hpp
View file @
9d29aaa0
...
...
@@ -132,6 +132,8 @@ class Run : public KeyValueContainer {
*/
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
override
;
bool
loadRawData
(
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
override
;
bool
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
override
;
bool
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
override
;
/**
* @brief Compares this Run with another Run. The Runs must point to
...
...
include/hepnos/SubRun.hpp
View file @
9d29aaa0
...
...
@@ -130,6 +130,8 @@ class SubRun : public KeyValueContainer {
*/
bool
loadRawData
(
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
override
;
bool
loadRawData
(
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
override
;
bool
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
override
;
bool
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
override
;
/**
* @brief Compares this SubRun with another SubRun. The SubRuns must point to
...
...
src/DataSet.cpp
View file @
9d29aaa0
...
...
@@ -7,12 +7,14 @@
#include
"hepnos/Run.hpp"
#include
"hepnos/RunSet.hpp"
#include
"hepnos/AsyncEngine.hpp"
#include
"hepnos/Prefetcher.hpp"
#include
"ItemImpl.hpp"
#include
"DataSetImpl.hpp"
#include
"EventSetImpl.hpp"
#include
"DataStoreImpl.hpp"
#include
"AsyncEngineImpl.hpp"
#include
"WriteBatchImpl.hpp"
#include
"PrefetcherImpl.hpp"
namespace
hepnos
{
...
...
@@ -97,6 +99,24 @@ bool DataSet::loadRawData(const std::string& key, char* value, size_t* vsize) co
return
m_impl
->
m_datastore
->
loadRawProduct
(
id
,
key
,
value
,
vsize
);
}
bool
DataSet
::
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
(
void
)
prefetcher
;
// prefetcher isn't usable with a DataSet
ItemDescriptor
id
(
m_impl
->
m_uuid
);
return
m_impl
->
m_datastore
->
loadRawProduct
(
id
,
key
,
buffer
);
}
bool
DataSet
::
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
(
void
)
prefetcher
;
// prefetcher isn't usable with a DataSet
ItemDescriptor
id
(
m_impl
->
m_uuid
);
return
m_impl
->
m_datastore
->
loadRawProduct
(
id
,
key
,
value
,
vsize
);
}
bool
DataSet
::
operator
==
(
const
DataSet
&
other
)
const
{
bool
v1
=
valid
();
bool
v2
=
other
.
valid
();
...
...
src/Event.cpp
View file @
9d29aaa0
...
...
@@ -5,10 +5,12 @@
*/
#include
"hepnos/Event.hpp"
#include
"hepnos/AsyncEngine.hpp"
#include
"hepnos/Prefetcher.hpp"
#include
"ItemImpl.hpp"
#include
"DataStoreImpl.hpp"
#include
"WriteBatchImpl.hpp"
#include
"AsyncEngineImpl.hpp"
#include
"PrefetcherImpl.hpp"
namespace
hepnos
{
...
...
@@ -104,6 +106,22 @@ bool Event::loadRawData(const std::string& key, char* value, size_t* vsize) cons
return
m_impl
->
m_datastore
->
loadRawProduct
(
id
,
key
,
value
,
vsize
);
}
bool
Event
::
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Event member function on an invalid Event object"
);
}
auto
&
id
=
m_impl
->
m_descriptor
;
return
prefetcher
.
m_impl
->
loadRawProduct
(
id
,
key
,
buffer
);
}
bool
Event
::
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
auto
&
id
=
m_impl
->
m_descriptor
;
return
prefetcher
.
m_impl
->
loadRawProduct
(
id
,
key
,
value
,
vsize
);
}
bool
Event
::
operator
==
(
const
Event
&
other
)
const
{
bool
v1
=
valid
();
bool
v2
=
other
.
valid
();
...
...
src/Prefetcher.cpp
View file @
9d29aaa0
...
...
@@ -34,4 +34,12 @@ void Prefetcher::setBatchSize(unsigned int size) {
m_impl
->
m_batch_size
=
size
;
}
void
Prefetcher
::
fetchProductImpl
(
const
std
::
string
&
label
,
bool
fetch
=
true
)
const
{
if
(
fetch
)
{
m_impl
->
m_active_product_keys
.
insert
(
label
);
}
else
{
m_impl
->
m_active_product_keys
.
erase
(
label
);
}
}
}
src/PrefetcherImpl.hpp
View file @
9d29aaa0
...
...
@@ -2,6 +2,8 @@
#define __HEPNOS_PREFETCHER_IMPL_HPP
#include
<set>
#include
<unordered_set>
#include
<unordered_map>
#include
"DataStoreImpl.hpp"
#include
"AsyncEngineImpl.hpp"
...
...
@@ -22,8 +24,10 @@ class PrefetcherImpl {
std
::
shared_ptr
<
AsyncEngineImpl
>
m_async
;
unsigned
int
m_cache_size
=
16
;
unsigned
int
m_batch_size
=
1
;
mutable
std
::
set
<
std
::
shared_ptr
<
ItemImpl
>
,
ItemPtrComparator
>
m_cache
;
bool
m_associated
=
false
;
std
::
unordered_set
<
std
::
string
>
m_active_product_keys
;
mutable
std
::
set
<
std
::
shared_ptr
<
ItemImpl
>
,
ItemPtrComparator
>
m_item_cache
;
mutable
std
::
unordered_map
<
std
::
string
,
std
::
string
>
m_product_cache
;
PrefetcherImpl
(
const
std
::
shared_ptr
<
DataStoreImpl
>&
ds
)
:
m_datastore
(
ds
)
{}
...
...
@@ -33,19 +37,35 @@ class PrefetcherImpl {
:
m_datastore
(
ds
)
,
m_async
(
async
)
{}
void
fetchRequestedProducts
(
const
std
::
shared_ptr
<
ItemImpl
>&
itemImpl
)
const
{
auto
&
descriptor
=
itemImpl
->
m_descriptor
;
for
(
auto
&
key
:
m_active_product_keys
)
{
auto
product_id
=
DataStoreImpl
::
buildProductID
(
descriptor
,
key
);
auto
it
=
m_product_cache
.
find
(
product_id
.
m_key
);
if
(
it
!=
m_product_cache
.
end
())
continue
;
std
::
string
data
;
bool
ok
=
m_datastore
->
loadRawProduct
(
product_id
,
data
);
if
(
ok
)
{
m_product_cache
[
product_id
.
m_key
]
=
std
::
move
(
data
);
}
}
}
void
prefetchFrom
(
const
ItemType
&
item_type
,
const
ItemType
&
prefix_type
,
const
std
::
shared_ptr
<
ItemImpl
>&
current
,
int
target
=-
1
)
const
{
auto
last
=
current
;
while
(
m_cache
.
size
()
!=
m_cache_size
)
{
while
(
m_
item_
cache
.
size
()
!=
m_cache_size
)
{
std
::
vector
<
std
::
shared_ptr
<
ItemImpl
>>
items
;
size_t
s
=
m_datastore
->
nextItems
(
item_type
,
prefix_type
,
last
,
items
,
m_batch_size
,
target
);
if
(
s
!=
0
)
last
=
items
[
items
.
size
()
-
1
];
for
(
auto
&
item
:
items
)
{
m_cache
.
insert
(
std
::
move
(
item
));
fetchRequestedProducts
(
item
);
m_item_cache
.
insert
(
std
::
move
(
item
));
}
if
(
s
<
m_batch_size
)
break
;
}
...
...
@@ -59,25 +79,53 @@ class PrefetcherImpl {
size_t
maxItems
,
int
target
=-
1
)
const
{
auto
ub
=
m_cache
.
upper_bound
(
current
);
if
(
ub
==
m_cache
.
end
())
{
m_cache
.
clear
();
auto
ub
=
m_
item_
cache
.
upper_bound
(
current
);
if
(
ub
==
m_
item_
cache
.
end
())
{
m_
item_
cache
.
clear
();
prefetchFrom
(
item_type
,
prefix_type
,
current
,
target
);
}
ub
=
m_cache
.
upper_bound
(
current
);
ub
=
m_
item_
cache
.
upper_bound
(
current
);
result
.
clear
();
if
(
ub
==
m_cache
.
end
())
{
if
(
ub
==
m_
item_
cache
.
end
())
{
return
0
;
}
else
{
auto
it
=
ub
;
result
.
clear
();
for
(
size_t
i
=
0
;
i
<
maxItems
&&
it
!=
m_cache
.
end
();
i
++
,
it
++
)
{
for
(
size_t
i
=
0
;
i
<
maxItems
&&
it
!=
m_
item_
cache
.
end
();
i
++
,
it
++
)
{
result
.
push_back
(
*
it
);
}
m_cache
.
erase
(
ub
,
it
);
m_
item_
cache
.
erase
(
ub
,
it
);
}
return
result
.
size
();
}
bool
loadRawProduct
(
const
ItemDescriptor
&
id
,
const
std
::
string
&
productName
,
std
::
string
&
data
)
const
{
auto
product_id
=
DataStoreImpl
::
buildProductID
(
id
,
productName
);
auto
it
=
m_product_cache
.
find
(
product_id
.
m_key
);
if
(
it
==
m_product_cache
.
end
())
{
return
m_datastore
->
loadRawProduct
(
product_id
,
data
);
}
else
{
data
=
std
::
move
(
it
->
second
);
m_product_cache
.
erase
(
it
);
return
true
;
}
}
bool
loadRawProduct
(
const
ItemDescriptor
&
id
,
const
std
::
string
&
productName
,
char
*
value
,
size_t
*
vsize
)
const
{
auto
product_id
=
DataStoreImpl
::
buildProductID
(
id
,
productName
);
auto
it
=
m_product_cache
.
find
(
product_id
.
m_key
);
if
(
it
==
m_product_cache
.
end
())
{
return
m_datastore
->
loadRawProduct
(
id
,
productName
,
value
,
vsize
);
}
else
{
*
vsize
=
it
->
second
.
size
();
std
::
memcpy
(
value
,
it
->
second
.
data
(),
*
vsize
);
return
true
;
}
}
};
}
...
...
src/Run.cpp
View file @
9d29aaa0
...
...
@@ -146,6 +146,22 @@ bool Run::loadRawData(const std::string& key, char* value, size_t* vsize) const
return
m_impl
->
m_datastore
->
loadRawProduct
(
id
,
key
,
value
,
vsize
);
}
bool
Run
::
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling Run member function on an invalid Run object"
);
}
const
ItemDescriptor
&
id
=
m_impl
->
m_descriptor
;
return
prefetcher
.
m_impl
->
loadRawProduct
(
id
,
key
,
buffer
);
}
bool
Run
::
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
const
ItemDescriptor
&
id
=
m_impl
->
m_descriptor
;
return
prefetcher
.
m_impl
->
loadRawProduct
(
id
,
key
,
value
,
vsize
);
}
bool
Run
::
operator
==
(
const
Run
&
other
)
const
{
bool
v1
=
valid
();
bool
v2
=
other
.
valid
();
...
...
@@ -228,6 +244,7 @@ Run::iterator Run::find(const SubRunNumber& subRunNumber, const Prefetcher& pref
auto
it
=
find
(
subRunNumber
);
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_subrun
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
SUBRUN
,
ItemType
::
RUN
,
it
.
m_impl
->
m_current_subrun
.
m_impl
);
}
return
it
;
...
...
@@ -256,6 +273,7 @@ Run::iterator Run::begin(const Prefetcher& prefetcher) {
auto
it
=
begin
();
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_subrun
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
SUBRUN
,
ItemType
::
RUN
,
it
.
m_impl
->
m_current_subrun
.
m_impl
);
}
return
it
;
...
...
@@ -330,6 +348,7 @@ Run::iterator Run::lower_bound(const SubRunNumber& lb, const Prefetcher& prefetc
auto
it
=
lower_bound
(
lb
);
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_subrun
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
SUBRUN
,
ItemType
::
RUN
,
it
.
m_impl
->
m_current_subrun
.
m_impl
);
}
return
it
;
...
...
@@ -359,6 +378,7 @@ Run::iterator Run::upper_bound(const SubRunNumber& ub, const Prefetcher& prefetc
auto
it
=
upper_bound
(
ub
);
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_subrun
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
SUBRUN
,
ItemType
::
RUN
,
it
.
m_impl
->
m_current_subrun
.
m_impl
);
}
return
it
;
...
...
src/RunSet.cpp
View file @
9d29aaa0
...
...
@@ -136,6 +136,7 @@ RunSet::iterator RunSet::begin(const Prefetcher& prefetcher) {
auto
it
=
begin
();
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_run
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
RUN
,
ItemType
::
DATASET
,
it
.
m_impl
->
m_current_run
.
m_impl
);
}
return
it
;
...
...
@@ -153,6 +154,7 @@ RunSet::const_iterator RunSet::cbegin(const Prefetcher& prefetcher) const {
auto
it
=
cbegin
();
if
(
it
!=
cend
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_run
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
RUN
,
ItemType
::
DATASET
,
it
.
m_impl
->
m_current_run
.
m_impl
);
}
return
it
;
...
...
@@ -170,6 +172,7 @@ RunSet::const_iterator RunSet::begin(const Prefetcher& prefetcher) const {
auto
it
=
const_iterator
(
const_cast
<
RunSet
*>
(
this
)
->
begin
());
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_run
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
RUN
,
ItemType
::
DATASET
,
it
.
m_impl
->
m_current_run
.
m_impl
);
}
return
it
;
...
...
@@ -211,6 +214,7 @@ RunSet::iterator RunSet::lower_bound(const RunNumber& lb, const Prefetcher& pref
auto
it
=
lower_bound
(
lb
);
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_run
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
RUN
,
ItemType
::
DATASET
,
it
.
m_impl
->
m_current_run
.
m_impl
);
}
return
it
;
...
...
@@ -225,6 +229,7 @@ RunSet::const_iterator RunSet::lower_bound(const RunNumber& lb, const Prefetcher
iterator
it
=
const_cast
<
RunSet
*>
(
this
)
->
lower_bound
(
lb
);
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_run
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
RUN
,
ItemType
::
DATASET
,
it
.
m_impl
->
m_current_run
.
m_impl
);
}
return
it
;
...
...
@@ -242,6 +247,7 @@ RunSet::iterator RunSet::upper_bound(const RunNumber& ub, const Prefetcher& pref
auto
it
=
upper_bound
(
ub
);
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_run
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
RUN
,
ItemType
::
DATASET
,
it
.
m_impl
->
m_current_run
.
m_impl
);
}
return
it
;
...
...
@@ -257,6 +263,7 @@ RunSet::const_iterator RunSet::upper_bound(const RunNumber& ub, const Prefetcher
iterator
it
=
const_cast
<
RunSet
*>
(
this
)
->
upper_bound
(
ub
);
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_run
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
RUN
,
ItemType
::
DATASET
,
it
.
m_impl
->
m_current_run
.
m_impl
);
}
return
it
;
...
...
src/SubRun.cpp
View file @
9d29aaa0
...
...
@@ -158,6 +158,22 @@ bool SubRun::loadRawData(const std::string& key, char* value, size_t* vsize) con
return
m_impl
->
m_datastore
->
loadRawProduct
(
id
,
key
,
value
,
vsize
);
}
bool
SubRun
::
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
std
::
string
&
buffer
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling SubRun member function on invalid SubRun object"
);
}
auto
&
id
=
m_impl
->
m_descriptor
;
return
prefetcher
.
m_impl
->
loadRawProduct
(
id
,
key
,
buffer
);
}
bool
SubRun
::
loadRawData
(
const
Prefetcher
&
prefetcher
,
const
std
::
string
&
key
,
char
*
value
,
size_t
*
vsize
)
const
{
if
(
!
valid
())
{
throw
Exception
(
"Calling DataSet member function on an invalid DataSet"
);
}
auto
&
id
=
m_impl
->
m_descriptor
;
return
prefetcher
.
m_impl
->
loadRawProduct
(
id
,
key
,
value
,
vsize
);
}
bool
SubRun
::
operator
==
(
const
SubRun
&
other
)
const
{
bool
v1
=
valid
();
bool
v2
=
other
.
valid
();
...
...
@@ -231,6 +247,7 @@ SubRun::iterator SubRun::find(const EventNumber& eventNumber, const Prefetcher&
auto
it
=
find
(
eventNumber
);
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_event
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
EVENT
,
ItemType
::
SUBRUN
,
it
.
m_impl
->
m_current_event
.
m_impl
);
}
return
it
;
...
...
@@ -261,6 +278,7 @@ SubRun::iterator SubRun::begin(const Prefetcher& prefetcher) {
auto
it
=
begin
();
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_event
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
EVENT
,
ItemType
::
SUBRUN
,
it
.
m_impl
->
m_current_event
.
m_impl
);
}
return
it
;
...
...
@@ -333,6 +351,7 @@ SubRun::iterator SubRun::lower_bound(const EventNumber& lb, const Prefetcher& pr
auto
it
=
lower_bound
(
lb
);
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);
prefetcher
.
m_impl
->
fetchRequestedProducts
(
it
.
m_impl
->
m_current_event
.
m_impl
);
prefetcher
.
m_impl
->
prefetchFrom
(
ItemType
::
EVENT
,
ItemType
::
SUBRUN
,
it
.
m_impl
->
m_current_event
.
m_impl
);
}
return
it
;
...
...
@@ -361,6 +380,7 @@ SubRun::iterator SubRun::upper_bound(const EventNumber& lb, const Prefetcher& pr
auto
it
=
upper_bound
(
lb
);
if
(
it
!=
end
())
{
it
.
m_impl
->
setPrefetcher
(
prefetcher
.
m_impl
);