Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
H
HEPnOS
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
sds
HEP
HEPnOS
Commits
c70bfcba
Commit
c70bfcba
authored
Mar 13, 2020
by
Matthieu Dorier
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
enabled statistics in prefetcher
parent
8669ee23
Changes
6
Show whitespace changes
Inline
Side-by-side
Showing
6 changed files
with
76 additions
and
10 deletions
+76
-10
include/hepnos/Prefetcher.hpp
include/hepnos/Prefetcher.hpp
+22
-0
src/Prefetcher.cpp
src/Prefetcher.cpp
+13
-0
src/PrefetcherImpl.hpp
src/PrefetcherImpl.hpp
+19
-0
src/SyncPrefetcherImpl.hpp
src/SyncPrefetcherImpl.hpp
+8
-1
src/WriteBatch.cpp
src/WriteBatch.cpp
+6
-1
src/WriteBatchImpl.hpp
src/WriteBatchImpl.hpp
+8
-8
No files found.
include/hepnos/Prefetcher.hpp
View file @
c70bfcba
...
...
@@ -9,6 +9,7 @@
#include <memory>
#include <hepnos/Demangle.hpp>
#include <hepnos/Prefetchable.hpp>
#include <hepnos/Statistics.hpp>
namespace
hepnos
{
...
...
@@ -21,6 +22,13 @@ class Run;
class
SubRun
;
class
Event
;
struct
PrefetcherStatistics
{
Statistics
<
size_t
,
double
>
batch_sizes
;
Statistics
<
size_t
,
double
>
product_sizes
;
size_t
product_cache_hit
=
0
;
size_t
product_cache_miss
=
0
;
};
/**
* @brief The Prefetcher object will actively try to prefetch
* items from the underlying DataStore when using iterators.
...
...
@@ -134,6 +142,20 @@ class Prefetcher {
fetchProductImpl
(
label
+
"#"
+
demangle
<
V
>
(),
fetch
);
}
/**
* @brief Activate statistics collection.
*
* @param activate Whether to activate statistics.
*/
void
activateStatistics
(
bool
activate
=
true
);
/**
* @brief Collects the usage statistics.
*
* @param stats PrefetcherStatistics object to fill.
*/
void
collectStatistics
(
PrefetcherStatistics
&
stats
)
const
;
private:
std
::
shared_ptr
<
PrefetcherImpl
>
m_impl
;
...
...
src/Prefetcher.cpp
View file @
c70bfcba
...
...
@@ -47,4 +47,17 @@ void Prefetcher::fetchProductImpl(const std::string& label, bool fetch=true) con
}
}
void
Prefetcher
::
activateStatistics
(
bool
activate
)
{
if
(
activate
)
{
if
(
m_impl
->
m_stats
)
return
;
m_impl
->
m_stats
=
std
::
make_unique
<
PrefetcherStatistics
>
();
}
else
{
m_impl
->
m_stats
.
reset
();
}
}
void
Prefetcher
::
collectStatistics
(
PrefetcherStatistics
&
stats
)
const
{
m_impl
->
collectStatistics
(
stats
);
}
}
src/PrefetcherImpl.hpp
View file @
c70bfcba
...
...
@@ -20,6 +20,9 @@ class PrefetcherImpl {
}
};
mutable
std
::
unique_ptr
<
PrefetcherStatistics
>
m_stats
;
mutable
tl
::
mutex
m_stats_mtx
;
std
::
shared_ptr
<
DataStoreImpl
>
m_datastore
;
unsigned
int
m_cache_size
=
16
;
unsigned
int
m_batch_size
=
1
;
...
...
@@ -33,6 +36,16 @@ class PrefetcherImpl {
virtual
~
PrefetcherImpl
()
=
default
;
void
update_batch_statistics
(
size_t
batch_size
)
const
{
if
(
!
m_stats
)
return
;
m_stats
->
batch_sizes
.
updateWith
(
batch_size
);
}
void
update_product_statistics
(
size_t
psize
)
const
{
if
(
!
m_stats
)
return
;
m_stats
->
product_sizes
.
updateWith
(
psize
);
}
virtual
void
fetchRequestedProducts
(
const
std
::
shared_ptr
<
ItemImpl
>&
itemImpl
)
const
=
0
;
virtual
void
prefetchFrom
(
const
ItemType
&
item_type
,
...
...
@@ -55,6 +68,12 @@ class PrefetcherImpl {
virtual
bool
loadRawProduct
(
const
ItemDescriptor
&
id
,
const
std
::
string
&
productName
,
char
*
value
,
size_t
*
vsize
)
const
=
0
;
void
collectStatistics
(
PrefetcherStatistics
&
stats
)
const
{
std
::
unique_lock
<
tl
::
mutex
>
lock
(
m_stats_mtx
);
if
(
m_stats
)
stats
=
*
m_stats
;
}
};
}
...
...
src/SyncPrefetcherImpl.hpp
View file @
c70bfcba
...
...
@@ -27,6 +27,7 @@ class SyncPrefetcherImpl : public PrefetcherImpl {
std
::
string
data
;
bool
ok
=
m_datastore
->
loadRawProduct
(
product_id
,
data
);
if
(
ok
)
{
update_product_statistics
(
data
.
size
());
m_product_cache
[
product_id
.
m_key
]
=
std
::
move
(
data
);
}
}
...
...
@@ -41,8 +42,10 @@ class SyncPrefetcherImpl : public PrefetcherImpl {
while
(
m_item_cache
.
size
()
!=
m_cache_size
)
{
std
::
vector
<
std
::
shared_ptr
<
ItemImpl
>>
items
;
size_t
s
=
m_datastore
->
nextItems
(
item_type
,
prefix_type
,
last
,
items
,
m_batch_size
,
target
);
if
(
s
!=
0
)
if
(
s
!=
0
)
{
update_batch_statistics
(
s
);
last
=
items
[
items
.
size
()
-
1
];
}
for
(
auto
&
item
:
items
)
{
fetchRequestedProducts
(
item
);
m_item_cache
.
insert
(
std
::
move
(
item
));
...
...
@@ -85,8 +88,10 @@ class SyncPrefetcherImpl : public PrefetcherImpl {
auto
product_id
=
DataStoreImpl
::
buildProductID
(
id
,
productName
);
auto
it
=
m_product_cache
.
find
(
product_id
.
m_key
);
if
(
it
==
m_product_cache
.
end
())
{
m_stats
->
product_cache_miss
+=
1
;
return
m_datastore
->
loadRawProduct
(
product_id
,
data
);
}
else
{
m_stats
->
product_cache_hit
+=
1
;
data
=
std
::
move
(
it
->
second
);
m_product_cache
.
erase
(
it
);
return
true
;
...
...
@@ -99,9 +104,11 @@ class SyncPrefetcherImpl : public PrefetcherImpl {
auto
product_id
=
DataStoreImpl
::
buildProductID
(
id
,
productName
);
auto
it
=
m_product_cache
.
find
(
product_id
.
m_key
);
if
(
it
==
m_product_cache
.
end
())
{
if
(
m_stats
)
m_stats
->
product_cache_miss
+=
1
;
return
m_datastore
->
loadRawProduct
(
id
,
productName
,
value
,
vsize
);
}
else
{
*
vsize
=
it
->
second
.
size
();
if
(
m_stats
)
m_stats
->
product_cache_hit
+=
1
;
std
::
memcpy
(
value
,
it
->
second
.
data
(),
*
vsize
);
return
true
;
}
...
...
src/WriteBatch.cpp
View file @
c70bfcba
...
...
@@ -22,7 +22,12 @@ void WriteBatch::flush() {
}
void
WriteBatch
::
activateStatistics
(
bool
activate
)
{
m_impl
->
m_stats_enabled
=
activate
;
if
(
activate
)
{
if
(
m_impl
->
m_stats
)
return
;
m_impl
->
m_stats
=
std
::
make_unique
<
WriteBatchStatistics
>
();
}
else
{
m_impl
->
m_stats
.
reset
();
}
}
void
WriteBatch
::
collectStatistics
(
WriteBatchStatistics
&
stats
)
const
{
...
...
src/WriteBatchImpl.hpp
View file @
c70bfcba
...
...
@@ -31,8 +31,7 @@ class WriteBatchImpl {
typedef
std
::
unordered_map
<
const
sdskv
::
database
*
,
keyvals
>
entries_type
;
WriteBatchStatistics
m_stats
;
bool
m_stats_enabled
;
std
::
unique_ptr
<
WriteBatchStatistics
>
m_stats
;
mutable
tl
::
mutex
m_stats_mtx
;
std
::
shared_ptr
<
DataStoreImpl
>
m_datastore
;
...
...
@@ -45,15 +44,15 @@ class WriteBatchImpl {
bool
m_async_thread_should_stop
=
false
;
void
update_keyval_statistics
(
size_t
ksize
,
size_t
vsize
)
{
if
(
!
m_stats
_enabled
)
return
;
m_stats
.
key_sizes
.
updateWith
(
ksize
);
if
(
!
m_stats
)
return
;
m_stats
->
key_sizes
.
updateWith
(
ksize
);
if
(
vsize
)
m_stats
.
value_sizes
.
updateWith
(
vsize
);
m_stats
->
value_sizes
.
updateWith
(
vsize
);
}
void
update_operation_statistics
(
size_t
batch_size
)
{
if
(
!
m_stats
_enabled
)
return
;
m_stats
.
batch_sizes
.
updateWith
(
batch_size
);
if
(
!
m_stats
)
return
;
m_stats
->
batch_sizes
.
updateWith
(
batch_size
);
}
static
void
writer_thread
(
WriteBatchImpl
&
wb
,
...
...
@@ -229,7 +228,8 @@ class WriteBatchImpl {
void
collectStatistics
(
WriteBatchStatistics
&
stats
)
const
{
std
::
unique_lock
<
tl
::
mutex
>
lock
(
m_stats_mtx
);
stats
=
m_stats
;
if
(
m_stats
)
stats
=
*
m_stats
;
}
};
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment