Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
H
HEPnOS
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
0
Issues
0
List
Boards
Labels
Milestones
Merge Requests
0
Merge Requests
0
CI / CD
CI / CD
Pipelines
Jobs
Schedules
Analytics
Analytics
CI / CD
Repository
Value Stream
Wiki
Wiki
Snippets
Snippets
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Jobs
Commits
Issue Boards
Open sidebar
sds
HEP
HEPnOS
Commits
3f69eaf5
Commit
3f69eaf5
authored
Mar 12, 2020
by
Matthieu Dorier
Browse files
Options
Browse Files
Download
Email Patches
Plain Diff
implemented and tested AsyncPrefetcher
parent
9d29aaa0
Changes
9
Expand all
Hide whitespace changes
Inline
Side-by-side
Showing
9 changed files
with
473 additions
and
92 deletions
+473
-92
include/hepnos/ProductID.hpp
include/hepnos/ProductID.hpp
+2
-1
src/AsyncEngineImpl.hpp
src/AsyncEngineImpl.hpp
+3
-0
src/AsyncPrefetcherImpl.hpp
src/AsyncPrefetcherImpl.hpp
+270
-0
src/ItemDescriptor.hpp
src/ItemDescriptor.hpp
+11
-4
src/Prefetcher.cpp
src/Prefetcher.cpp
+10
-5
src/PrefetcherImpl.hpp
src/PrefetcherImpl.hpp
+11
-82
src/SyncPrefetcherImpl.hpp
src/SyncPrefetcherImpl.hpp
+113
-0
test/RunTest.cpp
test/RunTest.cpp
+51
-0
test/RunTest.hpp
test/RunTest.hpp
+2
-0
No files found.
include/hepnos/ProductID.hpp
View file @
3f69eaf5
...
...
@@ -20,7 +20,8 @@ class ProductID {
friend
class
DataStoreImpl
;
friend
class
AsyncEngineImpl
;
friend
class
WriteBatchImpl
;
friend
class
PrefetcherImpl
;
friend
class
AsyncPrefetcherImpl
;
friend
class
SyncPrefetcherImpl
;
friend
class
boost
::
serialization
::
access
;
public:
...
...
src/AsyncEngineImpl.hpp
View file @
3f69eaf5
...
...
@@ -10,11 +10,13 @@ namespace tl = thallium;
namespace
hepnos
{
class
WriteBatchImpl
;
class
AsyncPrefetcherImpl
;
class
AsyncEngineImpl
{
friend
class
WriteBatchImpl
;
friend
class
AsyncEngine
;
friend
class
AsyncPrefetcherImpl
;
std
::
shared_ptr
<
DataStoreImpl
>
m_datastore
;
tl
::
pool
m_pool
;
...
...
@@ -45,6 +47,7 @@ class AsyncEngineImpl {
if
(
pools
.
size
()
!=
1
)
{
throw
Exception
(
"Could not get current execution stream's main Argobots pool"
);
}
m_pool
=
pools
[
0
];
}
}
...
...
src/AsyncPrefetcherImpl.hpp
0 → 100644
View file @
3f69eaf5
This diff is collapsed.
Click to expand it.
src/ItemDescriptor.hpp
View file @
3f69eaf5
...
...
@@ -85,10 +85,17 @@ struct ItemDescriptor {
template
<
typename
S
>
S
&
operator
<<
(
S
&
s
,
const
ItemDescriptor
&
d
)
{
s
<<
"["
<<
d
.
dataset
.
to_string
()
<<
", "
<<
d
.
run
<<
", "
<<
d
.
subrun
<<
", "
<<
d
.
event
<<
"]"
;
s
<<
"["
<<
d
.
dataset
.
to_string
();
if
(
d
.
run
!=
InvalidRunNumber
)
{
s
<<
", "
<<
d
.
run
;
if
(
d
.
subrun
!=
InvalidSubRunNumber
)
{
s
<<
", "
<<
d
.
subrun
;
if
(
d
.
event
!=
InvalidEventNumber
)
{
s
<<
", "
<<
d
.
event
;
}
}
}
s
<<
"]"
;
return
s
;
}
...
...
src/Prefetcher.cpp
View file @
3f69eaf5
#include "hepnos/Prefetcher.hpp"
#include "hepnos/AsyncEngine.hpp"
#include "PrefetcherImpl.hpp"
#include "SyncPrefetcherImpl.hpp"
#include "AsyncPrefetcherImpl.hpp"
namespace
hepnos
{
Prefetcher
::
Prefetcher
(
const
DataStore
&
ds
,
unsigned
int
cache_size
,
unsigned
int
batch_size
)
:
m_impl
(
std
::
make_shared
<
PrefetcherImpl
>
(
ds
.
m_impl
))
{
:
m_impl
(
std
::
make_shared
<
Sync
PrefetcherImpl
>
(
ds
.
m_impl
))
{
m_impl
->
m_cache_size
=
cache_size
;
m_impl
->
m_batch_size
=
batch_size
;
}
Prefetcher
::
Prefetcher
(
const
DataStore
&
ds
,
const
AsyncEngine
&
async
,
unsigned
int
cache_size
,
unsigned
int
batch_size
)
:
m_impl
(
std
::
make_shared
<
PrefetcherImpl
>
(
ds
.
m_impl
,
async
.
m_impl
))
{
:
m_impl
(
std
::
make_shared
<
Async
PrefetcherImpl
>
(
ds
.
m_impl
,
async
.
m_impl
))
{
m_impl
->
m_cache_size
=
cache_size
;
m_impl
->
m_batch_size
=
batch_size
;
}
...
...
@@ -35,10 +36,14 @@ void Prefetcher::setBatchSize(unsigned int size) {
}
void
Prefetcher
::
fetchProductImpl
(
const
std
::
string
&
label
,
bool
fetch
=
true
)
const
{
auto
&
v
=
m_impl
->
m_active_product_keys
;
auto
it
=
std
::
find
(
v
.
begin
(),
v
.
end
(),
label
);
if
(
fetch
)
{
m_impl
->
m_active_product_keys
.
insert
(
label
);
if
(
it
!=
v
.
end
())
v
.
push_back
(
label
);
}
else
{
m_impl
->
m_active_product_keys
.
erase
(
label
);
if
(
it
!=
v
.
end
())
v
.
erase
(
it
);
}
}
...
...
src/PrefetcherImpl.hpp
View file @
3f69eaf5
...
...
@@ -21,111 +21,40 @@ class PrefetcherImpl {
};
std
::
shared_ptr
<
DataStoreImpl
>
m_datastore
;
std
::
shared_ptr
<
AsyncEngineImpl
>
m_async
;
unsigned
int
m_cache_size
=
16
;
unsigned
int
m_batch_size
=
1
;
bool
m_associated
=
false
;
std
::
unordered_set
<
std
::
string
>
m_active_product_keys
;
std
::
vector
<
std
::
string
>
m_active_product_keys
;
mutable
std
::
set
<
std
::
shared_ptr
<
ItemImpl
>
,
ItemPtrComparator
>
m_item_cache
;
mutable
std
::
unordered_map
<
std
::
string
,
std
::
string
>
m_product_cache
;
PrefetcherImpl
(
const
std
::
shared_ptr
<
DataStoreImpl
>&
ds
)
:
m_datastore
(
ds
)
{}
PrefetcherImpl
(
const
std
::
shared_ptr
<
DataStoreImpl
>&
ds
,
const
std
::
shared_ptr
<
AsyncEngineImpl
>&
async
)
:
m_datastore
(
ds
)
,
m_async
(
async
)
{}
virtual
~
PrefetcherImpl
()
=
default
;
void
fetchRequestedProducts
(
const
std
::
shared_ptr
<
ItemImpl
>&
itemImpl
)
const
{
auto
&
descriptor
=
itemImpl
->
m_descriptor
;
for
(
auto
&
key
:
m_active_product_keys
)
{
auto
product_id
=
DataStoreImpl
::
buildProductID
(
descriptor
,
key
);
auto
it
=
m_product_cache
.
find
(
product_id
.
m_key
);
if
(
it
!=
m_product_cache
.
end
())
continue
;
std
::
string
data
;
bool
ok
=
m_datastore
->
loadRawProduct
(
product_id
,
data
);
if
(
ok
)
{
m_product_cache
[
product_id
.
m_key
]
=
std
::
move
(
data
);
}
}
}
virtual
void
fetchRequestedProducts
(
const
std
::
shared_ptr
<
ItemImpl
>&
itemImpl
)
const
=
0
;
void
prefetchFrom
(
const
ItemType
&
item_type
,
v
irtual
v
oid
prefetchFrom
(
const
ItemType
&
item_type
,
const
ItemType
&
prefix_type
,
const
std
::
shared_ptr
<
ItemImpl
>&
current
,
int
target
=-
1
)
const
{
auto
last
=
current
;
while
(
m_item_cache
.
size
()
!=
m_cache_size
)
{
std
::
vector
<
std
::
shared_ptr
<
ItemImpl
>>
items
;
size_t
s
=
m_datastore
->
nextItems
(
item_type
,
prefix_type
,
last
,
items
,
m_batch_size
,
target
);
if
(
s
!=
0
)
last
=
items
[
items
.
size
()
-
1
];
for
(
auto
&
item
:
items
)
{
fetchRequestedProducts
(
item
);
m_item_cache
.
insert
(
std
::
move
(
item
));
}
if
(
s
<
m_batch_size
)
break
;
}
}
int
target
=-
1
)
const
=
0
;
size_t
nextItems
(
virtual
size_t
nextItems
(
const
ItemType
&
item_type
,
const
ItemType
&
prefix_type
,
const
std
::
shared_ptr
<
ItemImpl
>&
current
,
std
::
vector
<
std
::
shared_ptr
<
ItemImpl
>>&
result
,
size_t
maxItems
,
int
target
=-
1
)
const
{
auto
ub
=
m_item_cache
.
upper_bound
(
current
);
if
(
ub
==
m_item_cache
.
end
())
{
m_item_cache
.
clear
();
prefetchFrom
(
item_type
,
prefix_type
,
current
,
target
);
}
ub
=
m_item_cache
.
upper_bound
(
current
);
result
.
clear
();
if
(
ub
==
m_item_cache
.
end
())
{
return
0
;
}
else
{
auto
it
=
ub
;
result
.
clear
();
for
(
size_t
i
=
0
;
i
<
maxItems
&&
it
!=
m_item_cache
.
end
();
i
++
,
it
++
)
{
result
.
push_back
(
*
it
);
}
m_item_cache
.
erase
(
ub
,
it
);
}
return
result
.
size
();
}
int
target
=-
1
)
const
=
0
;
bool
loadRawProduct
(
const
ItemDescriptor
&
id
,
virtual
bool
loadRawProduct
(
const
ItemDescriptor
&
id
,
const
std
::
string
&
productName
,
std
::
string
&
data
)
const
{
auto
product_id
=
DataStoreImpl
::
buildProductID
(
id
,
productName
);
auto
it
=
m_product_cache
.
find
(
product_id
.
m_key
);
if
(
it
==
m_product_cache
.
end
())
{
return
m_datastore
->
loadRawProduct
(
product_id
,
data
);
}
else
{
data
=
std
::
move
(
it
->
second
);
m_product_cache
.
erase
(
it
);
return
true
;
}
}
std
::
string
&
data
)
const
=
0
;
bool
loadRawProduct
(
const
ItemDescriptor
&
id
,
virtual
bool
loadRawProduct
(
const
ItemDescriptor
&
id
,
const
std
::
string
&
productName
,
char
*
value
,
size_t
*
vsize
)
const
{
auto
product_id
=
DataStoreImpl
::
buildProductID
(
id
,
productName
);
auto
it
=
m_product_cache
.
find
(
product_id
.
m_key
);
if
(
it
==
m_product_cache
.
end
())
{
return
m_datastore
->
loadRawProduct
(
id
,
productName
,
value
,
vsize
);
}
else
{
*
vsize
=
it
->
second
.
size
();
std
::
memcpy
(
value
,
it
->
second
.
data
(),
*
vsize
);
return
true
;
}
}
char
*
value
,
size_t
*
vsize
)
const
=
0
;
};
}
...
...
src/SyncPrefetcherImpl.hpp
0 → 100644
View file @
3f69eaf5
#ifndef __HEPNOS_SYNC_PREFETCHER_IMPL_HPP
#define __HEPNOS_SYNC_PREFETCHER_IMPL_HPP
#include <set>
#include <unordered_set>
#include <unordered_map>
#include "DataStoreImpl.hpp"
#include "AsyncEngineImpl.hpp"
#include "PrefetcherImpl.hpp"
namespace
hepnos
{
class
SyncPrefetcherImpl
:
public
PrefetcherImpl
{
public:
SyncPrefetcherImpl
(
const
std
::
shared_ptr
<
DataStoreImpl
>&
ds
)
:
PrefetcherImpl
(
ds
)
{}
void
fetchRequestedProducts
(
const
std
::
shared_ptr
<
ItemImpl
>&
itemImpl
)
const
override
{
auto
&
descriptor
=
itemImpl
->
m_descriptor
;
for
(
auto
&
key
:
m_active_product_keys
)
{
auto
product_id
=
DataStoreImpl
::
buildProductID
(
descriptor
,
key
);
auto
it
=
m_product_cache
.
find
(
product_id
.
m_key
);
if
(
it
!=
m_product_cache
.
end
())
continue
;
std
::
string
data
;
bool
ok
=
m_datastore
->
loadRawProduct
(
product_id
,
data
);
if
(
ok
)
{
m_product_cache
[
product_id
.
m_key
]
=
std
::
move
(
data
);
}
}
}
void
prefetchFrom
(
const
ItemType
&
item_type
,
const
ItemType
&
prefix_type
,
const
std
::
shared_ptr
<
ItemImpl
>&
current
,
int
target
=-
1
)
const
override
{
auto
last
=
current
;
while
(
m_item_cache
.
size
()
!=
m_cache_size
)
{
std
::
vector
<
std
::
shared_ptr
<
ItemImpl
>>
items
;
size_t
s
=
m_datastore
->
nextItems
(
item_type
,
prefix_type
,
last
,
items
,
m_batch_size
,
target
);
if
(
s
!=
0
)
last
=
items
[
items
.
size
()
-
1
];
for
(
auto
&
item
:
items
)
{
fetchRequestedProducts
(
item
);
m_item_cache
.
insert
(
std
::
move
(
item
));
}
if
(
s
<
m_batch_size
)
break
;
}
}
size_t
nextItems
(
const
ItemType
&
item_type
,
const
ItemType
&
prefix_type
,
const
std
::
shared_ptr
<
ItemImpl
>&
current
,
std
::
vector
<
std
::
shared_ptr
<
ItemImpl
>>&
result
,
size_t
maxItems
,
int
target
=-
1
)
const
override
{
auto
ub
=
m_item_cache
.
upper_bound
(
current
);
if
(
ub
==
m_item_cache
.
end
())
{
m_item_cache
.
clear
();
prefetchFrom
(
item_type
,
prefix_type
,
current
,
target
);
}
ub
=
m_item_cache
.
upper_bound
(
current
);
result
.
clear
();
if
(
ub
==
m_item_cache
.
end
())
{
return
0
;
}
else
{
auto
it
=
ub
;
result
.
clear
();
for
(
size_t
i
=
0
;
i
<
maxItems
&&
it
!=
m_item_cache
.
end
();
i
++
,
it
++
)
{
result
.
push_back
(
*
it
);
}
m_item_cache
.
erase
(
ub
,
it
);
}
return
result
.
size
();
}
bool
loadRawProduct
(
const
ItemDescriptor
&
id
,
const
std
::
string
&
productName
,
std
::
string
&
data
)
const
override
{
auto
product_id
=
DataStoreImpl
::
buildProductID
(
id
,
productName
);
auto
it
=
m_product_cache
.
find
(
product_id
.
m_key
);
if
(
it
==
m_product_cache
.
end
())
{
return
m_datastore
->
loadRawProduct
(
product_id
,
data
);
}
else
{
data
=
std
::
move
(
it
->
second
);
m_product_cache
.
erase
(
it
);
return
true
;
}
}
bool
loadRawProduct
(
const
ItemDescriptor
&
id
,
const
std
::
string
&
productName
,
char
*
value
,
size_t
*
vsize
)
const
override
{
auto
product_id
=
DataStoreImpl
::
buildProductID
(
id
,
productName
);
auto
it
=
m_product_cache
.
find
(
product_id
.
m_key
);
if
(
it
==
m_product_cache
.
end
())
{
return
m_datastore
->
loadRawProduct
(
id
,
productName
,
value
,
vsize
);
}
else
{
*
vsize
=
it
->
second
.
size
();
std
::
memcpy
(
value
,
it
->
second
.
data
(),
*
vsize
);
return
true
;
}
}
};
}
#endif
test/RunTest.cpp
View file @
3f69eaf5
...
...
@@ -243,3 +243,54 @@ void RunTest::testPrefetcher() {
}
}
}
void
RunTest
::
testAsyncPrefetcher
()
{
auto
root
=
datastore
->
root
();
DataSet
mds
=
root
.
createDataSet
(
"matthieu_prefetch"
);
CPPUNIT_ASSERT
(
mds
.
valid
());
Run
r
=
mds
.
createRun
(
42
);
CPPUNIT_ASSERT
(
r
.
valid
());
for
(
unsigned
i
=
0
;
i
<
20
;
i
++
)
{
SubRun
sr
=
r
.
createSubRun
(
i
);
CPPUNIT_ASSERT
(
sr
.
valid
());
}
// test begin/end
{
AsyncEngine
async
(
*
datastore
,
1
);
Prefetcher
prefetcher
(
*
datastore
,
async
);
unsigned
i
=
0
;
for
(
auto
it
=
r
.
begin
(
prefetcher
);
it
!=
r
.
end
();
it
++
)
{
CPPUNIT_ASSERT
(
it
->
valid
());
CPPUNIT_ASSERT
(
it
->
number
()
==
i
);
i
+=
1
;
}
CPPUNIT_ASSERT_EQUAL
(
20
,
(
int
)
i
);
}
// test lower_bound
{
AsyncEngine
async
(
*
datastore
,
1
);
Prefetcher
prefetcher
(
*
datastore
,
async
);
unsigned
i
=
5
;
auto
it
=
r
.
lower_bound
(
5
,
prefetcher
);
for
(;
it
!=
r
.
end
();
it
++
)
{
CPPUNIT_ASSERT
(
it
->
valid
());
CPPUNIT_ASSERT
(
it
->
number
()
==
i
);
i
+=
1
;
}
CPPUNIT_ASSERT
(
i
==
20
);
}
// test upper_bound
{
AsyncEngine
async
(
*
datastore
,
1
);
Prefetcher
prefetcher
(
*
datastore
,
async
);
unsigned
i
=
6
;
auto
it
=
r
.
upper_bound
(
5
,
prefetcher
);
for
(;
it
!=
r
.
end
();
it
++
)
{
CPPUNIT_ASSERT
(
it
->
valid
());
CPPUNIT_ASSERT
(
it
->
number
()
==
i
);
i
+=
1
;
}
CPPUNIT_ASSERT
(
i
==
20
);
}
}
test/RunTest.hpp
View file @
3f69eaf5
...
...
@@ -18,6 +18,7 @@ class RunTest : public CppUnit::TestFixture
CPPUNIT_TEST
(
testLowerUpperBounds
);
CPPUNIT_TEST
(
testAsync
);
CPPUNIT_TEST
(
testPrefetcher
);
CPPUNIT_TEST
(
testAsyncPrefetcher
);
CPPUNIT_TEST_SUITE_END
();
public:
...
...
@@ -34,6 +35,7 @@ class RunTest : public CppUnit::TestFixture
void
testLowerUpperBounds
();
void
testAsync
();
void
testPrefetcher
();
void
testAsyncPrefetcher
();
};
#endif
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment