Skip to content

H
HEPnOS
Commit 3f69eaf5 authored by Matthieu Dorier's avatar Matthieu Dorier
Browse files
Options

implemented and tested AsyncPrefetcher

parent 9d29aaa0
Expand all Hide whitespace changes
Inline Side-by-side
Showing with 473 additions and 92 deletions
include/hepnos/ProductID.hpp
View file @ 3f69eaf5
......@@ -20,7 +20,8 @@ class ProductID {
friend class DataStoreImpl;
friend class AsyncEngineImpl;
friend class WriteBatchImpl;
friend class PrefetcherImpl;
friend class AsyncPrefetcherImpl;
friend class SyncPrefetcherImpl;
friend class boost::serialization::access;
public:
......
src/AsyncEngineImpl.hpp
View file @ 3f69eaf5
......@@ -10,11 +10,13 @@ namespace tl = thallium;
namespace hepnos {
class WriteBatchImpl;
class AsyncPrefetcherImpl;
class AsyncEngineImpl {
friend class WriteBatchImpl;
friend class AsyncEngine;
friend class AsyncPrefetcherImpl;
std::shared_ptr<DataStoreImpl> m_datastore;
tl::pool m_pool;
......@@ -45,6 +47,7 @@ class AsyncEngineImpl {
if(pools.size() != 1) {
throw Exception("Could not get current execution stream's main Argobots pool");
}
m_pool = pools[0];
}
}
......
src/AsyncPrefetcherImpl.hpp 0 → 100644
View file @ 3f69eaf5
This diff is collapsed.
src/ItemDescriptor.hpp
View file @ 3f69eaf5
......@@ -85,10 +85,17 @@ struct ItemDescriptor {
template<typename S>
S& operator<<(S& s, const ItemDescriptor& d) {
s << "[" << d.dataset.to_string() << ", "
<< d.run << ", "
<< d.subrun << ", "
<< d.event << "]";
s << "[" << d.dataset.to_string();
if(d.run != InvalidRunNumber) {
s << ", " << d.run;
if(d.subrun != InvalidSubRunNumber) {
s << ", " << d.subrun;
if(d.event != InvalidEventNumber) {
s << ", " << d.event;
}
}
}
s << "]";
return s;
}
......
src/Prefetcher.cpp
View file @ 3f69eaf5
#include "hepnos/Prefetcher.hpp"
#include "hepnos/AsyncEngine.hpp"
#include "PrefetcherImpl.hpp"
#include "SyncPrefetcherImpl.hpp"
#include "AsyncPrefetcherImpl.hpp"
namespace hepnos {
Prefetcher::Prefetcher(const DataStore& ds, unsigned int cache_size, unsigned int batch_size)
: m_impl(std::make_shared<PrefetcherImpl>(ds.m_impl)) {
: m_impl(std::make_shared<SyncPrefetcherImpl>(ds.m_impl)) {
m_impl->m_cache_size = cache_size;
m_impl->m_batch_size = batch_size;
}
Prefetcher::Prefetcher(const DataStore& ds, const AsyncEngine& async, unsigned int cache_size, unsigned int batch_size)
: m_impl(std::make_shared<PrefetcherImpl>(ds.m_impl, async.m_impl)) {
: m_impl(std::make_shared<AsyncPrefetcherImpl>(ds.m_impl, async.m_impl)) {
m_impl->m_cache_size = cache_size;
m_impl->m_batch_size = batch_size;
}
......@@ -35,10 +36,14 @@ void Prefetcher::setBatchSize(unsigned int size) {
}
void Prefetcher::fetchProductImpl(const std::string& label, bool fetch=true) const {
auto& v = m_impl->m_active_product_keys;
auto it = std::find(v.begin(), v.end(), label);
if(fetch) {
m_impl->m_active_product_keys.insert(label);
if(it != v.end())
v.push_back(label);
} else {
m_impl->m_active_product_keys.erase(label);
if(it != v.end())
v.erase(it);
}
}
......
src/PrefetcherImpl.hpp
View file @ 3f69eaf5
......@@ -21,111 +21,40 @@ class PrefetcherImpl {
};
std::shared_ptr<DataStoreImpl> m_datastore;
std::shared_ptr<AsyncEngineImpl> m_async;
unsigned int m_cache_size = 16;
unsigned int m_batch_size = 1;
bool m_associated = false;
std::unordered_set<std::string> m_active_product_keys;
std::vector<std::string> m_active_product_keys;
mutable std::set<std::shared_ptr<ItemImpl>, ItemPtrComparator> m_item_cache;
mutable std::unordered_map<std::string, std::string> m_product_cache;
PrefetcherImpl(const std::shared_ptr<DataStoreImpl>& ds)
: m_datastore(ds) {}
PrefetcherImpl(const std::shared_ptr<DataStoreImpl>& ds,
const std::shared_ptr<AsyncEngineImpl>& async)
: m_datastore(ds)
, m_async(async) {}
virtual ~PrefetcherImpl() = default;
void fetchRequestedProducts(const std::shared_ptr<ItemImpl>& itemImpl) const {
auto& descriptor = itemImpl->m_descriptor;
for(auto& key : m_active_product_keys) {
auto product_id = DataStoreImpl::buildProductID(descriptor, key);
auto it = m_product_cache.find(product_id.m_key);
if(it != m_product_cache.end())
continue;
std::string data;
bool ok = m_datastore->loadRawProduct(product_id, data);
if(ok) {
m_product_cache[product_id.m_key] = std::move(data);
}
}
}
virtual void fetchRequestedProducts(const std::shared_ptr<ItemImpl>& itemImpl) const = 0;
void prefetchFrom(const ItemType& item_type,
virtual void prefetchFrom(const ItemType& item_type,
const ItemType& prefix_type,
const std::shared_ptr<ItemImpl>& current,
int target=-1) const
{
auto last = current;
while(m_item_cache.size() != m_cache_size) {
std::vector<std::shared_ptr<ItemImpl>> items;
size_t s = m_datastore->nextItems(item_type, prefix_type, last, items, m_batch_size, target);
if(s != 0)
last = items[items.size()-1];
for(auto& item : items) {
fetchRequestedProducts(item);
m_item_cache.insert(std::move(item));
}
if(s < m_batch_size) break;
}
}
int target=-1) const = 0;
size_t nextItems(
virtual size_t nextItems(
const ItemType& item_type,
const ItemType& prefix_type,
const std::shared_ptr<ItemImpl>& current,
std::vector<std::shared_ptr<ItemImpl>>& result,
size_t maxItems,
int target=-1) const
{
auto ub = m_item_cache.upper_bound(current);
if(ub == m_item_cache.end()) {
m_item_cache.clear();
prefetchFrom(item_type, prefix_type, current, target);
}
ub = m_item_cache.upper_bound(current);
result.clear();
if(ub == m_item_cache.end()) {
return 0;
} else {
auto it = ub;
result.clear();
for(size_t i=0; i < maxItems && it != m_item_cache.end(); i++, it++) {
result.push_back(*it);
}
m_item_cache.erase(ub, it);
}
return result.size();
}
int target=-1) const = 0;
bool loadRawProduct(const ItemDescriptor& id,
virtual bool loadRawProduct(const ItemDescriptor& id,
const std::string& productName,
std::string& data) const {
auto product_id = DataStoreImpl::buildProductID(id, productName);
auto it = m_product_cache.find(product_id.m_key);
if(it == m_product_cache.end()) {
return m_datastore->loadRawProduct(product_id, data);
} else {
data = std::move(it->second);
m_product_cache.erase(it);
return true;
}
}
std::string& data) const = 0;
bool loadRawProduct(const ItemDescriptor& id,
virtual bool loadRawProduct(const ItemDescriptor& id,
const std::string& productName,
char* value, size_t* vsize) const {
auto product_id = DataStoreImpl::buildProductID(id, productName);
auto it = m_product_cache.find(product_id.m_key);
if(it == m_product_cache.end()) {
return m_datastore->loadRawProduct(id, productName, value, vsize);
} else {
*vsize = it->second.size();
std::memcpy(value, it->second.data(), *vsize);
return true;
}
}
char* value, size_t* vsize) const = 0;
};
}
......
src/SyncPrefetcherImpl.hpp 0 → 100644
View file @ 3f69eaf5
#ifndef __HEPNOS_SYNC_PREFETCHER_IMPL_HPP
#define __HEPNOS_SYNC_PREFETCHER_IMPL_HPP
#include <set>
#include <unordered_set>
#include <unordered_map>
#include "DataStoreImpl.hpp"
#include "AsyncEngineImpl.hpp"
#include "PrefetcherImpl.hpp"
namespace hepnos {
class SyncPrefetcherImpl : public PrefetcherImpl {
public:
SyncPrefetcherImpl(const std::shared_ptr<DataStoreImpl>& ds)
: PrefetcherImpl(ds) {}
void fetchRequestedProducts(const std::shared_ptr<ItemImpl>& itemImpl) const override {
auto& descriptor = itemImpl->m_descriptor;
for(auto& key : m_active_product_keys) {
auto product_id = DataStoreImpl::buildProductID(descriptor, key);
auto it = m_product_cache.find(product_id.m_key);
if(it != m_product_cache.end())
continue;
std::string data;
bool ok = m_datastore->loadRawProduct(product_id, data);
if(ok) {
m_product_cache[product_id.m_key] = std::move(data);
}
}
}
void prefetchFrom(const ItemType& item_type,
const ItemType& prefix_type,
const std::shared_ptr<ItemImpl>& current,
int target=-1) const override
{
auto last = current;
while(m_item_cache.size() != m_cache_size) {
std::vector<std::shared_ptr<ItemImpl>> items;
size_t s = m_datastore->nextItems(item_type, prefix_type, last, items, m_batch_size, target);
if(s != 0)
last = items[items.size()-1];
for(auto& item : items) {
fetchRequestedProducts(item);
m_item_cache.insert(std::move(item));
}
if(s < m_batch_size) break;
}
}
size_t nextItems(
const ItemType& item_type,
const ItemType& prefix_type,
const std::shared_ptr<ItemImpl>& current,
std::vector<std::shared_ptr<ItemImpl>>& result,
size_t maxItems,
int target=-1) const override
{
auto ub = m_item_cache.upper_bound(current);
if(ub == m_item_cache.end()) {
m_item_cache.clear();
prefetchFrom(item_type, prefix_type, current, target);
}
ub = m_item_cache.upper_bound(current);
result.clear();
if(ub == m_item_cache.end()) {
return 0;
} else {
auto it = ub;
result.clear();
for(size_t i=0; i < maxItems && it != m_item_cache.end(); i++, it++) {
result.push_back(*it);
}
m_item_cache.erase(ub, it);
}
return result.size();
}
bool loadRawProduct(const ItemDescriptor& id,
const std::string& productName,
std::string& data) const override {
auto product_id = DataStoreImpl::buildProductID(id, productName);
auto it = m_product_cache.find(product_id.m_key);
if(it == m_product_cache.end()) {
return m_datastore->loadRawProduct(product_id, data);
} else {
data = std::move(it->second);
m_product_cache.erase(it);
return true;
}
}
bool loadRawProduct(const ItemDescriptor& id,
const std::string& productName,
char* value, size_t* vsize) const override {
auto product_id = DataStoreImpl::buildProductID(id, productName);
auto it = m_product_cache.find(product_id.m_key);
if(it == m_product_cache.end()) {
return m_datastore->loadRawProduct(id, productName, value, vsize);
} else {
*vsize = it->second.size();
std::memcpy(value, it->second.data(), *vsize);
return true;
}
}
};
}
#endif
test/RunTest.cpp
View file @ 3f69eaf5
......@@ -243,3 +243,54 @@ void RunTest::testPrefetcher() {
}
}
}
void RunTest::testAsyncPrefetcher() {
auto root = datastore->root();
DataSet mds = root.createDataSet("matthieu_prefetch");
CPPUNIT_ASSERT(mds.valid());
Run r = mds.createRun(42);
CPPUNIT_ASSERT(r.valid());
for(unsigned i=0; i < 20; i++) {
SubRun sr = r.createSubRun(i);
CPPUNIT_ASSERT(sr.valid());
}
// test begin/end
{
AsyncEngine async(*datastore, 1);
Prefetcher prefetcher(*datastore, async);
unsigned i=0;
for(auto it = r.begin(prefetcher); it != r.end(); it++) {
CPPUNIT_ASSERT(it->valid());
CPPUNIT_ASSERT(it->number() == i);
i += 1;
}
CPPUNIT_ASSERT_EQUAL(20, (int)i);
}
// test lower_bound
{
AsyncEngine async(*datastore, 1);
Prefetcher prefetcher(*datastore, async);
unsigned i=5;
auto it = r.lower_bound(5, prefetcher);
for(; it != r.end(); it++) {
CPPUNIT_ASSERT(it->valid());
CPPUNIT_ASSERT(it->number() == i);
i += 1;
}
CPPUNIT_ASSERT(i == 20);
}
// test upper_bound
{
AsyncEngine async(*datastore, 1);
Prefetcher prefetcher(*datastore, async);
unsigned i=6;
auto it = r.upper_bound(5, prefetcher);
for(; it != r.end(); it++) {
CPPUNIT_ASSERT(it->valid());
CPPUNIT_ASSERT(it->number() == i);
i += 1;
}
CPPUNIT_ASSERT(i == 20);
}
}
test/RunTest.hpp
View file @ 3f69eaf5
......@@ -18,6 +18,7 @@ class RunTest : public CppUnit::TestFixture
CPPUNIT_TEST( testLowerUpperBounds );
CPPUNIT_TEST( testAsync );
CPPUNIT_TEST( testPrefetcher );
CPPUNIT_TEST( testAsyncPrefetcher );
CPPUNIT_TEST_SUITE_END();
public:
......@@ -34,6 +35,7 @@ class RunTest : public CppUnit::TestFixture
void testLowerUpperBounds();
void testAsync();
void testPrefetcher();
void testAsyncPrefetcher();
};
#endif
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment