Commit 39c230b4 authored by Matthieu Dorier's avatar Matthieu Dorier

removed iterator functionalities from DataStore and put them in DataSet

parent 91903f4f
...@@ -37,7 +37,7 @@ int main(int argc, char* argv[]) ...@@ -37,7 +37,7 @@ int main(int argc, char* argv[])
} }
hepnos::DataStore datastore = hepnos::DataStore::connect(argv[1]); hepnos::DataStore datastore = hepnos::DataStore::connect(argv[1]);
for(auto& ds : datastore) { for(auto& ds : datastore.root()) {
navigate_dataset(0, ds); navigate_dataset(0, ds);
} }
......
...@@ -212,9 +212,6 @@ class DataSet : public KeyValueContainer { ...@@ -212,9 +212,6 @@ class DataSet : public KeyValueContainer {
Run createRun(const RunNumber& runNumber); Run createRun(const RunNumber& runNumber);
Run createRun(WriteBatch& batch, const RunNumber& runNumber); Run createRun(WriteBatch& batch, const RunNumber& runNumber);
typedef DataStore::const_iterator const_iterator;
typedef DataStore::iterator iterator;
/** /**
* @brief Accesses an existing DataSet using the [] * @brief Accesses an existing DataSet using the []
* operator. If no DataSet correspond to the provided name, * operator. If no DataSet correspond to the provided name,
...@@ -227,6 +224,20 @@ class DataSet : public KeyValueContainer { ...@@ -227,6 +224,20 @@ class DataSet : public KeyValueContainer {
*/ */
DataSet operator[](const std::string& datasetName) const; DataSet operator[](const std::string& datasetName) const;
/**
* @brief iterator class to navigate DataSets.
* This iterator is a forward iterator. DataSets are sorted
* alphabetically inside the DataStore.
*/
class iterator;
/**
* @brief const_iterator class to navigate DataSets.
* This iterator is a forward iterator. DataSets are sorted
* alphabetically inside the DataStore.
*/
class const_iterator;
/** /**
* @brief Searches this DataSet for a DataSet with * @brief Searches this DataSet for a DataSet with
* the provided path and returns an iterator to it if found, * the provided path and returns an iterator to it if found,
...@@ -373,6 +384,229 @@ class DataSet : public KeyValueContainer { ...@@ -373,6 +384,229 @@ class DataSet : public KeyValueContainer {
Run operator[](const RunNumber& runNumber) const; Run operator[](const RunNumber& runNumber) const;
}; };
class DataSet::const_iterator {
protected:
/**
* @brief Implementation of the class (using Pimpl idiom)
*/
class Impl;
std::unique_ptr<Impl> m_impl; /*!< Pointer to implementation */
public:
/**
* @brief Constructor. Creates a const_iterator pointing
* to an invalid DataSet.
*/
const_iterator();
/**
* @brief Constructor. Creates a const_iterator pointing
* to a given DataSet. The DataSet may or may not be valid.
*
* @param current DataSet to make the const_iterator point to.
*/
const_iterator(const DataSet& current);
/**
* @brief Constructor. Creates a const_iterator pointing
* to a given DataSet. The DataSet may or may not be valid.
*
* @param current DataSet to make the const_iterator point to.
*/
const_iterator(DataSet&& current);
typedef const_iterator self_type;
typedef DataSet value_type;
typedef DataSet& reference;
typedef DataSet* pointer;
typedef int difference_type;
typedef std::forward_iterator_tag iterator_category;
/**
* @brief Destructor. This destructor is virtual because
* the iterator class inherits from const_iterator.
*/
virtual ~const_iterator();
/**
* @brief Copy-constructor.
*
* @param other const_iterator to copy.
*/
const_iterator(const const_iterator& other);
/**
* @brief Move-constructor.
*
* @param other const_iterator to move.
*/
const_iterator(const_iterator&& other);
/**
* @brief Copy-assignment operator.
*
* @param other const_iterator to copy.
*
* @return this.
*/
const_iterator& operator=(const const_iterator&);
/**
* @brief Move-assignment operator.
*
* @param other const_iterator to move.
*
* @return this.
*/
const_iterator& operator=(const_iterator&&);
/**
* @brief Increments the const_iterator, returning
* a copy of the iterator after incrementation.
*
* @return a copy of the iterator after incrementation.
*/
self_type operator++();
/**
* @brief Increments the const_iterator, returning
* a copy of the iterator before incrementation.
*
* @return a copy of the iterator after incrementation.
*/
self_type operator++(int);
/**
* @brief Dereference operator. Returns a const reference
* to the DataSet this const_iterator points to.
*
* @return a const reference to the DataSet this
* const_iterator points to.
*/
const reference operator*();
/**
* @brief Returns a const pointer to the DataSet this
* const_iterator points to.
*
* @return a const pointer to the DataSet this
* const_iterator points to.
*/
const pointer operator->();
/**
* @brief Compares two const_iterators. The two const_iterators
* are equal if they point to the same DataSet or if both
* correspond to DataStore::cend().
*
* @param rhs const_iterator to compare with.
*
* @return true if the two const_iterators are equal, false otherwise.
*/
bool operator==(const self_type& rhs) const;
/**
* @brief Compares two const_iterators.
*
* @param rhs const_iterator to compare with.
*
* @return true if the two const_iterators are different, false otherwise.
*/
bool operator!=(const self_type& rhs) const;
};
class DataSet::iterator : public DataSet::const_iterator {
public:
/**
* @brief Constructor. Builds an iterator pointing to an
* invalid DataSet.
*/
iterator();
/**
* @brief Constructor. Builds an iterator pointing to
* an existing DataSet. The DataSet may or may not be
* valid.
*
* @param current DataSet to point to.
*/
iterator(const DataSet& current);
/**
* @brief Constructor. Builds an iterator pointing to
* an existing DataSet. The DataSet may or may not be
* valid.
*
* @param current DataSet to point to.
*/
iterator(DataSet&& current);
typedef iterator self_type;
typedef DataSet value_type;
typedef DataSet& reference;
typedef DataSet* pointer;
typedef int difference_type;
typedef std::forward_iterator_tag iterator_category;
/**
* @brief Destructor.
*/
~iterator();
/**
* @brief Copy constructor.
*
* @param other iterator to copy.
*/
iterator(const iterator& other);
/**
* @brief Move constructor.
*
* @param other iterator to move.
*/
iterator(iterator&& other);
/**
* @brief Copy-assignment operator.
*
* @param other iterator to copy.
*
* @return this.
*/
iterator& operator=(const iterator& other);
/**
* @brief Move-assignment operator.
*
* @param other iterator to move.
*
* @return this.
*/
iterator& operator=(iterator&& other);
/**
* @brief Dereference operator. Returns a reference
* to the DataSet this iterator points to.
*
* @return A reference to the DataSet this iterator
* points to.
*/
reference operator*();
/**
* @brief Returns a pointer to the DataSet this iterator
* points to.
*
* @return A pointer to the DataSet this iterator points to.
*/
pointer operator->();
};
} }
#endif #endif
This diff is collapsed.
...@@ -13,6 +13,8 @@ ...@@ -13,6 +13,8 @@
namespace hepnos { namespace hepnos {
DataSet::iterator DataSetImpl::m_end;
DataSet::DataSet() DataSet::DataSet()
: m_impl(std::make_shared<DataSetImpl>(nullptr, 0, std::make_shared<std::string>(""), "")) {} : m_impl(std::make_shared<DataSetImpl>(nullptr, 0, std::make_shared<std::string>(""), "")) {}
...@@ -171,7 +173,7 @@ DataSet::iterator DataSet::find(const std::string& datasetPath) { ...@@ -171,7 +173,7 @@ DataSet::iterator DataSet::find(const std::string& datasetPath) {
if(datasetPath.find('%') != std::string::npos) { if(datasetPath.find('%') != std::string::npos) {
throw Exception("Invalid character '%' in dataset name"); throw Exception("Invalid character '%' in dataset name");
} }
size_t slash_count = std::count(datasetPath.begin(), datasetPath.end(), '/'); size_t slash_count = std::count(datasetPath.begin(), datasetPath.end(), '/');
size_t level = m_impl->m_level + 1 + slash_count; size_t level = m_impl->m_level + 1 + slash_count;
std::string containerName; std::string containerName;
...@@ -184,13 +186,16 @@ DataSet::iterator DataSet::find(const std::string& datasetPath) { ...@@ -184,13 +186,16 @@ DataSet::iterator DataSet::find(const std::string& datasetPath) {
containerName = parent; containerName = parent;
} else { } else {
size_t c = datasetPath.find_last_of('/'); size_t c = datasetPath.find_last_of('/');
containerName = parent + "/" + datasetPath.substr(0,c); if(parent.size() > 0)
containerName = parent + "/" + datasetPath.substr(0,c);
else
containerName = datasetPath.substr(0,c);
datasetName = datasetPath.substr(c+1); datasetName = datasetPath.substr(c+1);
} }
bool b = m_impl->m_datastore->dataSetExists(level, containerName, datasetName); bool b = m_impl->m_datastore->dataSetExists(level, containerName, datasetName);
if(!b) { if(!b) {
return m_impl->m_datastore->m_end; return DataSetImpl::m_end;
} }
return iterator( return iterator(
DataSet( DataSet(
...@@ -228,7 +233,7 @@ DataSet::iterator DataSet::end() { ...@@ -228,7 +233,7 @@ DataSet::iterator DataSet::end() {
if(!valid()) { if(!valid()) {
throw Exception("Calling DataSet member function on an invalid DataSet"); throw Exception("Calling DataSet member function on an invalid DataSet");
} }
return m_impl->m_datastore->m_end; return DataSetImpl::m_end;
} }
DataSet::const_iterator DataSet::begin() const { DataSet::const_iterator DataSet::begin() const {
...@@ -239,7 +244,7 @@ DataSet::const_iterator DataSet::end() const { ...@@ -239,7 +244,7 @@ DataSet::const_iterator DataSet::end() const {
if(!valid()) { if(!valid()) {
throw Exception("Calling DataSet member function on an invalid DataSet"); throw Exception("Calling DataSet member function on an invalid DataSet");
} }
return m_impl->m_datastore->m_end; return DataSetImpl::m_end;
} }
DataSet::const_iterator DataSet::cbegin() const { DataSet::const_iterator DataSet::cbegin() const {
...@@ -250,7 +255,7 @@ DataSet::const_iterator DataSet::cend() const { ...@@ -250,7 +255,7 @@ DataSet::const_iterator DataSet::cend() const {
if(!valid()) { if(!valid()) {
throw Exception("Calling DataSet member function on an invalid DataSet"); throw Exception("Calling DataSet member function on an invalid DataSet");
} }
return m_impl->m_datastore->m_end; return DataSetImpl::m_end;
} }
DataSet::iterator DataSet::lower_bound(const std::string& lb) { DataSet::iterator DataSet::lower_bound(const std::string& lb) {
...@@ -310,4 +315,150 @@ RunSet DataSet::runs() const { ...@@ -310,4 +315,150 @@ RunSet DataSet::runs() const {
return RunSet(m_impl); return RunSet(m_impl);
} }
////////////////////////////////////////////////////////////////////////////////////////////
// DataSet::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////
class DataSet::const_iterator::Impl {
public:
DataSet m_current_dataset;
Impl()
: m_current_dataset()
{}
Impl(const DataSet& dataset)
: m_current_dataset(dataset)
{}
Impl(DataSet&& dataset)
: m_current_dataset(std::move(dataset))
{}
Impl(const Impl& other)
: m_current_dataset(other.m_current_dataset)
{}
bool operator==(const Impl& other) const {
return m_current_dataset == other.m_current_dataset;
}
};
DataSet::const_iterator::const_iterator()
: m_impl(std::make_unique<Impl>()) {}
DataSet::const_iterator::const_iterator(const DataSet& dataset)
: m_impl(std::make_unique<Impl>(dataset)) {}
DataSet::const_iterator::const_iterator(DataSet&& dataset)
: m_impl(std::make_unique<Impl>(std::move(dataset))) {}
DataSet::const_iterator::~const_iterator() {}
DataSet::const_iterator::const_iterator(const DataSet::const_iterator& other) {
if(other.m_impl) {
m_impl = std::make_unique<Impl>(*other.m_impl);
}
}
DataSet::const_iterator::const_iterator(DataSet::const_iterator&& other)
: m_impl(std::move(other.m_impl)) {}
DataSet::const_iterator& DataSet::const_iterator::operator=(const DataSet::const_iterator& other) {
if(&other == this) return *this;
if(other.m_impl)
m_impl = std::make_unique<Impl>(*other.m_impl);
else
m_impl.reset();
return *this;
}
DataSet::const_iterator& DataSet::const_iterator::operator=(DataSet::const_iterator&& other) {
if(&other == this) return *this;
m_impl = std::move(other.m_impl);
return *this;
}
DataSet::const_iterator::self_type DataSet::const_iterator::operator++() {
if(!m_impl) {
throw Exception("Trying to increment an invalid iterator");
}
m_impl->m_current_dataset = m_impl->m_current_dataset.next();
return *this;
}
DataSet::const_iterator::self_type DataSet::const_iterator::operator++(int) {
const_iterator copy = *this;
++(*this);
return copy;
}
const DataSet::const_iterator::reference DataSet::const_iterator::operator*() {
if(!m_impl) {
throw Exception("Trying to dereference an invalid iterator");
}
return m_impl->m_current_dataset;
}
const DataSet::const_iterator::pointer DataSet::const_iterator::operator->() {
if(!m_impl) {
throw Exception("Trying to dereference an invalid iterator");
}
return &(m_impl->m_current_dataset);
}
bool DataSet::const_iterator::operator==(const self_type& rhs) const {
if(!m_impl && !rhs.m_impl) return true;
if(m_impl && !rhs.m_impl) return false;
if(!m_impl && rhs.m_impl) return false;
return *m_impl == *(rhs.m_impl);
}
bool DataSet::const_iterator::operator!=(const self_type& rhs) const {
return !(*this == rhs);
}
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::iterator implementation
////////////////////////////////////////////////////////////////////////////////////////////
DataSet::iterator::iterator(const DataSet& current)
: const_iterator(current) {}
DataSet::iterator::iterator(DataSet&& current)
: const_iterator(std::move(current)) {}
DataSet::iterator::iterator()
: const_iterator() {}
DataSet::iterator::~iterator() {}
DataSet::iterator::iterator(const DataSet::iterator& other)
: const_iterator(other) {}
DataSet::iterator::iterator(DataSet::iterator&& other)
: const_iterator(std::move(other)) {}
DataSet::iterator& DataSet::iterator::operator=(const DataSet::iterator& other) {
if(this == &other) return *this;
if(other.m_impl)
m_impl = std::make_unique<Impl>(*other.m_impl);
else
m_impl.reset();
return *this;
}
DataSet::iterator& DataSet::iterator::operator=(DataSet::iterator&& other) {
if(this == &other) return *this;
m_impl = std::move(other.m_impl);
return *this;
}
DataSet::iterator::reference DataSet::iterator::operator*() {
return const_cast<reference>(const_iterator::operator*());
}
DataSet::iterator::pointer DataSet::iterator::operator->() {
return const_cast<pointer>(const_iterator::operator->());
}
} }
...@@ -22,6 +22,8 @@ class DataSetImpl { ...@@ -22,6 +22,8 @@ class DataSetImpl {
std::shared_ptr<std::string> m_container; std::shared_ptr<std::string> m_container;
std::string m_name; std::string m_name;
static DataSet::iterator m_end;
DataSetImpl(const std::shared_ptr<DataStoreImpl>& ds, DataSetImpl(const std::shared_ptr<DataStoreImpl>& ds,
uint8_t level, uint8_t level,
const std::shared_ptr<std::string>& container, const std::shared_ptr<std::string>& container,
......
This diff is collapsed.
...@@ -39,11 +39,10 @@ class DataStoreImpl { ...@@ -39,11 +39,10 @@ class DataStoreImpl {
sdskv::client m_sdskv_client; // SDSKV client sdskv::client m_sdskv_client; // SDSKV client
DistributedDBInfo m_databases; // list of SDSKV databases DistributedDBInfo m_databases; // list of SDSKV databases
DistributedDBInfo m_dataset_dbs; // list of SDSKV databases for DataSets DistributedDBInfo m_dataset_dbs; // list of SDSKV databases for DataSets
const DataStore::iterator m_end; // iterator for the end() of the DataStore
DataStoreImpl() DataStoreImpl()
: m_mid(MARGO_INSTANCE_NULL) : m_mid(MARGO_INSTANCE_NULL)
, m_end() {} {}
void populateDatabases(DistributedDBInfo& db_info, const YAML::Node& db_config) { void populateDatabases(DistributedDBInfo& db_info, const YAML::Node& db_config) {
int ret; int ret;
...@@ -118,6 +117,8 @@ class DataStoreImpl { ...@@ -118,6 +117,8 @@ class DataStoreImpl {
m_sdskv_client = sdskv::client(); m_sdskv_client = sdskv::client();
if(m_databases.chi) if(m_databases.chi)
ch_placement_finalize(m_databases.chi); ch_placement_finalize(m_databases.chi);
if(m_dataset_dbs.chi)
ch_placement_finalize(m_dataset_dbs.chi);
for(auto& addr : m_addrs) { for(auto& addr : m_addrs) {
margo_addr_free(m_mid, addr.second); margo_addr_free(m_mid, addr.second);
} }
...@@ -433,13 +434,17 @@ class DataStoreImpl { ...@@ -433,13 +434,17 @@ class DataStoreImpl {
* @brief Checks if a particular dataset exists. * @brief Checks if a particular dataset exists.
*/ */
bool dataSetExists(uint8_t level, const std::string& containerName, const std::string& objectName) const { bool dataSetExists(uint8_t level, const std::string& containerName, const std::string& objectName) const {
std::cerr << "Checking if dataset [" << (int)level
<< ", " << containerName << ", " << objectName << "] exists ... ";
int ret; int ret;
// build key // build key
auto key = buildKey(level, containerName, objectName); auto key = buildKey(level, containerName, objectName);
// find out which DB to access // find out which DB to access
auto& db = _locateDataSetDb(containerName); auto& db = _locateDataSetDb(containerName);
try { try {
return db.exists(key); bool b = db.exists(key);
std::cerr << b << std::endl;
return b;
} catch(sdskv::exception& ex) { } catch(sdskv::exception& ex) {
throw Exception("Error occured when calling sdskv::database::exists (SDSKV error="+std::to_string(ex.error())+")"); throw Exception("Error occured when calling sdskv::database::exists (SDSKV error="+std::to_string(ex.error())+")");
} }
...@@ -450,6 +455,8 @@ class DataStoreImpl { ...@@ -450,6 +455,8 @@ class DataStoreImpl {
* Creates a DataSet * Creates a DataSet
*/ */
bool createDataSet(uint8_t level, const std::string& containerName, const std::string& objectName) { bool createDataSet(uint8_t level, const std::string& containerName, const std::string& objectName) {
std::cerr << "Creating dataset [" << (int)level
<< ", " << containerName << ", " << objectName << "]" << std::endl;
// build full name // build full name
auto key = _buildDataSetKey(level, containerName, objectName); auto key = _buildDataSetKey(level, containerName, objectName);
// find out which DB to access // find out which DB to access
......
...@@ -11,10 +11,11 @@ void DataSetTest::tearDown() {} ...@@ -11,10 +11,11 @@ void DataSetTest::tearDown() {}
void DataSetTest::testFillDataStore() { void DataSetTest::testFillDataStore() {
auto mds = datastore->createDataSet("matthieu"); auto root = datastore->root();
datastore->createDataSet("shane"); auto mds = root.createDataSet("matthieu");
datastore->createDataSet("phil"); root.createDataSet("shane");
datastore->createDataSet("rob"); root.createDataSet("phil");
root.createDataSet("rob");
// erroneous dataset creations // erroneous dataset creations
// "/" is forbidden in the name, will throw an exception // "/" is forbidden in the name, will throw an exception
...@@ -27,22 +28,29 @@ void DataSetTest::testFillDataStore() { ...@@ -27,22 +28,29 @@ void DataSetTest::testFillDataStore() {
hepnos::Exception); hepnos::Exception);
// correct dataset creation // correct dataset creation
DataSet ds1 = mds.createDataSet("ds1"); DataSet ds1 = mds.createDataSet("ds1");
std::cerr << "AAAA" << std::endl;
// assert the characteristics of the created dataset // assert the characteristics of the created dataset
CPPUNIT_ASSERT(ds1.valid()); CPPUNIT_ASSERT(ds1.valid());
CPPUNIT_ASSERT_EQUAL_STR("ds1", ds1.name()); CPPUNIT_ASSERT_EQUAL_STR("ds1", ds1.name());
CPPUNIT_ASSERT_EQUAL_STR("matthieu", ds1.container()); CPPUNIT_ASSERT_EQUAL_STR("matthieu", ds1.container());
CPPUNIT_ASSERT_EQUAL_STR("matthieu/ds1", ds1.fullname()); CPPUNIT_ASSERT_EQUAL_STR("matthieu/ds1", ds1.fullname());
// assert access from DataStore using full path // assert access from DataStore using full path
DataSet matthieu_ds1 = (*datastore)["matthieu/ds1"]; DataSet matthieu_ds1 = root["matthieu/ds1"];
std::cerr << "BBBB" << std::endl;
CPPUNIT_ASSERT(matthieu_ds1.valid()); CPPUNIT_ASSERT(matthieu_ds1.valid());
CPPUNIT_ASSERT(matthieu_ds1 == ds1); CPPUNIT_ASSERT(matthieu_ds1 == ds1);
std::cerr << "CCCC" << std::endl;
// create a dataset inside ds1 // create a dataset inside ds1
DataSet ds11 = ds1.createDataSet("ds11"); DataSet ds11 = ds1.createDataSet("ds11");
std::cerr << "DDDD" << std::endl;
CPPUNIT_ASSERT(ds11.valid()); CPPUNIT_ASSERT(ds11.valid());
std::cerr << "EEEE" << std::endl;
// access ds11 using path from "matthieu" // access ds11 using path from "matthieu"
DataSet ds1_ds11 = mds["ds1/ds11"]; DataSet ds1_ds11 = mds["ds1/ds11"];
std::cerr << "FFFF" << std::endl;
CPPUNIT_ASSERT(ds1_ds11.valid()); CPPUNIT_ASSERT(ds1_ds11.valid());
CPPUNIT_ASSERT(ds1_ds11 == ds11); CPPUNIT_ASSERT(ds1_ds11 == ds11);
std::cerr << "GGGG" << std::endl;
// assert comparison with a default-constructed dataset // assert comparison with a default-constructed dataset
DataSet ds0; DataSet ds0;
CPPUNIT_ASSERT(ds0 != ds1); CPPUNIT_ASSERT(ds0 != ds1);
...@@ -51,8 +59,11 @@ void DataSetTest::testFillDataStore() { ...@@ -51,8 +59,11 @@ void DataSetTest::testFillDataStore() {
DataSet ds2 = ds1.next(); DataSet ds2 = ds1.next();
CPPUNIT_ASSERT(!ds2.valid()); CPPUNIT_ASSERT(!ds2.valid());
// create more datasets // create more datasets
std::cerr << "HHHH" << std::endl;
DataSet ds3 = mds.createDataSet("ds3"); DataSet ds3 = mds.createDataSet("ds3");
std::cerr << "IIII" << std::endl;
ds2 = mds.createDataSet("ds2"); ds2 = mds.createDataSet("ds2");
std::cerr << "JJJJ" << std::endl;
// assert that these are valid // assert that these are valid
CPPUNIT_ASSERT(ds2.valid()); CPPUNIT_ASSERT(ds2.valid());
CPPUNIT_ASSERT(ds3.valid()); CPPUNIT_ASSERT(ds3.valid());
...@@ -61,13 +72,16 @@ void DataSetTest::testFillDataStore() { ...@@ -61,13 +72,16 @@ void DataSetTest::testFillDataStore() {
CPPUNIT_ASSERT(ds3 == ds2.next()); CPPUNIT_ASSERT(ds3 == ds2.next());
// create more datasets for future tests // create more datasets for future tests
DataSet ds4 = mds.createDataSet("dsB"); DataSet ds4 = mds.createDataSet("dsB");
std::cerr << "KKKK" << std::endl;
DataSet ds5 = mds.createDataSet("dsD"); DataSet ds5 = mds.createDataSet("dsD");
std::cerr << "LLLL" << std::endl;