Commit d7048475 authored by Matthieu Dorier's avatar Matthieu Dorier

batching works

parent 2f12aef2
......@@ -225,6 +225,7 @@ class DataSet : public KeyValueContainer {
* @return A Run instance pointing to the created run.
*/
Run createRun(const RunNumber& runNumber);
Run createRun(WriteBatch& batch, const RunNumber& runNumber);
typedef DataStore::const_iterator const_iterator;
typedef DataStore::iterator iterator;
......
......@@ -313,6 +313,7 @@ class Run : public KeyValueContainer {
* @return a handle to the created or existing SubRun.
*/
SubRun createSubRun(const SubRunNumber& subRunNumber);
SubRun createSubRun(WriteBatch& batch, const SubRunNumber& subRunNumber);
};
class Run::const_iterator {
......
......@@ -303,6 +303,7 @@ class SubRun : public KeyValueContainer {
* @return a handle to the created or existing Event.
*/
Event createEvent(const EventNumber& eventNumber);
Event createEvent(WriteBatch& batch, const EventNumber& eventNumber);
};
class SubRun::const_iterator {
......
......@@ -188,6 +188,16 @@ Run DataSet::createRun(const RunNumber& runNumber) {
return Run(m_impl->m_datastore, m_impl->m_level+1, parent, runNumber);
}
Run DataSet::createRun(WriteBatch& batch, const RunNumber& runNumber) {
if(InvalidRunNumber == runNumber) {
throw Exception("Trying to create a Run with InvalidRunNumber");
}
std::string parent = fullname();
std::string runStr = Run::Impl::makeKeyStringFromRunNumber(runNumber);
batch.m_impl->store(m_impl->m_level+1, parent, runStr, std::string());
return Run(m_impl->m_datastore, m_impl->m_level+1, parent, runNumber);
}
DataSet DataSet::operator[](const std::string& datasetName) const {
auto it = find(datasetName);
if(!it->valid())
......
......@@ -143,6 +143,16 @@ SubRun Run::createSubRun(const SubRunNumber& subRunNumber) {
return SubRun(m_impl->m_datastore, m_impl->m_level+1, parent, subRunNumber);
}
SubRun Run::createSubRun(WriteBatch& batch, const SubRunNumber& subRunNumber) {
if(!valid()) {
throw Exception("Calling Run member function on an invalid Run object");
}
std::string parent = m_impl->fullpath();
std::string subRunStr = SubRun::Impl::makeKeyStringFromSubRunNumber(subRunNumber);
batch.m_impl->store(m_impl->m_level+1, parent, subRunStr, std::string());
return SubRun(m_impl->m_datastore, m_impl->m_level+1, parent, subRunNumber);
}
SubRun Run::operator[](const SubRunNumber& subRunNumber) const {
auto it = find(subRunNumber);
if(!it->valid())
......
......@@ -134,6 +134,16 @@ Event SubRun::createEvent(const EventNumber& eventNumber) {
return Event(m_impl->m_datastore, m_impl->m_level+1, parent, eventNumber);
}
Event SubRun::createEvent(WriteBatch& batch, const EventNumber& eventNumber) {
if(!valid()) {
throw Exception("Calling SubRun member function on invalid SubRun object");
}
std::string parent = m_impl->fullpath();
std::string eventStr = Event::Impl::makeKeyStringFromEventNumber(eventNumber);
batch.m_impl->store(m_impl->m_level+1, parent, eventStr, std::string());
return Event(m_impl->m_datastore, m_impl->m_level+1, parent, eventNumber);
}
Event SubRun::operator[](const EventNumber& eventNumber) const {
auto it = find(eventNumber);
if(!it->valid())
......
......@@ -14,10 +14,10 @@
#include <yaml-cpp/yaml.h>
#include <sdskv-client.hpp>
#include <ch-placement.h>
//#include "KeyTypes.hpp"
#include "hepnos/Exception.hpp"
#include "hepnos/DataStore.hpp"
#include "hepnos/DataSet.hpp"
#include "StringHash.hpp"
namespace hepnos {
......@@ -176,10 +176,10 @@ class DataStore::Impl {
long unsigned sdskv_db_idx = 0;
uint64_t name_hash;
if(level != 0) {
name_hash = std::hash<std::string>()(containerName);
name_hash = hashString(containerName);
} else {
// use the complete name for final objects (level 0)
name_hash = std::hash<std::string>()(key);
name_hash = hashString(key);
}
ch_placement_find_closest(m_chi_sdskv, name_hash, 1, &sdskv_db_idx);
return sdskv_db_idx;
......@@ -264,7 +264,7 @@ class DataStore::Impl {
if(level == 0) return 0; // cannot iterate at object level
// hash the name to get the provider id
long unsigned db_idx = 0;
uint64_t h = std::hash<std::string>()(containerName);
uint64_t h = hashString(containerName);
ch_placement_find_closest(m_chi_sdskv, h, 1, &db_idx);
// make an entry for the lower bound
auto lb_entry = buildKey(level, containerName, lower);
......
#ifndef __HEPNOS_STRING_HASH_HPP
#define __HEPNOS_STRING_HASH_HPP
namespace hepnos {
inline size_t hashString(const std::string& str) {
size_t hash = 14695981039346656037ULL;
size_t prime = 1099511628211ULL;
for(const auto& c : str) {
hash = hash ^ c;
hash *= prime;
}
return hash;
}
}
#endif
......@@ -11,24 +11,6 @@ void WriteBatchTest::setUp() {}
void WriteBatchTest::tearDown() {}
void WriteBatchTest::testFillDataStore() {
/*
for(auto i = 10; i < 30; i++)
datastore->createDataSet("matthieu" + std::to_string(i));
auto ds = (*datastore)["matthieu15"];
CPPUNIT_ASSERT(ds.valid());
for(auto i = 0; i < 20; i++)
ds.createRun(i);
auto r = ds[12];
CPPUNIT_ASSERT(r.valid());
for(auto i = 0; i < 20; i++)
r.createSubRun(i);
auto sr = r[5];
CPPUNIT_ASSERT(sr.valid());
for(auto i = 0; i < 20; i++)
sr.createEvent(i);
auto ev = sr[2];
CPPUNIT_ASSERT(ev.valid());
*/
}
void WriteBatchTest::testWriteBatchDataSet() {
......@@ -67,13 +49,106 @@ void WriteBatchTest::testWriteBatchDataSet() {
}
void WriteBatchTest::testWriteBatchRun() {
TestObjectA out_obj_a;
out_obj_a.x() = 44;
out_obj_a.y() = 1.2;
TestObjectB out_obj_b;
out_obj_b.a() = 33;
out_obj_b.b() = "you";
std::string key1 = "mykey";
auto dataset = datastore->createDataSet("testWriteBatchRun");
{
hepnos::WriteBatch batch(*datastore);
for(auto i = 0; i < 10; i++) {
auto run = dataset.createRun(batch, i);
CPPUNIT_ASSERT(run.store(batch, key1, out_obj_a));
CPPUNIT_ASSERT(run.store(batch, key1, out_obj_b));
}
}
TestObjectA in_obj_a;
TestObjectB in_obj_b;
for(auto i = 0; i < 10; i++) {
auto run = dataset.runs()[i];
CPPUNIT_ASSERT(run.valid());
CPPUNIT_ASSERT(run.load(key1, in_obj_a));
CPPUNIT_ASSERT(run.load(key1, in_obj_b));
CPPUNIT_ASSERT(out_obj_a == in_obj_a);
CPPUNIT_ASSERT(out_obj_b == in_obj_b);
}
}
void WriteBatchTest::testWriteBatchSubRun() {
TestObjectA out_obj_a;
out_obj_a.x() = 44;
out_obj_a.y() = 1.2;
TestObjectB out_obj_b;
out_obj_b.a() = 33;
out_obj_b.b() = "you";
std::string key1 = "mykey";
auto dataset = datastore->createDataSet("testWriteBatchSubRun");
auto run = dataset.createRun(42);
{
hepnos::WriteBatch batch(*datastore);
for(auto i = 0; i < 10; i++) {
auto sr = run.createSubRun(batch, i);
CPPUNIT_ASSERT(sr.store(batch, key1, out_obj_a));
CPPUNIT_ASSERT(sr.store(batch, key1, out_obj_b));
}
}
TestObjectA in_obj_a;
TestObjectB in_obj_b;
for(auto i = 0; i < 10; i++) {
auto sr = run[i];
CPPUNIT_ASSERT(sr.valid());
CPPUNIT_ASSERT(sr.load(key1, in_obj_a));
CPPUNIT_ASSERT(sr.load(key1, in_obj_b));
CPPUNIT_ASSERT(out_obj_a == in_obj_a);
CPPUNIT_ASSERT(out_obj_b == in_obj_b);
}
}
void WriteBatchTest::testWriteBatchEvent() {
TestObjectA out_obj_a;
out_obj_a.x() = 44;
out_obj_a.y() = 1.2;
TestObjectB out_obj_b;
out_obj_b.a() = 33;
out_obj_b.b() = "you";
std::string key1 = "mykey";
auto dataset = datastore->createDataSet("testWriteBatchEvent");
auto run = dataset.createRun(42);
auto subrun = run.createSubRun(2);
{
hepnos::WriteBatch batch(*datastore);
for(auto i = 0; i < 10; i++) {
auto e = subrun.createEvent(batch, i);
CPPUNIT_ASSERT(e.store(batch, key1, out_obj_a));
CPPUNIT_ASSERT(e.store(batch, key1, out_obj_b));
}
}
TestObjectA in_obj_a;
TestObjectB in_obj_b;
for(auto i = 0; i < 10; i++) {
auto e = subrun[i];
CPPUNIT_ASSERT(e.valid());
CPPUNIT_ASSERT(e.load(key1, in_obj_a));
CPPUNIT_ASSERT(e.load(key1, in_obj_b));
CPPUNIT_ASSERT(out_obj_a == in_obj_a);
CPPUNIT_ASSERT(out_obj_b == in_obj_b);
}
}
Markdown is supported
0%
or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment