DataSet.cpp 15.4 KB
Newer Older
Matthieu Dorier's avatar
Matthieu Dorier committed
1 2 3 4 5
/*
 * (C) 2018 The University of Chicago
 * 
 * See COPYRIGHT in top-level directory.
 */
6
#include "hepnos/DataSet.hpp"
7 8
#include "hepnos/Run.hpp"
#include "hepnos/RunSet.hpp"
9
#include "hepnos/AsyncEngine.hpp"
Matthieu Dorier's avatar
Matthieu Dorier committed
10
#include "ItemImpl.hpp"
11
#include "DataSetImpl.hpp"
12
#include "EventSetImpl.hpp"
13
#include "DataStoreImpl.hpp"
14
#include "AsyncEngineImpl.hpp"
15
#include "WriteBatchImpl.hpp"
16 17 18

namespace hepnos {

19 20
DataSet::iterator DataSetImpl::m_end;

21
DataSet::DataSet()
22
: m_impl(std::make_shared<DataSetImpl>(nullptr, 0, std::make_shared<std::string>(""), "")) {}
23

24
DataSet::DataSet(const std::shared_ptr<DataSetImpl>& impl)
25
: m_impl(impl) {}
26

27
DataSet::DataSet(std::shared_ptr<DataSetImpl>&& impl)
28
: m_impl(std::move(impl)) {}
29

30
DataStore DataSet::datastore() const {
31 32 33
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
34
    return DataStore(m_impl->m_datastore);
35 36
}

37 38
DataSet DataSet::next() const {
    if(!valid()) return DataSet();
39 40
    std::vector<std::shared_ptr<DataSetImpl>> result;
    size_t s = m_impl->m_datastore->nextDataSets(m_impl, result, 1);
41
    if(s == 0) return DataSet();
42
    else return DataSet(std::move(result[0]));
43 44 45
}

bool DataSet::valid() const {
46
    return m_impl && m_impl->m_datastore; 
47 48
}

49
ProductID DataSet::storeRawData(const std::string& key, const char* value, size_t vsize) {
50
    if(!valid()) {
51
        throw Exception("Calling DataSet member function on an invalid DataSet");
52 53
    }
    // forward the call to the datastore's store function
Matthieu Dorier's avatar
Matthieu Dorier committed
54 55
    ItemDescriptor id(m_impl->m_uuid);
    return m_impl->m_datastore->storeRawProduct(id, key, value, vsize);
56 57
}

58
ProductID DataSet::storeRawData(WriteBatch& batch, const std::string& key, const char* value, size_t vsize) {
59 60 61 62
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
    // forward the call to the datastore's store function
Matthieu Dorier's avatar
Matthieu Dorier committed
63 64
    ItemDescriptor id(m_impl->m_uuid);
    return batch.m_impl->storeRawProduct(id, key, value, vsize);
65 66
}

67 68 69 70 71 72 73 74 75
ProductID DataSet::storeRawData(AsyncEngine& async, const std::string& key, const char* value, size_t vsize) {
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
    // forward the call to the async engine's store function
    ItemDescriptor id(m_impl->m_uuid);
    return async.m_impl->storeRawProduct(id, key, value, vsize);
}

76
bool DataSet::loadRawData(const std::string& key, std::string& buffer) const {
77 78 79
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
80
    // forward the call to the datastore's load function
Matthieu Dorier's avatar
Matthieu Dorier committed
81 82
    ItemDescriptor id(m_impl->m_uuid);
    return m_impl->m_datastore->loadRawProduct(id, key, buffer);
83 84
}

85
bool DataSet::loadRawData(const std::string& key, char* value, size_t* vsize) const {
86
    if(!valid()) {
87
        throw Exception("Calling DataSet member function on an invalid DataSet");
88 89
    }
    // forward the call to the datastore's load function
Matthieu Dorier's avatar
Matthieu Dorier committed
90 91
    ItemDescriptor id(m_impl->m_uuid);
    return m_impl->m_datastore->loadRawProduct(id, key, value, vsize);
92 93 94
}

bool DataSet::operator==(const DataSet& other) const {
95 96 97 98
    bool v1 = valid();
    bool v2 = other.valid();
    if(!v1 && !v2) return true;
    if(v1  && !v2) return false;
99
    if(!v1 &&  v2) return false;
100 101
    return m_impl->m_datastore  == other.m_impl->m_datastore
        && m_impl->m_level      == other.m_impl->m_level
102 103
        && (m_impl->m_container == other.m_impl->m_container
            || *m_impl->m_container == *other.m_impl->m_container )
104
        && m_impl->m_name       == other.m_impl->m_name;
105 106
}

107 108 109 110
bool DataSet::operator!=(const DataSet& other) const {
    return !(*this == other);
}

111
const std::string& DataSet::name() const {
112 113 114
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
115 116 117 118
    return m_impl->m_name;
}

const std::string& DataSet::container() const {
119 120 121
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
122
    return *m_impl->m_container;
123 124 125
}

std::string DataSet::fullname() const {
126 127 128 129
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
    return m_impl->fullname();
130 131
}

132
DataSet DataSet::createDataSet(const std::string& name) {
133 134 135
    if(name.find('/') != std::string::npos
    || name.find('%') != std::string::npos) {
        throw Exception("Invalid character '/' or '%' in dataset name");
136
    }
137
    std::string parent = fullname();
Matthieu Dorier's avatar
Matthieu Dorier committed
138 139 140 141 142
    auto new_dataset_impl = std::make_shared<DataSetImpl>(
            m_impl->m_datastore, m_impl->m_level+1,
            std::make_shared<std::string>(parent), name);
    m_impl->m_datastore->createDataSet(m_impl->m_level+1, parent, name, new_dataset_impl->m_uuid);
    return DataSet(new_dataset_impl);
143
}
144

145
Run DataSet::createRun(const RunNumber& runNumber) {
146 147 148
    if(InvalidRunNumber == runNumber) {
        throw Exception("Trying to create a Run with InvalidRunNumber");
    }
Matthieu Dorier's avatar
Matthieu Dorier committed
149 150
    m_impl->m_datastore->createItem(m_impl->m_uuid, runNumber);
    return Run(std::make_shared<ItemImpl>(
151 152 153
                    m_impl->m_datastore,
                    m_impl->m_uuid,
                    runNumber));
154 155
}

Matthieu Dorier's avatar
Matthieu Dorier committed
156 157 158 159
Run DataSet::createRun(WriteBatch& batch, const RunNumber& runNumber) {
    if(InvalidRunNumber == runNumber) {
        throw Exception("Trying to create a Run with InvalidRunNumber");
    }
Matthieu Dorier's avatar
Matthieu Dorier committed
160 161 162 163 164
    batch.m_impl->createItem(m_impl->m_uuid, runNumber);
    return Run(std::make_shared<ItemImpl>(
                    m_impl->m_datastore,
                    m_impl->m_uuid,
                    runNumber));
Matthieu Dorier's avatar
Matthieu Dorier committed
165 166
}

167 168 169 170 171 172 173 174 175 176 177
Run DataSet::createRun(AsyncEngine& async, const RunNumber& runNumber) {
    if(InvalidRunNumber == runNumber) {
        throw Exception("Trying to create a Run with InvalidRunNumber");
    }
    async.m_impl->createItem(m_impl->m_uuid, runNumber);
    return Run(std::make_shared<ItemImpl>(
                    m_impl->m_datastore,
                    m_impl->m_uuid,
                    runNumber));
}

178 179
DataSet DataSet::operator[](const std::string& datasetName) const {
    auto it = find(datasetName);
180 181
    if(!it->valid())
        throw Exception("Requested DataSet does not exist");
182 183 184
    return std::move(*it);
}

185
Run DataSet::operator[](const RunNumber& runNumber) const {
186
    auto it = runs().find(runNumber);
187 188
    if(!it->valid())
        throw Exception("Requested Run does not exist");
189 190 191
    return std::move(*it);
}

192
DataSet::iterator DataSet::find(const std::string& datasetPath) {
193 194 195
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
196
    int ret;
197 198
    if(datasetPath.find('%') != std::string::npos) {
        throw Exception("Invalid character '%' in dataset name");
199
    }
200
    
201 202 203 204 205
    size_t slash_count = std::count(datasetPath.begin(), datasetPath.end(), '/');
    size_t level = m_impl->m_level + 1 + slash_count;
    std::string containerName;
    std::string datasetName;

206
    std::string parent = fullname();
207 208 209 210 211 212

    if(slash_count == 0) {
        datasetName = datasetPath;
        containerName = parent;
    } else {
        size_t c = datasetPath.find_last_of('/');
213
        containerName = parent + "/" + datasetPath.substr(0,c);
214 215 216
        datasetName   = datasetPath.substr(c+1);
    }

Matthieu Dorier's avatar
Matthieu Dorier committed
217 218
    UUID uuid;
    bool b = m_impl->m_datastore->loadDataSet(level, containerName, datasetName, uuid);
219
    if(!b) {
220
        return DataSetImpl::m_end;
221
    }
222 223
    return iterator(
                DataSet(
224
                    std::make_shared<DataSetImpl>(
225 226 227
                        m_impl->m_datastore,
                        level,
                        std::make_shared<std::string>(containerName),
Matthieu Dorier's avatar
Matthieu Dorier committed
228 229
                        datasetName,
                        uuid)));
230 231 232 233 234 235 236 237
}

DataSet::const_iterator DataSet::find(const std::string& datasetName) const {
    iterator it = const_cast<DataSet*>(this)->find(datasetName);
    return it;
}

DataSet::iterator DataSet::begin() {
238 239 240
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
Matthieu Dorier's avatar
Matthieu Dorier committed
241 242
    // we use the prefix "&" because we need something that comes after "%"
    // (which represents runs) and is not going to be in a dataset name
243
    DataSet ds(
244
            std::make_shared<DataSetImpl>(
245 246 247 248 249
                m_impl->m_datastore, 
                m_impl->m_level+1,
                std::make_shared<std::string>(fullname()),
                "&"));

250
    ds = ds.next();
251
    if(ds.valid()) return iterator(ds);
252 253 254 255
    else return end();
}

DataSet::iterator DataSet::end() {
256 257 258
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
259
    return DataSetImpl::m_end;
260 261
}

262 263 264 265 266
DataSet::const_iterator DataSet::begin() const {
    return const_iterator(const_cast<DataSet*>(this)->begin());
}

DataSet::const_iterator DataSet::end() const {
267 268 269
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
270
    return DataSetImpl::m_end;
271 272
}

273 274 275 276 277
DataSet::const_iterator DataSet::cbegin() const {
    return const_iterator(const_cast<DataSet*>(this)->begin());
}

DataSet::const_iterator DataSet::cend() const {
278 279 280
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
281
    return DataSetImpl::m_end;
282 283 284
}

DataSet::iterator DataSet::lower_bound(const std::string& lb) {
285 286 287
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
288 289 290 291 292 293 294 295 296
    std::string lb2 = lb;
    size_t s = lb2.size();
    lb2[s-1] -= 1; // sdskv_list_keys's start_key is exclusive
    iterator it = find(lb2);
    if(it != end()) {
        // we found something before the specified lower bound
        ++it;
        return it;
    }
297
    DataSet ds(
298
            std::make_shared<DataSetImpl>(
299 300 301 302 303
                m_impl->m_datastore, 
                m_impl->m_level+1,
                std::make_shared<std::string>(fullname()),
                lb2));

304 305
    ds = ds.next();
    if(!ds.valid()) return end();
306
    else return iterator(ds);
307 308 309 310 311 312 313 314
}

DataSet::const_iterator DataSet::lower_bound(const std::string& lb) const {
    iterator it = const_cast<DataSet*>(this)->lower_bound(lb);
    return it;
}

DataSet::iterator DataSet::upper_bound(const std::string& ub) {
315 316 317
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
318
    DataSet ds(
319
            std::make_shared<DataSetImpl>(
320 321 322 323
                m_impl->m_datastore,
                m_impl->m_level+1,
                std::make_shared<std::string>(fullname()), ub));

324 325
    ds = ds.next();
    if(!ds.valid()) return end();
326
    else return iterator(ds);
327 328 329 330 331 332 333
}

DataSet::const_iterator DataSet::upper_bound(const std::string& ub) const {
    iterator it = const_cast<DataSet*>(this)->upper_bound(ub);
    return it;
}

334
RunSet DataSet::runs() const {
335 336 337
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
338
    return RunSet(m_impl);
339 340
}

341 342 343 344 345 346 347 348 349 350 351 352
EventSet DataSet::events(int target) const {
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
    if(target >= 0) 
        return EventSet(std::make_shared<EventSetImpl>(*m_impl, target));
    else {
        auto numTargets = m_impl->m_datastore->numTargets(ItemType::EVENT);
        return EventSet(std::make_shared<EventSetImpl>(*m_impl, 0, numTargets));
    }
}

353 354 355 356 357 358 359 360 361 362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379 380 381 382 383 384 385 386 387 388 389 390 391 392 393 394 395 396 397 398 399 400 401 402 403 404 405 406 407 408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423 424 425 426 427 428 429 430 431 432 433 434 435 436 437 438 439 440 441 442 443 444 445 446 447 448 449 450 451 452 453 454 455 456 457 458 459 460 461 462 463 464 465 466 467 468 469 470 471 472 473 474 475 476 477 478 479 480 481 482 483 484 485 486 487 488 489 490 491 492 493 494 495 496 497 498
////////////////////////////////////////////////////////////////////////////////////////////
// DataSet::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////

class DataSet::const_iterator::Impl {
    public:
        DataSet    m_current_dataset;

        Impl()
        : m_current_dataset()
        {}

        Impl(const DataSet& dataset)
        : m_current_dataset(dataset)
        {}

        Impl(DataSet&& dataset)
        : m_current_dataset(std::move(dataset))
        {}

        Impl(const Impl& other)
        : m_current_dataset(other.m_current_dataset)
        {}

        bool operator==(const Impl& other) const {
            return m_current_dataset == other.m_current_dataset;
        }
};

DataSet::const_iterator::const_iterator()
: m_impl(std::make_unique<Impl>()) {}

DataSet::const_iterator::const_iterator(const DataSet& dataset)
: m_impl(std::make_unique<Impl>(dataset)) {}

DataSet::const_iterator::const_iterator(DataSet&& dataset)
: m_impl(std::make_unique<Impl>(std::move(dataset))) {}

DataSet::const_iterator::~const_iterator() {}

DataSet::const_iterator::const_iterator(const DataSet::const_iterator& other) {
    if(other.m_impl) {
        m_impl = std::make_unique<Impl>(*other.m_impl);
    }
}

DataSet::const_iterator::const_iterator(DataSet::const_iterator&& other)
: m_impl(std::move(other.m_impl)) {}

DataSet::const_iterator& DataSet::const_iterator::operator=(const DataSet::const_iterator& other) {
    if(&other == this) return *this;
    if(other.m_impl)
        m_impl = std::make_unique<Impl>(*other.m_impl);
    else
        m_impl.reset();
    return *this;
}

DataSet::const_iterator& DataSet::const_iterator::operator=(DataSet::const_iterator&& other) {
    if(&other == this) return *this;
    m_impl = std::move(other.m_impl);
    return *this;
}

DataSet::const_iterator::self_type DataSet::const_iterator::operator++() {
    if(!m_impl) {
        throw Exception("Trying to increment an invalid iterator");
    }
    m_impl->m_current_dataset = m_impl->m_current_dataset.next();
    return *this;
}

DataSet::const_iterator::self_type DataSet::const_iterator::operator++(int) {
    const_iterator copy = *this;
    ++(*this);
    return copy;
}

const DataSet::const_iterator::reference DataSet::const_iterator::operator*() {
    if(!m_impl) {
        throw Exception("Trying to dereference an invalid iterator");
    }
    return m_impl->m_current_dataset;
}
const DataSet::const_iterator::pointer DataSet::const_iterator::operator->() {
    if(!m_impl) {
        throw Exception("Trying to dereference an invalid iterator");
    }
    return &(m_impl->m_current_dataset);
}

bool DataSet::const_iterator::operator==(const self_type& rhs) const {
    if(!m_impl && !rhs.m_impl)  return true;
    if(m_impl  && !rhs.m_impl)  return false;
    if(!m_impl && rhs.m_impl)   return false;
    return *m_impl == *(rhs.m_impl);
}

bool DataSet::const_iterator::operator!=(const self_type& rhs) const {
    return !(*this == rhs);
}

////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::iterator implementation
////////////////////////////////////////////////////////////////////////////////////////////

DataSet::iterator::iterator(const DataSet& current)
: const_iterator(current) {}

DataSet::iterator::iterator(DataSet&& current)
: const_iterator(std::move(current)) {}

DataSet::iterator::iterator()
: const_iterator() {}

DataSet::iterator::~iterator() {}

DataSet::iterator::iterator(const DataSet::iterator& other)
: const_iterator(other) {}

DataSet::iterator::iterator(DataSet::iterator&& other)
: const_iterator(std::move(other)) {}

DataSet::iterator& DataSet::iterator::operator=(const DataSet::iterator& other) {
    if(this == &other) return *this;
    if(other.m_impl)
        m_impl = std::make_unique<Impl>(*other.m_impl);
    else
        m_impl.reset();
    return *this;
}

DataSet::iterator& DataSet::iterator::operator=(DataSet::iterator&& other) {
    if(this == &other) return *this;
    m_impl = std::move(other.m_impl);
    return *this;
}

DataSet::iterator::reference DataSet::iterator::operator*() {
    return const_cast<reference>(const_iterator::operator*());
}

DataSet::iterator::pointer DataSet::iterator::operator->() {
    return const_cast<pointer>(const_iterator::operator->());
}

499
}