DataSet.cpp 16.1 KB
Newer Older
Matthieu Dorier's avatar
Matthieu Dorier committed
1
2
3
4
5
/*
 * (C) 2018 The University of Chicago
 * 
 * See COPYRIGHT in top-level directory.
 */
6
#include "hepnos/DataSet.hpp"
7
8
#include "hepnos/Run.hpp"
#include "hepnos/RunSet.hpp"
9
#include "hepnos/AsyncEngine.hpp"
Matthieu Dorier's avatar
Matthieu Dorier committed
10
#include "ItemImpl.hpp"
11
#include "DataSetImpl.hpp"
12
#include "EventSetImpl.hpp"
13
#include "DataStoreImpl.hpp"
14
#include "AsyncEngineImpl.hpp"
15
#include "WriteBatchImpl.hpp"
16
17
18

namespace hepnos {

19
20
DataSet::iterator DataSetImpl::m_end;

21
DataSet::DataSet()
22
: m_impl(std::make_shared<DataSetImpl>(nullptr, 0, std::make_shared<std::string>(""), "")) {}
23

24
DataSet::DataSet(const std::shared_ptr<DataSetImpl>& impl)
25
: m_impl(impl) {}
26

27
DataSet::DataSet(std::shared_ptr<DataSetImpl>&& impl)
28
: m_impl(std::move(impl)) {}
29

30
DataStore DataSet::datastore() const {
31
32
33
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
34
    return DataStore(m_impl->m_datastore);
35
36
}

37
38
DataSet DataSet::next() const {
    if(!valid()) return DataSet();
39
40
    std::vector<std::shared_ptr<DataSetImpl>> result;
    size_t s = m_impl->m_datastore->nextDataSets(m_impl, result, 1);
41
    if(s == 0) return DataSet();
42
    else return DataSet(std::move(result[0]));
43
44
45
}

bool DataSet::valid() const {
46
    return m_impl && m_impl->m_datastore; 
47
48
}

49
ProductID DataSet::storeRawData(const std::string& key, const char* value, size_t vsize) {
50
    if(!valid()) {
51
        throw Exception("Calling DataSet member function on an invalid DataSet");
52
53
    }
    // forward the call to the datastore's store function
Matthieu Dorier's avatar
Matthieu Dorier committed
54
55
    ItemDescriptor id(m_impl->m_uuid);
    return m_impl->m_datastore->storeRawProduct(id, key, value, vsize);
56
57
}

58
ProductID DataSet::storeRawData(WriteBatch& batch, const std::string& key, const char* value, size_t vsize) {
59
60
61
62
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
    // forward the call to the datastore's store function
Matthieu Dorier's avatar
Matthieu Dorier committed
63
    ItemDescriptor id(m_impl->m_uuid);
64
65
66
67
    if(batch.m_impl)
        return batch.m_impl->storeRawProduct(id, key, value, vsize);
    else
        return m_impl->m_datastore->storeRawProduct(id, key, value, vsize);
68
69
}

70
71
72
73
74
75
ProductID DataSet::storeRawData(AsyncEngine& async, const std::string& key, const char* value, size_t vsize) {
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
    // forward the call to the async engine's store function
    ItemDescriptor id(m_impl->m_uuid);
76
77
78
79
    if(async.m_impl)
        return async.m_impl->storeRawProduct(id, key, value, vsize);
    else
        return m_impl->m_datastore->storeRawProduct(id, key, value, vsize);
80
81
}

82
bool DataSet::loadRawData(const std::string& key, std::string& buffer) const {
83
84
85
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
86
    // forward the call to the datastore's load function
Matthieu Dorier's avatar
Matthieu Dorier committed
87
88
    ItemDescriptor id(m_impl->m_uuid);
    return m_impl->m_datastore->loadRawProduct(id, key, buffer);
89
90
}

91
bool DataSet::loadRawData(const std::string& key, char* value, size_t* vsize) const {
92
    if(!valid()) {
93
        throw Exception("Calling DataSet member function on an invalid DataSet");
94
95
    }
    // forward the call to the datastore's load function
Matthieu Dorier's avatar
Matthieu Dorier committed
96
97
    ItemDescriptor id(m_impl->m_uuid);
    return m_impl->m_datastore->loadRawProduct(id, key, value, vsize);
98
99
100
}

bool DataSet::operator==(const DataSet& other) const {
101
102
103
104
    bool v1 = valid();
    bool v2 = other.valid();
    if(!v1 && !v2) return true;
    if(v1  && !v2) return false;
105
    if(!v1 &&  v2) return false;
106
107
    return m_impl->m_datastore  == other.m_impl->m_datastore
        && m_impl->m_level      == other.m_impl->m_level
108
109
        && (m_impl->m_container == other.m_impl->m_container
            || *m_impl->m_container == *other.m_impl->m_container )
110
        && m_impl->m_name       == other.m_impl->m_name;
111
112
}

113
114
115
116
bool DataSet::operator!=(const DataSet& other) const {
    return !(*this == other);
}

117
const std::string& DataSet::name() const {
118
119
120
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
121
122
123
124
    return m_impl->m_name;
}

const std::string& DataSet::container() const {
125
126
127
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
128
    return *m_impl->m_container;
129
130
131
}

std::string DataSet::fullname() const {
132
133
134
135
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
    return m_impl->fullname();
136
137
}

138
DataSet DataSet::createDataSet(const std::string& name) {
139
140
141
    if(name.find('/') != std::string::npos
    || name.find('%') != std::string::npos) {
        throw Exception("Invalid character '/' or '%' in dataset name");
142
    }
143
    std::string parent = fullname();
Matthieu Dorier's avatar
Matthieu Dorier committed
144
145
146
147
148
    auto new_dataset_impl = std::make_shared<DataSetImpl>(
            m_impl->m_datastore, m_impl->m_level+1,
            std::make_shared<std::string>(parent), name);
    m_impl->m_datastore->createDataSet(m_impl->m_level+1, parent, name, new_dataset_impl->m_uuid);
    return DataSet(new_dataset_impl);
149
}
150

151
Run DataSet::createRun(const RunNumber& runNumber) {
152
153
154
    if(InvalidRunNumber == runNumber) {
        throw Exception("Trying to create a Run with InvalidRunNumber");
    }
Matthieu Dorier's avatar
Matthieu Dorier committed
155
156
    m_impl->m_datastore->createItem(m_impl->m_uuid, runNumber);
    return Run(std::make_shared<ItemImpl>(
157
158
159
                    m_impl->m_datastore,
                    m_impl->m_uuid,
                    runNumber));
160
161
}

Matthieu Dorier's avatar
Matthieu Dorier committed
162
163
164
165
Run DataSet::createRun(WriteBatch& batch, const RunNumber& runNumber) {
    if(InvalidRunNumber == runNumber) {
        throw Exception("Trying to create a Run with InvalidRunNumber");
    }
166
167
168
169
    if(batch.m_impl)
        batch.m_impl->createItem(m_impl->m_uuid, runNumber);
    else
        m_impl->m_datastore->createItem(m_impl->m_uuid, runNumber);
Matthieu Dorier's avatar
Matthieu Dorier committed
170
171
172
173
    return Run(std::make_shared<ItemImpl>(
                    m_impl->m_datastore,
                    m_impl->m_uuid,
                    runNumber));
Matthieu Dorier's avatar
Matthieu Dorier committed
174
175
}

176
177
178
179
Run DataSet::createRun(AsyncEngine& async, const RunNumber& runNumber) {
    if(InvalidRunNumber == runNumber) {
        throw Exception("Trying to create a Run with InvalidRunNumber");
    }
180
181
182
183
    if(async.m_impl)
        async.m_impl->createItem(m_impl->m_uuid, runNumber);
    else
        m_impl->m_datastore->createItem(m_impl->m_uuid, runNumber);
184
185
186
187
188
189
    return Run(std::make_shared<ItemImpl>(
                    m_impl->m_datastore,
                    m_impl->m_uuid,
                    runNumber));
}

190
191
DataSet DataSet::operator[](const std::string& datasetName) const {
    auto it = find(datasetName);
192
193
    if(!it->valid())
        throw Exception("Requested DataSet does not exist");
194
195
196
    return std::move(*it);
}

197
Run DataSet::operator[](const RunNumber& runNumber) const {
198
    auto it = runs().find(runNumber);
199
200
    if(!it->valid())
        throw Exception("Requested Run does not exist");
201
202
203
    return std::move(*it);
}

204
DataSet::iterator DataSet::find(const std::string& datasetPath) {
205
206
207
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
208
    int ret;
209
210
    if(datasetPath.find('%') != std::string::npos) {
        throw Exception("Invalid character '%' in dataset name");
211
    }
212
    
213
214
215
216
217
    size_t slash_count = std::count(datasetPath.begin(), datasetPath.end(), '/');
    size_t level = m_impl->m_level + 1 + slash_count;
    std::string containerName;
    std::string datasetName;

218
    std::string parent = fullname();
219
220
221
222
223
224

    if(slash_count == 0) {
        datasetName = datasetPath;
        containerName = parent;
    } else {
        size_t c = datasetPath.find_last_of('/');
225
        containerName = parent + "/" + datasetPath.substr(0,c);
226
227
228
        datasetName   = datasetPath.substr(c+1);
    }

Matthieu Dorier's avatar
Matthieu Dorier committed
229
230
    UUID uuid;
    bool b = m_impl->m_datastore->loadDataSet(level, containerName, datasetName, uuid);
231
    if(!b) {
232
        return DataSetImpl::m_end;
233
    }
234
235
    return iterator(
                DataSet(
236
                    std::make_shared<DataSetImpl>(
237
238
239
                        m_impl->m_datastore,
                        level,
                        std::make_shared<std::string>(containerName),
Matthieu Dorier's avatar
Matthieu Dorier committed
240
241
                        datasetName,
                        uuid)));
242
243
244
245
246
247
248
249
}

DataSet::const_iterator DataSet::find(const std::string& datasetName) const {
    iterator it = const_cast<DataSet*>(this)->find(datasetName);
    return it;
}

DataSet::iterator DataSet::begin() {
250
251
252
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
Matthieu Dorier's avatar
Matthieu Dorier committed
253
254
    // we use the prefix "&" because we need something that comes after "%"
    // (which represents runs) and is not going to be in a dataset name
255
    DataSet ds(
256
            std::make_shared<DataSetImpl>(
257
258
259
260
261
                m_impl->m_datastore, 
                m_impl->m_level+1,
                std::make_shared<std::string>(fullname()),
                "&"));

262
    ds = ds.next();
263
    if(ds.valid()) return iterator(ds);
264
265
266
267
    else return end();
}

DataSet::iterator DataSet::end() {
268
269
270
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
271
    return DataSetImpl::m_end;
272
273
}

274
275
276
277
278
DataSet::const_iterator DataSet::begin() const {
    return const_iterator(const_cast<DataSet*>(this)->begin());
}

DataSet::const_iterator DataSet::end() const {
279
280
281
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
282
    return DataSetImpl::m_end;
283
284
}

285
286
287
288
289
DataSet::const_iterator DataSet::cbegin() const {
    return const_iterator(const_cast<DataSet*>(this)->begin());
}

DataSet::const_iterator DataSet::cend() const {
290
291
292
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
293
    return DataSetImpl::m_end;
294
295
296
}

DataSet::iterator DataSet::lower_bound(const std::string& lb) {
297
298
299
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
300
301
302
303
304
305
306
307
308
    std::string lb2 = lb;
    size_t s = lb2.size();
    lb2[s-1] -= 1; // sdskv_list_keys's start_key is exclusive
    iterator it = find(lb2);
    if(it != end()) {
        // we found something before the specified lower bound
        ++it;
        return it;
    }
309
    DataSet ds(
310
            std::make_shared<DataSetImpl>(
311
312
313
314
315
                m_impl->m_datastore, 
                m_impl->m_level+1,
                std::make_shared<std::string>(fullname()),
                lb2));

316
317
    ds = ds.next();
    if(!ds.valid()) return end();
318
    else return iterator(ds);
319
320
321
322
323
324
325
326
}

DataSet::const_iterator DataSet::lower_bound(const std::string& lb) const {
    iterator it = const_cast<DataSet*>(this)->lower_bound(lb);
    return it;
}

DataSet::iterator DataSet::upper_bound(const std::string& ub) {
327
328
329
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
330
    DataSet ds(
331
            std::make_shared<DataSetImpl>(
332
333
334
335
                m_impl->m_datastore,
                m_impl->m_level+1,
                std::make_shared<std::string>(fullname()), ub));

336
337
    ds = ds.next();
    if(!ds.valid()) return end();
338
    else return iterator(ds);
339
340
341
342
343
344
345
}

DataSet::const_iterator DataSet::upper_bound(const std::string& ub) const {
    iterator it = const_cast<DataSet*>(this)->upper_bound(ub);
    return it;
}

346
RunSet DataSet::runs() const {
347
348
349
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
350
    return RunSet(m_impl);
351
352
}

353
354
355
356
EventSet DataSet::events(int target) const {
    if(!valid()) {
        throw Exception("Calling DataSet member function on an invalid DataSet");
    }
357
358
359
360
361
362
363
    auto numTargets = m_impl->m_datastore->numTargets(ItemType::EVENT);
    if(target >= (int)numTargets) {
        throw Exception(std::string("Invalid target number ")
                        +std::to_string(target) 
                        +" for EventSet (>= "
                        +std::to_string(numTargets)+")");
    }
364
365
366
367
368
369
370
    if(target >= 0) 
        return EventSet(std::make_shared<EventSetImpl>(*m_impl, target));
    else {
        return EventSet(std::make_shared<EventSetImpl>(*m_impl, 0, numTargets));
    }
}

371
372
373
374
375
376
377
378
379
380
381
382
383
384
385
386
387
388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
417
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
440
441
442
443
444
445
446
447
448
449
450
451
452
453
454
455
456
457
458
459
460
461
462
463
464
465
466
467
468
469
470
471
472
473
474
475
476
477
478
479
480
481
482
483
484
485
486
487
488
489
490
491
492
493
494
495
496
497
498
499
500
501
502
503
504
505
506
507
508
509
510
511
512
513
514
515
516
////////////////////////////////////////////////////////////////////////////////////////////
// DataSet::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////

class DataSet::const_iterator::Impl {
    public:
        DataSet    m_current_dataset;

        Impl()
        : m_current_dataset()
        {}

        Impl(const DataSet& dataset)
        : m_current_dataset(dataset)
        {}

        Impl(DataSet&& dataset)
        : m_current_dataset(std::move(dataset))
        {}

        Impl(const Impl& other)
        : m_current_dataset(other.m_current_dataset)
        {}

        bool operator==(const Impl& other) const {
            return m_current_dataset == other.m_current_dataset;
        }
};

DataSet::const_iterator::const_iterator()
: m_impl(std::make_unique<Impl>()) {}

DataSet::const_iterator::const_iterator(const DataSet& dataset)
: m_impl(std::make_unique<Impl>(dataset)) {}

DataSet::const_iterator::const_iterator(DataSet&& dataset)
: m_impl(std::make_unique<Impl>(std::move(dataset))) {}

DataSet::const_iterator::~const_iterator() {}

DataSet::const_iterator::const_iterator(const DataSet::const_iterator& other) {
    if(other.m_impl) {
        m_impl = std::make_unique<Impl>(*other.m_impl);
    }
}

DataSet::const_iterator::const_iterator(DataSet::const_iterator&& other)
: m_impl(std::move(other.m_impl)) {}

DataSet::const_iterator& DataSet::const_iterator::operator=(const DataSet::const_iterator& other) {
    if(&other == this) return *this;
    if(other.m_impl)
        m_impl = std::make_unique<Impl>(*other.m_impl);
    else
        m_impl.reset();
    return *this;
}

DataSet::const_iterator& DataSet::const_iterator::operator=(DataSet::const_iterator&& other) {
    if(&other == this) return *this;
    m_impl = std::move(other.m_impl);
    return *this;
}

DataSet::const_iterator::self_type DataSet::const_iterator::operator++() {
    if(!m_impl) {
        throw Exception("Trying to increment an invalid iterator");
    }
    m_impl->m_current_dataset = m_impl->m_current_dataset.next();
    return *this;
}

DataSet::const_iterator::self_type DataSet::const_iterator::operator++(int) {
    const_iterator copy = *this;
    ++(*this);
    return copy;
}

const DataSet::const_iterator::reference DataSet::const_iterator::operator*() {
    if(!m_impl) {
        throw Exception("Trying to dereference an invalid iterator");
    }
    return m_impl->m_current_dataset;
}
const DataSet::const_iterator::pointer DataSet::const_iterator::operator->() {
    if(!m_impl) {
        throw Exception("Trying to dereference an invalid iterator");
    }
    return &(m_impl->m_current_dataset);
}

bool DataSet::const_iterator::operator==(const self_type& rhs) const {
    if(!m_impl && !rhs.m_impl)  return true;
    if(m_impl  && !rhs.m_impl)  return false;
    if(!m_impl && rhs.m_impl)   return false;
    return *m_impl == *(rhs.m_impl);
}

bool DataSet::const_iterator::operator!=(const self_type& rhs) const {
    return !(*this == rhs);
}

////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::iterator implementation
////////////////////////////////////////////////////////////////////////////////////////////

DataSet::iterator::iterator(const DataSet& current)
: const_iterator(current) {}

DataSet::iterator::iterator(DataSet&& current)
: const_iterator(std::move(current)) {}

DataSet::iterator::iterator()
: const_iterator() {}

DataSet::iterator::~iterator() {}

DataSet::iterator::iterator(const DataSet::iterator& other)
: const_iterator(other) {}

DataSet::iterator::iterator(DataSet::iterator&& other)
: const_iterator(std::move(other)) {}

DataSet::iterator& DataSet::iterator::operator=(const DataSet::iterator& other) {
    if(this == &other) return *this;
    if(other.m_impl)
        m_impl = std::make_unique<Impl>(*other.m_impl);
    else
        m_impl.reset();
    return *this;
}

DataSet::iterator& DataSet::iterator::operator=(DataSet::iterator&& other) {
    if(this == &other) return *this;
    m_impl = std::move(other.m_impl);
    return *this;
}

DataSet::iterator::reference DataSet::iterator::operator*() {
    return const_cast<reference>(const_iterator::operator*());
}

DataSet::iterator::pointer DataSet::iterator::operator->() {
    return const_cast<pointer>(const_iterator::operator->());
}

517
}