DataStore.cpp 10.4 KB
Newer Older
Matthieu Dorier's avatar
Matthieu Dorier committed
1
2
3
4
5
/*
 * (C) 2018 The University of Chicago
 * 
 * See COPYRIGHT in top-level directory.
 */
6
7
8
9
10
11
12
13
14
15
#include <vector>
#include <functional>
#include <iostream>
#include <yaml-cpp/yaml.h>
#include <sdskv-client.h>
#include <bake-client.h>
#include <ch-placement.h>
#include "hepnos/Exception.hpp"
#include "hepnos/DataStore.hpp"
#include "hepnos/DataSet.hpp"
16
#include "private/DataStoreImpl.hpp"
17
18
19

namespace hepnos {

20
21
22
23
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore implementation
////////////////////////////////////////////////////////////////////////////////////////////

24
25
26
27
28
29
30
31
32
DataStore::DataStore() 
: m_impl(std::make_unique<DataStore::Impl>(this)) {
    char* file = getenv("HEPNOS_CONFIG_FILE");
    if(file == nullptr) 
        throw Exception("HEPNOS_CONFIG_FILE environment variable not set");
    std::string configFile(file);
    m_impl->init(configFile);
}

33
DataStore::DataStore(const std::string& configFile) 
34
: m_impl(std::make_unique<DataStore::Impl>(this)) {
35
36
37
38
39
40
41
42
43
44
45
46
    m_impl->init(configFile);
}

DataStore::DataStore(DataStore&& other)
: m_impl(std::move(other.m_impl)) {}

DataStore& DataStore::operator=(DataStore&& other) {
    if(&other == this) return *this;
    if(m_impl) {
        m_impl->cleanup();
    }
    m_impl = std::move(other.m_impl);
47
    return *this;
48
49
50
51
52
53
54
55
}
    
DataStore::~DataStore() {
    if(m_impl) {
        m_impl->cleanup();
    }
}

56
DataStore::iterator DataStore::find(const std::string& datasetPath) {
57
58
59
60
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }

61
    int ret;
62
63
64

    if(datasetPath.find('%') != std::string::npos) {
        throw Exception("Invalid character ('%') in dataset name");
65
    }
66
67
68
69
70
71
72
73
74
75
76
77
78
79
80

    size_t slash_count = std::count(datasetPath.begin(), datasetPath.end(), '/');
    size_t level = 1 + slash_count;
    std::string containerName;
    std::string datasetName;

    if(slash_count == 0) {
        datasetName = datasetPath;
        containerName = "";
    } else {
        size_t c = datasetPath.find_last_of('/');
        containerName = datasetPath.substr(0,c);
        datasetName   = datasetPath.substr(c+1);
    }

81
    std::vector<char> data;
82
    bool b = m_impl->load(level, containerName, datasetName, data);
83
84
    if(!b) {
        return m_impl->m_end;
85
    }
86
    return iterator(DataSet(this, level, containerName, datasetName));
87
88
}

Matthieu Dorier's avatar
Matthieu Dorier committed
89
90
DataSet DataStore::operator[](const std::string& datasetName) const {
    auto it = find(datasetName);
91
92
    if(!it->valid())
        throw Exception("Requested DataSet does not exist");
Matthieu Dorier's avatar
Matthieu Dorier committed
93
94
95
    return std::move(*it);
}

96
DataStore::const_iterator DataStore::find(const std::string& datasetName) const {
97
98
    DataStore::iterator it = const_cast<DataStore*>(this)->find(datasetName);
    return it;
99
100
101
}

DataStore::iterator DataStore::begin() {
102
103
104
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
105
    DataSet ds(this, 1, "", "");
106
    ds = ds.next();
107
    if(ds.valid()) return iterator(std::move(ds));
108
    else return end();
109
110
}

111
112
113
114
115
DataStore::const_iterator DataStore::begin() const {
    iterator it = const_cast<DataStore*>(this)->begin();
    return const_iterator(std::move(it));
}

116
DataStore::iterator DataStore::end() {
117
118
119
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
120
121
122
    return m_impl->m_end;
}

123
DataStore::const_iterator DataStore::end() const {
124
125
126
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
127
128
129
    return m_impl->m_end;
}

130
DataStore::const_iterator DataStore::cbegin() const {
131
    return const_cast<DataStore*>(this)->begin();
132
133
134
}

DataStore::const_iterator DataStore::cend() const {
135
    return const_cast<DataStore*>(this)->end();
136
137
138
}

DataStore::iterator DataStore::lower_bound(const std::string& lb) {
139
140
141
142
143
144
145
146
147
    std::string lb2 = lb;
    size_t s = lb2.size();
    lb2[s-1] -= 1; // sdskv_list_keys's start_key is exclusive
    iterator it = find(lb2);
    if(it != end()) {
        // we found something before the specified lower bound
        ++it;
        return it;
    }
148
    DataSet ds(this, 1, "", lb2);
149
150
    ds = ds.next();
    if(!ds.valid()) return end();
151
    else return iterator(std::move(ds));
152
153
154
}

DataStore::const_iterator DataStore::lower_bound(const std::string& lb) const {
155
156
    iterator it = const_cast<DataStore*>(this)->lower_bound(lb);
    return it;
157
158
159
}

DataStore::iterator DataStore::upper_bound(const std::string& ub) {
160
161
162
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
163
    DataSet ds(this, 1, "", ub);
164
165
    ds = ds.next();
    if(!ds.valid()) return end();
166
    else return iterator(std::move(ds));
167
168
169
}

DataStore::const_iterator DataStore::upper_bound(const std::string& ub) const {
170
171
    iterator it = const_cast<DataStore*>(this)->upper_bound(ub);
    return it;
172
173
174
}

DataSet DataStore::createDataSet(const std::string& name) {
175
176
177
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
178
179
180
    if(name.find('/') != std::string::npos
    || name.find('%') != std::string::npos) {
        throw Exception("Invalid character ('/' or '%') in dataset name");
181
    }
182
    m_impl->store(1, "", name, std::vector<char>());
183
    return DataSet(this, 1, "", name);
184
185
}

186
void DataStore::shutdown() {
187
188
189
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
190
    for(auto addr : m_impl->m_addrs) {
Matthieu Dorier's avatar
Matthieu Dorier committed
191
        margo_shutdown_remote_instance(m_impl->m_mid, addr.second);
192
193
194
    }
}

195
bool DataStore::loadRawProduct(const ProductID& productID, std::vector<char>& buffer) {
196
197
198
    if(!m_impl) {
        throw Exception("Calling DataStore member function on an invalid DataStore object");
    }
199
200
201
    return m_impl->load(productID.m_level, productID.m_containerName, productID.m_objectName, buffer);
}

202
203
204
205
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////

206
207
208
209
class DataStore::const_iterator::Impl {
    public:
        DataSet    m_current_dataset;

210
211
        Impl()
        : m_current_dataset()
212
        {}
213

214
215
216
217
218
219
        Impl(const DataSet& dataset)
        : m_current_dataset(dataset)
        {}

        Impl(DataSet&& dataset)
        : m_current_dataset(std::move(dataset))
220
        {}
221
222

        Impl(const Impl& other)
223
        : m_current_dataset(other.m_current_dataset) 
224
225
226
        {}

        bool operator==(const Impl& other) const {
227
            return m_current_dataset == other.m_current_dataset;
228
        }
229
230
};

231
232
233
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::const_iterator::Impl implementation
////////////////////////////////////////////////////////////////////////////////////////////
234

235
236
DataStore::const_iterator::const_iterator()
: m_impl(std::make_unique<Impl>()) {}
237

238
239
240
241
242
DataStore::const_iterator::const_iterator(const DataSet& dataset)
: m_impl(std::make_unique<Impl>(dataset)) {}

DataStore::const_iterator::const_iterator(DataSet&& dataset)
: m_impl(std::make_unique<Impl>(std::move(dataset))) {}
243

244
DataStore::const_iterator::~const_iterator() {}
245

246
247
248
249
250
DataStore::const_iterator::const_iterator(const DataStore::const_iterator& other) {
    if(other.m_impl) {
        m_impl = std::make_unique<Impl>(*other.m_impl);
    }
}
251
252

DataStore::const_iterator::const_iterator(DataStore::const_iterator&& other) 
253
: m_impl(std::move(other.m_impl)) {}
254

255
256
DataStore::const_iterator& DataStore::const_iterator::operator=(const DataStore::const_iterator& other) {
    if(&other == this) return *this;
257
258
259
260
    if(other.m_impl)
        m_impl = std::make_unique<Impl>(*other.m_impl);
    else
        m_impl.reset();
261
    return *this;
262
263
}

264
265
266
267
DataStore::const_iterator& DataStore::const_iterator::operator=(DataStore::const_iterator&& other) {
    if(&other == this) return *this;
    m_impl = std::move(other.m_impl);
    return *this;
268
269
270
}

DataStore::const_iterator::self_type DataStore::const_iterator::operator++() {
271
272
273
274
275
    if(!m_impl) {
        throw Exception("Trying to increment an invalid iterator");
    }
    m_impl->m_current_dataset = m_impl->m_current_dataset.next();
    return *this;
276
277
278
}

DataStore::const_iterator::self_type DataStore::const_iterator::operator++(int) {
279
280
281
    const_iterator copy = *this;
    ++(*this);
    return copy;
282
283
284
}

const DataStore::const_iterator::reference DataStore::const_iterator::operator*() {
285
286
287
288
    if(!m_impl) {
        throw Exception("Trying to dereference an invalid iterator");
    }
    return m_impl->m_current_dataset;
289
290
291
}

const DataStore::const_iterator::pointer DataStore::const_iterator::operator->() {
292
293
294
    if(!m_impl) {
        throw Exception("Trying to dereference an invalid iterator");
    }
295
    return &(m_impl->m_current_dataset);
296
297
298
}

bool DataStore::const_iterator::operator==(const self_type& rhs) const {
299
300
301
302
    if(!m_impl && !rhs.m_impl)  return true;
    if(m_impl  && !rhs.m_impl)  return false;
    if(!m_impl && rhs.m_impl)   return false;
    return *m_impl == *(rhs.m_impl);
303
304
305
}

bool DataStore::const_iterator::operator!=(const self_type& rhs) const {
306
    return !(*this == rhs);
307
308
}

309
310
311
312
////////////////////////////////////////////////////////////////////////////////////////////
// DataStore::iterator implementation
////////////////////////////////////////////////////////////////////////////////////////////

313
314
315
316
317
DataStore::iterator::iterator(const DataSet& current)
: const_iterator(current) {}

DataStore::iterator::iterator(DataSet&& current)
: const_iterator(std::move(current)) {}
318

319
320
DataStore::iterator::iterator()
: const_iterator() {}
321

322
DataStore::iterator::~iterator() {}
323
324

DataStore::iterator::iterator(const DataStore::iterator& other)
325
: const_iterator(other) {}
326
327

DataStore::iterator::iterator(DataStore::iterator&& other) 
328
: const_iterator(std::move(other)) {}
329
330

DataStore::iterator& DataStore::iterator::operator=(const DataStore::iterator& other) {
331
    if(this == &other) return *this;
332
333
334
335
    if(other.m_impl)
        m_impl = std::make_unique<Impl>(*other.m_impl);
    else
        m_impl.reset();
336
    return *this;
337
338
339
}

DataStore::iterator& DataStore::iterator::operator=(DataStore::iterator&& other) {
340
341
342
    if(this == &other) return *this;
    m_impl = std::move(other.m_impl);
    return *this;
343
344
345
}

DataStore::iterator::reference DataStore::iterator::operator*() {
346
    return const_cast<reference>(const_iterator::operator*());
347
348
349
}

DataStore::iterator::pointer DataStore::iterator::operator->() {
350
    return const_cast<pointer>(const_iterator::operator->());
351
352
353
354
}

}