leveldb_datastore.cc 7.4 KB
Newer Older
1 2 3
// Copyright (c) 2017, Los Alamos National Security, LLC.
// All rights reserved.
#include "leveldb_datastore.h"
4
#include "fs_util.h"
5
#include "kv-config.h"
Matthieu Dorier's avatar
Matthieu Dorier committed
6
#include <cstring>
7 8
#include <chrono>
#include <iostream>
Matthieu Dorier's avatar
Matthieu Dorier committed
9
#include <sstream>
10 11 12 13

using namespace std::chrono;

LevelDBDataStore::LevelDBDataStore() :
14
  AbstractDataStore(Duplicates::IGNORE, false, false), _less(nullptr), _keycmp(this) {
15 16 17 18
  _dbm = NULL;
};

LevelDBDataStore::LevelDBDataStore(Duplicates duplicates, bool eraseOnGet, bool debug) :
19
  AbstractDataStore(duplicates, eraseOnGet, debug), _less(nullptr), _keycmp(this) {
20 21 22 23 24 25 26 27 28 29 30 31 32 33 34 35 36 37
  _dbm = NULL;
};
  
std::string LevelDBDataStore::toString(const ds_bulk_t &bulk_val) {
  std::string str_val(bulk_val.begin(), bulk_val.end());
  return str_val;
};

ds_bulk_t LevelDBDataStore::fromString(const std::string &str_val) {
  ds_bulk_t bulk_val(str_val.begin(), str_val.end());
  return bulk_val;
};

LevelDBDataStore::~LevelDBDataStore() {
  delete _dbm;
  //leveldb::Env::Shutdown(); // Riak version only
};

38
bool LevelDBDataStore::openDatabase(const std::string& db_name, const std::string& db_path) {
39 40 41
  leveldb::Options options;
  leveldb::Status status;
  
42
  if (!db_path.empty()) {
43
    mkdirs(db_path.c_str());
44
  }
45
  options.comparator = &_keycmp;
46
  options.create_if_missing = true;
47 48 49 50
  std::string fullname = db_path;
  if(!fullname.empty()) fullname += std::string("/");
  fullname += db_name;
  status = leveldb::DB::Open(options, fullname, &_dbm);
51 52 53 54
  
  if (!status.ok()) {
    // error
    std::cerr << "LevelDBDataStore::createDatabase: LevelDB error on Open = " << status.ToString() << std::endl;
55
    return false;
56
  }
57
  return true;
58 59
};

60
void LevelDBDataStore::set_comparison_function(comparator_fn less) {
61
   _less = less; 
62 63
}

64 65 66
bool LevelDBDataStore::put(const ds_bulk_t &key, const ds_bulk_t &data) {
  leveldb::Status status;
  bool success = false;
67 68 69 70 71 72

  if(_no_overwrite) {
      if(exists(key)) return false;
  }

  //high_resolution_clock::time_point start = high_resolution_clock::now();
73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
  // IGNORE case deals with redundant puts (where key/value is the same). In LevelDB a
  // redundant put simply overwrites previous value which is fine when key/value is the same.
  if (_duplicates == Duplicates::IGNORE) {
    status = _dbm->Put(leveldb::WriteOptions(), toString(key), toString(data));
    if (status.ok()) {
      success = true;
    }
    else {
      std::cerr << "LevelDBDataStore::put: LevelDB error on Put = " << status.ToString() << std::endl;
    }
  }
  else if (_duplicates == Duplicates::ALLOW) {
    std::cerr << "LevelDBDataStore::put: Duplicates::ALLOW set, LevelDB does not support duplicates" << std::endl;
  }
  else {
    std::cerr << "LevelDBDataStore::put: Unexpected Duplicates option = " << int32_t(_duplicates) << std::endl;
  }
90 91
//  uint64_t elapsed = duration_cast<microseconds>(high_resolution_clock::now()-start).count();
//  std::cout << "LevelDBDataStore::put time = " << elapsed << " microseconds" << std::endl;
92 93 94 95

  return success;
};

Matthieu Dorier's avatar
Matthieu Dorier committed
96 97 98 99 100 101
bool LevelDBDataStore::erase(const ds_bulk_t &key) {
    leveldb::Status status;
    status = _dbm->Delete(leveldb::WriteOptions(), toString(key));
    return status.ok();
}

102 103 104 105 106 107 108
bool LevelDBDataStore::exists(const ds_bulk_t &key) {
    leveldb::Status status;
    std::string value;
    status = _dbm->Get(leveldb::ReadOptions(), toString(key), &value);
    return status.ok();
}

109 110 111 112
bool LevelDBDataStore::get(const ds_bulk_t &key, ds_bulk_t &data) {
  leveldb::Status status;
  bool success = false;

113
  //high_resolution_clock::time_point start = high_resolution_clock::now();
114 115 116 117 118 119 120 121 122 123
  data.clear();
  std::string value;
  status = _dbm->Get(leveldb::ReadOptions(), toString(key), &value);
  if (status.ok()) {
    data = fromString(value);
    success = true;
  }
  else if (!status.IsNotFound()) {
    std::cerr << "LevelDBDataStore::get: LevelDB error on Get = " << status.ToString() << std::endl;
  }
124 125
//  uint64_t elapsed = duration_cast<microseconds>(high_resolution_clock::now()-start).count();
//  std::cout << "LevelDBDataStore::get time = " << elapsed << " microseconds" << std::endl;
126 127 128 129 130 131 132 133 134 135 136 137 138 139 140 141 142

  return success;
};

bool LevelDBDataStore::get(const ds_bulk_t &key, std::vector<ds_bulk_t> &data) {
  bool success = false;

  data.clear();
  ds_bulk_t value;
  if (get(key, value)) {
    data.push_back(value);
    success = true;
  }
  
  return success;
};

143
void LevelDBDataStore::set_in_memory(bool enable)
144 145
{};

146 147
std::vector<ds_bulk_t> LevelDBDataStore::vlist_keys(
        const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix) const
148 149 150 151
{
    std::vector<ds_bulk_t> keys;

    leveldb::Iterator *it = _dbm->NewIterator(leveldb::ReadOptions());
152 153
    leveldb::Slice start_slice(start.data(), start.size());

154 155
    int c = 0;

156
    if (start.size() > 0) {
157 158 159 160 161 162 163
        it->Seek(start_slice);
        /* we treat 'start' the way RADOS treats it: excluding it from returned
         * keys. LevelDB treats start inclusively, so skip over it if we found
         * an exact match */
        if ( start.size() == it->key().size() &&
                (memcmp(it->key().data(), start.data(), start.size()) == 0))
            it->Next();
164
    } else {
165
        it->SeekToFirst();
166 167 168
    }
    /* note: iterator initialized above, not in for loop */
    for (; it->Valid() && keys.size() < count; it->Next() ) {
169 170
        ds_bulk_t k(it->key().size());
        memcpy(k.data(), it->key().data(), it->key().size() );
171 172 173 174 175 176
        c = std::memcmp(prefix.data(), k.data(), prefix.size());
        if(c == 0) {
            keys.push_back(std::move(k));
        } else if(c < 0) {
            break;
        }
177 178 179 180
    }
    delete it;
    return keys;
}
181

182 183
std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::vlist_keyvals(
        const ds_bulk_t &start, size_t count, const ds_bulk_t &prefix) const
184 185 186 187 188 189 190 191 192 193 194 195 196 197 198 199 200 201 202 203 204 205 206 207 208 209 210 211 212 213 214 215 216 217 218 219
{
    std::vector<std::pair<ds_bulk_t,ds_bulk_t>> result;

    leveldb::Iterator *it = _dbm->NewIterator(leveldb::ReadOptions());
    leveldb::Slice start_slice(start.data(), start.size());

    int c = 0;

    if (start.size() > 0) {
        it->Seek(start_slice);
        /* we treat 'start' the way RADOS treats it: excluding it from returned
         * keys. LevelDB treats start inclusively, so skip over it if we found
         * an exact match */
        if ( start.size() == it->key().size() &&
                (memcmp(it->key().data(), start.data(), start.size()) == 0))
            it->Next();
    } else {
        it->SeekToFirst();
    }
    /* note: iterator initialized above, not in for loop */
    for (; it->Valid() && result.size() < count; it->Next() ) {
        ds_bulk_t k(it->key().size());
        ds_bulk_t v(it->value().size());
        memcpy(k.data(), it->key().data(), it->key().size());
        memcpy(v.data(), it->value().data(), it->value().size());

        c = std::memcmp(prefix.data(), k.data(), prefix.size());
        if(c == 0) {
            result.push_back(std::make_pair(std::move(k), std::move(v)));
        } else if(c < 0) {
            break;
        }
    }
    delete it;
    return result;
}
220

221 222 223 224 225 226 227 228 229 230 231 232 233 234
std::vector<ds_bulk_t> LevelDBDataStore::vlist_key_range(
        const ds_bulk_t &lower_bound, const ds_bulk_t &upper_bound, size_t max_keys) const {
    std::vector<ds_bulk_t> result;
    // TODO implement this function
    throw SDSKV_OP_NOT_IMPL;
    return result;
}

std::vector<std::pair<ds_bulk_t,ds_bulk_t>> LevelDBDataStore::vlist_keyval_range(
        const ds_bulk_t &lower_bound, const ds_bulk_t &upper_bound, size_t max_keys) const {
    std::vector<std::pair<ds_bulk_t,ds_bulk_t>> result;
    // TODO implement this function
    throw SDSKV_OP_NOT_IMPL;
    return result;
235
}