Commit 9fb2d579 authored by Matthieu Dorier's avatar Matthieu Dorier

Merge branch 'dev-sdskv-integration' into 'master'

Dev sdskv integration

See merge request !4
parents 6f285fdd 935e2a52
......@@ -194,7 +194,7 @@ extern "C" {
void mobject_write_op_zero(
mobject_store_write_op_t write_op,
uint64_t offset,
uint64_t len);
size_t len);
/**
* Set key/value pairs on an object
......
......@@ -9,6 +9,7 @@
#include <margo.h>
#include <bake-client.h>
#include <sdskv-client.h>
/* server-side utilities and routines. Clients are looking for either
* libmobject-store.h or librados-mobject-store.h */
......@@ -24,6 +25,7 @@ typedef struct mobject_server_context* mobject_provider_t;
* @param[in] mplex_id multiplex id of the provider
* @param[in] pool Argobots pool for the provider
* @param[in] bake_ph Bake provider handle to use to write/read data
* @param[in] sdskv_ph SDSKV provider handle to use to access metadata
* @param[in] cluster_file file name to write cluster connect info to
* @param[out] provider resulting provider
*
......@@ -34,6 +36,7 @@ int mobject_provider_register(
uint8_t mplex_id,
ABT_pool pool,
bake_provider_handle_t bake_ph,
sdskv_provider_handle_t sdskv_ph,
const char *cluster_file,
mobject_provider_t* provider);
......
......@@ -66,8 +66,7 @@ src_server_libmobject_server_la_SOURCES = \
src/server/core/core-write-op.cpp \
src/server/core/core-read-op.cpp \
src/server/printer/print-write-op.c \
src/server/printer/print-read-op.c \
src/server/core/fake-kv.cpp
src/server/printer/print-read-op.c
src_server_libmobject_server_la_CPPFLAGS = ${AM_CPPFLAGS} ${SERVER_CPPFLAGS}
src_server_libmobject_server_la_CFLAGS = ${AM_CFLAGS} ${SERVER_CFLAGS}
src_server_libmobject_server_la_LIBADD = src/omap-iter/libomap-iter.la \
......
......@@ -386,7 +386,8 @@ int mobject_store_read_op_operate(mobject_store_read_op_t read_op,
if(r != 0) return r;
r = mobject_read_op_operate(mph,read_op, ioctx->pool_name, oid, flags);
return 0;
mobject_provider_handle_release(mph);
return r;
}
// send a shutdown signal to a server cluster
......
......@@ -42,8 +42,8 @@ void mobject_write_op_create(mobject_store_write_op_t write_op,
void mobject_write_op_write(mobject_store_write_op_t write_op,
const char *buffer,
size_t len,
uint64_t offset)
uint64_t offset,
size_t len)
{
MOBJECT_ASSERT(write_op != MOBJECT_WRITE_OP_NULL, "invalid mobject_store_write_op_t obect");
MOBJECT_ASSERT(!(write_op->ready), "can't modify a write_op that is ready to be processed");
......@@ -53,7 +53,7 @@ void mobject_write_op_write(mobject_store_write_op_t write_op,
action->buffer.as_pointer = buffer;
action->len = len;
action->offset = offset;
WRITE_ACTION_UPCAST(base, action);
DL_APPEND(write_op->actions, base);
......@@ -98,9 +98,9 @@ void mobject_write_op_write_full(mobject_store_write_op_t write_op,
void mobject_write_op_write_same(mobject_store_write_op_t write_op,
const char *buffer,
uint64_t offset,
size_t data_len,
size_t write_len,
uint64_t offset)
size_t write_len)
{
MOBJECT_ASSERT(write_op != MOBJECT_WRITE_OP_NULL, "invalid mobject_store_write_op_t obect");
MOBJECT_ASSERT(!(write_op->ready), "can't modify a write_op that is ready to be processed");
......@@ -186,7 +186,7 @@ void mobject_write_op_truncate(mobject_store_write_op_t write_op,
void mobject_write_op_zero(mobject_store_write_op_t write_op,
uint64_t offset,
uint64_t len)
size_t len)
{
MOBJECT_ASSERT(write_op != MOBJECT_WRITE_OP_NULL, "invalid mobject_store_write_op_t obect");
MOBJECT_ASSERT(!(write_op->ready), "can't modify a write_op that is ready to be processed");
......
This diff is collapsed.
This diff is collapsed.
#include "src/server/core/fake-kv.hpp"
bool operator<(const segment_key_t& s1, const segment_key_t& s2) {
// sort by oid first
if(s1.oid != s2.oid)
return s1.oid < s2.oid;
return s1.timestamp > s2.timestamp;
}
bool operator<(const omap_key_t& k1, const omap_key_t& k2) {
if(k1.oid != k2.oid)
return k1.oid < k2.oid;
return k1.key < k2.key;
}
std::map<oid_t, std::string> oid_map;
std::map<std::string, oid_t> name_map;
std::map<segment_key_t, bake_region_id_t> segment_map;
std::map<omap_key_t, std::vector<char>> omap_map;
#include <map>
#include <vector>
#include <string>
#ifndef __CORE_KEY_TYPES_H
#define __CORE_KEY_TYPES_H
#include <stdint.h>
#include <bake-client.h>
typedef uint64_t oid_t;
enum class seg_type_t : std::int32_t {
typedef enum seg_type_t {
ZERO = 0,
BAKE_REGION = 1,
SMALL_REGION = 2,
TOMBSTONE = 3
};
} seg_type_t;
struct segment_key_t {
typedef struct segment_key_t {
oid_t oid;
seg_type_t type;
uint32_t type; /* seg_type */
double timestamp;
uint64_t start_index; // first index, included
uint64_t end_index; // end index is not included
};
} segment_key_t;
struct omap_key_t {
typedef struct omap_key_t {
oid_t oid;
std::string key;
};
#define SMALL_REGION_THRESHOLD (sizeof(bake_region_id_t))
char key[1];
} omap_key_t;
bool operator<(const segment_key_t& s1, const segment_key_t& s2);
bool operator<(const omap_key_t& k1, const omap_key_t& k2);
#define MAX_OMAP_KEY_SIZE 128
#define MAX_OMAP_VAL_SIZE 256
extern std::map<oid_t, std::string> oid_map;
extern std::map<std::string, oid_t> name_map;
extern std::map<segment_key_t, bake_region_id_t> segment_map;
extern std::map<omap_key_t, std::vector<char>> omap_map;
//extern std::map<oid_t, std::size_t> size_map;
#define SMALL_REGION_THRESHOLD (sizeof(bake_region_id_t))
#endif
......@@ -8,8 +8,8 @@
#include <margo.h>
//#include <sds-keyval.h>
#include <bake-server.h>
#include <bake-client.h>
#include <sdskv-client.h>
#include <ssg-mpi.h>
#ifdef __cplusplus
......@@ -27,6 +27,12 @@ struct mobject_server_context
/* bake-related data */
bake_provider_handle_t bake_ph;
bake_target_id_t bake_tid;
/* sdskv-related data */
sdskv_provider_handle_t sdskv_ph;
sdskv_database_id_t oid_db_id;
sdskv_database_id_t name_db_id;
sdskv_database_id_t segment_db_id;
sdskv_database_id_t omap_db_id;
/* other data */
int ref_count;
};
......
......@@ -9,8 +9,11 @@
#include <ssg.h>
#include <bake-client.h>
#include <bake-server.h>
#include <sdskv-client.h>
#include <sdskv-server.h>
#include "mobject-server.h"
#include "src/server/core/key-types.h"
void usage(void)
{
......@@ -25,7 +28,19 @@ typedef struct {
bake_provider_handle_t provider_handle;
} bake_client_data;
typedef struct {
sdskv_client_t client;
sdskv_provider_handle_t provider_handle;
} sdskv_client_data;
static void finalize_bake_client_cb(void* data);
static void finalize_sdskv_client_cb(void* data);
/* comparison functions for SDSKV */
static int oid_map_compare(const void*, size_t, const void*, size_t);
static int name_map_compare(const void*, size_t, const void*, size_t);
static int seg_map_compare(const void*, size_t, const void*, size_t);
static int omap_map_compare(const void*, size_t, const void*, size_t);
int main(int argc, char *argv[])
{
......@@ -60,6 +75,8 @@ int main(int argc, char *argv[])
return -1;
}
/* TODO assert the return value of all the calls below */
/* Get self address */
hg_addr_t self_addr;
margo_addr_self(mid, &self_addr);
......@@ -72,7 +89,6 @@ int main(int argc, char *argv[])
bake_target_id_t bake_tid;
bake_provider_register(mid, bake_mplex_id, BAKE_ABT_POOL_DEFAULT, &bake_prov);
bake_provider_add_storage_target(bake_prov, bake_target_name, &bake_tid);
/* TODO check return value of above calls */
/* Bake provider handle initialization from self addr */
bake_client_data bake_clt_data;
......@@ -80,10 +96,30 @@ int main(int argc, char *argv[])
bake_provider_handle_create(bake_clt_data.client, self_addr, bake_mplex_id, &(bake_clt_data.provider_handle));
margo_push_finalize_callback(mid, &finalize_bake_client_cb, (void*)&bake_clt_data);
/* SDSKV provider initialization */
uint8_t sdskv_mplex_id = 1;
sdskv_provider_t sdskv_prov;
sdskv_database_id_t oid_map_id, name_map_id, seg_map_id, omap_map_id;
sdskv_provider_register(mid, sdskv_mplex_id, SDSKV_ABT_POOL_DEFAULT, &sdskv_prov);
sdskv_provider_add_database(sdskv_prov, "oid_map", KVDB_MAP, &oid_map_compare, &oid_map_id);
sdskv_provider_add_database(sdskv_prov, "name_map", KVDB_MAP, &name_map_compare, &name_map_id);
sdskv_provider_add_database(sdskv_prov, "seg_map", KVDB_MAP, &seg_map_compare, &seg_map_id);
sdskv_provider_add_database(sdskv_prov, "omap_map", KVDB_MAP, &omap_map_compare, &omap_map_id);
/* SDSKV provider handle initialization from self addr */
sdskv_client_data sdskv_clt_data;
sdskv_client_init(mid, &(sdskv_clt_data.client));
sdskv_provider_handle_create(sdskv_clt_data.client, self_addr, sdskv_mplex_id, &(sdskv_clt_data.provider_handle));
margo_push_finalize_callback(mid, &finalize_sdskv_client_cb, (void*)&sdskv_clt_data);
/* Mobject provider initialization */
mobject_provider_t mobject_prov;
ret = mobject_provider_register(mid, 1, MOBJECT_ABT_POOL_DEFAULT,
bake_clt_data.provider_handle, cluster_file, &mobject_prov);
ret = mobject_provider_register(mid, 1,
MOBJECT_ABT_POOL_DEFAULT,
bake_clt_data.provider_handle,
sdskv_clt_data.provider_handle,
cluster_file, &mobject_prov);
if (ret != 0)
{
fprintf(stderr, "Error: Unable to initialize mobject provider\n");
......@@ -91,6 +127,8 @@ int main(int argc, char *argv[])
return -1;
}
margo_addr_free(mid,self_addr);
margo_wait_for_finalize(mid);
MPI_Finalize();
......@@ -104,3 +142,63 @@ static void finalize_bake_client_cb(void* data)
bake_provider_handle_release(clt_data->provider_handle);
bake_client_finalize(clt_data->client);
}
static void finalize_sdskv_client_cb(void* data)
{
sdskv_client_data* clt_data = (sdskv_client_data*)data;
sdskv_provider_handle_release(clt_data->provider_handle);
sdskv_client_finalize(clt_data->client);
}
static int oid_map_compare(const void* k1, size_t sk1, const void* k2, size_t sk2)
{
// keys are oid_t (uint64_t)
oid_t x = *((oid_t*)k1);
oid_t y = *((oid_t*)k2);
if(x == y) return 0;
if(x < y) return -1;
return 1;
}
static int name_map_compare(const void* k1, size_t sk1, const void* k2, size_t sk2)
{
// names are strings (const char*)
const char* n1 = (const char*)k1;
const char* n2 = (const char*)k2;
return strcmp(n1,n2);
}
static int seg_map_compare(const void* k1, size_t sk1, const void* k2, size_t sk2)
{
// segments are as follows:
/*struct segment_key_t {
oid_t oid;
seg_type_t type;
double timestamp;
uint64_t start_index;
uint64_t end_index;
};
*/
const segment_key_t* seg1 = (const segment_key_t*)k1;
const segment_key_t* seg2 = (const segment_key_t*)k2;
if(seg1->oid < seg2->oid) return -1;
if(seg1->oid > seg2->oid) return 1;
if(seg1->timestamp > seg2->timestamp) return -1;
if(seg1->timestamp < seg2->timestamp) return 1;
return 0;
}
static int omap_map_compare(const void* k1, size_t sk1, const void* k2, size_t sk2)
{
// omap keys are as follows:
/* struct omap_key_t {
oid_t oid;
char key[1];
};
*/
const omap_key_t* ok1 = (const omap_key_t*)k1;
const omap_key_t* ok2 = (const omap_key_t*)k2;
if(ok1->oid < ok2->oid) return -1;
if(ok1->oid > ok2->oid) return 1;
return strcmp(ok1->key, ok2->key);
}
......@@ -10,9 +10,6 @@
#include <mpi.h>
#include <abt.h>
#include <margo.h>
//#include <sds-keyval.h>
#include <bake-server.h>
#include <bake-client.h>
#include <ssg-mpi.h>
#include "mobject-server.h"
......@@ -42,6 +39,7 @@ int mobject_provider_register(
uint8_t mplex_id,
ABT_pool pool,
bake_provider_handle_t bake_ph,
sdskv_provider_handle_t sdskv_ph,
const char *cluster_file,
mobject_provider_t* provider)
{
......@@ -113,6 +111,41 @@ int mobject_provider_register(
free(srv_ctx);
return -1;
}
/* SDSKV settings initialization */
sdskv_provider_handle_ref_incr(sdskv_ph);
srv_ctx->sdskv_ph = sdskv_ph;
ret = sdskv_open(sdskv_ph, "oid_map", &(srv_ctx->oid_db_id));
if(ret != SDSKV_SUCCESS) {
fprintf(stderr, "Error: unable to open oid_map from SDSKV provider\n");
ssg_group_destroy(srv_ctx->gid);
bake_provider_handle_release(srv_ctx->bake_ph);
sdskv_provider_handle_release(srv_ctx->sdskv_ph);
free(srv_ctx);
}
ret = sdskv_open(sdskv_ph, "name_map", &(srv_ctx->name_db_id));
if(ret != SDSKV_SUCCESS) {
fprintf(stderr, "Error: unable to open name_map from SDSKV provider\n");
bake_provider_handle_release(srv_ctx->bake_ph);
sdskv_provider_handle_release(srv_ctx->sdskv_ph);
ssg_group_destroy(srv_ctx->gid);
free(srv_ctx);
}
ret = sdskv_open(sdskv_ph, "seg_map", &(srv_ctx->segment_db_id));
if(ret != SDSKV_SUCCESS) {
fprintf(stderr, "Error: unable to open seg_map from SDSKV provider\n");
bake_provider_handle_release(srv_ctx->bake_ph);
sdskv_provider_handle_release(srv_ctx->sdskv_ph);
ssg_group_destroy(srv_ctx->gid);
free(srv_ctx);
}
ret = sdskv_open(sdskv_ph, "omap_map", &(srv_ctx->omap_db_id));
if(ret != SDSKV_SUCCESS) {
fprintf(stderr, "Error: unable to open omap_map from SDSKV provider\n");
bake_provider_handle_release(srv_ctx->bake_ph);
sdskv_provider_handle_release(srv_ctx->sdskv_ph);
ssg_group_destroy(srv_ctx->gid);
free(srv_ctx);
}
hg_id_t rpc_id;
......@@ -149,6 +182,7 @@ static hg_return_t mobject_write_op_ult(hg_handle_t h)
server_visitor_args vargs;
vargs.object_name = in.object_name;
vargs.oid = 0;
vargs.pool_name = in.pool_name;
vargs.srv_ctx = margo_registered_data_mplex(mid, info->id, info->target_id);
if(vargs.srv_ctx == NULL) return HG_OTHER_ERROR;
......@@ -201,6 +235,7 @@ static hg_return_t mobject_read_op_ult(hg_handle_t h)
server_visitor_args vargs;
vargs.object_name = in.object_name;
vargs.oid = 0;
vargs.pool_name = in.pool_name;
vargs.srv_ctx = margo_registered_data_mplex(mid, info->id, info->target_id);
if(vargs.srv_ctx == NULL) return HG_OTHER_ERROR;
......@@ -239,6 +274,7 @@ static void mobject_finalize_cb(void* data)
{
mobject_provider_t srv_ctx = (mobject_provider_t)data;
sdskv_provider_handle_release(srv_ctx->sdskv_ph);
bake_provider_handle_release(srv_ctx->bake_ph);
ssg_group_destroy(srv_ctx->gid);
......
......@@ -3,6 +3,7 @@
#include <margo.h>
#include "libmobject-store.h"
#include "src/server/core/key-types.h"
#include "src/server/mobject-server-context.h"
#ifdef __cplusplus
......@@ -11,6 +12,7 @@ extern "C" {
typedef struct {
const char* object_name;
oid_t oid;
const char* pool_name;
struct mobject_server_context* srv_ctx;
const char* client_addr_str;
......
......@@ -14,6 +14,7 @@ int main(int argc, char** argv)
mobject_store_ioctx_t ioctx;
mobject_store_ioctx_create(cluster, "my-object-pool", &ioctx);
fprintf(stderr, "********** WRITE PHASE **********\n");
{ // WRITE OP TEST
mobject_store_write_op_t write_op = mobject_store_create_write_op();
......@@ -22,23 +23,34 @@ int main(int argc, char** argv)
mobject_store_write_op_create(write_op, LIBMOBJECT_CREATE_EXCLUSIVE, NULL);
// Add a "write_full" operation to write "AAAABBBB"
mobject_store_write_op_write_full(write_op, content, 8);
// Add a "write" operation to write "CCCC"
// Add a "write" operation to write "CCCC", content should then be "AAAABBBBCCCC"
mobject_store_write_op_write(write_op, content+8, 4, 8);
// Add a "writesame" operation to write "DDDD" in two "DD"
// Add a "writesame" operation to write "DDDD" in two "DD",
// content should then be "AAAABBBBCCCCDDDD"
mobject_store_write_op_writesame(write_op, content+12, 2, 4, 12);
// Add a "append" operation to append "EEEEFFFF"
// content should then be "AAAABBBBCCCCDDDDEEEEFFFFF"
mobject_store_write_op_append(write_op, content+16, 8);
// Add a "remove" operation
// mobject_store_write_op_remove(write_op);
// Add a "truncate" operation to remove the "FFFF" part
// content should then be "AAAABBBBCCCCDDDDEEEE"
mobject_store_write_op_truncate(write_op, 20);
// Add a "zero" operation zero-ing the "BBBBCCCC"
// content should then be "AAAA********DDDDEEEE" where "*" represent 0s
mobject_store_write_op_zero(write_op, 4, 8);
// Add a "omap_set" operation
const char* keys[] = { "matthieu", "rob", "shane", "phil", "robl" };
const char* values[] = { "mdorier@anl.gov", "rross@anl.gov", "ssnyder@anl.gov", "carns@anl.gov", "robl@anl.gov" };
size_t val_sizes[] = { 16, 14, 16, 14, 13 };
mobject_store_write_op_omap_set(write_op, keys, values, val_sizes, 5);
// keys will be sorted and stored as follows:
/* matthieu => mdorier@anl.gov
phil => carns@anl.gov
rob => rross@anl.gov
robl => robl@anl.gov
shane => ssnyder@anl.gov
*/
// Add a omap_rm_keys" operation
// mobject_store_write_op_omap_rm_keys(write_op, keys, 5);
......@@ -48,6 +60,7 @@ int main(int argc, char** argv)
}
fprintf(stderr, "********** READ PHASE **********\n");
{ // READ OP TEST
mobject_store_read_op_t read_op = mobject_store_create_read_op();
......@@ -66,12 +79,14 @@ int main(int argc, char** argv)
const char* start_after1 = "rob";
mobject_store_omap_iter_t iter3;
int prval3;
// the following should return ["robl","shane"]
mobject_store_read_op_omap_get_keys(read_op, start_after1, 7, &iter3, &prval3);
// Add "omap_get_vals" operation
const char* start_after2 = "matthieu";
const char* filter_prefix2 = "p";
mobject_store_omap_iter_t iter4;
int prval4;
// the following should return ["phil"], and its associated value
mobject_store_read_op_omap_get_vals(read_op, start_after2, filter_prefix2, 3, &iter4, &prval4);
// Add "omap_get_vals_by_keys" operation
const char* keys[] = {"matthieu", "robl"};
......@@ -87,6 +102,7 @@ int main(int argc, char** argv)
printf("Client received the following results:\n");
printf("stat: psize=%ld pmtime=%lld prval=%d\n", psize, (long long)pmtime, prval1);
{
printf("read: bytes_read = %ld, prval=%d content: ", bytes_read, prval2);
unsigned i;
......@@ -124,7 +140,6 @@ int main(int argc, char** argv)
} while(key);
}
}
mobject_store_ioctx_destroy(ioctx);
mobject_store_shutdown(cluster);
......
......@@ -20,8 +20,12 @@ function mobject_test_start_servers()
startwait=${2:-15}
maxtime=${3:-120}
cfile=${4:-/tmp/mobject-connect-cluster.gid}
storage=${5:-/dev/shm/mobject.dat}
run_to $maxtime mpirun -np $nservers src/server/mobject-server-daemon na+sm:// $cfile &
rm -rf ${storage}
bake-mkpool -s 50M /dev/shm/mobject.dat
run_to $maxtime mpirun -np $nservers src/server/mobject-server-daemon tcp:// $cfile &
if [ $? -ne 0 ]; then
# TODO: this doesn't actually work; can't check return code of
# something executing in background. We have to rely on the
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment