Commit 051883a2 authored by Rob Latham's avatar Rob Latham
Browse files

rename from 'mochio' to 'benvolio'

Benvolio was Romeo's (ROMIO) friend.  Abbreviated 'bv'
parent 173e719c
......@@ -8,8 +8,8 @@ configure
compile
install-sh
missing
include/mochio-config.h.in
include/mochio-config.h.in~
include/bv-config.h.in
include/bv-config.h.in~
config.guess
config.sub
cscope.files
......
......@@ -3,22 +3,22 @@ ACLOCAL_AMFLAGS="-Im4"
AM_CPPFLAGS = -I$(top_srcdir)/include
bin_PROGRAMS = src/provider/mochio-server
bin_PROGRAMS = src/provider/bv-server
src_provider_mochio_server_SOURCES = src/provider/server.c
src_provider_mochio_server_LDADD = lib/libmochio-provider.la
src_provider_bv_server_SOURCES = src/provider/server.c
src_provider_bv_server_LDADD = lib/libbv-provider.la
lib_LTLIBRARIES = lib/libmochio-client.la \
lib/libmochio-provider.la
lib_LTLIBRARIES = lib/libbv-client.la \
lib/libbv-provider.la
lib_libmochio_client_la_SOURCES = src/client/mochio-client.cc \
lib_libbv_client_la_SOURCES = src/client/bv-client.cc \
src/client/calc-request.cc
lib_libmochio_provider_la_SOURCES = src/provider/mochio-provider.cc \
lib_libbv_provider_la_SOURCES = src/provider/bv-provider.cc \
src/provider/lustre-utils.c
include_HEADERS = include/mochio.h \
include/mochio-provider.h
include_HEADERS = include/bv.h \
include/bv-provider.h
noinst_HEADERS = include/io_stats.h \
include/access.h \
......@@ -28,8 +28,8 @@ noinst_HEADERS = include/io_stats.h \
include/lustre-utils.h
pkgconfigdir = $(libdir)/pkgconfig
pkgconfig_DATA = maint/mochio-provider.pc \
maint/mochio-client.pc
pkgconfig_DATA = maint/bv-provider.pc \
maint/bv-client.pc
SUFFIXES=
LDADD=
......
AC_PREREQ([2.59])
AC_INIT([mochio], [0.0], [robl@mcs.anl.gov])
AC_INIT([benvolio], [0.0], [robl@mcs.anl.gov])
AM_INIT_AUTOMAKE([-Wall -Werror foreign subdir-objects silent-rules])
AM_SILENT_RULES([yes])
AC_CONFIG_MACRO_DIR([m4])
AC_CONFIG_SRCDIR([src/client/mochio-client.cc])
AC_CONFIG_HEADERS([include/mochio-config.h])
AC_CONFIG_SRCDIR([src/client/bv-client.cc])
AC_CONFIG_HEADERS([include/bv-config.h])
#
# checks for programs
......@@ -66,5 +66,5 @@ AC_CHECK_HEADERS([lustre/lustreapi.h])
# checks for library functions
AC_CONFIG_FILES([Makefile maint/mochio-client.pc maint/mochio-provider.pc])
AC_CONFIG_FILES([Makefile maint/bv-client.pc maint/bv-provider.pc])
AC_OUTPUT
......@@ -24,14 +24,14 @@ server and another starts a client, it does mean that the protection domains
can end up hanging around longer than expected. You might get an error like
this:
apmgr pdomain create mochio-test failed: cannot allocate protection domain; reached limit of 10
apmgr pdomain create bv-test failed: cannot allocate protection domain; reached limit of 10
No way around that except to contact the support desk and ask for help cleaning up old ones.
## Building MPICH
Like any other external library, the 'mochio' driver (
https://github.com/roblatham00/mpich/tree/mochio
Like any other external library, the 'bv' driver (
https://github.com/roblatham00/mpich/tree/bv
) requires CPPFLAGS, LDFLAGS, and LIBS to be set. Furthermore, builidng on
the Cray requires a few extra steps, documented here:
https://wiki.mpich.org/mpich/index.php/Cray
......
# MOCHIO and MPICH
# Benvolio and MPICH
- check out https://github.com/roblatham00/mpich/tree/mochio
- check out https://github.com/roblatham00/mpich/tree/benvolio
- set CPPFLAGS and LDFLAGS to point to mochio installation
- set CPPFLAGS and LDFLAGS to point to benvolio installation
- add 'mochio' to the `--with-file-system` list: e.g. --with-file-system=lustre+mochio
- add 'bv' to the `--with-file-system` list: e.g. --with-file-system=lustre+bv
- Start however many MOCHIO targets you want.
- Start however many Benvolio targets you want.
- set the environemnt variable MOCHIO_STATEFILE to point to the server statefile.
- set the environemnt variable BV_STATEFILE to point to the server statefile.
- if you want extra information about mochio behavior, set the MOCHIO_SHOW_STATS
- if you want extra information about benvolio behavior, set the BV_SHOW_STATS
......@@ -5,12 +5,12 @@ extern "C"
{
#endif
#define MOCHIO_PROVIDER_GROUP_NAME "mochio-provider-group"
#define BV_PROVIDER_GROUP_NAME "bv-provider-group"
typedef struct mochio_svc_provider * mochio_svc_provider_t;
typedef struct bv_svc_provider * bv_svc_provider_t;
int mochio_svc_provider_register(margo_instance_id mid,
abt_io_instance_id abtio, ABT_pool pool, ssg_group_id_t gid, mochio_svc_provider_t *mochio_id);
int bv_svc_provider_register(margo_instance_id mid,
abt_io_instance_id abtio, ABT_pool pool, ssg_group_id_t gid, bv_svc_provider_t *bv_id);
#ifdef __cplusplus
}
......
#ifndef mochio_SVC_H
#define mochio_SVC_H
#ifndef BV_SVC_H
#define BV_SVC_H
#include <stdint.h>
#include <unistd.h>
......@@ -13,21 +13,21 @@ extern "C"
typedef struct mochio_client * mochio_client_t;
typedef struct bv_client * bv_client_t;
/* easy to imagine more sophisticated distribution schemes
* - even: hint a final file size and then divide that size across the N servers
* - block: first N bytes to server 0, next N bytes to server 1, etc
* - progressive: Lustre's Progressive File Layout strategy */
int mochio_setchunk(const char *file, ssize_t nbytes);
int bv_setchunk(const char *file, ssize_t nbytes);
/* "init" might be a place to pass in distribution information too? */
mochio_client_t mochio_init(MPI_Comm comm, const char * ssg_statefile);
int mochio_finalize(mochio_client_t client);
bv_client_t bv_init(MPI_Comm comm, const char * ssg_statefile);
int bv_finalize(bv_client_t client);
/* stateless api: always pass in a file name? */
ssize_t mochio_write(mochio_client_t client,
ssize_t bv_write(bv_client_t client,
const char *file,
const int64_t mem_count,
const char *mem_addresses[],
......@@ -36,7 +36,7 @@ ssize_t mochio_write(mochio_client_t client,
const off_t file_starts[],
const uint64_t file_sizes[]);
ssize_t mochio_read (mochio_client_t client,
ssize_t bv_read (bv_client_t client,
const char *file,
const int64_t mem_count,
const char *mem_addresses[],
......@@ -56,25 +56,25 @@ ssize_t mochio_read (mochio_client_t client,
* - queue depth
* ...
*/
struct mochio_stats {
struct bv_stats {
ssize_t blocksize;
int32_t stripe_size;
int32_t stripe_count;
};
int mochio_stat(mochio_client_t client, const char *filename, struct mochio_stats *stats);
int bv_stat(bv_client_t client, const char *filename, struct bv_stats *stats);
/**
* if `show_server` set, statistics will also include information from every remote target
* Use case: at end of MPI job, every process will want to show client statistis
* but only one process will want to show the server information
*/
int mochio_statistics(mochio_client_t client, int show_server);
int bv_statistics(bv_client_t client, int show_server);
/* flush: request all cached data written to disk */
int mochio_flush(mochio_client_t client, const char *filename);
int bv_flush(bv_client_t client, const char *filename);
/* delete: remove the file */
int mochio_delete(mochio_client_t client, const char *filename);
int bv_delete(bv_client_t client, const char *filename);
#ifdef __cplusplus
}
#endif
......
......@@ -36,10 +36,10 @@ class io_stats {
int64_t bytes_read; // bytes read from storage
double mutex_time; // time spent acquiring mutexes
/* client things */
int client_write_calls; // number of times "mochio_write" called
double client_write_time; // time client spent in "mochio_write",
int client_read_calls; // number of tiems "mochio_read" called
double client_read_time; // time client spent in "mochio_read
int client_write_calls; // number of times "bv_write" called
double client_write_time; // time client spent in "bv_write",
int client_read_calls; // number of tiems "bv_read" called
double client_read_time; // time client spent in "bv_read
double client_init_time; // how long does it take to set everything up
io_stats & operator = (const io_stats &rhs);
......
......@@ -5,8 +5,8 @@ includedir=@includedir@
Name: MOCHIO
Description: A mochiservice for I/O: client-side library
URL: https://xgitlab.cels.anl.gov/sds/mochio
URL: https://xgitlab.cels.anl.gov/sds/benvolio
Version: 0.0.1
Requires: thallium abt-io margo
Libs: -L${libdir} -lmochio-client
Libs: -L${libdir} -lbv-client
Cflags: -I${includedir}
......@@ -3,10 +3,10 @@ exec_prefix=@exec_prefix@
libdir=@libdir@
includedir=@includedir@
Name: MOCHIO
Name: bv
Description: A mochi service for I/O: library for provider-side
URL: https://xgitlab.cels.anl.gov/sds/mochio
URL: https://xgitlab.cels.anl.gov/sds/benvolio
Version: 0.0.1
Requires: thallium abt-io margo
Libs: -L${libdir} -lmochio-provider
Libs: -L${libdir} -lbv-provider
Cflags: -I${includedir}
#include <mochio.h>
#include <bv.h>
#include <thallium.hpp>
#include <utility>
#include <vector>
......@@ -16,7 +16,7 @@ namespace tl = thallium;
#define MAX_PROTO_LEN 24
struct mochio_client {
struct bv_client {
tl::engine *engine;
std::vector<tl::provider_handle> targets;
char proto[MAX_PROTO_LEN];
......@@ -37,11 +37,11 @@ struct mochio_client {
};
typedef enum {
MOCHIO_READ,
MOCHIO_WRITE
bv_READ,
bv_WRITE
} io_kind;
static int set_proto_from_addr(mochio_client_t client, char *addr_str)
static int set_proto_from_addr(bv_client_t client, char *addr_str)
{
int i;
for (i=0; i< MAX_PROTO_LEN; i++) {
......@@ -54,12 +54,12 @@ static int set_proto_from_addr(mochio_client_t client, char *addr_str)
if (client->proto[i] != '\0') return -1;
return 0;
}
mochio_client_t mochio_init(MPI_Comm comm, const char * cfg_file)
bv_client_t bv_init(MPI_Comm comm, const char * cfg_file)
{
char *addr_str;
int rank;
int ret, i, nr_targets;
struct mochio_client * client = (struct mochio_client *)calloc(1,sizeof(*client));
struct bv_client * client = (struct bv_client *)calloc(1,sizeof(*client));
char *ssg_group_buf;
double init_time = ABT_get_wtime();
MPI_Comm dupcomm;
......@@ -123,16 +123,16 @@ mochio_client_t mochio_init(MPI_Comm comm, const char * cfg_file)
/* need to patch up the API i think: we don't know what servers to talk to. Or
* do you talk to one and then that provider informs the others? */
int mochio_setchunk(const char *file, ssize_t nbytes)
int bv_setchunk(const char *file, ssize_t nbytes)
{
return 0;
}
int mochio_delete(mochio_client_t client, const char *file)
int bv_delete(bv_client_t client, const char *file)
{
return client->delete_op.on(client->targets[0])(std::string(file) );
}
int mochio_finalize(mochio_client_t client)
int bv_finalize(bv_client_t client)
{
ssg_group_detach(client->gid);
ssg_finalize();
......@@ -149,7 +149,7 @@ int mochio_finalize(mochio_client_t client)
// -- possibly splitting up anything that falls on a server boundary
// - are the file lists monotonically non-decreasing? Any benefit if we relax that requirement?
static size_t mochio_io(mochio_client_t client, const char *filename, io_kind op,
static size_t bv_io(bv_client_t client, const char *filename, io_kind op,
const int64_t mem_count, const char *mem_addresses[], const uint64_t mem_sizes[],
const int64_t file_count, const off_t file_starts[], const uint64_t file_sizes[])
{
......@@ -157,7 +157,7 @@ static size_t mochio_io(mochio_client_t client, const char *filename, io_kind op
size_t bytes_moved = 0;
/* How expensive is this? do we need to move this out of the I/O path?
* Maybe 'mochio_stat' can cache these values on the client struct? */
* Maybe 'bv_stat' can cache these values on the client struct? */
client->targets_used = client->targets.size();
compute_striping_info(client->stripe_size, client->stripe_count, &client->targets_used, 1);
......@@ -174,7 +174,7 @@ static size_t mochio_io(mochio_client_t client, const char *filename, io_kind op
auto mode = tl::bulk_mode::read_only;
auto rpc = client->write_op;
if (op == MOCHIO_READ) {
if (op == bv_READ) {
mode = tl::bulk_mode::write_only;
rpc = client->read_op;
}
......@@ -196,7 +196,7 @@ static size_t mochio_io(mochio_client_t client, const char *filename, io_kind op
return bytes_moved;
}
ssize_t mochio_write(mochio_client_t client, const char *filename,
ssize_t bv_write(bv_client_t client, const char *filename,
const int64_t mem_count, const char * mem_addresses[], const uint64_t mem_sizes[],
const int64_t file_count, const off_t file_starts[], const uint64_t file_sizes[])
{
......@@ -204,7 +204,7 @@ ssize_t mochio_write(mochio_client_t client, const char *filename,
double write_time = ABT_get_wtime();
client->statistics.client_write_calls++;
ret = mochio_io(client, filename, MOCHIO_WRITE, mem_count, mem_addresses, mem_sizes,
ret = bv_io(client, filename, bv_WRITE, mem_count, mem_addresses, mem_sizes,
file_count, file_starts, file_sizes);
write_time = ABT_get_wtime() - write_time;
......@@ -213,7 +213,7 @@ ssize_t mochio_write(mochio_client_t client, const char *filename,
return ret;
}
ssize_t mochio_read(mochio_client_t client, const char *filename,
ssize_t bv_read(bv_client_t client, const char *filename,
const int64_t mem_count, const char *mem_addresses[], const uint64_t mem_sizes[],
const int64_t file_count, const off_t file_starts[], const uint64_t file_sizes[])
{
......@@ -221,7 +221,7 @@ ssize_t mochio_read(mochio_client_t client, const char *filename,
double read_time = ABT_get_wtime();
client->statistics.client_read_calls++;
ret = mochio_io(client, filename, MOCHIO_READ, mem_count, mem_addresses, mem_sizes,
ret = bv_io(client, filename, bv_READ, mem_count, mem_addresses, mem_sizes,
file_count, file_starts, file_sizes);
read_time = ABT_get_wtime() - read_time;
......@@ -230,7 +230,7 @@ ssize_t mochio_read(mochio_client_t client, const char *filename,
return ret;
}
int mochio_stat(mochio_client_t client, const char *filename, struct mochio_stats *stats)
int bv_stat(bv_client_t client, const char *filename, struct bv_stats *stats)
{
struct file_stats response = client->stat_op.on(client->targets[0])(std::string(filename));
......@@ -245,7 +245,7 @@ int mochio_stat(mochio_client_t client, const char *filename, struct mochio_stat
return(1);
}
int mochio_statistics(mochio_client_t client, int show_server)
int bv_statistics(bv_client_t client, int show_server)
{
int ret =0;
if (show_server) {
......@@ -260,7 +260,7 @@ int mochio_statistics(mochio_client_t client, int show_server)
return ret;
}
int mochio_flush(mochio_client_t client, const char *filename)
int bv_flush(bv_client_t client, const char *filename)
{
int ret=0;
for (auto target : client->targets)
......
......@@ -15,7 +15,7 @@
#include <thallium/serialization/stl/string.hpp>
#include <thallium/serialization/stl/vector.hpp>
#include "mochio-provider.h"
#include "bv-provider.h"
#include "common.h"
......@@ -29,7 +29,7 @@ namespace tl = thallium;
#define BUFSIZE 1024
struct mochio_svc_provider : public tl::provider<mochio_svc_provider>
struct bv_svc_provider : public tl::provider<bv_svc_provider>
{
tl::engine * engine;
ssg_group_id_t gid;
......@@ -293,16 +293,16 @@ struct mochio_svc_provider : public tl::provider<mochio_svc_provider>
return (fsync(fd));
}
mochio_svc_provider(tl::engine *e, abt_io_instance_id abtio,
bv_svc_provider(tl::engine *e, abt_io_instance_id abtio,
ssg_group_id_t gid, uint16_t provider_id, tl::pool &pool)
: tl::provider<mochio_svc_provider>(*e, provider_id), engine(e), gid(gid), pool(pool), abt_id(abtio) {
: tl::provider<bv_svc_provider>(*e, provider_id), engine(e), gid(gid), pool(pool), abt_id(abtio) {
define("write", &mochio_svc_provider::process_write, pool);
define("read", &mochio_svc_provider::process_read, pool);
define("stat", &mochio_svc_provider::getstats);
define("delete", &mochio_svc_provider::del);
define("flush", &mochio_svc_provider::flush);
define("statistics", &mochio_svc_provider::statistics);
define("write", &bv_svc_provider::process_write, pool);
define("read", &bv_svc_provider::process_read, pool);
define("stat", &bv_svc_provider::getstats);
define("delete", &bv_svc_provider::del);
define("flush", &bv_svc_provider::flush);
define("statistics", &bv_svc_provider::statistics);
}
void dump_io_req(const std::string extra, tl::bulk &client_bulk, std::vector<off_t> &file_starts, std::vector<uint64_t> &file_sizes)
......@@ -317,22 +317,22 @@ struct mochio_svc_provider : public tl::provider<mochio_svc_provider>
std::cout << std::endl;
}
~mochio_svc_provider() {
~bv_svc_provider() {
wait_for_finalize();
}
};
int mochio_svc_provider_register(margo_instance_id mid,
int bv_svc_provider_register(margo_instance_id mid,
abt_io_instance_id abtio,
ABT_pool pool,
ssg_group_id_t gid,
mochio_svc_provider_t *mochio_id)
bv_svc_provider_t *bv_id)
{
uint16_t provider_id = 0xABC;
auto thallium_engine = new tl::engine(mid);
auto thallium_pool = tl::pool(pool);
auto mochio_provider = new mochio_svc_provider(thallium_engine, abtio, gid, provider_id, thallium_pool);
*mochio_id = mochio_provider;
auto bv_provider = new bv_svc_provider(thallium_engine, abtio, gid, provider_id, thallium_pool);
*bv_id = bv_provider;
return 0;
}
#include "mochio-config.h"
#include "bv-config.h"
#include <stdlib.h>
#include <errno.h>
......
......@@ -2,7 +2,7 @@
#include <abt-io.h>
#include <ssg.h>
#include <ssg-mpi.h>
#include "mochio-provider.h"
#include "bv-provider.h"
#define ASSERT(__cond, __msg, ...) { if(!(__cond)) { fprintf(stderr, "[%s:%d] " __msg, __FILE__, __LINE__, __VA_ARGS__); exit(-1); } }
......@@ -37,7 +37,7 @@ int main(int argc, char **argv)
{
margo_instance_id mid;
abt_io_instance_id abtio;
mochio_svc_provider_t mochio_id;
bv_svc_provider_t bv_id;
int ret;
int rank;
ssg_group_id_t gid;
......@@ -56,14 +56,14 @@ int main(int argc, char **argv)
ret = ssg_init(mid);
ASSERT(ret == 0, "ssg_init() failed (ret = %d)\n", ret);
gid = ssg_group_create_mpi(MOCHIO_PROVIDER_GROUP_NAME, MPI_COMM_WORLD, NULL, NULL);
gid = ssg_group_create_mpi(BV_PROVIDER_GROUP_NAME, MPI_COMM_WORLD, NULL, NULL);
ASSERT(gid != SSG_GROUP_ID_NULL, "ssg_group_create_mpi() failed (ret = %s)","SSG_GROUP_ID_NULL");
margo_push_finalize_callback(mid, &finalized_ssg_group_cb, (void*)&gid);
if (rank == 0)
service_config_store(argv[2], gid);
ret = mochio_svc_provider_register(mid, abtio, ABT_POOL_NULL, gid, &mochio_id);
ret = bv_svc_provider_register(mid, abtio, ABT_POOL_NULL, gid, &bv_id);
margo_wait_for_finalize(mid);
margo_finalize(mid);
......
......@@ -10,7 +10,7 @@ SUFFIXES += .o .c .cpp
AM_DEFAULT_SOURCE_EXT = .c
AM_CPPFLAGS += -I$(top_srcdir)/include
LDADD += $(top_srcdir)/lib/libmochio-client.la
LDADD += $(top_builddir)/lib/libbv-client.la
TESTPROGRAMS = tests/simple \
tests/strided \
......@@ -19,7 +19,7 @@ TESTPROGRAMS = tests/simple \
check_PROGRAMS += ${TESTPROGRAMS}
EXTRA_DIST += tests/wrap_runs.sh
EXTRA_DIST += $(top_srcdir)/tests/wrap_runs.sh
OUT_FILES = $(check_PROGRAMS:%=%.bin)
SERVER_FILES = $(check_PROGRAMS:%=%.svc)
......@@ -27,7 +27,7 @@ SERVER_FILES = $(check_PROGRAMS:%=%.svc)
CLEANFILES += $(OUT_FILES) tests/core tests/core.* \
$(SERVER_FILES)
LOG_COMPILER = tests/wrap_runs.sh
LOG_COMPILER = $(top_srcdir)/tests/wrap_runs.sh
TESTS += ${TESTPROGRAMS}
......
#include <mochio.h>
#include <bv.h>
#include <mpi.h>
#include <stdlib.h>
/* what if we send a null buffer to the server? */
int main(int argc, char **argv)
{
mochio_client_t client=NULL;
bv_client_t client=NULL;
int bufcount = 32768, ret;
uint64_t size = bufcount * sizeof(int);
......@@ -22,11 +22,11 @@ int main(int argc, char **argv)
else
filename = "dummy";
client = mochio_init(MPI_COMM_WORLD, argv[1]);
client = bv_init(MPI_COMM_WORLD, argv[1]);
writebuf = NULL;
ret = mochio_write(client, filename, 1, (const char **)&writebuf, &size, 1, &offset, &size);
ret = bv_write(client, filename, 1, (const char **)&writebuf, &size, 1, &offset, &size);
MPI_Finalize();
return (ret != 0 ? -1 : 0);
......
......@@ -2,7 +2,7 @@
#include <mpi.h>
#include <stdio.h>
#include <stdlib.h>
#include <mochio.h>
#include <bv.h>
#include <mpi.h>
#define VERBOSE 1
......@@ -13,8 +13,8 @@ int main(int argc, char **argv) {
int ret = 0;
int i, j;
int rank, np;
mochio_client_t client=NULL;
struct mochio_stats stats;
bv_client_t client=NULL;
struct bv_stats stats;
const char *write_address, *read_address;
uint64_t write_size, read_size;
int buf[NWORD];
......@@ -26,7 +26,7 @@ int main(int argc, char **argv) {
#if 0
printf("delete:\n");
mochio_delete(client, filename);
bv_delete(client, filename);
#endif
MPI_Init(&argc, &argv);
......@@ -39,10 +39,10 @@ int main(int argc, char **argv) {
MPI_Comm_rank(MPI_COMM_WORLD, &rank);
MPI_Comm_size(MPI_COMM_WORLD, &np);
client = mochio_init(MPI_COMM_WORLD, argv[1]);
client = bv_init(MPI_COMM_WORLD, argv[1]);
printf("stat:");
mochio_stat(client, filename, &stats);
bv_stat(client, filename, &stats);
printf("got blocksize %ld stripe_count: %d stripe_size: %d from provider\n",
stats.blocksize, stats.stripe_count, stats.stripe_size);
......@@ -56,10 +56,10 @@ int main(int argc, char **argv) {
len = NWORD * sizeof(int);
printf("writing\n");
mochio_write(client, filename, 1, &write_address, &write_size, 1, &offset, &len);
bv_write(client, filename, 1, &write_address, &write_size, 1, &offset, &len);
printf("flushing\n");
mochio_flush(client, filename);
bv_flush(client, filename);
printf("init read\n");
......@@ -73,7 +73,7 @@ int main(int argc, char **argv) {
len = NWORD * sizeof(int);
printf("reading\n");
mochio_read(client, filename, 1, &read_address, &read_size, 1, &offset, &len);
bv_read(client