Commit e07aac2f authored by Rob Latham's avatar Rob Latham
Browse files

provider file cache: discriminate on flags + path

Some strage problems with lustre getstripe tracked down to trying to look up a
file with different access flags. Now a file cache only matches if both name
and access flags are the same.
parent fd7b9a37
......@@ -5,11 +5,14 @@ class file_stats {
file_stats() : stripe_size(4096), stripe_count(1), blocksize(4096) {};
template<typename A> void serialize(A &ar) {
ar & status;
ar & stripe_size;
ar & stripe_count;
ar & blocksize;
}
/* status */
int status;
/* parallel file system info */
int32_t stripe_size;
int32_t stripe_count;
......
......@@ -60,7 +60,7 @@ typedef enum {
} io_kind;
/* note: alters addr_str */
char * get_proto_from_addr(char *addr_str)
static char * get_proto_from_addr(char *addr_str)
{
char *p = addr_str;
char *q = strchr(addr_str, ':');
......
......@@ -27,6 +27,11 @@
#include "lustre-utils.h"
namespace tl = thallium;
struct file_info {
int fd;
int flags;
};
struct bv_svc_provider : public tl::provider<bv_svc_provider>
{
tl::engine * engine;
......@@ -34,7 +39,7 @@ struct bv_svc_provider : public tl::provider<bv_svc_provider>
tl::pool pool;
abt_io_instance_id abt_id;
ssize_t blocksize=1024*8; // todo: some kind of general distribution function perhaps
std::map<std::string, int> filetable; // filename to file id mapping
std::map<std::string, file_info> filetable; // filename to file id mapping
// probably needs to be larger and registered with mercury somehow
char *buffer; // intermediate buffer for read/write operations
unsigned int bufsize;
......@@ -45,14 +50,23 @@ struct bv_svc_provider : public tl::provider<bv_svc_provider>
// server will maintain a cache of open files
// std::map not great for LRU
// if we see a request for a file with a different 'flags' we will close and reopen
int getfd(const std::string &file, int flags, int mode=default_mode) {
int fd=-1;
auto entry = filetable.find(file);
if (entry == filetable.end() ) {
// no 'file' in table
fd = abt_io_open(abt_id, file.c_str(), flags, mode);
if (fd > 0) filetable[file] = fd;
if (fd > 0) filetable[file] = {fd, flags};
} else {
fd = entry->second;
// found the file but we will close and reopen if different flags requested
if (entry->second.flags == flags)
fd = entry->second.fd;
else {
abt_io_close(abt_id, entry->second.fd);
fd = abt_io_open(abt_id, file.c_str(), flags, mode);
if (fd > 0) filetable[file] = {fd, flags};
}
}
return fd;
}
......@@ -217,7 +231,9 @@ struct bv_svc_provider : public tl::provider<bv_svc_provider>
rd_mutex_time = ABT_get_wtime() - rd_mutex_time;
/* like with write, open for both read and write in case file opened
* first for read then written to */
* first for read then written to. can omit O_CREAT here because
* reading a non-existent file would be an error */
int flags = O_RDWR;
double getfd_time = ABT_get_wtime();
......@@ -323,7 +339,7 @@ struct bv_svc_provider : public tl::provider<bv_svc_provider>
/* lustre header incompatible with c++ , so need to stuff the lustre
* bits into a c-compiled object */
lustre_getstripe(file.c_str(), &(ret.stripe_size), &(ret.stripe_count));
ret.status = lustre_getstripe(file.c_str(), &(ret.stripe_size), &(ret.stripe_count));
return ret;
}
......@@ -339,13 +355,15 @@ struct bv_svc_provider : public tl::provider<bv_svc_provider>
return ret;
}
int flush(const std::string &file) {
/* omiting O_CREAT: what would it mean to flush a nonexistent file ? */
int fd = getfd(file, O_RDWR);
return (fsync(fd));
}
ssize_t getsize(const std::string &file) {
off_t oldpos=-1, pos=-1;
int fd = getfd(file, O_RDONLY);
/* have to open read-write in case subsequent write call comes in */
int fd = getfd(file, O_CREAT|O_RDONLY);
oldpos = lseek(fd, 0, SEEK_CUR);
if (oldpos == -1)
return -errno;
......
......@@ -31,6 +31,7 @@ static size_t get_lumsize()
int lustre_getstripe(const char * filename, int32_t *stripe_size, int32_t *stripe_count)
{
int ret = 0;
int status = 0;
/* guess some reasonable defaults for non-lustre systems */
*stripe_size = 4096;
*stripe_count = 1;
......@@ -43,25 +44,30 @@ int lustre_getstripe(const char * filename, int32_t *stripe_size, int32_t *strip
* - check parent directory
* - maybe the file is not lustre? */
ret = llapi_file_get_stripe(filename, lov);
switch(errno) {
if (ret != 0) status = errno;
switch(status) {
char *dup, *parent;
case ENOENT:
case ENOTTY: /* a valid file, but not a lustre file */
ret = 0;
goto fn_exit;
case ENOENT: /* given file didn't exist, but can use parent's stripe info */
dup = strdup(filename);
parent = dirname(dup);
ret = llapi_file_get_stripe(parent, lov);
/* if still enoent, we give up */
free(dup);
if (errno == ENOENT)
if (ret != 0 && errno == ENOENT) {
free(dup);
goto fn_exit;
}
/* fall through */
case 0: /* success */
*stripe_size = lov->lmm_stripe_size;
*stripe_count = lov->lmm_stripe_count;
break;
case ENOTTY:
ret = 0;
goto fn_exit;
default:
if (ret != 0) perror("Unable to get Lustre stripe info");
perror("Unable to get Lustre stripe info");
};
*stripe_size = lov->lmm_stripe_size;
*stripe_count = lov->lmm_stripe_count;
#endif
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment