Commit 09a1222a authored by robl's avatar robl
Browse files

from panasas: i don't know how to test this but it looks ok to me

parent d2e5d205
......@@ -6,14 +6,14 @@ srcdir = @srcdir@
CC_SHL = @CC_SHL@
SHLIBNAME = @SHLIBNAME@
INCLUDE_DIR = -I@MPI_INCLUDE_DIR@ -I${srcdir}/../include -I../include -I../../include -I/opt/panfs/include
INCLUDE_DIR = -I@MPI_INCLUDE_DIR@ -I${srcdir}/../include -I../include -I../../include -I${srcdir}/../../../../include -I../../../../include -I/opt/panfs/include
CFLAGS = @CFLAGS@ $(INCLUDE_DIR)
C_COMPILE_SHL = $(CC_SHL)
@VPATH@
AD_PANFS_OBJECTS = ad_panfs.o ad_panfs_open.o ad_panfs_hints.o
AD_PANFS_OBJECTS = ad_panfs.o ad_panfs_open.o ad_panfs_hints.o ad_panfs_read.o ad_panfs_resize.o ad_panfs_write.o
default: $(LIBNAME)
......
......@@ -13,8 +13,8 @@
struct ADIOI_Fns_struct ADIO_PANFS_operations = {
ADIOI_PANFS_Open, /* Open */
ADIOI_GEN_ReadContig, /* ReadContig */
ADIOI_GEN_WriteContig, /* WriteContig */
ADIOI_PANFS_ReadContig, /* ReadContig */
ADIOI_PANFS_WriteContig, /* WriteContig */
ADIOI_GEN_ReadStridedColl, /* ReadStridedColl */
ADIOI_GEN_WriteStridedColl, /* WriteStridedColl */
ADIOI_GEN_SeekIndividual, /* SeekIndividual */
......@@ -37,6 +37,6 @@ struct ADIOI_Fns_struct ADIO_PANFS_operations = {
ADIOI_GEN_IreadStrided, /* IreadStrided */
ADIOI_GEN_IwriteStrided, /* IwriteStrided */
ADIOI_GEN_Flush, /* Flush */
ADIOI_GEN_Resize, /* Resize */
ADIOI_PANFS_Resize, /* Resize */
ADIOI_GEN_Delete, /* Delete */
};
......@@ -6,8 +6,8 @@
* See COPYRIGHT notice in top-level directory.
*/
#ifndef AD_UNIX_INCLUDE
#define AD_UNIX_INCLUDE
#ifndef AD_PANFS_INCLUDE
#define AD_PANFS_INCLUDE
#include <unistd.h>
#include <sys/types.h>
......@@ -25,28 +25,32 @@ typedef struct adiocb adiocb_t;
#endif
#endif
int ADIOI_PANFS_aio(ADIO_File fd, void *buf, int len, ADIO_Offset offset,
int wr, void *handle);
void ADIOI_PANFS_Open(ADIO_File fd, int *error_code);
void ADIOI_PANFS_IwriteContig(ADIO_File fd, void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, ADIO_Request *request, int
*error_code);
void ADIOI_PANFS_IreadContig(ADIO_File fd, void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, ADIO_Request *request, int
*error_code);
int ADIOI_PANFS_ReadDone(ADIO_Request *request, ADIO_Status *status, int
*error_code);
int ADIOI_PANFS_WriteDone(ADIO_Request *request, ADIO_Status *status, int
*error_code);
void ADIOI_PANFS_ReadComplete(ADIO_Request *request, ADIO_Status *status, int
*error_code);
void ADIOI_PANFS_WriteComplete(ADIO_Request *request, ADIO_Status *status,
int *error_code);
void ADIOI_PANFS_Fcntl(ADIO_File fd, int flag, ADIO_Fcntl_t *fcntl_struct, int
*error_code);
void ADIOI_PANFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code);
void ADIOI_PANFS_ReadContig(ADIO_File fd, void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, ADIO_Status *status,
int *error_code);
void ADIOI_PANFS_Resize(ADIO_File fd, ADIO_Offset size, int *error_code);
void ADIOI_PANFS_WriteContig(ADIO_File fd, void *buf, int count,
MPI_Datatype datatype, int file_ptr_type,
ADIO_Offset offset, ADIO_Status *status,
int *error_code);
/* Delay 1 ms */
#define AD_PANFS_RETRY_DELAY 1000
#define AD_PANFS_RETRY(_op_,_rc_) \
{ \
_rc_ = (_op_); \
while(_rc_ == -1 && errno == EAGAIN) \
{ \
if(usleep(AD_PANFS_RETRY_DELAY) == -1) \
{ \
break; \
} \
_rc_ = (_op_); \
} \
}
#endif
......@@ -116,7 +116,7 @@ void ADIOI_PANFS_SetInfo(ADIO_File fd, MPI_Info users_info, int *error_code)
MPI_Info_get(users_info, "panfs_layout_visit_policy", MPI_MAX_INFO_VAL,
value, &flag);
if (flag && (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE)) {
if (flag && (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE || layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID10)) {
layout_visit_policy = strtoul(value,NULL,10);
tmp_val = layout_visit_policy;
MPI_Bcast(&tmp_val, 1, MPI_INT, 0, fd->comm);
......
......@@ -33,6 +33,9 @@ void ADIOI_PANFS_Open(ADIO_File fd, int *error_code)
unsigned long int layout_parity_stripe_depth = 0;
unsigned long int layout_total_num_comps = 0;
pan_fs_client_layout_visit_t layout_visit_policy = PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN;
int myrank;
MPI_Comm_rank(fd->comm, &myrank);
*error_code = MPI_SUCCESS;
value = (char *) ADIOI_Malloc((MPI_MAX_INFO_VAL+1)*sizeof(char));
......@@ -68,10 +71,10 @@ void ADIOI_PANFS_Open(ADIO_File fd, int *error_code)
}
ADIOI_Free(value);
amode = amode | O_CREAT;
amode = amode | O_CREAT;
/* Check for valid set of hints */
if ((layout_type < PAN_FS_CLIENT_LAYOUT_TYPE__DEFAULT) ||
(layout_type > PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE))
(layout_type > PAN_FS_CLIENT_LAYOUT_TYPE__RAID10))
{
FPRINTF(stderr, "%s: panfs_layout_type is not a valid value: %u.\n", myname, layout_type);
MPI_Abort(MPI_COMM_WORLD, 1);
......@@ -121,99 +124,136 @@ void ADIOI_PANFS_Open(ADIO_File fd, int *error_code)
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
if ((layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID0) ||
(layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE))
{
int myrank;
MPI_Comm_rank(fd->comm, &myrank);
if (myrank == 0) {
pan_fs_client_layout_create_args_t file_create_args;
int fd_dir;
char* slash;
struct stat stat_buf;
int err;
char *value, *path, *file_name_ptr;
/* Check that the file does not exist before
* trying to create it. The ioctl itself should
* be able to handle this condition. Currently,
* the ioctl will return successfully if the file
* has been previously created. Filed bug 33862
* to track the problem.
*/
err = stat(fd->filename,&stat_buf);
if((err == -1) && (errno != ENOENT))
if (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID10)
{
if ((layout_stripe_unit == 0) || (layout_total_num_comps == 0))
{
if(layout_stripe_unit == 0)
{
FPRINTF(stderr,"%s: Unexpected I/O Error calling stat() on PanFS file: %s.\n", myname, strerror(errno));
MPI_Abort(MPI_COMM_WORLD, 1);
FPRINTF(stderr, "%s: MPI_Info does not contain the panfs_layout_stripe_unit hint which is necessary to specify a valid RAID10 layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n", myname);
}
else if (err == 0)
if(layout_total_num_comps == 0)
{
FPRINTF(stderr,"%s: Cannot create PanFS file with ioctl when file already exists.\n", myname);
FPRINTF(stderr, "%s: MPI_Info does not contain the panfs_layout_total_num_comps hint which is necessary to specify a valid RAID10 layout to the PAN_FS_CLIENT_LAYOUT_CREATE_FILE ioctl.\n", myname);
}
MPI_Abort(MPI_COMM_WORLD, 1);
}
if ((layout_visit_policy < PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN) ||
(layout_visit_policy > PAN_FS_CLIENT_LAYOUT_VISIT__ROUND_ROBIN_WITH_HASHED_OFFSET))
{
FPRINTF(stderr, "%s: panfs_layout_visit_policy is not a valid value: %u.\n", myname, layout_visit_policy);
MPI_Abort(MPI_COMM_WORLD, 1);
}
}
/* Create the file via ioctl() or open(). ADIOI_PANFS_Open's caller
* already optimizes performance by only calling this function with
* ADIO_CREATE on rank 0. Therefore, we don't need to worry about
* implementing that optimization here. */
if((layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID0) || (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE)
|| (layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID10)) {
pan_fs_client_layout_create_args_t file_create_args;
int fd_dir;
char* slash;
struct stat stat_buf;
int err;
char *value, *path, *file_name_ptr;
/* Check that the file does not exist before
* trying to create it. The ioctl itself should
* be able to handle this condition. Currently,
* the ioctl will return successfully if the file
* has been previously created. Filed bug 33862
* to track the problem.
*/
err = stat(fd->filename,&stat_buf);
if((err == -1) && (errno != ENOENT))
{
FPRINTF(stderr,"%s: Unexpected I/O Error calling stat() on PanFS file: %s.\n", myname, strerror(errno));
MPI_Abort(MPI_COMM_WORLD, 1);
}
else if (err == 0)
{
FPRINTF(stderr,"%s: Cannot create PanFS file with ioctl when file already exists.\n", myname);
MPI_Abort(MPI_COMM_WORLD, 1);
}
else
{
/* (err == -1) && (errno == ENOENT) */
/* File does not exist */
path = ADIOI_Strdup(fd->filename);
slash = strrchr(path, '/');
if (!slash)
ADIOI_Strncpy(path, ".", 2);
else {
if (slash == path)
*(path + 1) = '\0';
else *slash = '\0';
}
/* create PanFS object */
bzero(&file_create_args,sizeof(pan_fs_client_layout_create_args_t));
/* open directory */
fd_dir = open(path, O_RDONLY);
if (fd_dir < 0) {
FPRINTF(stderr, "%s: I/O Error opening parent directory to create PanFS file using ioctl: %s.\n", myname, strerror(errno));
MPI_Abort(MPI_COMM_WORLD, 1);
}
else
{
/* (err == -1) && (errno == ENOENT) */
/* File does not exist */
path = ADIOI_Strdup(fd->filename);
slash = strrchr(path, '/');
if (!slash)
ADIOI_Strncpy(path, ".", 2);
else {
if (slash == path)
*(path + 1) = '\0';
else *slash = '\0';
char *file_name_ptr = fd->filename;
slash = strrchr(fd->filename, '/');
if (slash)
{
file_name_ptr = slash + 1;
}
/* create PanFS object */
bzero(&file_create_args,sizeof(pan_fs_client_layout_create_args_t));
/* open directory */
fd_dir = open(path, O_RDONLY);
if (fd_dir < 0) {
FPRINTF(stderr, "%s: I/O Error opening parent directory to create PanFS file using ioctl: %s.\n", myname, strerror(errno));
MPI_Abort(MPI_COMM_WORLD, 1);
/* create file in the directory */
file_create_args.mode = perm;
file_create_args.version = PAN_FS_CLIENT_LAYOUT_VERSION;
file_create_args.flags = PAN_FS_CLIENT_LAYOUT_CREATE_F__NONE;
ADIOI_Strncpy(file_create_args.filename, file_name_ptr, strlen(fd->filename)+1);
file_create_args.layout.agg_type = layout_type;
file_create_args.layout.layout_is_valid = 1;
if(layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE)
{
file_create_args.layout.u.raid1_5_parity_stripe.total_num_comps = layout_total_num_comps;
file_create_args.layout.u.raid1_5_parity_stripe.parity_stripe_width = layout_parity_stripe_width;
file_create_args.layout.u.raid1_5_parity_stripe.parity_stripe_depth = layout_parity_stripe_depth;
file_create_args.layout.u.raid1_5_parity_stripe.stripe_unit = layout_stripe_unit;
file_create_args.layout.u.raid1_5_parity_stripe.layout_visit_policy = layout_visit_policy;
}
else if(layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID0)
{
file_create_args.layout.u.raid0.total_num_comps = layout_total_num_comps;
file_create_args.layout.u.raid0.stripe_unit = layout_stripe_unit;
}
else
else if(layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID10)
{
char *file_name_ptr = fd->filename;
slash = strrchr(fd->filename, '/');
if (slash)
{
file_name_ptr = slash + 1;
}
/* create file in the directory */
file_create_args.mode = perm;
file_create_args.version = PAN_FS_CLIENT_LAYOUT_VERSION;
file_create_args.flags = PAN_FS_CLIENT_LAYOUT_CREATE_F__NONE;
ADIOI_Strncpy(file_create_args.filename, file_name_ptr, strlen(fd->filename)+1);
file_create_args.layout.agg_type = layout_type;
file_create_args.layout.layout_is_valid = 1;
if(layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID1_5_PARITY_STRIPE)
{
file_create_args.layout.u.raid1_5_parity_stripe.total_num_comps = layout_total_num_comps;
file_create_args.layout.u.raid1_5_parity_stripe.parity_stripe_width = layout_parity_stripe_width;
file_create_args.layout.u.raid1_5_parity_stripe.parity_stripe_depth = layout_parity_stripe_depth;
file_create_args.layout.u.raid1_5_parity_stripe.stripe_unit = layout_stripe_unit;
file_create_args.layout.u.raid1_5_parity_stripe.layout_visit_policy = layout_visit_policy;
}
else if(layout_type == PAN_FS_CLIENT_LAYOUT_TYPE__RAID0)
{
file_create_args.layout.u.raid0.total_num_comps = layout_total_num_comps;
file_create_args.layout.u.raid0.stripe_unit = layout_stripe_unit;
}
err = ioctl(fd_dir, PAN_FS_CLIENT_LAYOUT_CREATE_FILE, &file_create_args);
if (err < 0) {
FPRINTF(stderr, "%s: I/O Error doing ioctl on parent directory to create PanFS file using ioctl: %s.\n", myname, strerror(errno));
MPI_Abort(MPI_COMM_WORLD, 1);
}
err = close(fd_dir);
file_create_args.layout.u.raid10.total_num_comps = layout_total_num_comps;
file_create_args.layout.u.raid10.stripe_unit = layout_stripe_unit;
file_create_args.layout.u.raid10.layout_visit_policy = layout_visit_policy;
}
err = ioctl(fd_dir, PAN_FS_CLIENT_LAYOUT_CREATE_FILE, &file_create_args);
if (err < 0) {
FPRINTF(stderr, "%s: I/O Error doing ioctl on parent directory to create PanFS file using ioctl: %s.\n", myname, strerror(errno));
MPI_Abort(MPI_COMM_WORLD, 1);
}
ADIOI_Free(path);
err = close(fd_dir);
}
ADIOI_Free(path);
}
}
else
{
int create_fd = open(fd->filename,amode,perm);
if(create_fd != -1)
{
close(create_fd);
}
else
{
FPRINTF(stderr, "%s: I/O Error creating PanFS file using open: %s.\n", myname, strerror(errno));
MPI_Abort(MPI_COMM_WORLD, 1);
}
MPI_Barrier(fd->comm);
}
}
if (fd->access_mode & ADIO_RDONLY)
......@@ -279,6 +319,14 @@ void ADIOI_PANFS_Open(ADIO_File fd, int *error_code)
ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid1_5_parity_stripe.layout_visit_policy);
MPI_Info_set(fd->info, "panfs_layout_visit_policy", temp_buffer);
break;
case PAN_FS_CLIENT_LAYOUT_TYPE__RAID10:
ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid10.stripe_unit);
MPI_Info_set(fd->info, "panfs_layout_stripe_unit", temp_buffer);
ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid10.total_num_comps);
MPI_Info_set(fd->info, "panfs_layout_total_num_comps", temp_buffer);
ADIOI_Snprintf(temp_buffer,TEMP_BUFFER_SIZE,"%u",file_query_args.layout.u.raid10.layout_visit_policy);
MPI_Info_set(fd->info, "panfs_layout_visit_policy", temp_buffer);
break;
}
}
}
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment