Commit 73dc8387 authored by Rob Latham's avatar Rob Latham
Browse files

[svn-r8550] further rework of datatype hashing

- hash the datatype in set view so we can get at least one thing out of the i/o path
- use a jenkins hash: it's not cryptographically secure but offers low
  collisions, which is what we really care about
parent d2a3d71d
......@@ -36,7 +36,7 @@ AD_OBJECTS = ad_close.o ad_init.o ad_end.o ad_open.o flatten.o \
ad_coll_exch_new.o ad_coll_build_req_new.o ad_io_coll.o \
ad_aggregate_new.o heap-sort.o \
ad_opencoll.o ad_opencoll_scalable.o ad_opencoll_failsafe.o ad_features.o \
same_dtype.o sha1.o lookup3.o
hash_type.o same_dtype.o lookup3.o
all: $(LIBNAME)
@if [ "@ENABLE_SHLIB@" != "none" ] ; then \
......
......@@ -102,6 +102,12 @@ void ADIO_Close(ADIO_File fd, int *error_code)
MPI_Type_get_envelope(fd->filetype, &i, &j, &k, &combiner);
if (combiner != MPI_COMBINER_NAMED) MPI_Type_free(&(fd->filetype));
if (fd->etype_hash_key != MPI_KEYVAL_INVALID)
MPI_Keyval_free(&(fd->etype_hash_key));
if (fd->ftype_hash_key != MPI_KEYVAL_INVALID)
MPI_Keyval_free(&(fd->ftype_hash_key));
MPI_Info_free(&(fd->info));
/* memory for fd is freed in MPI_File_close */
......
......@@ -59,6 +59,9 @@ MPI_File ADIO_Open(MPI_Comm orig_comm,
fd->filetype = filetype; /* MPI_BYTE by default */
fd->etype_size = 1; /* default etype is MPI_BYTE */
fd->etype_hash_key = MPI_KEYVAL_INVALID; /* will be set in file view (if called) */
fd->ftype_hash_key = MPI_KEYVAL_INVALID;
fd->file_realm_st_offs = NULL;
fd->file_realm_types = NULL;
......
......@@ -172,7 +172,7 @@ void ADIOI_GEN_ReadStridedColl(ADIO_File fd, void *buf, int count,
* config file, an initial mesh dataset, etc), then the full two-phase
* algorithm is overkill and we should just read-and-broadcast */
if (ADIOI_allsame_access(fd, count, datatype)) {
if (ADIOI_allsame_access(fd, count, datatype, offset)) {
*error_code = MPI_SUCCESS;
if (myrank == fd->hints->ranklist[0]) {
ADIO_ReadContig(fd, buf, count, datatype, file_ptr_type,
......
......@@ -8,6 +8,26 @@
#include "adio.h"
#include "adio_extern.h"
int hash_copy_fn(MPI_Datatype oldtype, int type_keyval, void *extra_state,
void * attr_val_in, void *attr_val_out, int *flag)
{
((ADIOI_hashed_dtype *)attr_val_in)->ref_count +=1;
*((ADIOI_hashed_dtype **)attr_val_out) = (ADIOI_hashed_dtype *)attr_val_in;
*flag = 1;
return MPI_SUCCESS;
}
int hash_delete_fn(MPI_Datatype type, int type_keyval,
void *attr_val, void *extra_state)
{
ADIOI_hashed_dtype *hash;
hash = (ADIOI_hashed_dtype *)attr_val;
hash->ref_count -= 1;
if (hash->ref_count == 0) {
ADIOI_Free(hash);
}
}
/* this used to be implemented in every file system as an fcntl. It makes
* deferred open easier if we know ADIO_Fcntl will always need a file to really
* be open. set_view doesn't modify anything related to the open files.
......@@ -16,6 +36,8 @@ void ADIO_Set_view(ADIO_File fd, ADIO_Offset disp, MPI_Datatype etype,
MPI_Datatype filetype, MPI_Info info, int *error_code)
{
int combiner, i, j, k, err, filetype_is_contig;
int key;
ADIOI_hashed_dtype *etype_hash, *ftype_hash;
MPI_Datatype copy_etype, copy_filetype;
ADIOI_Flatlist_node *flat_file;
/* free copies of old etypes and filetypes and delete flattened
......@@ -57,6 +79,23 @@ void ADIO_Set_view(ADIO_File fd, ADIO_Offset disp, MPI_Datatype etype,
MPI_Type_size(fd->etype, &(fd->etype_size));
fd->disp = disp;
etype_hash = ADIOI_Calloc(1, sizeof(ADIOI_hashed_dtype));
ADIOI_dtype_hash(fd->etype, etype_hash);
MPI_Type_create_keyval(hash_copy_fn, hash_delete_fn,
&key, etype_hash);
fd->etype_hash_key = key;
MPI_Type_set_attr(fd->etype, key, etype_hash);
ftype_hash = ADIOI_Calloc(1, sizeof(ADIOI_hashed_dtype));
ADIOI_dtype_hash(fd->filetype, ftype_hash);
MPI_Type_create_keyval(hash_copy_fn, hash_delete_fn,
&key, ftype_hash);
fd->ftype_hash_key = key;
MPI_Type_set_attr(fd->filetype, key, ftype_hash);
/* reset MPI-IO file pointer to point to the first byte that can
be accessed in this view. */
......
This diff is collapsed.
This diff is collapsed.
This diff is collapsed.
......@@ -233,6 +233,11 @@ typedef struct ADIOI_FileD {
ADIO_Offset *file_realm_st_offs; /* file realm starting offsets */
MPI_Datatype *file_realm_types; /* file realm datatypes */
int my_cb_nodes_index; /* my index into cb_config_list. -1 if N/A */
/* datatype hashes */
int etype_hash_key;
int ftype_hash_key;
} ADIOI_FileD;
typedef struct ADIOI_FileD *ADIO_File;
......
......@@ -110,6 +110,20 @@ typedef struct ADIOI_Fl_node {
struct ADIOI_Fl_node *next; /* pointer to next node */
} ADIOI_Flatlist_node;
/* hashed datatypes. */
#ifdef HAVE_STDINT_H
#include <stdint.h>
#else
typedef unsigned int uint32_t;
#endif
#define ADIOI_HASH_COUNT 2
#define ADIOI_HASH_BYTES (ADIOI_HASH_COUNT*sizeof(uint32_t))
typedef struct dtype_hash_t{
uint32_t hash[ADIOI_HASH_COUNT];
int ref_count;
} ADIOI_hashed_dtype;
#ifdef ROMIO_PVFS2
#include <pvfs2.h>
#endif
......@@ -561,7 +575,8 @@ int ADIOI_End_call(MPI_Comm comm, int keyval, void *attribute_val, void *extra_s
int MPIR_Status_set_bytes(MPI_Status *status, MPI_Datatype datatype, int nbytes);
int ADIOI_Uses_generic_read(ADIO_File fd);
int ADIOI_Uses_generic_write(ADIO_File fd);
int ADIOI_allsame_access(ADIO_File fd, int count, MPI_Datatype datatype);
int ADIOI_allsame_access(ADIO_File fd,
int count, MPI_Datatype datatype, ADIO_Offset offset);
int ADIOI_FAKE_IODone(ADIO_Request *request, ADIO_Status *status,
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment