let's get the PVFS async interface back in shape

src/mpi/romio/adio/ad_pvfs2/ad_pvfs2_aio.c

@@ -48,7 +48,6 @@ void ADIOI_PVFS2_AIO_contig(ADIO_File fd, void *buf, int count,
 {
 
     int ret, datatype_size, len;
-    PVFS_Request file_req, mem_req;
     ADIOI_PVFS2_fs *pvfs_fs;
     ADIOI_AIO_Request *aio_req;
     static char myname[] = "ADIOI_PVFS2_AIO_contig";
@@ -60,7 +59,7 @@ void ADIOI_PVFS2_AIO_contig(ADIO_File fd, void *buf, int count,
     MPI_Type_size(datatype, &datatype_size);
     len = datatype_size * count;
 
-    ret = PVFS_Request_contiguous(len, PVFS_BYTE, &mem_req);
+    ret = PVFS_Request_contiguous(len, PVFS_BYTE, &(aio_req->mem_req));
     /* --BEGIN ERROR HANDLING-- */
     if (ret != 0) {
 	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
@@ -72,7 +71,7 @@ void ADIOI_PVFS2_AIO_contig(ADIO_File fd, void *buf, int count,
     }
     /* --END ERROR HANDLING-- */
 
-    ret = PVFS_Request_contiguous(len, PVFS_BYTE, &file_req);
+    ret = PVFS_Request_contiguous(len, PVFS_BYTE, &(aio_req->file_req));
     /* --BEGIN ERROR HANDLING-- */
     if (ret != 0) {
 	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
@@ -92,9 +91,9 @@ void ADIOI_PVFS2_AIO_contig(ADIO_File fd, void *buf, int count,
 #ifdef ADIOI_MPE_LOGGING
 	MPE_Log_event( ADIOI_MPE_iread_a, 0, NULL );
 #endif
-	ret = PVFS_isys_read(pvfs_fs->object_ref, file_req, offset, buf, 
-			mem_req, &(pvfs_fs->credentials), &(aio_req->resp_io),
-			aio_req->op_id, NULL);
+	ret = PVFS_isys_read(pvfs_fs->object_ref, aio_req->file_req, offset, 
+		buf, aio_req->mem_req, &(pvfs_fs->credentials), 
+		&(aio_req->resp_io), &(aio_req->op_id), NULL);
 #ifdef ADIOI_MPE_LOGGING
 	MPE_Log_event( ADIOI_MPE_iread_b, 0, NULL );
 #endif
@@ -102,9 +101,9 @@ void ADIOI_PVFS2_AIO_contig(ADIO_File fd, void *buf, int count,
 #ifdef ADIOI_MPE_LOGGING
 	MPE_Log_event( ADIOI_MPE_iwrite_a, 0, NULL );
 #endif
-	ret = PVFS_isys_write(pvfs_fs->object_ref, file_req, offset, buf, 
-			mem_req, &(pvfs_fs->credentials), &(aio_req->resp_io),
-			aio_req->op_id, aio_req);
+	ret = PVFS_isys_write(pvfs_fs->object_ref, aio_req->file_req, offset, 
+		buf, aio_req->mem_req, &(pvfs_fs->credentials), 
+		&(aio_req->resp_io), &(aio_req->op_id), NULL);
 #ifdef ADIOI_MPE_LOGGING
 	MPE_Log_event( ADIOI_MPE_iwrite_b, 0, NULL );
 #endif
@@ -137,8 +136,6 @@ void ADIOI_PVFS2_AIO_contig(ADIO_File fd, void *buf, int count,
 
     *error_code = MPI_SUCCESS;
 fn_exit:
-    PVFS_Request_free(&mem_req);
-    PVFS_Request_free(&file_req);
     return;
 }
 
@@ -147,6 +144,8 @@ int ADIOI_PVFS2_aio_free_fn(void *extra_state)
     ADIOI_AIO_Request *aio_req;
     aio_req = (ADIOI_AIO_Request*)extra_state;
 
+    PVFS_Request_free(&(aio_req->mem_req));
+    PVFS_Request_free(&(aio_req->file_req));
     ADIOI_Free(aio_req);
 
     return MPI_SUCCESS;
@@ -177,35 +176,35 @@ int ADIOI_PVFS2_aio_wait_fn(int count, void ** array_of_states,
 {
 
     ADIOI_AIO_Request **aio_reqlist;
-    ADIOI_AIO_Request *tmp_req;
     PVFS_sys_op_id *op_id_array;
-    int i;
+    int i,j, greq_count;
     int *error_array;
 
     aio_reqlist = (ADIOI_AIO_Request **)array_of_states;
 
     op_id_array = (PVFS_sys_op_id*)ADIOI_Calloc(count, sizeof(PVFS_sys_op_id));
     error_array = (int *)ADIOI_Calloc(count, sizeof(int));
+    greq_count = count;
 
-    PVFS_sys_testsome(op_id_array, &count, (void *)aio_reqlist, 
-	    error_array, INT_MAX);
+    /* PVFS-2.6: testsome actually tests all requests and fills in op_id_array
+     * with the ones that have completed.  count is an in/out parameter.
+     * returns with the number of completed operations.  what a mess! */
+    PVFS_sys_testsome(op_id_array, &count, NULL, error_array, INT_MAX);
     for (i=0; i< count; i++) {
-	tmp_req = aio_reqlist[i];
-	if (tmp_req->op_id != op_id_array[i])
-	    continue;
-	tmp_req->nbytes = tmp_req->resp_io.total_completed;
-	MPIR_Nest_incr();
-	MPI_Grequest_complete(tmp_req->req);
-	MPIR_Nest_decr();
+	for (j=0; j<greq_count; j++) {
+	    if (op_id_array[i] == aio_reqlist[j]->op_id) {
+		aio_reqlist[j]->nbytes = 
+		    aio_reqlist[j]->resp_io.total_completed;
+		MPIR_Nest_incr();
+		MPI_Grequest_complete(aio_reqlist[j]->req);
+		MPIR_Nest_decr();
+	    }
+	}
     }
     return MPI_SUCCESS; /* TODO: no idea how to deal with errors */
 }
 
 
-
-
-
-
 /*
  * vim: ts=8 sts=4 sw=4 noexpandtab 
  */

src/mpi/romio/adio/include/adioi.h

@@ -113,6 +113,8 @@ typedef struct ADIOI_AIO_req_str {
 #ifdef ROMIO_PVFS2
 	PVFS_sys_op_id op_id;
 	PVFS_sysresp_io resp_io;
+	PVFS_Request file_req;
+	PVFS_Request mem_req;
 #endif
 #ifdef ROMIO_NTFS
     /* Ptr to Overlapped struct */