ad_lustre_rwcontig.c 5.93 KB
Newer Older
1
/* -*- Mode: C; c-basic-offset:4 ; indent-tabs-mode:nil ; -*- */
2
3
/*
 *   Copyright (C) 1997 University of Chicago.
4
5
6
 *   See COPYRIGHT notice in top-level directory.
 *
 *   Copyright (C) 2007 Oak Ridge National Laboratory
7
8
 *
 *   Copyright (C) 2008 Sun Microsystems, Lustre group
9
10
 */

11
12
#include <unistd.h>

13
14
15
16
17
18
#include <stdlib.h>
#include <malloc.h>
#include "ad_lustre.h"

#define LUSTRE_MEMALIGN (1<<12)  /* to use page_shift */

19
static void ADIOI_LUSTRE_Aligned_Mem_File_Write(ADIO_File fd, const void *buf, int len,
20
              ADIO_Offset offset, int *err);
21
static void ADIOI_LUSTRE_Aligned_Mem_File_Write(ADIO_File fd, const void *buf, int len,
22
23
              ADIO_Offset offset, int *err)
{
24
    int rem, size, nbytes;
25
26
27
28
29
30
31
32
33
34
35
36
37
    if (!(len % fd->d_miniosz) && (len >= fd->d_miniosz)) {
	*err = pwrite(fd->fd_direct, buf, len, offset);
    } else if (len < fd->d_miniosz) {
	*err = pwrite(fd->fd_sys, buf, len, offset);
    } else {
	rem = len % fd->d_miniosz;
	size = len - rem;
	nbytes = pwrite(fd->fd_direct, buf, size, offset);
	nbytes += pwrite(fd->fd_sys, ((char *)buf) + size, rem, offset+size);
	*err = nbytes;
    }
}

38
static void ADIOI_LUSTRE_Aligned_Mem_File_Read(ADIO_File fd, const void *buf, int len,
39
              ADIO_Offset offset, int *err);
40
static void ADIOI_LUSTRE_Aligned_Mem_File_Read(ADIO_File fd, const void *buf, int len,
41
42
              ADIO_Offset offset, int *err)
{
43
    int rem, size, nbytes;
44
    if (!(len % fd->d_miniosz) && (len >= fd->d_miniosz))
45
	*err = pread(fd->fd_direct, (void *)buf, len, offset);
46
    else if (len < fd->d_miniosz)
47
	*err = pread(fd->fd_sys, (void *)buf, len, offset);
48
49
50
    else {
	rem = len % fd->d_miniosz;
	size = len - rem;
51
	nbytes = pread(fd->fd_direct, (void *)buf, size, offset);
52
53
54
55
56
57
	nbytes += pread(fd->fd_sys, ((char *)buf) + size, rem, offset+size);
	*err = nbytes;
    }
}


58
static int ADIOI_LUSTRE_Directio(ADIO_File fd, const void *buf, int len,
59
			   off_t offset, int rw);
60
static int ADIOI_LUSTRE_Directio(ADIO_File fd, const void *buf, int len,
61
62
63
64
65
66
67
68
69
			   off_t offset, int rw)
{
    int err=-1, diff, size=len, nbytes = 0;
    void *newbuf;

    if (offset % fd->d_miniosz) {
	diff = fd->d_miniosz - (offset % fd->d_miniosz);
	diff = ADIOI_MIN(diff, len);
	if (rw)
70
	    nbytes = pwrite(fd->fd_sys, (void *)buf, diff, offset);
71
	else
72
	    nbytes = pread(fd->fd_sys, (void *)buf, diff, offset);
73
74
75
76
77
78
79
80
81
82
83
84
85
86
87
88
89
90
91
	buf = ((char *) buf) + diff;
	offset += diff;
	size = len - diff;
    }

    if (!size) {
	return diff;
    }

    if (rw) { /* direct I/O enabled */
	if (!(((long) buf) % fd->d_mem)) {
	    ADIOI_LUSTRE_Aligned_Mem_File_Write(fd, buf, size, offset, &err);
	    nbytes += err;
	} else {
	    newbuf = (void *) memalign(LUSTRE_MEMALIGN, size);
	    if (newbuf) {
		memcpy(newbuf, buf, size);
		ADIOI_LUSTRE_Aligned_Mem_File_Write(fd, newbuf, size, offset, &err);
		nbytes += err;
92
		ADIOI_Free(newbuf);
93
94
95
96
97
98
99
100
101
102
103
104
	    }
	    else nbytes += pwrite(fd->fd_sys, buf, size, offset);
	}
	err = nbytes;
    } else {       
	if (!(((long) buf) % fd->d_mem)) {
	    ADIOI_LUSTRE_Aligned_Mem_File_Read(fd, buf, size, offset, &err);
	    nbytes += err;
	} else {
	    newbuf = (void *) memalign(LUSTRE_MEMALIGN, size);
	    if (newbuf) {
		ADIOI_LUSTRE_Aligned_Mem_File_Read(fd, newbuf, size, offset, &err);
105
		if (err > 0) memcpy((void *)buf, newbuf, err);
106
		nbytes += err;
107
		ADIOI_Free(newbuf);
108
	    }
109
	    else nbytes += pread(fd->fd_sys, (void *)buf, size, offset);
110
111
112
113
114
115
	}
	err = nbytes;
    }
    return err;
}

116
static void ADIOI_LUSTRE_IOContig(ADIO_File fd, const void *buf, int count,
117
118
119
                   MPI_Datatype datatype, int file_ptr_type,
	           ADIO_Offset offset, ADIO_Status *status, 
		   int io_mode, int *error_code);
120
static void ADIOI_LUSTRE_IOContig(ADIO_File fd, const void *buf, int count,
121
122
123
124
                   MPI_Datatype datatype, int file_ptr_type,
	           ADIO_Offset offset, ADIO_Status *status, 
		   int io_mode, int *error_code)
{
Gilles Gouaillardet's avatar
Gilles Gouaillardet committed
125
126
    int err=-1;
    MPI_Count datatype_size, len;
127
128
    static char myname[] = "ADIOI_LUSTRE_IOCONTIG";

Gilles Gouaillardet's avatar
Gilles Gouaillardet committed
129
    MPI_Type_size_x(datatype, &datatype_size);
130
131
132
133
134
135
136
137
138
139
140
141
    len = datatype_size * count;

    if (file_ptr_type == ADIO_INDIVIDUAL) {
	offset = fd->fp_ind;
    }

    if (!(fd->direct_read || fd->direct_write)) {
	if (fd->fp_sys_posn != offset) {
	    err = lseek(fd->fd_sys, offset, SEEK_SET);
	    if (err == -1) goto ioerr;
	}
	
142
143
144
145
	if (io_mode) {
#ifdef ADIOI_MPE_LOGGING
        MPE_Log_event(ADIOI_MPE_write_a, 0, NULL);
#endif
146
	    err = write(fd->fd_sys, buf, len);
147
148
149
150
151
152
153
#ifdef ADIOI_MPE_LOGGING
        MPE_Log_event(ADIOI_MPE_write_b, 0, NULL);
#endif
        } else {
#ifdef ADIOI_MPE_LOGGING
        MPE_Log_event(ADIOI_MPE_read_a, 0, NULL);
#endif
154
	    err = read(fd->fd_sys, (void *)buf, len);
155
156
157
158
#ifdef ADIOI_MPE_LOGGING
        MPE_Log_event(ADIOI_MPE_read_b, 0, NULL);
#endif
        }
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
    } else {
	err = ADIOI_LUSTRE_Directio(fd, buf, len, offset, io_mode);
    }

    if (err == -1) goto ioerr;
    fd->fp_sys_posn = offset + err;

    if (file_ptr_type == ADIO_INDIVIDUAL) {
	fd->fp_ind += err; 
    }

#ifdef HAVE_STATUS_SET_BYTES
    if (status) MPIR_Status_set_bytes(status, datatype, err);
#endif
    *error_code = MPI_SUCCESS;

ioerr:
    /* --BEGIN ERROR HANDLING-- */
    if (err == -1) {
	*error_code = MPIO_Err_create_code(MPI_SUCCESS,
					   MPIR_ERR_RECOVERABLE,
					   myname, __LINE__,
					   MPI_ERR_IO, "**io",
					   "**io %s", strerror(errno));
	fd->fp_sys_posn = -1;
	return;
    }
    /* --END ERROR HANDLING-- */
}

189
void ADIOI_LUSTRE_WriteContig(ADIO_File fd, const void *buf, int count,
190
191
192
193
194
195
196
197
198
199
200
201
202
203
                   MPI_Datatype datatype, int file_ptr_type,
	           ADIO_Offset offset, ADIO_Status *status, int *error_code)
{
    ADIOI_LUSTRE_IOContig(fd, buf, count, datatype, file_ptr_type,
	           offset, status, 1, error_code);
}

void ADIOI_LUSTRE_ReadContig(ADIO_File fd, void *buf, int count, 
                   MPI_Datatype datatype, int file_ptr_type,
	           ADIO_Offset offset, ADIO_Status *status, int *error_code)
{
    ADIOI_LUSTRE_IOContig(fd, buf, count, datatype, file_ptr_type,
	           offset, status, 0, error_code);
}