dma_linux_par.c 7.88 KB
Newer Older
Swann Perarnau's avatar
Swann Perarnau committed
1 2 3 4 5 6 7 8 9 10
/*******************************************************************************
 * Copyright 2019 UChicago Argonne, LLC.
 * (c.f. AUTHORS, LICENSE)
 *
 * This file is part of the AML project.
 * For more info, see https://xgitlab.cels.anl.gov/argo/aml
 *
 * SPDX-License-Identifier: BSD-3-Clause
*******************************************************************************/

11
#include "aml.h"
12
#include "aml/dma/linux-par.h"
13
#include "aml/layout/dense.h"
14

15 16 17 18 19 20 21
#include <assert.h>
#include <errno.h>
#include <sys/mman.h>

/*******************************************************************************
 * Linux-backed, paruential dma
 * The dma itself is organized into several different components
22
 * - request types: copy
23 24 25 26 27 28 29 30
 * - implementation of the request
 * - user API (i.e. generic request creation and call)
 * - how to init the dma
 ******************************************************************************/

/*******************************************************************************
 * Requests:
 ******************************************************************************/
31 32
int aml_dma_request_linux_par_create(struct aml_dma_request_linux_par **req,
				     int uuid)
33 34
{
	assert(req != NULL);
35 36 37 38
	*req = calloc(1, sizeof(struct aml_dma_request_linux_par));
	if (*req == NULL)
		return -AML_ENOMEM;
	(*req)->uuid = uuid;
39 40 41
	return 0;
}

42
void aml_dma_request_linux_par_destroy(struct aml_dma_request_linux_par **req)
43
{
44 45 46 47 48 49 50 51 52 53 54 55 56 57 58
	assert(req != NULL);
	free(*req);
	*req = NULL;
}

int aml_dma_linux_par_request_data_init(
				struct aml_dma_linux_par_request_data *req,
				int type,
				struct aml_layout *dest,
				struct aml_layout *src)
{
	assert(req != NULL);
	req->type = type;
	req->dest = dest;
	req->src = src;
59 60 61 62 63 64 65 66 67
	return 0;
}

/*******************************************************************************
 * Internal functions
 ******************************************************************************/

void *aml_dma_linux_par_do_thread(void *arg)
{
68 69
	struct aml_dma_linux_par_request_data *req =
		(struct aml_dma_linux_par_request_data *)arg;
70

Swann Perarnau's avatar
Swann Perarnau committed
71
	pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
72 73
	if (req->type != AML_DMA_REQUEST_TYPE_INVALID)
		aml_copy_layout_generic(req->dest, req->src);
74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92
	return NULL;
}

struct aml_dma_linux_par_ops aml_dma_linux_par_inner_ops = {
	aml_dma_linux_par_do_thread,
};

/*******************************************************************************
 * Public API
 ******************************************************************************/

int aml_dma_linux_par_create_request(struct aml_dma_data *d,
				     struct aml_dma_request **r,
				     int type, va_list ap)
{
	assert(d != NULL);
	assert(r != NULL);
	struct aml_dma_linux_par *dma =
		(struct aml_dma_linux_par *)d;
93 94
	struct aml_dma_request_linux_par *ret;
	struct aml_dma_linux_par_request_data *req;
Swann Perarnau's avatar
Swann Perarnau committed
95
	int err = AML_SUCCESS;
96

97
	pthread_mutex_lock(&dma->data.lock);
98
	req = aml_vector_add(dma->data.requests);
99 100

	/* init the request */
101 102 103 104 105
	if (type == AML_DMA_REQUEST_TYPE_LAYOUT) {
		struct aml_layout *dl, *sl;

		dl = va_arg(ap, struct aml_layout *);
		sl = va_arg(ap, struct aml_layout *);
Swann Perarnau's avatar
Swann Perarnau committed
106 107 108 109
		if (dl == NULL || sl == NULL) {
			err = -AML_EINVAL;
			goto unlock;
		}
110
		aml_dma_linux_par_request_data_init(req,
111 112 113 114 115 116 117 118 119 120
						    AML_DMA_REQUEST_TYPE_LAYOUT,
						    dl, sl);
	} else if (type == AML_DMA_REQUEST_TYPE_PTR) {
		struct aml_layout *dl, *sl;
		void *dp, *sp;
		size_t sz;

		dp = va_arg(ap, void *);
		sp = va_arg(ap, void *);
		sz = va_arg(ap, size_t);
Swann Perarnau's avatar
Swann Perarnau committed
121 122 123 124
		if (dp == NULL || sp == NULL || sz == 0) {
			err = -AML_EINVAL;
			goto unlock;
		}
125 126 127
		/* simple 1D layout, none of the parameters really matter, as
		 * long as the copy generates a single memcpy.
		 */
Swann Perarnau's avatar
Swann Perarnau committed
128 129
		aml_layout_dense_create(&dl, dp, 0, 1, 1, &sz, NULL, NULL);
		aml_layout_dense_create(&sl, sp, 0, 1, 1, &sz, NULL, NULL);
130
		aml_dma_linux_par_request_data_init(req,
131 132
						    AML_DMA_REQUEST_TYPE_PTR,
						    dl, sl);
Swann Perarnau's avatar
Swann Perarnau committed
133 134 135
	} else
		err = -AML_EINVAL;
unlock:
136
	pthread_mutex_unlock(&dma->data.lock);
137 138
	if (req->type != AML_DMA_REQUEST_TYPE_INVALID) {
		int uuid = aml_vector_getid(dma->data.requests, req);
139

140
		pthread_create(&req->thread, NULL, dma->ops.do_thread, req);
141 142 143
		aml_dma_request_linux_par_create(&ret, uuid);
		*r = (struct aml_dma_request *)ret;
	}
Swann Perarnau's avatar
Swann Perarnau committed
144
	return err;
145 146 147
}

int aml_dma_linux_par_destroy_request(struct aml_dma_data *d,
Swann Perarnau's avatar
Swann Perarnau committed
148
				      struct aml_dma_request **r)
149 150 151 152 153
{
	assert(d != NULL);
	assert(r != NULL);
	struct aml_dma_linux_par *dma =
		(struct aml_dma_linux_par *)d;
Swann Perarnau's avatar
Swann Perarnau committed
154
	struct aml_dma_request_linux_par *req;
155 156
	struct aml_dma_linux_par_request_data *inner_req;

Swann Perarnau's avatar
Swann Perarnau committed
157 158 159 160
	if (*r == NULL)
		return -AML_EINVAL;
	req = (struct aml_dma_request_linux_par *)*r;

161 162 163
	inner_req = aml_vector_get(dma->data.requests, req->uuid);
	if (inner_req == NULL)
		return -AML_EINVAL;
164 165

	/* we cancel and join, instead of killing, for a cleaner result */
166 167 168
	if (inner_req->type != AML_DMA_REQUEST_TYPE_INVALID) {
		pthread_cancel(inner_req->thread);
		pthread_join(inner_req->thread, NULL);
Swann Perarnau's avatar
Swann Perarnau committed
169 170 171 172 173
	}

	if (inner_req->type == AML_DMA_REQUEST_TYPE_PTR) {
		aml_layout_dense_destroy(&inner_req->dest);
		aml_layout_dense_destroy(&inner_req->src);
174
	}
175
	pthread_mutex_lock(&dma->data.lock);
176
	aml_vector_remove(dma->data.requests, inner_req);
177
	pthread_mutex_unlock(&dma->data.lock);
178
	aml_dma_request_linux_par_destroy(&req);
Swann Perarnau's avatar
Swann Perarnau committed
179
	*r = NULL;
180 181 182 183
	return 0;
}

int aml_dma_linux_par_wait_request(struct aml_dma_data *d,
Swann Perarnau's avatar
Swann Perarnau committed
184
				   struct aml_dma_request **r)
185 186 187 188
{
	assert(d != NULL);
	assert(r != NULL);
	struct aml_dma_linux_par *dma = (struct aml_dma_linux_par *)d;
Swann Perarnau's avatar
Swann Perarnau committed
189
	struct aml_dma_request_linux_par *req;
190
	struct aml_dma_linux_par_request_data *inner_req;
191

Swann Perarnau's avatar
Swann Perarnau committed
192 193 194 195
	if (*r == NULL)
		return -AML_EINVAL;
	req = (struct aml_dma_request_linux_par *)*r;

196 197 198 199
	inner_req = aml_vector_get(dma->data.requests, req->uuid);
	if (inner_req == NULL)
		return -AML_EINVAL;

Swann Perarnau's avatar
Swann Perarnau committed
200
	if (inner_req->type != AML_DMA_REQUEST_TYPE_INVALID)
201
		pthread_join(inner_req->thread, NULL);
202

Swann Perarnau's avatar
Swann Perarnau committed
203 204 205 206
	if (inner_req->type == AML_DMA_REQUEST_TYPE_PTR) {
		aml_layout_dense_destroy(&inner_req->dest);
		aml_layout_dense_destroy(&inner_req->src);
	}
207
	pthread_mutex_lock(&dma->data.lock);
208
	aml_vector_remove(dma->data.requests, inner_req);
209
	pthread_mutex_unlock(&dma->data.lock);
210
	aml_dma_request_linux_par_destroy(&req);
Swann Perarnau's avatar
Swann Perarnau committed
211
	*r = NULL;
212 213 214 215 216 217 218 219 220 221 222 223 224
	return 0;
}

struct aml_dma_ops aml_dma_linux_par_ops = {
	aml_dma_linux_par_create_request,
	aml_dma_linux_par_destroy_request,
	aml_dma_linux_par_wait_request,
};

/*******************************************************************************
 * Init functions:
 ******************************************************************************/

225
int aml_dma_linux_par_create(struct aml_dma **dma, size_t nbreqs)
226 227
{
	struct aml_dma *ret = NULL;
228
	struct aml_dma_linux_par *d;
229

230
	if (dma == NULL)
231
		return -AML_EINVAL;
232

233 234
	*dma = NULL;

235
	ret = AML_INNER_MALLOC_2(struct aml_dma, struct aml_dma_linux_par);
236
	if (ret == NULL)
237
		return -AML_ENOMEM;
238

239 240
	ret->data = AML_INNER_MALLOC_NEXTPTR(ret, struct aml_dma,
					     struct aml_dma_linux_par);
241
	ret->ops = &aml_dma_linux_par_ops;
242 243
	d = (struct aml_dma_linux_par *)ret->data;
	d->ops = aml_dma_linux_par_inner_ops;
244

245
	/* allocate request array */
246
	aml_vector_create(&d->data.requests, nbreqs,
247 248
			  sizeof(struct aml_dma_linux_par_request_data),
			  offsetof(struct aml_dma_linux_par_request_data, type),
249 250 251 252
			  AML_DMA_REQUEST_TYPE_INVALID);
	pthread_mutex_init(&d->data.lock, NULL);

	*dma = ret;
253 254 255
	return 0;
}

Swann Perarnau's avatar
Swann Perarnau committed
256
void aml_dma_linux_par_destroy(struct aml_dma **d)
257
{
Swann Perarnau's avatar
Swann Perarnau committed
258
	struct aml_dma_linux_par *dma;
259

Swann Perarnau's avatar
Swann Perarnau committed
260
	if (d == NULL || *d == NULL)
261
		return;
Swann Perarnau's avatar
Swann Perarnau committed
262 263 264 265 266 267 268 269 270 271 272 273 274 275 276 277 278 279
	dma = (struct aml_dma_linux_par *)(*d)->data;
	for (size_t i = 0; i < aml_vector_size(dma->data.requests); i++) {
		struct aml_dma_linux_par_request_data *req;

		req = aml_vector_get(dma->data.requests, i);
		if (req->type != AML_DMA_REQUEST_TYPE_INVALID) {
			pthread_cancel(req->thread);
			pthread_join(req->thread, NULL);
		}
		if (req->type == AML_DMA_REQUEST_TYPE_PTR) {
			aml_layout_dense_destroy(&req->dest);
			aml_layout_dense_destroy(&req->src);
		}
	}
	aml_vector_destroy(&dma->data.requests);
	pthread_mutex_destroy(&dma->data.lock);
	free(*d);
	*d = NULL;
280
}