scratch_seq.c 9.18 KB
Newer Older
Swann Perarnau's avatar
Swann Perarnau committed
1
2
3
4
5
6
7
8
9
10
/*******************************************************************************
 * Copyright 2019 UChicago Argonne, LLC.
 * (c.f. AUTHORS, LICENSE)
 *
 * This file is part of the AML project.
 * For more info, see https://xgitlab.cels.anl.gov/argo/aml
 *
 * SPDX-License-Identifier: BSD-3-Clause
*******************************************************************************/

11
#include "aml.h"
12
#include "aml/layout/dense.h"
13
#include "aml/scratch/seq.h"
14
15
16
17
18
19
20
21
22
23
24
25
26
27
28
#include <assert.h>

/*******************************************************************************
 * Sequential scratchpad
 * The scratch itself is organized into several different components
 * - request types: push and pull
 * - implementation of the request
 * - user API (i.e. generic request creation and call)
 * - how to init the scratch
 ******************************************************************************/

/*******************************************************************************
 * Requests:
 ******************************************************************************/

29
30
31
int aml_scratch_request_seq_init(struct aml_scratch_request_seq *req, int type,
				 struct aml_tiling *t, void *dstptr, int dstid,
				 void *srcptr, int srcid)
32
33
34

{
	assert(req != NULL);
35
36
37
	void *dp, *sp;
	size_t size;

38
	req->type = type;
39
	req->tiling = t;
40
41
	req->srcid = srcid;
	req->dstid = dstid;
42
43
44
45
46
	dp = aml_tiling_tilestart(req->tiling, dstptr, dstid);
	sp = aml_tiling_tilestart(req->tiling, srcptr, srcid);
	size = aml_tiling_tilesize(req->tiling, srcid);
	aml_layout_dense_create(&req->dst, dp, 0, 1, 1, &size, NULL, NULL);
	aml_layout_dense_create(&req->src, sp, 0, 1, 1, &size, NULL, NULL);
47
48
49
50
51
52
	return 0;
}

int aml_scratch_request_seq_destroy(struct aml_scratch_request_seq *r)
{
	assert(r != NULL);
53
54
	aml_layout_dense_destroy(&r->dst);
	aml_layout_dense_destroy(&r->src);
55
	return 0;
56
57
58
59
60
61
}

/*******************************************************************************
 * Internal functions
 ******************************************************************************/
int aml_scratch_seq_doit(struct aml_scratch_seq_data *scratch,
62
			      struct aml_scratch_request_seq *req)
63
64
65
66
{
	assert(scratch != NULL);
	assert(req != NULL);
	return aml_dma_async_copy(scratch->dma, &req->dma_req,
67
68
				  AML_DMA_REQUEST_TYPE_LAYOUT,
				  req->dst, req->src);
69
70
71
72
73
74
75
76
77
78
}

struct aml_scratch_seq_ops aml_scratch_seq_inner_ops = {
	aml_scratch_seq_doit,
};

/*******************************************************************************
 * Public API
 ******************************************************************************/

79
80
/* TODO: not thread-safe */

81
82
83
84
85
86
87
88
int aml_scratch_seq_create_request(struct aml_scratch_data *d,
				   struct aml_scratch_request **r,
				   int type, va_list ap)
{
	assert(d != NULL);
	assert(r != NULL);
	struct aml_scratch_seq *scratch =
		(struct aml_scratch_seq *)d;
89
90

	struct aml_scratch_request_seq *req;
91

92
	pthread_mutex_lock(&scratch->data.lock);
93
	req = aml_vector_add(scratch->data.requests);
94
	/* init the request */
95
	if (type == AML_SCRATCH_REQUEST_TYPE_PUSH) {
96
97
98
99
100
101
102
103
104
105
106
		int scratchid;
		int *srcid;
		void *srcptr;
		void *scratchptr;

		srcptr = va_arg(ap, void *);
		srcid = va_arg(ap, int *);
		scratchptr = va_arg(ap, void *);
		scratchid = va_arg(ap, int);

		/* find destination tile */
107
		int *slot = aml_vector_get(scratch->data.tilemap, scratchid);
108

109
110
		assert(slot != NULL);
		*srcid = *slot;
111
112

		/* init request */
113
		aml_scratch_request_seq_init(req, type,
114
					     scratch->data.tiling,
115
116
					     srcptr, *srcid,
					     scratchptr, scratchid);
117
	} else if (type == AML_SCRATCH_REQUEST_TYPE_PULL) {
118
119
120
121
		int *scratchid;
		int srcid;
		void *srcptr;
		void *scratchptr;
122
		int slot, *tile;
123
124
125
126
127
128

		scratchptr = va_arg(ap, void *);
		scratchid = va_arg(ap, int *);
		srcptr = va_arg(ap, void *);
		srcid = va_arg(ap, int);

129
130
131
132
133
134
		/* find destination tile
		 * We don't use add here because adding a tile means allocating
		 * new tiles on the sch_area too. */
		/* TODO: this is kind of a bug: we reuse a tile, instead of
		 * creating a no-op request
		 */
135
		slot = aml_vector_find(scratch->data.tilemap, srcid);
136
		if (slot == -1) {
137
			slot = aml_vector_find(scratch->data.tilemap, -1);
138
			assert(slot != -1);
139
			tile = aml_vector_get(scratch->data.tilemap, slot);
140
			*tile = srcid;
141
		} else
142
143
144
			type = AML_SCRATCH_REQUEST_TYPE_NOOP;

		/* save the key */
145
		*scratchid = slot;
146
147

		/* init request */
148
149
150
		aml_scratch_request_seq_init(req, type, scratch->data.tiling,
					     scratchptr, *scratchid,
					     srcptr, srcid);
151
	}
152
	pthread_mutex_unlock(&scratch->data.lock);
153
	if (req->type != AML_SCRATCH_REQUEST_TYPE_NOOP)
154
		scratch->ops.doit(&scratch->data, req);
155
156

	*r = (struct aml_scratch_request *)req;
157
158
159
160
161
162
163
164
165
166
	return 0;
}

int aml_scratch_seq_destroy_request(struct aml_scratch_data *d,
					 struct aml_scratch_request *r)
{
	assert(d != NULL);
	assert(r != NULL);
	struct aml_scratch_seq *scratch =
		(struct aml_scratch_seq *)d;
167

168
169
	struct aml_scratch_request_seq *req =
		(struct aml_scratch_request_seq *)r;
170
	int *tile;
171

172
173
174
	if (req->type != AML_SCRATCH_REQUEST_TYPE_NOOP)
		aml_dma_cancel(scratch->data.dma, &req->dma_req);
	aml_scratch_request_seq_destroy(req);
175
176

	/* destroy removes the tile from the scratch */
177
	pthread_mutex_lock(&scratch->data.lock);
178
179
180
181
	if (req->type == AML_SCRATCH_REQUEST_TYPE_PUSH)
		tile = aml_vector_get(scratch->data.tilemap, req->srcid);
	else if (req->type == AML_SCRATCH_REQUEST_TYPE_PULL)
		tile = aml_vector_get(scratch->data.tilemap, req->dstid);
182
	aml_vector_remove(scratch->data.tilemap, tile);
183
	aml_vector_remove(scratch->data.requests, req);
184
	pthread_mutex_unlock(&scratch->data.lock);
185
186
187
188
189
190
191
192
193
194
195
	return 0;
}

int aml_scratch_seq_wait_request(struct aml_scratch_data *d,
				   struct aml_scratch_request *r)
{
	assert(d != NULL);
	assert(r != NULL);
	struct aml_scratch_seq *scratch = (struct aml_scratch_seq *)d;
	struct aml_scratch_request_seq *req =
		(struct aml_scratch_request_seq *)r;
196
	int *tile;
197
198

	/* wait for completion of the request */
199
200
	if (req->type != AML_SCRATCH_REQUEST_TYPE_NOOP)
		aml_dma_wait(scratch->data.dma, &req->dma_req);
201

202
	/* cleanup a completed request. In case of push, free up the tile */
203
	aml_scratch_request_seq_destroy(req);
204
	pthread_mutex_lock(&scratch->data.lock);
205
206
	if (req->type == AML_SCRATCH_REQUEST_TYPE_PUSH) {
		tile = aml_vector_get(scratch->data.tilemap, req->srcid);
207
		aml_vector_remove(scratch->data.tilemap, tile);
208
	}
209
	aml_vector_remove(scratch->data.requests, req);
210
	pthread_mutex_unlock(&scratch->data.lock);
211
212
213
	return 0;
}

214
void *aml_scratch_seq_baseptr(const struct aml_scratch_data *d)
215
216
{
	assert(d != NULL);
217
218
219
	const struct aml_scratch_seq *scratch =
		(const struct aml_scratch_seq *)d;

220
221
222
	return scratch->data.sch_ptr;
}

223
224
225
226
227
228
229
int aml_scratch_seq_release(struct aml_scratch_data *d, int scratchid)
{
	assert(d != NULL);
	struct aml_scratch_seq *scratch = (struct aml_scratch_seq *)d;
	int *tile;

	pthread_mutex_lock(&scratch->data.lock);
230
	tile = aml_vector_get(scratch->data.tilemap, scratchid);
231
	if (tile != NULL)
232
		aml_vector_remove(scratch->data.tilemap, tile);
233
234
235
236
	pthread_mutex_unlock(&scratch->data.lock);
	return 0;
}

237
238
239
240
struct aml_scratch_ops aml_scratch_seq_ops = {
	aml_scratch_seq_create_request,
	aml_scratch_seq_destroy_request,
	aml_scratch_seq_wait_request,
241
	aml_scratch_seq_baseptr,
242
	aml_scratch_seq_release,
243
244
245
246
247
248
};

/*******************************************************************************
 * Init functions:
 ******************************************************************************/

249
int aml_scratch_seq_create(struct aml_scratch **scratch,
250
251
252
253
			   struct aml_area *scratch_area,
			   struct aml_area *src_area,
			   struct aml_dma *dma, struct aml_tiling *tiling,
			   size_t nbtiles, size_t nbreqs)
254
255
{
	struct aml_scratch *ret = NULL;
256
	struct aml_scratch_seq *s;
257

258
259
260
	if (scratch == NULL
	    || scratch_area == NULL || src_area == NULL
	    || dma == NULL || tiling == NULL)
261
		return -AML_EINVAL;
262

263
264
	*scratch = NULL;

265
	ret = AML_INNER_MALLOC_2(struct aml_scratch, struct aml_scratch_seq);
266
	if (ret == NULL)
267
		return -AML_ENOMEM;
268

269
	ret->ops = &aml_scratch_seq_ops;
270
271
	ret->data = AML_INNER_MALLOC_NEXTPTR(ret, struct aml_scratch,
					     struct aml_scratch_seq);
272
273
	s = (struct aml_scratch_seq *)ret->data;
	s->ops = aml_scratch_seq_inner_ops;
274

275
276
277
278
	s->data.sch_area = scratch_area;
	s->data.src_area = src_area;
	s->data.dma = dma;
	s->data.tiling = tiling;
279

280
	/* allocate request array */
281
	aml_vector_create(&s->data.requests, nbreqs,
282
283
			  sizeof(struct aml_scratch_request_seq),
			  offsetof(struct aml_scratch_request_seq, type),
284
			  AML_SCRATCH_REQUEST_TYPE_INVALID);
285

286
287
288
	/* s init */
	aml_vector_create(&s->data.tilemap, nbtiles, sizeof(int), 0, -1);
	size_t tilesize = aml_tiling_tilesize(s->data.tiling, 0);
289

290
291
	s->data.scratch_size = nbtiles * tilesize;
	s->data.sch_ptr = aml_area_mmap(s->data.sch_area,
Nicolas Denoyelle's avatar
Nicolas Denoyelle committed
292
					      NULL,
293
294
295
296
					      s->data.scratch_size);
	pthread_mutex_init(&s->data.lock, NULL);

	*scratch = ret;
297
298
299
	return 0;
}

300
void aml_scratch_seq_destroy(struct aml_scratch **scratch)
301
{
302
303
	struct aml_scratch *s;
	struct aml_scratch_seq *inner;
304

305
	if (scratch == NULL)
306
		return;
307
	s = *scratch;
308
	if (s == NULL)
309
		return;
310
311

	assert(s->data != NULL);
312
313
314
315
316
317
318
319
320
	inner = (struct aml_scratch_seq *)s->data;
	aml_vector_destroy(&inner->data.requests);
	aml_vector_destroy(&inner->data.tilemap);
	aml_area_munmap(inner->data.sch_area,
			inner->data.sch_ptr,
			inner->data.scratch_size);
	pthread_mutex_destroy(&inner->data.lock);
	free(s);
	*scratch = NULL;
321
}