[feature] add sequential, copy-based scratchpad

This is the initial implementation and validation of a scratchpad: a
logic unit that handles tracking data across a "main" area and a
"scratch" area.

The API and internals will probably change again soon, as there's no
clear way to implement a move based scratchpad on this one.

Note that this implementation doesn't do any tracking, not really, and
that's the next step.

src/Makefile.am

@@ -20,13 +20,17 @@ DMA_CSOURCES = dma.c \
 	       dma_linux_par.c \
 	       dma_linux_seq.c
 
+SCRATCH_CSOURCES = scratch.c \
+		   scratch_seq.c
+
 LIBCSOURCES = aml.c area.c arena.c \
 	      $(ARENA_JEMALLOC_CSOURCES) \
 	      $(AREA_LINUX_CSOURCES) \
 	      $(AREA_POSIX_CSOURCES) \
 	      $(TILING_CSOURCES) \
 	      $(BINDING_CSOURCES) \
-	      $(DMA_CSOURCES)
+	      $(DMA_CSOURCES) \
+	      $(SCRATCH_CSOURCES)
 
 LIBHSOURCES = aml.h
 

src/aml.h

@@ -643,6 +643,100 @@ int aml_dma_linux_par_init(struct aml_dma *, ...);
 int aml_dma_linux_par_vinit(struct aml_dma *, va_list);
 int aml_dma_linux_par_destroy(struct aml_dma *);
 
+/*******************************************************************************
+ * Scratchpad:
+ * Use an area to stage data from an another area in and out.
+ * A dma handles the movement itself.
+ ******************************************************************************/
+
+struct aml_scratch_request;
+struct aml_scratch_data;
+
+#define AML_SCRATCH_REQUEST_TYPE_INVALID -1
+#define AML_SCRATCH_REQUEST_TYPE_PUSH 0
+#define AML_SCRATCH_REQUEST_TYPE_PULL 1
+
+struct aml_scratch_ops {
+	int (*create_request)(struct aml_scratch_data *scratch,
+			      struct aml_scratch_request **req, int type,
+			      va_list args);
+	int (*destroy_request)(struct aml_scratch_data *scratch,
+			       struct aml_scratch_request *req);
+	int (*wait_request)(struct aml_scratch_data *scratch,
+			    struct aml_scratch_request *req);
+};
+
+struct aml_scratch {
+	struct aml_scratch_ops *ops;
+	struct aml_scratch_data *data;
+};
+
+int aml_scratch_pull(struct aml_scratch *scratch, ...);
+int aml_scratch_async_pull(struct aml_scratch *scratch,
+			   struct aml_scratch_request **req, ...);
+int aml_scratch_push(struct aml_scratch *scratch, ...);
+int aml_scratch_async_push(struct aml_scratch *scratch,
+			   struct aml_scratch_request **req, ...);
+int aml_scratch_wait(struct aml_scratch *scratch,
+		     struct aml_scratch_request *req);
+int aml_scratch_cancel(struct aml_scratch *scratch,
+		       struct aml_scratch_request *req);
+
+/*******************************************************************************
+ * Sequential scratchpad API:
+ * Scratchpad uses calling thread to trigger dma movements.
+ ******************************************************************************/
+
+extern struct aml_scratch_ops aml_scratch_seq_ops;
+
+struct aml_scratch_request_seq {
+	int type;
+	struct aml_tiling *stiling;
+	void *srcptr;
+	int srcid;
+	struct aml_tiling *dtiling;
+	void *dstptr;
+	int dstid;
+	struct aml_dma_request *dma_req;
+};
+
+struct aml_scratch_seq_data {
+	struct aml_area *srcarea, *scratcharea;
+	struct aml_tiling *srctiling, *scratchtiling;
+	struct aml_dma *dma;
+	size_t nbrequests;
+	struct aml_scratch_request_seq *requests;
+};
+
+struct aml_scratch_seq_ops {
+	int (*doit)(struct aml_scratch_seq_data *scratch,
+		    struct aml_scratch_request_seq *req);
+	int (*add_request)(struct aml_scratch_seq_data *scratch,
+			   struct aml_scratch_request_seq **req);
+	int (*remove_request)(struct aml_scratch_seq_data *scratch,
+			      struct aml_scratch_request_seq **req);
+};
+
+struct aml_scratch_seq {
+	struct aml_scratch_seq_ops ops;
+	struct aml_scratch_seq_data data;
+};
+
+#define AML_SCRATCH_SEQ_DECL(name) \
+	struct aml_scratch_seq __ ##name## _inner_data; \
+	struct aml_scratch name = { \
+		&aml_scratch_seq_ops, \
+		(struct aml_scratch_data *)&__ ## name ## _inner_data, \
+	};
+
+#define AML_SCRATCH_SEQ_ALLOCSIZE \
+	(sizeof(struct aml_scratch_seq) + \
+	 sizeof(struct aml_scratch))
+
+int aml_scratch_seq_create(struct aml_scratch **scratch, ...);
+int aml_scratch_seq_init(struct aml_scratch *scratch, ...);
+int aml_scratch_seq_vinit(struct aml_scratch *scratch, va_list args);
+int aml_scratch_seq_destroy(struct aml_scratch *scratch);
 
 /*******************************************************************************
  * General functions:

src/scratch.c

+#include <aml.h>
+#include <assert.h>
+
+/*******************************************************************************
+ * Generic Scratchpad API:
+ * Most of the stuff is dispatched to a different layer, using type-specific
+ * functions.
+ *
+ * As for DMA, the API is slightly different than the functions below, as we
+ * abstract the request creation after this layer.
+ ******************************************************************************/
+
+int aml_scratch_pull(struct aml_scratch *scratch, ...)
+{
+	assert(scratch != NULL);
+	va_list ap;
+	int ret;
+	struct aml_scratch_request *req;
+	va_start(ap, scratch);
+	ret = scratch->ops->create_request(scratch->data, &req,
+				       AML_SCRATCH_REQUEST_TYPE_PULL, ap);
+	va_end(ap);
+	ret = scratch->ops->wait_request(scratch->data, req);
+	return ret;
+}
+
+int aml_scratch_async_pull(struct aml_scratch *scratch,
+			   struct aml_scratch_request **req, ...)
+{
+	assert(scratch != NULL);
+	assert(req != NULL);
+	va_list ap;
+	int ret;
+	va_start(ap, req);
+	ret = scratch->ops->create_request(scratch->data, req,
+				       AML_SCRATCH_REQUEST_TYPE_PULL, ap);
+	va_end(ap);
+	return ret;
+}
+
+int aml_scratch_push(struct aml_scratch *scratch, ...)
+{
+	assert(scratch != NULL);
+	struct aml_scratch_request *req;
+	va_list ap;
+	int ret;
+	va_start(ap, scratch);
+	ret = scratch->ops->create_request(scratch->data, &req,
+				       AML_SCRATCH_REQUEST_TYPE_PUSH, ap);
+	va_end(ap);
+	ret = scratch->ops->wait_request(scratch->data, req);
+	return ret;
+}
+
+
+int aml_scratch_async_push(struct aml_scratch *scratch, struct aml_scratch_request **req, ...)
+{
+	assert(scratch != NULL);
+	assert(req != NULL);
+	va_list ap;
+	int ret;
+	va_start(ap, req);
+	ret = scratch->ops->create_request(scratch->data, req,
+				       AML_SCRATCH_REQUEST_TYPE_PUSH, ap);
+	va_end(ap);
+	return ret;
+}
+
+int aml_scratch_cancel(struct aml_scratch *scratch, struct aml_scratch_request *req)
+{
+	assert(scratch != NULL);
+	assert(req != NULL);
+	return scratch->ops->destroy_request(scratch->data, req);
+}
+
+int aml_scratch_wait(struct aml_scratch *scratch, struct aml_scratch_request *req)
+{
+	assert(scratch != NULL);
+	assert(req != NULL);
+	return scratch->ops->wait_request(scratch->data, req);
+}

src/scratch_seq.c

+#include <aml.h>
+#include <assert.h>
+
+/*******************************************************************************
+ * Sequential scratchpad
+ * The scratch itself is organized into several different components
+ * - request types: push and pull
+ * - implementation of the request
+ * - user API (i.e. generic request creation and call)
+ * - how to init the scratch
+ ******************************************************************************/
+
+/*******************************************************************************
+ * Requests:
+ ******************************************************************************/
+
+int aml_scratch_request_seq_init(struct aml_scratch_request_seq *req, int type,
+				 struct aml_tiling *dt, void *dstptr, int dstid,
+				 struct aml_tiling *st, void *srcptr, int srcid)
+
+{
+	assert(req != NULL);
+	req->type = type;
+	req->stiling = st;
+	req->srcptr = srcptr;
+	req->srcid = srcid;
+	req->dtiling = dt;
+	req->dstptr = dstptr;
+	req->dstid = dstid;
+	return 0;
+}
+
+int aml_scratch_request_seq_destroy(struct aml_scratch_request_seq *r)
+{
+	assert(r != NULL);
+	return 0;
+}
+
+/*******************************************************************************
+ * Internal functions
+ ******************************************************************************/
+int aml_scratch_seq_doit(struct aml_scratch_seq_data *scratch,
+			      struct aml_scratch_request_seq *req)
+{
+	assert(scratch != NULL);
+	assert(req != NULL);
+	return aml_dma_async_copy(scratch->dma, &req->dma_req,
+				  req->dtiling, req->dstptr, req->dstid,
+				  req->stiling, req->srcptr, req->srcid);
+}
+
+int aml_scratch_seq_add_request(struct aml_scratch_seq_data *data,
+				struct aml_scratch_request_seq **req)
+{
+	for(int i = 0; i < data->nbrequests; i++)
+	{
+		if(data->requests[i].type == AML_SCRATCH_REQUEST_TYPE_INVALID)
+		{
+			*req = &data->requests[i];
+			return 0;
+		}
+	}
+	/* TODO: slow path, need to resize the array */
+	return 0;
+}
+
+int aml_scratch_seq_remove_request(struct aml_scratch_seq_data *data,
+				     struct aml_scratch_request_seq **req)
+{
+	/* TODO: assert that the pointer is in the right place */
+	(*req)->type = AML_SCRATCH_REQUEST_TYPE_INVALID;
+	return 0;
+}
+
+struct aml_scratch_seq_ops aml_scratch_seq_inner_ops = {
+	aml_scratch_seq_doit,
+	aml_scratch_seq_add_request,
+	aml_scratch_seq_remove_request,
+};
+
+/*******************************************************************************
+ * Public API
+ ******************************************************************************/
+
+int aml_scratch_seq_create_request(struct aml_scratch_data *d,
+				   struct aml_scratch_request **r,
+				   int type, va_list ap)
+{
+	assert(d != NULL);
+	assert(r != NULL);
+	struct aml_scratch_seq *scratch =
+		(struct aml_scratch_seq *)d;
+
+	struct aml_scratch_request_seq *req;
+
+	/* find an available request slot */
+	scratch->ops.add_request(&scratch->data, &req);
+
+	/* init the request */
+	if(type == AML_SCRATCH_REQUEST_TYPE_PUSH)
+	{
+		int scratchid;
+		int *srcid;
+		void *srcptr;
+		void *scratchptr;
+
+		srcptr = va_arg(ap, void *);
+		srcid = va_arg(ap, int *);
+		scratchptr = va_arg(ap, void *);
+		scratchid = va_arg(ap, int);
+
+		/* find destination tile */
+		//scratch->ops.scratch2src(scratch->data, scratchid, srcid);
+		*srcid = scratchid;
+
+		/* init request */
+		aml_scratch_request_seq_init(req, type, scratch->data.srctiling,
+					     srcptr, *srcid,
+					     scratch->data.scratchtiling,
+					     scratchptr, scratchid);
+	}
+	else if(type == AML_SCRATCH_REQUEST_TYPE_PULL)
+	{
+		int *scratchid;
+		int srcid;
+		void *srcptr;
+		void *scratchptr;
+
+		scratchptr = va_arg(ap, void *);
+		scratchid = va_arg(ap, int *);
+		srcptr = va_arg(ap, void *);
+		srcid = va_arg(ap, int);
+
+		/* find destination tile */
+		//scratch->ops.src2scratch(scratch->data, scratchid, srcid);
+		*scratchid = srcid;
+
+		/* init request */
+		aml_scratch_request_seq_init(req, type,
+					     scratch->data.scratchtiling,
+					     scratchptr, *scratchid,
+					     scratch->data.srctiling,
+					     srcptr, srcid);
+	}
+	scratch->ops.doit(&scratch->data, req);
+
+	*r = (struct aml_scratch_request *)req;
+	return 0;
+}
+
+int aml_scratch_seq_destroy_request(struct aml_scratch_data *d,
+					 struct aml_scratch_request *r)
+{
+	assert(d != NULL);
+	assert(r != NULL);
+	struct aml_scratch_seq *scratch =
+		(struct aml_scratch_seq *)d;
+
+	struct aml_scratch_request_seq *req =
+		(struct aml_scratch_request_seq *)r;
+
+	aml_dma_cancel(scratch->data.dma, req->dma_req);
+	aml_scratch_request_seq_destroy(req);
+	scratch->ops.remove_request(&scratch->data, &req);
+	return 0;
+}
+
+int aml_scratch_seq_wait_request(struct aml_scratch_data *d,
+				   struct aml_scratch_request *r)
+{
+	assert(d != NULL);
+	assert(r != NULL);
+	struct aml_scratch_seq *scratch = (struct aml_scratch_seq *)d;
+	struct aml_scratch_request_seq *req =
+		(struct aml_scratch_request_seq *)r;
+
+	/* wait for completion of the request */
+	aml_dma_wait(scratch->data.dma, req->dma_req);
+
+	/* destroy a completed request */
+	aml_scratch_request_seq_destroy(req);
+	scratch->ops.remove_request(&scratch->data, &req);
+	return 0;
+}
+
+struct aml_scratch_ops aml_scratch_seq_ops = {
+	aml_scratch_seq_create_request,
+	aml_scratch_seq_destroy_request,
+	aml_scratch_seq_wait_request,
+};
+
+/*******************************************************************************
+ * Init functions:
+ ******************************************************************************/
+
+int aml_scratch_seq_create(struct aml_scratch **d, ...)
+{
+	va_list ap;
+	struct aml_scratch *ret = NULL;
+	intptr_t baseptr, dataptr;
+	va_start(ap, d);
+
+	/* alloc */
+	baseptr = (intptr_t) calloc(1, AML_SCRATCH_SEQ_ALLOCSIZE);
+	dataptr = baseptr + sizeof(struct aml_scratch);
+
+	ret = (struct aml_scratch *)baseptr;
+	ret->data = (struct aml_scratch_data *)dataptr;
+
+	aml_scratch_seq_vinit(ret, ap);
+
+	va_end(ap);
+	*d = ret;
+	return 0;
+}
+int aml_scratch_seq_vinit(struct aml_scratch *d, va_list ap)
+{
+	d->ops = &aml_scratch_seq_ops;
+	struct aml_scratch_seq *scratch = (struct aml_scratch_seq *)d->data;
+
+	scratch->ops = aml_scratch_seq_inner_ops;
+
+	scratch->data.scratcharea = va_arg(ap, struct aml_area *);
+	scratch->data.scratchtiling = va_arg(ap, struct aml_tiling *);
+	scratch->data.srcarea = va_arg(ap, struct aml_area *);
+	scratch->data.srctiling = va_arg(ap, struct aml_tiling *);
+	scratch->data.dma = va_arg(ap, struct aml_dma *);
+	
+	/* allocate request array */
+	scratch->data.nbrequests = va_arg(ap, size_t);
+	scratch->data.requests = calloc(scratch->data.nbrequests,
+				     sizeof(struct aml_scratch_request_seq));
+	for(int i = 0; i < scratch->data.nbrequests; i++)
+		scratch->data.requests[i].type = AML_SCRATCH_REQUEST_TYPE_INVALID;
+	return 0;
+}
+int aml_scratch_seq_init(struct aml_scratch *d, ...)
+{
+	int err;
+	va_list ap;
+	va_start(ap, d);
+	err = aml_scratch_seq_vinit(d, ap);
+	va_end(ap);
+	return err;
+}
+
+int aml_scratch_seq_destroy(struct aml_scratch *d)
+{
+	struct aml_scratch_seq *scratch = (struct aml_scratch_seq *)d->data;
+	free(scratch->data.requests);
+	return 0;
+}

tests/Makefile.am

@@ -25,13 +25,16 @@ BINDING_TESTS = binding
 DMA_LINUX_TESTS = dma_linux_seq \
 		  dma_linux_par
 
+SCRATCH_TESTS = scratch_seq
+
 # unit tests
 UNIT_TESTS = $(ARENA_JEMALLOC_TESTS) \
 	     $(TILING_TESTS) \
 	     $(BINDING_TESTS) \
 	     $(AREA_LINUX_TESTS) \
 	     $(AREA_POSIX_TESTS) \
-	     $(DMA_LINUX_TESTS)
+	     $(DMA_LINUX_TESTS) \
+	     $(SCRATCH_TESTS)
 
 # fonctional tests
 FUNC_TESTS = stream_add_pth stream_add_omp stream_vanilla

tests/scratch_seq.c

+#include <aml.h>
+#include <assert.h>
+
+#define TILESIZE (2)
+#define NBTILES (4)
+
+int main(int argc, char *argv[])
+{
+	AML_BINDING_SINGLE_DECL(binding);
+	AML_TILING_1D_DECL(tiling);
+	AML_ARENA_JEMALLOC_DECL(arena);
+	AML_AREA_LINUX_DECL(area);
+	AML_DMA_LINUX_PAR_DECL(dma);
+	AML_SCRATCH_SEQ_DECL(scratch);
+	unsigned long nodemask[AML_NODEMASK_SZ];
+	void *dst, *src;
+
+	/* library initialization */
+	aml_init(&argc, &argv);
+
+	/* initialize all the supporting struct */
+	assert(!aml_binding_init(&binding, AML_BINDING_TYPE_SINGLE, 0));
+	assert(!aml_tiling_init(&tiling, AML_TILING_TYPE_1D, TILESIZE*PAGE_SIZE,
+				TILESIZE*PAGE_SIZE*NBTILES));
+	AML_NODEMASK_ZERO(nodemask);
+	AML_NODEMASK_SET(nodemask, 0);
+	assert(!aml_arena_jemalloc_init(&arena, AML_ARENA_JEMALLOC_TYPE_REGULAR));
+
+	assert(!aml_area_linux_init(&area,
+				    AML_AREA_LINUX_MANAGER_TYPE_SINGLE,
+				    AML_AREA_LINUX_MBIND_TYPE_REGULAR,
+				    AML_AREA_LINUX_MMAP_TYPE_ANONYMOUS,
+				    &arena, MPOL_BIND, nodemask));
+
+	size_t maxrequests = NBTILES;
+	size_t maxthreads = 4;
+	assert(!aml_dma_linux_par_init(&dma, maxrequests, maxthreads));
+
+	/* allocate some memory */
+	src = aml_area_malloc(&area, TILESIZE*PAGE_SIZE*NBTILES);
+	assert(src != NULL);
+	dst = aml_area_malloc(&area, TILESIZE*PAGE_SIZE*NBTILES);
+	assert(dst != NULL);
+
+	memset(src, 42, TILESIZE*PAGE_SIZE*NBTILES);
+	memset(dst, 24, TILESIZE*PAGE_SIZE*NBTILES);
+
+	/* create scratchpad */
+	assert(!aml_scratch_seq_init(&scratch, &area, &tiling, &area, &tiling,
+				     &dma, NBTILES));
+
+	/* move some stuff */
+	for(int i = 0; i < NBTILES; i++)
+	{
+		int di, si;
+		void *dp, *sp;
+		aml_scratch_pull(&scratch, dst, &di, src, i);
+	
+		dp = aml_tiling_tilestart(&tiling, dst, di);
+		sp = aml_tiling_tilestart(&tiling, src, i);
+
+		assert(!memcmp(sp, dp, TILESIZE*PAGE_SIZE));
+
+		memset(dp, 33, TILESIZE*PAGE_SIZE);
+	
+		aml_scratch_push(&scratch, src, &si, dst, di);
+		assert(si == i);
+
+		sp = aml_tiling_tilestart(&tiling, src, si);
+
+		assert(!memcmp(sp, dp, TILESIZE*PAGE_SIZE));
+	}
+
+	/* delete everything */
+	aml_scratch_seq_destroy(&scratch);
+	aml_dma_linux_par_destroy(&dma);
+	aml_area_free(&area, dst);
+	aml_area_free(&area, src);
+	aml_area_linux_destroy(&area);
+	aml_tiling_destroy(&tiling, AML_TILING_TYPE_1D);
+	aml_binding_destroy(&binding, AML_BINDING_TYPE_SINGLE);
+
+	aml_finalize();
+	return 0;
+}