[feature] add extra argument to dma operator

Some operators will need an extra argument (like transforms).

benchmarks/dgemm_prefetch.c

@@ -110,7 +110,7 @@ int main(int argc, char* argv[])
 	aml_area_linux_create(&fast, &fastb, AML_AREA_LINUX_POLICY_BIND);
 	assert(fast != NULL);
 	
-	assert(!aml_dma_linux_seq_create(&dma, 2, NULL));
+	assert(!aml_dma_linux_seq_create(&dma, 2, NULL, NULL));
 	assert(!aml_scratch_par_create(&sa, fast, slow, dma, tiling_prefetch, (size_t)2, (size_t)2));
 	assert(!aml_scratch_par_create(&sb, fast, slow, dma, tiling_prefetch, (size_t)2, (size_t)2));
 	/* allocation */

benchmarks/stream_add_omp.c

@@ -39,7 +39,7 @@ int main(int argc, char *argv[])
 	struct aml_area *slow = &aml_area_linux, *fast = aml_area_linux;
 
 	struct aml_dma *dma;
-	assert(!aml_dma_create(&dma, 0, NULL));
+	assert(!aml_dma_create(&dma, 0, NULL, NULL));
 
 	void *a, *b, *c;
 

benchmarks/stream_add_omp_mt.c

@@ -68,7 +68,7 @@ int main(int argc, char *argv[])
 	assert(slow != NULL);
 	aml_area_linux_create(&fast, &fastb, AML_AREA_LINUX_POLICY_BIND);
 	assert(fast != NULL);
-	assert(!aml_dma_linux_par_create(&dma, numthreads*2, NULL));
+	assert(!aml_dma_linux_par_create(&dma, numthreads*2, NULL, NULL));
 	assert(!aml_scratch_seq_create(&sa, fast, slow, dma, tiling,
 				     (size_t)2*numthreads, (size_t)1));
 	assert(!aml_scratch_seq_create(&sb, fast, slow, dma, tiling,

benchmarks/stream_add_omp_st.c

@@ -99,7 +99,7 @@ int main(int argc, char *argv[])
 	assert(slow != NULL);
 	aml_area_linux_create(&fast, &fastb, AML_AREA_LINUX_POLICY_BIND);
 	assert(fast != NULL);
-	assert(!aml_dma_linux_seq_create(&dma, numthreads*2, NULL));
+	assert(!aml_dma_linux_seq_create(&dma, numthreads*2, NULL, NULL));
 	assert(!aml_scratch_par_create(&sa, fast, slow, dma, tiling,
 				     2*numthreads, numthreads));
 	assert(!aml_scratch_par_create(&sb, fast, slow, dma, tiling,

benchmarks/stream_add_pth_st.c

@@ -105,7 +105,7 @@ int main(int argc, char *argv[])
 	assert(slow != NULL);
 	aml_area_linux_create(&fast, &fastb, AML_AREA_LINUX_POLICY_BIND);
 	assert(fast != NULL);
-	assert(!aml_dma_linux_seq_create(dma, (size_t)numthreads*4, NULL));
+	assert(!aml_dma_linux_seq_create(dma, (size_t)numthreads*4, NULL, NULL));
 	assert(!aml_scratch_par_create(&sa, fast, slow, dma, tiling,
 				     (size_t)2*numthreads, (size_t)numthreads));
 	assert(!aml_scratch_par_create(&sb, fast, slow, dma, tiling,

include/aml.h

@@ -848,10 +848,10 @@ struct aml_dma_data;
  * Type of the function used to perform the DMA between two layouts.
  * @param dst: destination layout
  * @param src: source layout
- * @param extra_arg: extra argument needed by the operator
+ * @param arg: extra argument needed by the operator
  **/
 typedef int (*aml_dma_operator)(struct aml_layout *dst,
-				const struct aml_layout *src);
+				const struct aml_layout *src, void *arg);
 
 /**
    aml_dma_ops is a structure containing operations for a specific
@@ -880,7 +880,7 @@ struct aml_dma_ops {
 			      struct aml_dma_request **req,
 			      struct aml_layout *dest,
 			      struct aml_layout *src,
-			      aml_dma_operator op);
+			      aml_dma_operator op, void *op_arg);
 
 	/**
 	 * Destroy the request handle. If the data movement is still ongoing,
@@ -928,7 +928,7 @@ struct aml_dma {
  * @return 0 if successful; an error code otherwise.
  **/
 int aml_dma_copy_custom(struct aml_dma *dma, struct aml_layout *dest,
-		 struct aml_layout *src, aml_dma_operator op);
+		 struct aml_layout *src, aml_dma_operator op, void *op_arg);
 
 /**
  * Requests a data copy between two different buffers.This is an asynchronous
@@ -944,11 +944,11 @@ int aml_dma_copy_custom(struct aml_dma *dma, struct aml_layout *dest,
 int aml_dma_async_copy_custom(struct aml_dma *dma, struct aml_dma_request **req,
 		       struct aml_layout *dest,
 		       struct aml_layout *src,
-		       aml_dma_operator op);
+		       aml_dma_operator op, void *op_arg);
 
-#define aml_dma_copy(dma, d, s) aml_dma_copy_custom(dma, d, s, NULL)
+#define aml_dma_copy(dma, d, s) aml_dma_copy_custom(dma, d, s, NULL, NULL)
 #define aml_dma_async_copy(dma, r, d, s) \
-	aml_dma_async_copy_custom(dma, r, d, s, NULL)
+	aml_dma_async_copy_custom(dma, r, d, s, NULL, NULL)
 
 /**
  * Waits for an asynchronous DMA request to complete.
@@ -970,9 +970,10 @@ int aml_dma_cancel(struct aml_dma *dma, struct aml_dma_request **req);
  * Generic helper to copy from one layout to another.
  * @param dst[out]: destination layout
  * @param src[in]: source layout
+ * @param arg: unused
  */
 int aml_copy_layout_generic(struct aml_layout *dst,
-			    const struct aml_layout *src);
+			    const struct aml_layout *src, void *arg);
 
 
 ////////////////////////////////////////////////////////////////////////////////

include/aml/dma/linux-par.h

@@ -44,6 +44,8 @@ struct aml_dma_request_linux_par {
 	pthread_t thread;
 	/** operator for this request **/
 	aml_dma_operator op;
+	/** operator argument for this request **/
+	void *op_arg;
 };
 
 /** Inside of a parallel dma for linux movement. **/
@@ -52,6 +54,8 @@ struct aml_dma_linux_par_data {
 	pthread_mutex_t lock;
 	/** default operator for this dma **/
 	aml_dma_operator default_op;
+	/** default operator arg for this dma **/
+	void *default_op_arg;
 };
 
 /** Declaration of linux parallel dma operations **/
@@ -81,7 +85,7 @@ struct aml_dma_linux_par {
  * @return 0 if successful; an error code otherwise.
  **/
 int aml_dma_linux_par_create(struct aml_dma **dma, size_t nbreqs,
-			     aml_dma_operator op);
+			     aml_dma_operator op, void *op_arg);
 
 /**
  * Tears down a parallel DMA created with aml_dma_linux_par_create.

include/aml/dma/linux-seq.h

@@ -40,6 +40,8 @@ struct aml_dma_request_linux_seq {
 	struct aml_layout *src;
 	/** The operator being used **/
 	aml_dma_operator op;
+	/** Argument for operator **/
+	void *op_arg;
 };
 
 /** Inner data of sequential linux aml_dma implementation **/
@@ -54,6 +56,8 @@ struct aml_dma_linux_seq_data {
 	pthread_mutex_t lock;
 	/** default operator **/
 	aml_dma_operator default_op;
+	/** default op_arg **/
+	void *default_op_arg;
 };
 
 /** Declaration of available linux sequential dma operations **/
@@ -90,7 +94,7 @@ struct aml_dma_linux_seq {
  * @return 0 if successful; an error code otherwise.
  **/
 int aml_dma_linux_seq_create(struct aml_dma **dma, size_t nbreqs,
-			     aml_dma_operator op);
+			     aml_dma_operator op, void *op_arg);
 
 /**
  * Tears down a sequential DMA created with aml_dma_linux_seq_create.

src/dma/dma.c

@@ -44,10 +44,11 @@ static inline void aml_copy_layout_generic_helper(size_t d,
 }
 
 int aml_copy_layout_generic(struct aml_layout *dst,
-			    const struct aml_layout *src)
+			    const struct aml_layout *src, void *arg)
 {
 	size_t d;
 	size_t elem_size;
+	(void)arg;
 
 	assert(aml_layout_ndims(dst) == aml_layout_ndims(src));
 	d = aml_layout_ndims(dst);
@@ -77,7 +78,7 @@ int aml_copy_layout_generic(struct aml_layout *dst,
  ******************************************************************************/
 
 int aml_dma_copy_custom(struct aml_dma *dma, struct aml_layout *dest,
-		 struct aml_layout *src, aml_dma_operator op)
+		 struct aml_layout *src, aml_dma_operator op, void *op_arg)
 {
 	int ret;
 	struct aml_dma_request *req;
@@ -85,7 +86,7 @@ int aml_dma_copy_custom(struct aml_dma *dma, struct aml_layout *dest,
 	if (dma == NULL || dest == NULL || src == NULL)
 		return -AML_EINVAL;
 
-	ret = dma->ops->create_request(dma->data, &req, dest, src, op);
+	ret = dma->ops->create_request(dma->data, &req, dest, src, op, op_arg);
 	if (ret != AML_SUCCESS)
 		return ret;
 	ret = dma->ops->wait_request(dma->data, &req);
@@ -94,12 +95,12 @@ int aml_dma_copy_custom(struct aml_dma *dma, struct aml_layout *dest,
 
 int aml_dma_async_copy_custom(struct aml_dma *dma, struct aml_dma_request **req,
 		       struct aml_layout *dest, struct aml_layout *src,
-		       aml_dma_operator op)
+		       aml_dma_operator op, void *op_arg)
 {
 	if (dma == NULL || req == NULL || dest == NULL || src == NULL)
 		return -AML_EINVAL;
 
-	return dma->ops->create_request(dma->data, req, dest, src, op);
+	return dma->ops->create_request(dma->data, req, dest, src, op, op_arg);
 }
 
 int aml_dma_cancel(struct aml_dma *dma, struct aml_dma_request **req)

src/dma/dma_linux_par.c

@@ -32,13 +32,14 @@
 int aml_dma_request_linux_par_copy_init(struct aml_dma_request_linux_par *req,
 					struct aml_layout *dest,
 					struct aml_layout *src,
-					aml_dma_operator op)
+					aml_dma_operator op, void *op_arg)
 {
 	assert(req != NULL);
 	req->type = AML_DMA_REQUEST_TYPE_LAYOUT;
 	req->dest = dest;
 	req->src = src;
 	req->op = op;
+	req->op_arg = op_arg;
 	return 0;
 }
 
@@ -59,7 +60,7 @@ void *aml_dma_linux_par_do_thread(void *arg)
 
 	pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
 	if (req->type != AML_DMA_REQUEST_TYPE_INVALID)
-		req->op(req->dest, req->src);
+		req->op(req->dest, req->src, req->op_arg);
 	return NULL;
 }
 
@@ -75,7 +76,7 @@ int aml_dma_linux_par_create_request(struct aml_dma_data *d,
 				     struct aml_dma_request **r,
 				     struct aml_layout *dest,
 				     struct aml_layout *src,
-				     aml_dma_operator op)
+				     aml_dma_operator op, void *op_arg)
 {
 	/* NULL checks done by the generic API */
 	assert(d != NULL);
@@ -88,10 +89,12 @@ int aml_dma_linux_par_create_request(struct aml_dma_data *d,
 
 	if (op == NULL)
 		op = dma->data.default_op;
+	if (op_arg == NULL)
+		op_arg = dma->data.default_op_arg;
 
 	pthread_mutex_lock(&dma->data.lock);
 	req = aml_vector_add(dma->data.requests);
-	aml_dma_request_linux_par_copy_init(req, dest, src, op);
+	aml_dma_request_linux_par_copy_init(req, dest, src, op, op_arg);
 	pthread_mutex_unlock(&dma->data.lock);
 	pthread_create(&req->thread, NULL, dma->ops.do_thread, req);
 	*r = (struct aml_dma_request *)req;
@@ -162,7 +165,7 @@ struct aml_dma_ops aml_dma_linux_par_ops = {
  ******************************************************************************/
 
 int aml_dma_linux_par_create(struct aml_dma **dma, size_t nbreqs,
-			     aml_dma_operator op)
+			     aml_dma_operator op, void *op_arg)
 {
 	struct aml_dma *ret = NULL;
 	struct aml_dma_linux_par *d;
@@ -182,9 +185,12 @@ int aml_dma_linux_par_create(struct aml_dma **dma, size_t nbreqs,
 	d = (struct aml_dma_linux_par *)ret->data;
 	d->ops = aml_dma_linux_par_inner_ops;
 
-	if (op == NULL)
+	if (op == NULL) {
 		op = aml_copy_layout_generic;
+		op_arg = NULL;
+	}
 	d->data.default_op = op;
+	d->data.default_op_arg = op_arg;
 
 	/* allocate request array */
 	aml_vector_create(&d->data.requests, nbreqs,

src/dma/dma_linux_seq.c

@@ -32,13 +32,15 @@
 int aml_dma_request_linux_seq_copy_init(struct aml_dma_request_linux_seq *req,
 					struct aml_layout *dest,
 					struct aml_layout *src,
-					aml_dma_operator op)
+					aml_dma_operator op,
+					void *op_arg)
 {
 	assert(req != NULL);
 	req->type = AML_DMA_REQUEST_TYPE_LAYOUT;
 	req->dest = dest;
 	req->src = src;
 	req->op = op;
+	req->op_arg = op_arg;
 	return 0;
 }
 
@@ -58,7 +60,7 @@ int aml_dma_linux_seq_do_copy(struct aml_dma_linux_seq_data *dma,
 	assert(dma != NULL);
 	assert(req != NULL);
 	assert(req->op != NULL);
-	return req->op(req->dest, req->src);
+	return req->op(req->dest, req->src, req->op_arg);
 }
 
 struct aml_dma_linux_seq_ops aml_dma_linux_seq_inner_ops = {
@@ -73,7 +75,7 @@ int aml_dma_linux_seq_create_request(struct aml_dma_data *d,
 				     struct aml_dma_request **r,
 				     struct aml_layout *dest,
 				     struct aml_layout *src,
-				     aml_dma_operator op)
+				     aml_dma_operator op, void *op_arg)
 {
 	/* NULL checks done by the generic API */
 	assert(d != NULL);
@@ -86,10 +88,12 @@ int aml_dma_linux_seq_create_request(struct aml_dma_data *d,
 
 	if (op == NULL)
 		op = dma->data.default_op;
+	if (op_arg == NULL)
+		op_arg = dma->data.default_op_arg;
 
 	pthread_mutex_lock(&dma->data.lock);
 	req = aml_vector_add(dma->data.requests);
-	aml_dma_request_linux_seq_copy_init(req, dest, src, op);
+	aml_dma_request_linux_seq_copy_init(req, dest, src, op, op_arg);
 	pthread_mutex_unlock(&dma->data.lock);
 	*r = (struct aml_dma_request *)req;
 	return 0;
@@ -98,8 +102,7 @@ int aml_dma_linux_seq_create_request(struct aml_dma_data *d,
 int aml_dma_linux_seq_destroy_request(struct aml_dma_data *d,
 				      struct aml_dma_request **r)
 {
-	assert(d != NULL);
-	assert(r != NULL);
+	assert(d != NULL); assert(r != NULL);
 	struct aml_dma_linux_seq *dma =
 		(struct aml_dma_linux_seq *)d;
 	struct aml_dma_request_linux_seq *req;
@@ -148,7 +151,7 @@ struct aml_dma_ops aml_dma_linux_seq_ops = {
  ******************************************************************************/
 
 int aml_dma_linux_seq_create(struct aml_dma **dma, size_t nbreqs,
-			     aml_dma_operator op)
+			     aml_dma_operator op, void *op_arg)
 {
 	struct aml_dma *ret = NULL;
 	struct aml_dma_linux_seq *d;
@@ -169,9 +172,12 @@ int aml_dma_linux_seq_create(struct aml_dma **dma, size_t nbreqs,
 
 	d->ops = aml_dma_linux_seq_inner_ops;
 
-	if (op == NULL)
+	if (op == NULL) {
 		op = aml_copy_layout_generic;
+		op_arg = NULL;
+	}
 	d->data.default_op = op;
+	d->data.default_op_arg = op_arg;
 
 	aml_vector_create(&d->data.requests, nbreqs,
 			  sizeof(struct aml_dma_request_linux_seq),

tests/dma/test_dma_linux_par.c

@@ -34,10 +34,10 @@ int main(int argc, char *argv[])
 		isrc[i] = 24;
 	}
 	/* invalid create input */
-	assert(aml_dma_linux_par_create(NULL, 1, NULL) == -AML_EINVAL);
+	assert(aml_dma_linux_par_create(NULL, 1, NULL, NULL) == -AML_EINVAL);
 
 	/* invalid requests */
-	assert(!aml_dma_linux_par_create(&dma, 1, NULL));
+	assert(!aml_dma_linux_par_create(&dma, 1, NULL, NULL));
 	assert(aml_dma_copy(dma, NULL, isl) == -AML_EINVAL);
 	assert(aml_dma_copy(dma, idl, NULL) == -AML_EINVAL);
 
@@ -52,7 +52,7 @@ int main(int argc, char *argv[])
 	aml_dma_linux_par_destroy(&dma);
 
 	/* cancel a request on the fly */
-	assert(!aml_dma_linux_par_create(&dma, 1, NULL));
+	assert(!aml_dma_linux_par_create(&dma, 1, NULL, NULL));
 	assert(aml_dma_cancel(dma, NULL) == -AML_EINVAL);
 	assert(!aml_dma_async_copy(dma, &r1, idl, isl));
 	assert(!aml_dma_cancel(dma, &r1));
@@ -62,7 +62,7 @@ int main(int argc, char *argv[])
 	aml_dma_linux_par_destroy(&dma);
 
 	/* move data around */
-	assert(!aml_dma_linux_par_create(&dma, 1, NULL));
+	assert(!aml_dma_linux_par_create(&dma, 1, NULL, NULL));
 	struct aml_dma_request *requests[16];
 	struct aml_layout *layouts[16][2];
 	for (int i = 0; i < 16; i++) {

tests/dma/test_dma_linux_seq.c

@@ -34,10 +34,10 @@ int main(int argc, char *argv[])
 		isrc[i] = 24;
 	}
 	/* invalid create input */
-	assert(aml_dma_linux_seq_create(NULL, 1, NULL) == -AML_EINVAL);
+	assert(aml_dma_linux_seq_create(NULL, 1, NULL, NULL) == -AML_EINVAL);
 
 	/* invalid requests */
-	assert(!aml_dma_linux_seq_create(&dma, 1, NULL));
+	assert(!aml_dma_linux_seq_create(&dma, 1, NULL, NULL));
 	assert(aml_dma_copy(dma, NULL, isl) == -AML_EINVAL);
 	assert(aml_dma_copy(dma, idl, NULL) == -AML_EINVAL);
 
@@ -61,7 +61,7 @@ int main(int argc, char *argv[])
 	aml_dma_linux_seq_destroy(&dma);
 
 	/* move data around */
-	assert(!aml_dma_linux_seq_create(&dma, 1, NULL));
+	assert(!aml_dma_linux_seq_create(&dma, 1, NULL, NULL));
 	struct aml_dma_request *requests[16];
 	struct aml_layout *layouts[16][2];
 	for (int i = 0; i < 16; i++) {

tests/scratch/test_scratch_par.c

@@ -33,7 +33,7 @@ int main(int argc, char *argv[])
 				     TILESIZE*_SC_PAGE_SIZE*NBTILES));
 
 	size_t maxrequests = NBTILES;
-	assert(!aml_dma_linux_seq_create(&dma, maxrequests, NULL));
+	assert(!aml_dma_linux_seq_create(&dma, maxrequests, NULL, NULL));
 
 	/* allocate some memory */
 	src = aml_area_mmap(&aml_area_linux, TILESIZE*_SC_PAGE_SIZE*NBTILES, NULL);

tests/scratch/test_scratch_seq.c

@@ -33,7 +33,7 @@ int main(int argc, char *argv[])
 				     TILESIZE*_SC_PAGE_SIZE*NBTILES));
 
 	size_t maxrequests = NBTILES;
-	assert(!aml_dma_linux_par_create(&dma, maxrequests, NULL));
+	assert(!aml_dma_linux_par_create(&dma, maxrequests, NULL, NULL));
 
 	/* allocate some memory */
 	src = aml_area_mmap(&aml_area_linux, TILESIZE*_SC_PAGE_SIZE*NBTILES, NULL);