Commit 12fee91a authored by Swann Perarnau's avatar Swann Perarnau

Merge branch 'dma-copy-arg' into 'master'

[feature] extend dma to support custom operators

See merge request !79
parents a466aaa7 7eb3d520
Pipeline #8381 passed with stages
in 6 minutes and 21 seconds
......@@ -110,7 +110,7 @@ int main(int argc, char* argv[])
aml_area_linux_create(&fast, &fastb, AML_AREA_LINUX_POLICY_BIND);
assert(fast != NULL);
assert(!aml_dma_linux_seq_create(&dma, 2));
assert(!aml_dma_linux_seq_create(&dma, 2, NULL, NULL));
assert(!aml_scratch_par_create(&sa, fast, slow, dma, tiling_prefetch, (size_t)2, (size_t)2));
assert(!aml_scratch_par_create(&sb, fast, slow, dma, tiling_prefetch, (size_t)2, (size_t)2));
/* allocation */
......
......@@ -39,7 +39,7 @@ int main(int argc, char *argv[])
struct aml_area *slow = &aml_area_linux, *fast = aml_area_linux;
struct aml_dma *dma;
assert(!aml_dma_create(&dma, 0));
assert(!aml_dma_create(&dma, 0, NULL, NULL));
void *a, *b, *c;
......
......@@ -68,7 +68,7 @@ int main(int argc, char *argv[])
assert(slow != NULL);
aml_area_linux_create(&fast, &fastb, AML_AREA_LINUX_POLICY_BIND);
assert(fast != NULL);
assert(!aml_dma_linux_par_create(&dma, numthreads*2));
assert(!aml_dma_linux_par_create(&dma, numthreads*2, NULL, NULL));
assert(!aml_scratch_seq_create(&sa, fast, slow, dma, tiling,
(size_t)2*numthreads, (size_t)1));
assert(!aml_scratch_seq_create(&sb, fast, slow, dma, tiling,
......
......@@ -99,7 +99,7 @@ int main(int argc, char *argv[])
assert(slow != NULL);
aml_area_linux_create(&fast, &fastb, AML_AREA_LINUX_POLICY_BIND);
assert(fast != NULL);
assert(!aml_dma_linux_seq_create(&dma, numthreads*2));
assert(!aml_dma_linux_seq_create(&dma, numthreads*2, NULL, NULL));
assert(!aml_scratch_par_create(&sa, fast, slow, dma, tiling,
2*numthreads, numthreads));
assert(!aml_scratch_par_create(&sb, fast, slow, dma, tiling,
......
......@@ -105,7 +105,7 @@ int main(int argc, char *argv[])
assert(slow != NULL);
aml_area_linux_create(&fast, &fastb, AML_AREA_LINUX_POLICY_BIND);
assert(fast != NULL);
assert(!aml_dma_linux_seq_create(dma, (size_t)numthreads*4));
assert(!aml_dma_linux_seq_create(dma, (size_t)numthreads*4, NULL, NULL));
assert(!aml_scratch_par_create(&sa, fast, slow, dma, tiling,
(size_t)2*numthreads, (size_t)numthreads));
assert(!aml_scratch_par_create(&sb, fast, slow, dma, tiling,
......
......@@ -844,6 +844,15 @@ struct aml_dma_request;
**/
struct aml_dma_data;
/**
* Type of the function used to perform the DMA between two layouts.
* @param dst: destination layout
* @param src: source layout
* @param arg: extra argument needed by the operator
**/
typedef int (*aml_dma_operator)(struct aml_layout *dst,
const struct aml_layout *src, void *arg);
/**
aml_dma_ops is a structure containing operations for a specific
* aml_dma implementation.
......@@ -870,7 +879,8 @@ struct aml_dma_ops {
int (*create_request)(struct aml_dma_data *dma,
struct aml_dma_request **req,
struct aml_layout *dest,
struct aml_layout *src);
struct aml_layout *src,
aml_dma_operator op, void *op_arg);
/**
* Destroy the request handle. If the data movement is still ongoing,
......@@ -914,10 +924,11 @@ struct aml_dma {
* @param dma: an initialized DMA structure.
* @param dest: layout describing the destination.
* @param src: layout describing the source.
* @param op: optional custom operator for this dma
* @return 0 if successful; an error code otherwise.
**/
int aml_dma_copy(struct aml_dma *dma, struct aml_layout *dest,
struct aml_layout *src);
int aml_dma_copy_custom(struct aml_dma *dma, struct aml_layout *dest,
struct aml_layout *src, aml_dma_operator op, void *op_arg);
/**
* Requests a data copy between two different buffers.This is an asynchronous
......@@ -927,11 +938,17 @@ int aml_dma_copy(struct aml_dma *dma, struct aml_layout *dest,
* will be stored.
* @param dest: layout describing the destination.
* @param src: layout describing the source.
* @param op: optional custom operator for this dma
* @return 0 if successful; an error code otherwise.
**/
int aml_dma_async_copy(struct aml_dma *dma, struct aml_dma_request **req,
int aml_dma_async_copy_custom(struct aml_dma *dma, struct aml_dma_request **req,
struct aml_layout *dest,
struct aml_layout *src);
struct aml_layout *src,
aml_dma_operator op, void *op_arg);
#define aml_dma_copy(dma, d, s) aml_dma_copy_custom(dma, d, s, NULL, NULL)
#define aml_dma_async_copy(dma, r, d, s) \
aml_dma_async_copy_custom(dma, r, d, s, NULL, NULL)
/**
* Waits for an asynchronous DMA request to complete.
......@@ -953,9 +970,10 @@ int aml_dma_cancel(struct aml_dma *dma, struct aml_dma_request **req);
* Generic helper to copy from one layout to another.
* @param dst[out]: destination layout
* @param src[in]: source layout
* @param arg: unused
*/
int aml_copy_layout_generic(struct aml_layout *dst,
const struct aml_layout *src);
const struct aml_layout *src, void *arg);
////////////////////////////////////////////////////////////////////////////////
......
......@@ -42,12 +42,20 @@ struct aml_dma_request_linux_par {
struct aml_dma_linux_par *dma;
/** The actual thread in charge for the request progress**/
pthread_t thread;
/** operator for this request **/
aml_dma_operator op;
/** operator argument for this request **/
void *op_arg;
};
/** Inside of a parallel dma for linux movement. **/
struct aml_dma_linux_par_data {
struct aml_vector *requests;
pthread_mutex_t lock;
/** default operator for this dma **/
aml_dma_operator default_op;
/** default operator arg for this dma **/
void *default_op_arg;
};
/** Declaration of linux parallel dma operations **/
......@@ -73,10 +81,11 @@ struct aml_dma_linux_par {
* @param nbreqs the initial number of slots for asynchronous requests that are
* in-flight (will be increased automatically if necessary).
* @param nbthreads the number of threads to launch for each request.
*
* @param op: default operator
* @return 0 if successful; an error code otherwise.
**/
int aml_dma_linux_par_create(struct aml_dma **dma, size_t nbreqs);
int aml_dma_linux_par_create(struct aml_dma **dma, size_t nbreqs,
aml_dma_operator op, void *op_arg);
/**
* Tears down a parallel DMA created with aml_dma_linux_par_create.
......
......@@ -38,6 +38,10 @@ struct aml_dma_request_linux_seq {
struct aml_layout *dest;
/** The source pointer of the data movement **/
struct aml_layout *src;
/** The operator being used **/
aml_dma_operator op;
/** Argument for operator **/
void *op_arg;
};
/** Inner data of sequential linux aml_dma implementation **/
......@@ -50,6 +54,10 @@ struct aml_dma_linux_seq_data {
struct aml_vector *requests;
/** Lock for queuing requests concurrently **/
pthread_mutex_t lock;
/** default operator **/
aml_dma_operator default_op;
/** default op_arg **/
void *default_op_arg;
};
/** Declaration of available linux sequential dma operations **/
......@@ -81,10 +89,12 @@ struct aml_dma_linux_seq {
* will be stored.
* @param nbreqs the initial number of slots for asynchronous requests that are
* in-flight (will be increased automatically if necessary).
* @param op: default operator
*
* @return 0 if successful; an error code otherwise.
**/
int aml_dma_linux_seq_create(struct aml_dma **dma, size_t nbreqs);
int aml_dma_linux_seq_create(struct aml_dma **dma, size_t nbreqs,
aml_dma_operator op, void *op_arg);
/**
* Tears down a sequential DMA created with aml_dma_linux_seq_create.
......
......@@ -44,10 +44,11 @@ static inline void aml_copy_layout_generic_helper(size_t d,
}
int aml_copy_layout_generic(struct aml_layout *dst,
const struct aml_layout *src)
const struct aml_layout *src, void *arg)
{
size_t d;
size_t elem_size;
(void)arg;
assert(aml_layout_ndims(dst) == aml_layout_ndims(src));
d = aml_layout_ndims(dst);
......@@ -76,8 +77,8 @@ int aml_copy_layout_generic(struct aml_layout *dst,
* abstract the request creation after this layer.
******************************************************************************/
int aml_dma_copy(struct aml_dma *dma, struct aml_layout *dest,
struct aml_layout *src)
int aml_dma_copy_custom(struct aml_dma *dma, struct aml_layout *dest,
struct aml_layout *src, aml_dma_operator op, void *op_arg)
{
int ret;
struct aml_dma_request *req;
......@@ -85,20 +86,21 @@ int aml_dma_copy(struct aml_dma *dma, struct aml_layout *dest,
if (dma == NULL || dest == NULL || src == NULL)
return -AML_EINVAL;
ret = dma->ops->create_request(dma->data, &req, dest, src);
ret = dma->ops->create_request(dma->data, &req, dest, src, op, op_arg);
if (ret != AML_SUCCESS)
return ret;
ret = dma->ops->wait_request(dma->data, &req);
return ret;
}
int aml_dma_async_copy(struct aml_dma *dma, struct aml_dma_request **req,
struct aml_layout *dest, struct aml_layout *src)
int aml_dma_async_copy_custom(struct aml_dma *dma, struct aml_dma_request **req,
struct aml_layout *dest, struct aml_layout *src,
aml_dma_operator op, void *op_arg)
{
if (dma == NULL || req == NULL || dest == NULL || src == NULL)
return -AML_EINVAL;
return dma->ops->create_request(dma->data, req, dest, src);
return dma->ops->create_request(dma->data, req, dest, src, op, op_arg);
}
int aml_dma_cancel(struct aml_dma *dma, struct aml_dma_request **req)
......
......@@ -31,12 +31,15 @@
int aml_dma_request_linux_par_copy_init(struct aml_dma_request_linux_par *req,
struct aml_layout *dest,
struct aml_layout *src)
struct aml_layout *src,
aml_dma_operator op, void *op_arg)
{
assert(req != NULL);
req->type = AML_DMA_REQUEST_TYPE_LAYOUT;
req->dest = dest;
req->src = src;
req->op = op;
req->op_arg = op_arg;
return 0;
}
......@@ -57,7 +60,7 @@ void *aml_dma_linux_par_do_thread(void *arg)
pthread_setcanceltype(PTHREAD_CANCEL_ASYNCHRONOUS, NULL);
if (req->type != AML_DMA_REQUEST_TYPE_INVALID)
aml_copy_layout_generic(req->dest, req->src);
req->op(req->dest, req->src, req->op_arg);
return NULL;
}
......@@ -72,7 +75,8 @@ struct aml_dma_linux_par_ops aml_dma_linux_par_inner_ops = {
int aml_dma_linux_par_create_request(struct aml_dma_data *d,
struct aml_dma_request **r,
struct aml_layout *dest,
struct aml_layout *src)
struct aml_layout *src,
aml_dma_operator op, void *op_arg)
{
/* NULL checks done by the generic API */
assert(d != NULL);
......@@ -83,9 +87,14 @@ int aml_dma_linux_par_create_request(struct aml_dma_data *d,
(struct aml_dma_linux_par *)d;
struct aml_dma_request_linux_par *req;
if (op == NULL)
op = dma->data.default_op;
if (op_arg == NULL)
op_arg = dma->data.default_op_arg;
pthread_mutex_lock(&dma->data.lock);
req = aml_vector_add(dma->data.requests);
aml_dma_request_linux_par_copy_init(req, dest, src);
aml_dma_request_linux_par_copy_init(req, dest, src, op, op_arg);
pthread_mutex_unlock(&dma->data.lock);
pthread_create(&req->thread, NULL, dma->ops.do_thread, req);
*r = (struct aml_dma_request *)req;
......@@ -155,7 +164,8 @@ struct aml_dma_ops aml_dma_linux_par_ops = {
* Init functions:
******************************************************************************/
int aml_dma_linux_par_create(struct aml_dma **dma, size_t nbreqs)
int aml_dma_linux_par_create(struct aml_dma **dma, size_t nbreqs,
aml_dma_operator op, void *op_arg)
{
struct aml_dma *ret = NULL;
struct aml_dma_linux_par *d;
......@@ -175,6 +185,13 @@ int aml_dma_linux_par_create(struct aml_dma **dma, size_t nbreqs)
d = (struct aml_dma_linux_par *)ret->data;
d->ops = aml_dma_linux_par_inner_ops;
if (op == NULL) {
op = aml_copy_layout_generic;
op_arg = NULL;
}
d->data.default_op = op;
d->data.default_op_arg = op_arg;
/* allocate request array */
aml_vector_create(&d->data.requests, nbreqs,
sizeof(struct aml_dma_request_linux_par),
......
......@@ -31,12 +31,16 @@
int aml_dma_request_linux_seq_copy_init(struct aml_dma_request_linux_seq *req,
struct aml_layout *dest,
struct aml_layout *src)
struct aml_layout *src,
aml_dma_operator op,
void *op_arg)
{
assert(req != NULL);
req->type = AML_DMA_REQUEST_TYPE_LAYOUT;
req->dest = dest;
req->src = src;
req->op = op;
req->op_arg = op_arg;
return 0;
}
......@@ -55,8 +59,8 @@ int aml_dma_linux_seq_do_copy(struct aml_dma_linux_seq_data *dma,
{
assert(dma != NULL);
assert(req != NULL);
aml_copy_layout_generic(req->dest, req->src);
return 0;
assert(req->op != NULL);
return req->op(req->dest, req->src, req->op_arg);
}
struct aml_dma_linux_seq_ops aml_dma_linux_seq_inner_ops = {
......@@ -70,7 +74,8 @@ struct aml_dma_linux_seq_ops aml_dma_linux_seq_inner_ops = {
int aml_dma_linux_seq_create_request(struct aml_dma_data *d,
struct aml_dma_request **r,
struct aml_layout *dest,
struct aml_layout *src)
struct aml_layout *src,
aml_dma_operator op, void *op_arg)
{
/* NULL checks done by the generic API */
assert(d != NULL);
......@@ -81,9 +86,14 @@ int aml_dma_linux_seq_create_request(struct aml_dma_data *d,
(struct aml_dma_linux_seq *)d;
struct aml_dma_request_linux_seq *req;
if (op == NULL)
op = dma->data.default_op;
if (op_arg == NULL)
op_arg = dma->data.default_op_arg;
pthread_mutex_lock(&dma->data.lock);
req = aml_vector_add(dma->data.requests);
aml_dma_request_linux_seq_copy_init(req, dest, src);
aml_dma_request_linux_seq_copy_init(req, dest, src, op, op_arg);
pthread_mutex_unlock(&dma->data.lock);
*r = (struct aml_dma_request *)req;
return 0;
......@@ -92,8 +102,7 @@ int aml_dma_linux_seq_create_request(struct aml_dma_data *d,
int aml_dma_linux_seq_destroy_request(struct aml_dma_data *d,
struct aml_dma_request **r)
{
assert(d != NULL);
assert(r != NULL);
assert(d != NULL); assert(r != NULL);
struct aml_dma_linux_seq *dma =
(struct aml_dma_linux_seq *)d;
struct aml_dma_request_linux_seq *req;
......@@ -141,7 +150,8 @@ struct aml_dma_ops aml_dma_linux_seq_ops = {
* Init functions:
******************************************************************************/
int aml_dma_linux_seq_create(struct aml_dma **dma, size_t nbreqs)
int aml_dma_linux_seq_create(struct aml_dma **dma, size_t nbreqs,
aml_dma_operator op, void *op_arg)
{
struct aml_dma *ret = NULL;
struct aml_dma_linux_seq *d;
......@@ -161,6 +171,14 @@ int aml_dma_linux_seq_create(struct aml_dma **dma, size_t nbreqs)
d = (struct aml_dma_linux_seq *)ret->data;
d->ops = aml_dma_linux_seq_inner_ops;
if (op == NULL) {
op = aml_copy_layout_generic;
op_arg = NULL;
}
d->data.default_op = op;
d->data.default_op_arg = op_arg;
aml_vector_create(&d->data.requests, nbreqs,
sizeof(struct aml_dma_request_linux_seq),
offsetof(struct aml_dma_request_linux_seq, type),
......
......@@ -34,10 +34,10 @@ int main(int argc, char *argv[])
isrc[i] = 24;
}
/* invalid create input */
assert(aml_dma_linux_par_create(NULL, 1) == -AML_EINVAL);
assert(aml_dma_linux_par_create(NULL, 1, NULL, NULL) == -AML_EINVAL);
/* invalid requests */
assert(!aml_dma_linux_par_create(&dma, 1));
assert(!aml_dma_linux_par_create(&dma, 1, NULL, NULL));
assert(aml_dma_copy(dma, NULL, isl) == -AML_EINVAL);
assert(aml_dma_copy(dma, idl, NULL) == -AML_EINVAL);
......@@ -52,7 +52,7 @@ int main(int argc, char *argv[])
aml_dma_linux_par_destroy(&dma);
/* cancel a request on the fly */
assert(!aml_dma_linux_par_create(&dma, 1));
assert(!aml_dma_linux_par_create(&dma, 1, NULL, NULL));
assert(aml_dma_cancel(dma, NULL) == -AML_EINVAL);
assert(!aml_dma_async_copy(dma, &r1, idl, isl));
assert(!aml_dma_cancel(dma, &r1));
......@@ -62,7 +62,7 @@ int main(int argc, char *argv[])
aml_dma_linux_par_destroy(&dma);
/* move data around */
assert(!aml_dma_linux_par_create(&dma, 1));
assert(!aml_dma_linux_par_create(&dma, 1, NULL, NULL));
struct aml_dma_request *requests[16];
struct aml_layout *layouts[16][2];
for (int i = 0; i < 16; i++) {
......
......@@ -34,10 +34,10 @@ int main(int argc, char *argv[])
isrc[i] = 24;
}
/* invalid create input */
assert(aml_dma_linux_seq_create(NULL, 1) == -AML_EINVAL);
assert(aml_dma_linux_seq_create(NULL, 1, NULL, NULL) == -AML_EINVAL);
/* invalid requests */
assert(!aml_dma_linux_seq_create(&dma, 1));
assert(!aml_dma_linux_seq_create(&dma, 1, NULL, NULL));
assert(aml_dma_copy(dma, NULL, isl) == -AML_EINVAL);
assert(aml_dma_copy(dma, idl, NULL) == -AML_EINVAL);
......@@ -61,7 +61,7 @@ int main(int argc, char *argv[])
aml_dma_linux_seq_destroy(&dma);
/* move data around */
assert(!aml_dma_linux_seq_create(&dma, 1));
assert(!aml_dma_linux_seq_create(&dma, 1, NULL, NULL));
struct aml_dma_request *requests[16];
struct aml_layout *layouts[16][2];
for (int i = 0; i < 16; i++) {
......
......@@ -33,7 +33,7 @@ int main(int argc, char *argv[])
TILESIZE*_SC_PAGE_SIZE*NBTILES));
size_t maxrequests = NBTILES;
assert(!aml_dma_linux_seq_create(&dma, maxrequests));
assert(!aml_dma_linux_seq_create(&dma, maxrequests, NULL, NULL));
/* allocate some memory */
src = aml_area_mmap(&aml_area_linux, TILESIZE*_SC_PAGE_SIZE*NBTILES, NULL);
......
......@@ -33,7 +33,7 @@ int main(int argc, char *argv[])
TILESIZE*_SC_PAGE_SIZE*NBTILES));
size_t maxrequests = NBTILES;
assert(!aml_dma_linux_par_create(&dma, maxrequests));
assert(!aml_dma_linux_par_create(&dma, maxrequests, NULL, NULL));
/* allocate some memory */
src = aml_area_mmap(&aml_area_linux, TILESIZE*_SC_PAGE_SIZE*NBTILES, NULL);
......
Markdown is supported
0% or
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment