From f7ec77621ad882e82dfa3b5607df0782a29b882d Mon Sep 17 00:00:00 2001 From: Swann Perarnau Date: Thu, 5 Sep 2019 09:34:08 -0500 Subject: [PATCH] [refactor] make scratch work with new tilings Only works for scratch_seq right now. --- include/aml.h | 23 +++++---- include/aml/scratch/seq.h | 33 +++--------- src/scratch/scratch.c | 99 ++++++++++++++++++------------------ src/scratch/scratch_seq.c | 104 ++++++++++---------------------------- 4 files changed, 99 insertions(+), 160 deletions(-) diff --git a/include/aml.h b/include/aml.h index dc3791f..9442bc2 100644 --- a/include/aml.h +++ b/include/aml.h @@ -957,7 +957,8 @@ struct aml_scratch_ops { **/ int (*create_request)(struct aml_scratch_data *scratch, struct aml_scratch_request **req, int type, - va_list args); + struct aml_layout **dest, int *destid, + struct aml_layout *src, int srcid); /** * \todo Doc **/ @@ -968,10 +969,6 @@ struct aml_scratch_ops { **/ int (*wait_request)(struct aml_scratch_data *scratch, struct aml_scratch_request *req); - /** - * \todo Doc - **/ - void *(*baseptr)(const struct aml_scratch_data *scratch); /** * \todo Doc **/ @@ -1003,7 +1000,9 @@ struct aml_scratch { * @see aml_scratch_baseptr() * @return 0 if successful; an error code otherwise. **/ -int aml_scratch_pull(struct aml_scratch *scratch, ...); +int aml_scratch_pull(struct aml_scratch *scratch, + struct aml_layout **dest, int *scratchid, + struct aml_layout *src, int srcid); /** * Requests a pull from regular memory to the scratchpad. This is an @@ -1016,7 +1015,9 @@ int aml_scratch_pull(struct aml_scratch *scratch, ...); * @see aml_scratch_pull() **/ int aml_scratch_async_pull(struct aml_scratch *scratch, - struct aml_scratch_request **req, ...); + struct aml_scratch_request **req, + struct aml_layout **scratch_layout, int *scratchid, + struct aml_layout *src_layout, int srcid); /** * Requests a synchronous push from the scratchpad to regular memory. * @param scratch: an initialized scratchpad structure. @@ -1030,7 +1031,9 @@ int aml_scratch_async_pull(struct aml_scratch *scratch, * @return 0 if successful; an error code otherwise. * @see aml_scratch_baseptr() **/ -int aml_scratch_push(struct aml_scratch *scratch, ...); +int aml_scratch_push(struct aml_scratch *scratch, + struct aml_layout **dest_layout, int *destid, + struct aml_layout *scratch_layout, int scratchid); /** * Requests a push from the scratchpad to regular memory. This is an @@ -1043,7 +1046,9 @@ int aml_scratch_push(struct aml_scratch *scratch, ...); * @see aml_scratch_push() **/ int aml_scratch_async_push(struct aml_scratch *scratch, - struct aml_scratch_request **req, ...); + struct aml_scratch_request **req, + struct aml_layout **dest_layout, int *destid, + struct aml_layout *scratch_layout, int scratchid); /** * Waits for an asynchronous scratch request to complete. * @param scratch: an initialized scratchpad structure. diff --git a/include/aml/scratch/seq.h b/include/aml/scratch/seq.h index 95d532b..3edc9d3 100644 --- a/include/aml/scratch/seq.h +++ b/include/aml/scratch/seq.h @@ -32,15 +32,11 @@ struct aml_scratch_request_seq { * @see **/ int type; - /** The tiling used for data organization in source and destination **/ - struct aml_tiling *tiling; /** The source layout of the data movement **/ struct aml_layout *src; - /** The identifier of the source tile **/ int srcid; /** The destination pointer of the data movement **/ struct aml_layout *dst; - /** The identifier of the destination tile **/ int dstid; /** The request used for movement **/ struct aml_dma_request *dma_req; @@ -48,22 +44,11 @@ struct aml_scratch_request_seq { /** Inner data of the sequential scratchpad implementation **/ struct aml_scratch_seq_data { - /** The source area where data comes from **/ - struct aml_area *src_area; - /** The destination area where data temporariliy goes to **/ - struct aml_area *sch_area; - /** - * The data organisation. - * /todo why can't source and destination tiling vary? - **/ - struct aml_tiling *tiling; - /** \todo What is this? **/ - size_t scratch_size; + struct aml_tiling *src_tiling; + struct aml_tiling *scratch_tiling; /** The dma engine in charge of the transfer **/ struct aml_dma *dma; - /** Pointer to data in scratch destination **/ - void *sch_ptr; - /** The tilings involved in ongoing scratch requests **/ + /** Map of tiles src layouts to scratch ids **/ struct aml_vector *tilemap; /** The set of dma requests submitted to the dma to mode data **/ struct aml_vector *requests; @@ -96,21 +81,17 @@ struct aml_scratch_seq { * @param scratch an address where the pointer to the newly allocated scratchpad * structure will be stored. * - * @param scratch_area the memory area where the scratchpad will be allocated. - * @param source_area the memory area containing the user data structure. * @param dma the DMA that will be used for migrating data to and from * the scratchpad. - * @param tiling the tiling to use on the user data structure and the scratch. - * @param nbtiles number of tiles to divide the scratchpad into. + * @param src_tiling the tiling on the source memory + * @param scratch_tiling the tiling to use on the scratch * @param nbreqs the initial number of slots for asynchronous request that * are in-flight (will be increased automatically if necessary). * @return 0 if successful; an error code otherwise. **/ int aml_scratch_seq_create(struct aml_scratch **scratch, - struct aml_area *scratch_area, - struct aml_area *src_area, - struct aml_dma *dma, struct aml_tiling *tiling, - size_t nbtiles, size_t nbreqs); + struct aml_dma *dma, struct aml_tiling *src_tiling, + struct aml_tiling *scratch_tiling, size_t nbreqs); /** * Tears down an initialized sequential scratchpad. diff --git a/src/scratch/scratch.c b/src/scratch/scratch.c index 9af6be5..9a74764 100644 --- a/src/scratch/scratch.c +++ b/src/scratch/scratch.c @@ -20,91 +20,94 @@ * abstract the request creation after this layer. ******************************************************************************/ -int aml_scratch_pull(struct aml_scratch *scratch, ...) +int aml_scratch_pull(struct aml_scratch *scratch, + struct aml_layout **dest, int *scratchid, + struct aml_layout *src, int srcid) { - assert(scratch != NULL); - va_list ap; - int ret; struct aml_scratch_request *req; + int ret; - va_start(ap, scratch); + if (scratch == NULL || dest == NULL || scratchid == NULL + || src == NULL) + return -AML_EINVAL; + ret = scratch->ops->create_request(scratch->data, &req, - AML_SCRATCH_REQUEST_TYPE_PULL, ap); - va_end(ap); - ret = scratch->ops->wait_request(scratch->data, req); - return ret; + AML_SCRATCH_REQUEST_TYPE_PULL, + dest, scratchid, src, srcid); + if (ret) + return ret; + + return scratch->ops->wait_request(scratch->data, req); } int aml_scratch_async_pull(struct aml_scratch *scratch, - struct aml_scratch_request **req, ...) + struct aml_scratch_request **req, + struct aml_layout **dest, int *scratchid, + struct aml_layout *src, int srcid) { - assert(scratch != NULL); - assert(req != NULL); - va_list ap; - int ret; + if (scratch == NULL || dest == NULL || scratchid == NULL + || src == NULL) + return -AML_EINVAL; - va_start(ap, req); - ret = scratch->ops->create_request(scratch->data, req, - AML_SCRATCH_REQUEST_TYPE_PULL, ap); - va_end(ap); - return ret; + return scratch->ops->create_request(scratch->data, req, + AML_SCRATCH_REQUEST_TYPE_PULL, + dest, scratchid, src, srcid); } -int aml_scratch_push(struct aml_scratch *scratch, ...) +int aml_scratch_push(struct aml_scratch *scratch, + struct aml_layout **dest, int *destid, + struct aml_layout *src, int srcid) { - assert(scratch != NULL); struct aml_scratch_request *req; - va_list ap; int ret; - va_start(ap, scratch); + if (scratch == NULL || dest == NULL || destid == NULL + || src == NULL) + return -AML_EINVAL; + ret = scratch->ops->create_request(scratch->data, &req, - AML_SCRATCH_REQUEST_TYPE_PUSH, ap); - va_end(ap); - ret = scratch->ops->wait_request(scratch->data, req); - return ret; + AML_SCRATCH_REQUEST_TYPE_PUSH, + dest, destid, src, srcid); + if (ret) + return ret; + + return scratch->ops->wait_request(scratch->data, req); } int aml_scratch_async_push(struct aml_scratch *scratch, - struct aml_scratch_request **req, ...) + struct aml_scratch_request **req, + struct aml_layout **dest, int *destid, + struct aml_layout *src, int srcid) { - assert(scratch != NULL); - assert(req != NULL); - va_list ap; - int ret; + if (scratch == NULL || dest == NULL || destid == NULL + || src == NULL) + return -AML_EINVAL; - va_start(ap, req); - ret = scratch->ops->create_request(scratch->data, req, - AML_SCRATCH_REQUEST_TYPE_PUSH, ap); - va_end(ap); - return ret; + return scratch->ops->create_request(scratch->data, req, + AML_SCRATCH_REQUEST_TYPE_PUSH, + dest, destid, src, srcid); } int aml_scratch_cancel(struct aml_scratch *scratch, struct aml_scratch_request *req) { - assert(scratch != NULL); - assert(req != NULL); + if (scratch == NULL || req == NULL) + return -AML_EINVAL; return scratch->ops->destroy_request(scratch->data, req); } int aml_scratch_wait(struct aml_scratch *scratch, struct aml_scratch_request *req) { - assert(scratch != NULL); - assert(req != NULL); + if (scratch == NULL || req == NULL) + return -AML_EINVAL; return scratch->ops->wait_request(scratch->data, req); } -void *aml_scratch_baseptr(const struct aml_scratch *scratch) -{ - assert(scratch != NULL); - return scratch->ops->baseptr(scratch->data); -} - int aml_scratch_release(struct aml_scratch *scratch, int scratchid) { - assert(scratch != NULL); + if (scratch == NULL) + return -AML_EINVAL; return scratch->ops->release(scratch->data, scratchid); } diff --git a/src/scratch/scratch_seq.c b/src/scratch/scratch_seq.c index 43589f8..bff8446 100644 --- a/src/scratch/scratch_seq.c +++ b/src/scratch/scratch_seq.c @@ -27,31 +27,23 @@ ******************************************************************************/ int aml_scratch_request_seq_init(struct aml_scratch_request_seq *req, int type, - struct aml_tiling *t, void *dstptr, int dstid, - void *srcptr, int srcid) + struct aml_layout *dest, int destid, + struct aml_layout *src, int srcid) { assert(req != NULL); - void *dp, *sp; - size_t size; req->type = type; - req->tiling = t; + req->src = src; req->srcid = srcid; - req->dstid = dstid; - dp = aml_tiling_tilestart(req->tiling, dstptr, dstid); - sp = aml_tiling_tilestart(req->tiling, srcptr, srcid); - size = aml_tiling_tilesize(req->tiling, srcid); - aml_layout_dense_create(&req->dst, dp, 0, 1, 1, &size, NULL, NULL); - aml_layout_dense_create(&req->src, sp, 0, 1, 1, &size, NULL, NULL); + req->dst = dest; + req->dstid = destid; return 0; } int aml_scratch_request_seq_destroy(struct aml_scratch_request_seq *r) { assert(r != NULL); - aml_layout_dense_destroy(&r->dst); - aml_layout_dense_destroy(&r->src); return 0; } @@ -75,11 +67,11 @@ struct aml_scratch_seq_ops aml_scratch_seq_inner_ops = { * Public API ******************************************************************************/ -/* TODO: not thread-safe */ - int aml_scratch_seq_create_request(struct aml_scratch_data *d, struct aml_scratch_request **r, - int type, va_list ap) + int type, + struct aml_layout **dest, int *destid, + struct aml_layout *src, int srcid) { assert(d != NULL); assert(r != NULL); @@ -92,45 +84,24 @@ int aml_scratch_seq_create_request(struct aml_scratch_data *d, req = aml_vector_add(scratch->data.requests); /* init the request */ if (type == AML_SCRATCH_REQUEST_TYPE_PUSH) { - int scratchid; - int *srcid; - void *srcptr; - void *scratchptr; - - srcptr = va_arg(ap, void *); - srcid = va_arg(ap, int *); - scratchptr = va_arg(ap, void *); - scratchid = va_arg(ap, int); /* find destination tile */ - int *slot = aml_vector_get(scratch->data.tilemap, scratchid); + int *slot = aml_vector_get(scratch->data.tilemap, srcid); assert(slot != NULL); - *srcid = *slot; + *destid = *slot; + *dest = aml_tiling_index_linear(scratch->data.src_tiling, + *destid); /* init request */ - aml_scratch_request_seq_init(req, type, - scratch->data.tiling, - srcptr, *srcid, - scratchptr, scratchid); + aml_scratch_request_seq_init(req, type, *dest, *destid, + src, srcid); } else if (type == AML_SCRATCH_REQUEST_TYPE_PULL) { - int *scratchid; - int srcid; - void *srcptr; - void *scratchptr; int slot, *tile; - scratchptr = va_arg(ap, void *); - scratchid = va_arg(ap, int *); - srcptr = va_arg(ap, void *); - srcid = va_arg(ap, int); - /* find destination tile * We don't use add here because adding a tile means allocating * new tiles on the sch_area too. */ - /* TODO: this is kind of a bug: we reuse a tile, instead of - * creating a no-op request - */ slot = aml_vector_find(scratch->data.tilemap, srcid); if (slot == -1) { slot = aml_vector_find(scratch->data.tilemap, -1); @@ -141,12 +112,13 @@ int aml_scratch_seq_create_request(struct aml_scratch_data *d, type = AML_SCRATCH_REQUEST_TYPE_NOOP; /* save the key */ - *scratchid = slot; + *destid = slot; + *dest = aml_tiling_index_linear(scratch->data.scratch_tiling, + slot); /* init request */ - aml_scratch_request_seq_init(req, type, scratch->data.tiling, - scratchptr, *scratchid, - srcptr, srcid); + aml_scratch_request_seq_init(req, type, *dest, *destid, + src, srcid); } pthread_mutex_unlock(&scratch->data.lock); if (req->type != AML_SCRATCH_REQUEST_TYPE_NOOP) @@ -210,15 +182,6 @@ int aml_scratch_seq_wait_request(struct aml_scratch_data *d, return 0; } -void *aml_scratch_seq_baseptr(const struct aml_scratch_data *d) -{ - assert(d != NULL); - const struct aml_scratch_seq *scratch = - (const struct aml_scratch_seq *)d; - - return scratch->data.sch_ptr; -} - int aml_scratch_seq_release(struct aml_scratch_data *d, int scratchid) { assert(d != NULL); @@ -237,7 +200,6 @@ struct aml_scratch_ops aml_scratch_seq_ops = { aml_scratch_seq_create_request, aml_scratch_seq_destroy_request, aml_scratch_seq_wait_request, - aml_scratch_seq_baseptr, aml_scratch_seq_release, }; @@ -246,17 +208,14 @@ struct aml_scratch_ops aml_scratch_seq_ops = { ******************************************************************************/ int aml_scratch_seq_create(struct aml_scratch **scratch, - struct aml_area *scratch_area, - struct aml_area *src_area, - struct aml_dma *dma, struct aml_tiling *tiling, - size_t nbtiles, size_t nbreqs) + struct aml_dma *dma, struct aml_tiling *src_tiling, + struct aml_tiling *scratch_tiling, size_t nbreqs) { struct aml_scratch *ret = NULL; struct aml_scratch_seq *s; - if (scratch == NULL - || scratch_area == NULL || src_area == NULL - || dma == NULL || tiling == NULL) + if (scratch == NULL || dma == NULL || src_tiling == NULL + || scratch_tiling == NULL) return -AML_EINVAL; *scratch = NULL; @@ -271,10 +230,9 @@ int aml_scratch_seq_create(struct aml_scratch **scratch, s = (struct aml_scratch_seq *)ret->data; s->ops = aml_scratch_seq_inner_ops; - s->data.sch_area = scratch_area; - s->data.src_area = src_area; s->data.dma = dma; - s->data.tiling = tiling; + s->data.src_tiling = src_tiling; + s->data.scratch_tiling = scratch_tiling; /* allocate request array */ aml_vector_create(&s->data.requests, nbreqs, @@ -282,14 +240,9 @@ int aml_scratch_seq_create(struct aml_scratch **scratch, offsetof(struct aml_scratch_request_seq, type), AML_SCRATCH_REQUEST_TYPE_INVALID); - /* s init */ + /* "hashmap for src to scratch tiles */ + size_t nbtiles = aml_tiling_ntiles(scratch_tiling); aml_vector_create(&s->data.tilemap, nbtiles, sizeof(int), 0, -1); - size_t tilesize = aml_tiling_tilesize(s->data.tiling, 0); - - s->data.scratch_size = nbtiles * tilesize; - s->data.sch_ptr = aml_area_mmap(s->data.sch_area, - s->data.scratch_size, - NULL); pthread_mutex_init(&s->data.lock, NULL); *scratch = ret; @@ -311,9 +264,6 @@ void aml_scratch_seq_destroy(struct aml_scratch **scratch) inner = (struct aml_scratch_seq *)s->data; aml_vector_destroy(&inner->data.requests); aml_vector_destroy(&inner->data.tilemap); - aml_area_munmap(inner->data.sch_area, - inner->data.sch_ptr, - inner->data.scratch_size); pthread_mutex_destroy(&inner->data.lock); free(s); *scratch = NULL; -- 2.26.2