aml.h 19.3 KB
Newer Older
Swann Perarnau's avatar
Swann Perarnau committed
1 2 3 4 5 6 7 8 9 10
/*******************************************************************************
 * Copyright 2019 UChicago Argonne, LLC.
 * (c.f. AUTHORS, LICENSE)
 *
 * This file is part of the AML project.
 * For more info, see https://xgitlab.cels.anl.gov/argo/aml
 *
 * SPDX-License-Identifier: BSD-3-Clause
*******************************************************************************/

11 12 13
#ifndef AML_H
#define AML_H 1

14
#include <assert.h>
15 16 17 18 19 20 21
#include <inttypes.h>
#include <numa.h>
#include <numaif.h>
#include <pthread.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdio.h>
22
#include <stdlib.h>
23 24 25
#include <sys/mman.h>
#include <unistd.h>

26
/* Used by mbind */
27 28 29 30
#ifndef PAGE_SIZE
#define PAGE_SIZE 4096
#endif

31
#include "aml/utils/bitmap.h"
Nicolas Denoyelle's avatar
Nicolas Denoyelle committed
32
#include "aml/utils/error.h"
33
#include "aml/utils/vector.h"
Nicolas Denoyelle's avatar
Nicolas Denoyelle committed
34
#include "aml/utils/version.h"
35

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
/*******************************************************************************
 * General functions:
 * Initialize internal structures, cleanup everything at the end.
 ******************************************************************************/

/*
 * Initializes the library.
 * "argc": pointer to the main()'s argc argument; contents can get modified.
 * "argv": pointer to the main()'s argv argument; contents can get modified.
 * Returns 0 if successful; an error code otherwise.
 */
int aml_init(int *argc, char **argv[]);
/*
 * Terminates the library.
 * Returns 0 if successful; an error code otherwise.
 */
int aml_finalize(void);
53

54 55
/*******************************************************************************
 * Areas:
56
 * embeds information about a byte-addressable physical memory location as well
57 58
 * as binding policies over it.
 ******************************************************************************/
59

Nicolas Denoyelle's avatar
Nicolas Denoyelle committed
60
/**
61 62
 * An AML area is an implementation of memory operations for several type of
 * devices through a consistent abstraction.
Nicolas Denoyelle's avatar
Nicolas Denoyelle committed
63 64 65
 * This abstraction is meant to be implemented for several kind of devices,
 * i.e the same function calls allocate different kinds of devices depending
 * on the area implementation provided.
66
 * List of aml area implementations each may provide additional functionalities:
Nicolas Denoyelle's avatar
Nicolas Denoyelle committed
67 68 69
 * - <aml/area/linux.h>
 **/

70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89
/* Opaque handle to areas data. Defined by implementations */
struct aml_area_data;

/** Implementation specific operations. **/
struct aml_area_ops {
	/**
	 * Coarse grain allocator of virtual memory.
	 *
	 * "area_data": Opaque handle to implementation specific data.
	 * "ptr": A virtual address to be used by underlying implementation.
	 *        Can be NULL.
	 * "size": The minimum size of allocation.
	 *         Is greater than 0. Must not fail unless not enough
	 *         memory is available, or ptr argument does not point to a
	 *         suitable address.
	 *         In case of failure, aml_errno must be set to an appropriate
	 *         value.
	 *
	 * Returns a pointer to allocated memory object.
	 **/
90
	void* (*mmap)(const struct aml_area_data  *area_data,
91 92 93 94 95 96 97 98 99 100 101 102
		      void                        *ptr,
		      size_t                       size);

	/**
	 * Unmapping of virtual memory mapped with map().
	 *
	 * "area_data": An opaque handle to implementation specific data.
	 * "ptr": Pointer to data mapped in physical memory. Cannot be NULL.
	 * "size": The size of data. Cannot be 0.
	 *
	 * Returns AML_AREA_* error code.
	 **/
103
	int (*munmap)(const struct aml_area_data *area_data,
104 105 106 107 108 109 110 111 112 113
		      void                       *ptr,
		      size_t                      size);
};

struct aml_area {
	/* Basic memory operations implementation */
	struct aml_area_ops *ops;
	/* Implementation specific data. Set to NULL at creation. */
	struct aml_area_data *data;
};
Nicolas Denoyelle's avatar
Nicolas Denoyelle committed
114 115 116 117 118 119 120 121 122 123 124 125 126 127 128

/**
 * Returns virtual memory from this area with at least queried size bytes.
 * "area": A valid area implementing access to target memory.
 * "ptr":  Implementation specific argument. See specific header.
 * "size": The usable size of memory returned.
 **/
void*
aml_area_mmap(const struct aml_area *area,
	      void                 **ptr,
	      size_t                 size);

/**
 * Release data provided with aml_area_mmap() and the same area.
 * "area": A valid area implementing access to target memory.
129
 * "ptr": A pointer to memory address provided with aml_area_mmap()
Nicolas Denoyelle's avatar
Nicolas Denoyelle committed
130 131 132 133 134 135
 *        by same area and size.
 **/
int
aml_area_munmap(const struct aml_area *area,
		void                  *ptr,
		size_t                 size);
136

137 138 139 140 141 142 143 144 145 146 147 148 149 150 151
/*******************************************************************************
 * Tiling:
 * Representation of a data structure organization in memory.
 ******************************************************************************/

/* opaque handle to all tilings */
struct aml_tiling_data;
struct aml_tiling_iterator_data;

/*forward declarations */
struct aml_tiling_iterator_ops;
struct aml_tiling_iterator;


struct aml_tiling_ops {
152 153 154 155 156 157 158
	int (*create_iterator)(struct aml_tiling_data *tiling,
			       struct aml_tiling_iterator **iterator,
			       int flags);
	int (*init_iterator)(struct aml_tiling_data *tiling,
			     struct aml_tiling_iterator *iterator, int flags);
	int (*destroy_iterator)(struct aml_tiling_data *tiling,
				struct aml_tiling_iterator *iterator);
159
	int (*tileid)(const struct aml_tiling_data *tiling, va_list coords);
Kamil Iskra's avatar
Kamil Iskra committed
160 161 162
	size_t (*tilesize)(const struct aml_tiling_data *tiling, int tileid);
	void* (*tilestart)(const struct aml_tiling_data *tiling,
			   const void *ptr, int tileid);
163
	int (*ndims)(const struct aml_tiling_data *tiling, va_list results);
164 165 166 167 168 169 170
};

struct aml_tiling {
	struct aml_tiling_ops *ops;
	struct aml_tiling_data *data;
};

171 172 173 174 175 176 177 178 179 180 181
/*
 * Provides the tile id of a tile.
 * "tiling": an initialized tiling structure.
 * Variadic arguments:
 *  - a list of size_t coordinates, one per dimension of the tiling.
 * Returns the id of the tile (that is, its order in memory), to use with other
 * functions.
 * Returns -1 in case of invalid coordinates.
 */
int aml_tiling_tileid(const struct aml_tiling *tiling, ...);

182 183 184 185 186 187 188
/*
 * Provides the information on the size of a tile.
 * "tiling": an initialized tiling structure.
 * "tileid": an identifier of a tile (a value between 0 and the number of tiles
 *           minus 1).
 * Returns the size of a tile.
 */
Kamil Iskra's avatar
Kamil Iskra committed
189
size_t aml_tiling_tilesize(const struct aml_tiling *tiling, int tileid);
190

191 192 193 194 195 196 197 198 199 200
/*
 * Provides the information on the location of a tile in memory.
 * "tiling": an initialized tiling structure.
 * "ptr": an address of the start of the complete user data structure that this
 *        tiling describes.
 * "tileid": an identifier of a tile (a value between 0 and the number of tiles
 *           minus 1).
 * Returns the address of the start of the tile identified by "tileid", within
 * the provided user data structure.
 */
201
void *aml_tiling_tilestart(const struct aml_tiling *tiling, const void *ptr,
Kamil Iskra's avatar
Kamil Iskra committed
202
			   int tileid);
203

204 205 206 207 208 209 210 211 212 213
/*
 * Provides the dimensions of the entire tiling in tiles.
 * "tiling": an initialized tiling structure.
 * Variadic arguments:
 *  - a list of (size_t *), one per dimension of the tiling.
 *  Will contain the size of each dimension in tiles upon return.
 * Returns 0 if successful, an error code otherwise.
 */
int aml_tiling_ndims(const struct aml_tiling *tiling, ...);

214 215 216 217 218 219 220 221
/*
 * Allocates and initializes a new tiling iterator.
 * "tiling": an initialized tiling structure.
 * "iterator": an address where the pointer to the newly allocated iterator
 *             structure will be stored.
 * "flags": reserved for future use; pass 0 for now.
 * Returns 0 if successful; an error code otherwise.
 */
222 223 224
int aml_tiling_create_iterator(struct aml_tiling *tiling,
			       struct aml_tiling_iterator **iterator,
			       int flags);
225 226 227 228 229 230 231
/*
 * Initializes a tiling iterator.
 * "tiling": an initialized tiling structure.
 * "iterator": an allocated tiling iterator structure.
 * "flags": reserved for future use; pass 0 for now.
 * Returns 0 if successful; an error code otherwise.
 */
232 233
int aml_tiling_init_iterator(struct aml_tiling *tiling,
			     struct aml_tiling_iterator *iterator, int flags);
234 235 236
/*
 * Tears down an initialized tiling iterator.
 * "tiling": an initialized tiling structure.
237
 * "iterator": an initialized tiling iterator structure.
238 239
 * Returns 0 if successful; an error code otherwise.
 */
240 241
int aml_tiling_destroy_iterator(struct aml_tiling *tiling,
				struct aml_tiling_iterator *iterator);
242 243

struct aml_tiling_iterator_ops {
244 245
	int (*reset)(struct aml_tiling_iterator_data *iterator);
	int (*next)(struct aml_tiling_iterator_data *iterator);
Kamil Iskra's avatar
Kamil Iskra committed
246 247 248
	int (*end)(const struct aml_tiling_iterator_data *iterator);
	int (*get)(const struct aml_tiling_iterator_data *iterator,
		   va_list args);
249 250 251 252 253 254 255
};

struct aml_tiling_iterator {
	struct aml_tiling_iterator_ops *ops;
	struct aml_tiling_iterator_data *data;
};

256 257
/*
 * Resets a tiling iterator to the first tile.
258
 * "iterator": an initialized tiling iterator structure.
259 260
 * Returns 0 if successful; an error code otherwise.
 */
261
int aml_tiling_iterator_reset(struct aml_tiling_iterator *iterator);
262 263
/*
 * Advances a tiling iterator to the next tile.
264
 * "iterator": an initialized tiling iterator structure.
265 266
 * Returns 0 if successful; an error code otherwise.
 */
267
int aml_tiling_iterator_next(struct aml_tiling_iterator *iterator);
268 269
/*
 * Checks whether the iterator is past the last tile.
270
 * "iterator": an initialized tiling iterator structure.
271 272 273
 * Returns 0 if the iterator points at a valid tile; 1 if it's past the last
 * tile.
 */
Kamil Iskra's avatar
Kamil Iskra committed
274
int aml_tiling_iterator_end(const struct aml_tiling_iterator *iterator);
275 276
/*
 * Queries the iterator.
277
 * "iterator": an initialized tiling iterator structure.
278 279 280 281 282
 * Variadic arguments:
 * - "x": an argument of type unsigned long*; on return gets filled with the
 *        identifier of the tile currently pointed to.
 * Returns 0 if successful; an error code otherwise.
 */
Kamil Iskra's avatar
Kamil Iskra committed
283
int aml_tiling_iterator_get(const struct aml_tiling_iterator *iterator, ...);
284

285 286
/* Tiling types passed to the tiling create()/init()/vinit() routines.  */
/* Regular, linear tiling with uniform tile sizes.  */
287
#define AML_TILING_TYPE_1D 0
288 289
#define AML_TILING_TYPE_2D_ROWMAJOR 1
#define AML_TILING_TYPE_2D_COLMAJOR 2
290

291 292 293 294 295 296 297 298 299 300
/*
 * Allocates and initializes a new tiling.
 * "tiling": an address where the pointer to the newly allocated tiling
 *           structure will be stored.
 * "type": see AML_TILING_TYPE_*.
 * Variadic arguments:
 * - if "type" equals AML_TILING_TYPE_1D, two additional arguments are needed:
 *   - "tilesize": an argument of type size_t; provides the size of each tile.
 *   - "totalsize": an argument of type size_t; provides the size of the
 *                  complete user data structure to be tiled.
301 302
 * - if "type" equals AML_TILING_TYPE_2D, four additional arguments are needed:
 *   - "tilesize": an argument of type size_t; provides the size of a tile.
303 304
 *   - "totalsize": an argument of type size_t; provides the size of the
 *                  complete user data structure to be tiled.
305 306
 *   - "rowsize": an argument of type size_t; the number of tiles in a row
 *   - "colsize": an argument of type size_t; the number of tiles in a column
307 308
 * Returns 0 if successful; an error code otherwise.
 */
309
int aml_tiling_create(struct aml_tiling **tiling, int type, ...);
310 311 312 313 314 315 316 317
/*
 * Initializes a tiling.  This is a varargs-variant of the aml_tiling_vinit()
 * routine.
 * "tiling": an allocated tiling structure.
 * "type": see aml_tiling_create().
 * Variadic arguments: see aml_tiling_create().
 * Returns 0 if successful; an error code otherwise.
 */
318
int aml_tiling_init(struct aml_tiling *tiling, int type, ...);
319 320 321 322 323 324 325
/*
 * Initializes a tiling.
 * "tiling": an allocated tiling structure.
 * "type": see aml_tiling_create().
 * "args": see the variadic arguments of aml_tiling_create().
 * Returns 0 if successful; an error code otherwise.
 */
326
int aml_tiling_vinit(struct aml_tiling *tiling, int type, va_list args);
327 328 329 330 331 332
/*
 * Tears down an initialized tiling.
 * "tiling": an initialized tiling structure.
 * "type": see aml_tiling_create().
 * Returns 0 if successful; an error code otherwise.
 */
333
int aml_tiling_destroy(struct aml_tiling *tiling, int type);
334

335 336 337 338 339
/*******************************************************************************
 * DMA:
 * Management of low-level movement of memory.
 ******************************************************************************/

340 341
/* Internal macros used for tracking DMA request types.  */
/* Invalid request type.  Used for marking inactive requests in the vector.  */
342
#define AML_DMA_REQUEST_TYPE_INVALID -1
343
/* Copy request type.  Uses memcpy() for data migration.  */
344 345
#define AML_DMA_REQUEST_TYPE_COPY 0

346
struct aml_dma_request;
347 348 349
struct aml_dma_data;

struct aml_dma_ops {
350
	int (*create_request)(struct aml_dma_data *dma,
351
			      struct aml_dma_request **req, int type,
352 353 354 355 356
			      va_list args);
	int (*destroy_request)(struct aml_dma_data *dma,
			       struct aml_dma_request *req);
	int (*wait_request)(struct aml_dma_data *dma,
			    struct aml_dma_request *req);
357 358 359 360 361 362 363
};

struct aml_dma {
	struct aml_dma_ops *ops;
	struct aml_dma_data *data;
};

364 365 366 367 368 369 370 371 372 373 374 375 376 377 378 379
/*
 * Requests a synchronous data copy between two different tiles, using
 * memcpy() or equivalent.
 * "dma": an initialized DMA structure.
 * Variadic arguments:
 * - "dt": an argument of type struct aml_tiling*; the destination tiling
 *         structure.
 * - "dptr": an argument of type void*; the start address of the complete
 *           destination user data structure.
 * - "dtid": an argument of type int; the destination tile identifier.
 * - "st": an argument of type struct aml_tiling*; the source tiling structure.
 * - "sptr": an argument of type void*; the start address of the complete
 *           source user data structure.
 * - "stid": an argument of type int; the source tile identifier.
 * Returns 0 if successful; an error code otherwise.
 */
380
int aml_dma_copy(struct aml_dma *dma, ...);
381 382 383 384 385 386 387 388 389
/*
 * Requests a data copy between two different tiles.  This is an asynchronous
 * version of aml_dma_copy().
 * "dma": an initialized DMA structure.
 * "req": an address where the pointer to the newly assigned DMA request will be
 *        stored.
 * Variadic arguments: see aml_dma_copy().
 * Returns 0 if successful; an error code otherwise.
 */
390
int aml_dma_async_copy(struct aml_dma *dma, struct aml_dma_request **req, ...);
391 392 393 394 395 396
/*
 * Waits for an asynchronous DMA request to complete.
 * "dma": an initialized DMA structure.
 * "req": a DMA request obtained using aml_dma_async_*() calls.
 * Returns 0 if successful; an error code otherwise.
 */
397
int aml_dma_wait(struct aml_dma *dma, struct aml_dma_request *req);
398 399 400 401 402 403
/*
 * Tears down an asynchronous DMA request before it completes.
 * "dma": an initialized DMA structure.
 * "req": a DMA request obtained using aml_dma_async_*() calls.
 * Returns 0 if successful; an error code otherwise.
 */
404
int aml_dma_cancel(struct aml_dma *dma, struct aml_dma_request *req);
405

406 407
/*******************************************************************************
 * Scratchpad:
408
 * Use an area to stage data from another area in and out.
409 410 411 412 413 414
 * A dma handles the movement itself.
 ******************************************************************************/

struct aml_scratch_request;
struct aml_scratch_data;

415 416
/* Internal macros used for tracking scratchpad request types.  */
/* Invalid request type.  Used for marking inactive requests in the vector.  */
417
#define AML_SCRATCH_REQUEST_TYPE_INVALID -1
418
/* Push from the scratchpad to regular memory.  */
419
#define AML_SCRATCH_REQUEST_TYPE_PUSH 0
420
/* Pull from regular memory to the scratchpad.  */
421
#define AML_SCRATCH_REQUEST_TYPE_PULL 1
422 423
/* No-op/empty request */
#define AML_SCRATCH_REQUEST_TYPE_NOOP 2
424 425 426 427 428 429 430 431 432

struct aml_scratch_ops {
	int (*create_request)(struct aml_scratch_data *scratch,
			      struct aml_scratch_request **req, int type,
			      va_list args);
	int (*destroy_request)(struct aml_scratch_data *scratch,
			       struct aml_scratch_request *req);
	int (*wait_request)(struct aml_scratch_data *scratch,
			    struct aml_scratch_request *req);
433
	void *(*baseptr)(const struct aml_scratch_data *scratch);
434
	int (*release)(struct aml_scratch_data *scratch, int scratchid);
435 436 437 438 439 440 441
};

struct aml_scratch {
	struct aml_scratch_ops *ops;
	struct aml_scratch_data *data;
};

442 443 444 445 446 447 448 449 450 451 452 453 454
/*
 * Requests a synchronous pull from regular memory to the scratchpad.
 * "scratch": an initialized scratchpad structure.
 * Variadic arguments:
 * - "scratchptr": an argument of type void*; the scratchpad base pointer (see
 *                 aml_scratch_baseptr()).
 * - "scratchid": an argument of type int*; gets filled with the scratch tile
 *                identifier where the data will be pulled into.
 * - "srcptr": an argument of type void*; the start address of the complete
 *             source user data structure.
 * - "srcid": an argument of type int; the source tile identifier.
 * Returns 0 if successful; an error code otherwise.
 */
455
int aml_scratch_pull(struct aml_scratch *scratch, ...);
456 457 458 459 460 461 462 463 464
/*
 * Requests a pull from regular memory to the scratchpad.  This is an
 * asynchronous version of aml_scratch_pull().
 * "scratch": an initialized scratchpad structure.
 * "req": an address where the pointer to the newly assigned scratch request
 *        will be stored.
 * Variadic arguments: see aml_scratch_pull().
 * Returns 0 if successful; an error code otherwise.
 */
465 466
int aml_scratch_async_pull(struct aml_scratch *scratch,
			   struct aml_scratch_request **req, ...);
467 468 469 470
/*
 * Requests a synchronous push from the scratchpad to regular memory.
 * "scratch": an initialized scratchpad structure.
 * Variadic arguments:
471 472 473
 * - "dstptr": an argument of type void*; the start address of the complete
 *             destination user data structure.
 * - "dstid": an argument of type int*; gets filled with the destination tile
474 475 476 477 478 479 480
 *            identifier where the data will be pushed into (and where it was
 *            pulled from in the first place).
 * - "scratchptr": an argument of type void*; the scratchpad base pointer (see
 *                 aml_scratch_baseptr()).
 * - "scratchid": an argument of type int; the scratchpad tile identifier.
 * Returns 0 if successful; an error code otherwise.
 */
481
int aml_scratch_push(struct aml_scratch *scratch, ...);
482 483 484 485 486 487 488 489 490
/*
 * Requests a push from the scratchpad to regular memory.  This is an
 * asynchronous version of aml_scratch_push().
 * "scratch": an initialized scratchpad structure.
 * "req": an address where the pointer to the newly assigned scratch request
 *        will be stored.
 * Variadic arguments: see aml_scratch_push().
 * Returns 0 if successful; an error code otherwise.
 */
491 492
int aml_scratch_async_push(struct aml_scratch *scratch,
			   struct aml_scratch_request **req, ...);
493 494 495 496 497 498
/*
 * Waits for an asynchronous scratch request to complete.
 * "scratch": an initialized scratchpad structure.
 * "req": a scratch request obtained using aml_scratch_async_*() calls.
 * Returns 0 if successful; an error code otherwise.
 */
499 500
int aml_scratch_wait(struct aml_scratch *scratch,
		     struct aml_scratch_request *req);
501

502 503
/*
 * Tears down an asynchronous scratch request before it completes.
504
 * "scratch": an initialized scratchpad structure.
505 506 507
 * "req": a scratch request obtained using aml_scratch_async_*() calls.
 * Returns 0 if successful; an error code otherwise.
 */
508 509
int aml_scratch_cancel(struct aml_scratch *scratch,
		       struct aml_scratch_request *req);
510 511
/*
 * Provides the location of the scratchpad.
512
 * "scratch": an initialized scratchpad structure.
513 514
 * Returns a base pointer to the scratchpad memory buffer.
 */
515
void *aml_scratch_baseptr(const struct aml_scratch *scratch);
516

517 518 519
/*
 * Release a scratch tile for immediate reuse.
 * "scratch": an initialized scratchpad structure.
520 521
 * "scratchid": a scratchpad tile identifier.
 * Returns 0 if successful; an error code otherwise.
522 523 524
 */
int aml_scratch_release(struct aml_scratch *scratch, int scratchid);

525
#endif