aml.h 20.6 KB
Newer Older
Swann Perarnau's avatar
Swann Perarnau committed
1 2 3 4 5 6 7 8 9 10
/*******************************************************************************
 * Copyright 2019 UChicago Argonne, LLC.
 * (c.f. AUTHORS, LICENSE)
 *
 * This file is part of the AML project.
 * For more info, see https://xgitlab.cels.anl.gov/argo/aml
 *
 * SPDX-License-Identifier: BSD-3-Clause
*******************************************************************************/

11 12 13
#ifndef AML_H
#define AML_H 1

14
#include <assert.h>
15 16 17 18 19 20 21
#include <inttypes.h>
#include <numa.h>
#include <numaif.h>
#include <pthread.h>
#include <stdarg.h>
#include <stddef.h>
#include <stdio.h>
22
#include <stdlib.h>
23 24 25
#include <sys/mman.h>
#include <unistd.h>

26
/* Used by mbind */
27 28 29 30
#ifndef PAGE_SIZE
#define PAGE_SIZE 4096
#endif

31
#include "aml/utils/bitmap.h"
Nicolas Denoyelle's avatar
Nicolas Denoyelle committed
32
#include "aml/utils/error.h"
33
#include "aml/utils/vector.h"
Nicolas Denoyelle's avatar
Nicolas Denoyelle committed
34
#include "aml/utils/version.h"
35

36 37 38 39 40 41 42 43 44 45 46 47 48 49 50 51 52
/*******************************************************************************
 * General functions:
 * Initialize internal structures, cleanup everything at the end.
 ******************************************************************************/

/*
 * Initializes the library.
 * "argc": pointer to the main()'s argc argument; contents can get modified.
 * "argv": pointer to the main()'s argv argument; contents can get modified.
 * Returns 0 if successful; an error code otherwise.
 */
int aml_init(int *argc, char **argv[]);
/*
 * Terminates the library.
 * Returns 0 if successful; an error code otherwise.
 */
int aml_finalize(void);
53

54 55
/*******************************************************************************
 * Areas:
56
 * embeds information about a byte-addressable physical memory location as well
57 58
 * as binding policies over it.
 ******************************************************************************/
59

Nicolas Denoyelle's avatar
Nicolas Denoyelle committed
60 61 62 63 64 65 66 67 68 69 70 71 72 73 74 75 76 77 78 79 80 81 82 83 84 85 86 87 88 89 90 91 92 93
/**
 * An AML area is an implementation of memory operations for several type of devices
 * through a consistent abstraction.
 * This abstraction is meant to be implemented for several kind of devices,
 * i.e the same function calls allocate different kinds of devices depending
 * on the area implementation provided.
 * List of aml area implementations each may provide additional functionnalities:
 * - <aml/area/linux.h>
 **/

/* Implementation details of area building block */
#include "aml/area/area.h"

/**
 * Returns virtual memory from this area with at least queried size bytes.
 * "area": A valid area implementing access to target memory.
 * "ptr":  Implementation specific argument. See specific header.
 * "size": The usable size of memory returned.
 **/
void*
aml_area_mmap(const struct aml_area *area,
	      void                 **ptr,
	      size_t                 size);

/**
 * Release data provided with aml_area_mmap() and the same area.
 * "area": A valid area implementing access to target memory.
 * "ptr": A pointer to memory address provided with aml_area_mmap() 
 *        by same area and size.
 **/
int
aml_area_munmap(const struct aml_area *area,
		void                  *ptr,
		size_t                 size);
94

95 96 97 98 99 100 101 102 103 104 105 106 107 108 109
/*******************************************************************************
 * Tiling:
 * Representation of a data structure organization in memory.
 ******************************************************************************/

/* opaque handle to all tilings */
struct aml_tiling_data;
struct aml_tiling_iterator_data;

/*forward declarations */
struct aml_tiling_iterator_ops;
struct aml_tiling_iterator;


struct aml_tiling_ops {
110 111 112 113 114 115 116
	int (*create_iterator)(struct aml_tiling_data *tiling,
			       struct aml_tiling_iterator **iterator,
			       int flags);
	int (*init_iterator)(struct aml_tiling_data *tiling,
			     struct aml_tiling_iterator *iterator, int flags);
	int (*destroy_iterator)(struct aml_tiling_data *tiling,
				struct aml_tiling_iterator *iterator);
117
	int (*tileid)(const struct aml_tiling_data *tiling, va_list);
Kamil Iskra's avatar
Kamil Iskra committed
118 119 120
	size_t (*tilesize)(const struct aml_tiling_data *tiling, int tileid);
	void* (*tilestart)(const struct aml_tiling_data *tiling,
			   const void *ptr, int tileid);
121
	int (*ndims)(const struct aml_tiling_data *tiling, va_list);
122 123 124 125 126 127 128
};

struct aml_tiling {
	struct aml_tiling_ops *ops;
	struct aml_tiling_data *data;
};

129 130 131 132 133 134 135 136 137 138 139
/*
 * Provides the tile id of a tile.
 * "tiling": an initialized tiling structure.
 * Variadic arguments:
 *  - a list of size_t coordinates, one per dimension of the tiling.
 * Returns the id of the tile (that is, its order in memory), to use with other
 * functions.
 * Returns -1 in case of invalid coordinates.
 */
int aml_tiling_tileid(const struct aml_tiling *tiling, ...);

140 141 142 143 144 145 146
/*
 * Provides the information on the size of a tile.
 * "tiling": an initialized tiling structure.
 * "tileid": an identifier of a tile (a value between 0 and the number of tiles
 *           minus 1).
 * Returns the size of a tile.
 */
Kamil Iskra's avatar
Kamil Iskra committed
147
size_t aml_tiling_tilesize(const struct aml_tiling *tiling, int tileid);
148

149 150 151 152 153 154 155 156 157 158
/*
 * Provides the information on the location of a tile in memory.
 * "tiling": an initialized tiling structure.
 * "ptr": an address of the start of the complete user data structure that this
 *        tiling describes.
 * "tileid": an identifier of a tile (a value between 0 and the number of tiles
 *           minus 1).
 * Returns the address of the start of the tile identified by "tileid", within
 * the provided user data structure.
 */
Kamil Iskra's avatar
Kamil Iskra committed
159 160
void* aml_tiling_tilestart(const struct aml_tiling *tiling, const void *ptr,
			   int tileid);
161

162 163 164 165 166 167 168 169 170 171
/*
 * Provides the dimensions of the entire tiling in tiles.
 * "tiling": an initialized tiling structure.
 * Variadic arguments:
 *  - a list of (size_t *), one per dimension of the tiling.
 *  Will contain the size of each dimension in tiles upon return.
 * Returns 0 if successful, an error code otherwise.
 */
int aml_tiling_ndims(const struct aml_tiling *tiling, ...);

172 173 174 175 176 177 178 179
/*
 * Allocates and initializes a new tiling iterator.
 * "tiling": an initialized tiling structure.
 * "iterator": an address where the pointer to the newly allocated iterator
 *             structure will be stored.
 * "flags": reserved for future use; pass 0 for now.
 * Returns 0 if successful; an error code otherwise.
 */
180 181 182
int aml_tiling_create_iterator(struct aml_tiling *tiling,
			       struct aml_tiling_iterator **iterator,
			       int flags);
183 184 185 186 187 188 189
/*
 * Initializes a tiling iterator.
 * "tiling": an initialized tiling structure.
 * "iterator": an allocated tiling iterator structure.
 * "flags": reserved for future use; pass 0 for now.
 * Returns 0 if successful; an error code otherwise.
 */
190 191
int aml_tiling_init_iterator(struct aml_tiling *tiling,
			     struct aml_tiling_iterator *iterator, int flags);
192 193 194
/*
 * Tears down an initialized tiling iterator.
 * "tiling": an initialized tiling structure.
195
 * "iterator": an initialized tiling iterator structure.
196 197
 * Returns 0 if successful; an error code otherwise.
 */
198 199
int aml_tiling_destroy_iterator(struct aml_tiling *tiling,
				struct aml_tiling_iterator *iterator);
200 201

struct aml_tiling_iterator_ops {
202 203
	int (*reset)(struct aml_tiling_iterator_data *iterator);
	int (*next)(struct aml_tiling_iterator_data *iterator);
Kamil Iskra's avatar
Kamil Iskra committed
204 205 206
	int (*end)(const struct aml_tiling_iterator_data *iterator);
	int (*get)(const struct aml_tiling_iterator_data *iterator,
		   va_list args);
207 208 209 210 211 212 213
};

struct aml_tiling_iterator {
	struct aml_tiling_iterator_ops *ops;
	struct aml_tiling_iterator_data *data;
};

214 215
/*
 * Resets a tiling iterator to the first tile.
216
 * "iterator": an initialized tiling iterator structure.
217 218
 * Returns 0 if successful; an error code otherwise.
 */
219
int aml_tiling_iterator_reset(struct aml_tiling_iterator *iterator);
220 221
/*
 * Advances a tiling iterator to the next tile.
222
 * "iterator": an initialized tiling iterator structure.
223 224
 * Returns 0 if successful; an error code otherwise.
 */
225
int aml_tiling_iterator_next(struct aml_tiling_iterator *iterator);
226 227
/*
 * Checks whether the iterator is past the last tile.
228
 * "iterator": an initialized tiling iterator structure.
229 230 231
 * Returns 0 if the iterator points at a valid tile; 1 if it's past the last
 * tile.
 */
Kamil Iskra's avatar
Kamil Iskra committed
232
int aml_tiling_iterator_end(const struct aml_tiling_iterator *iterator);
233 234
/*
 * Queries the iterator.
235
 * "iterator": an initialized tiling iterator structure.
236 237 238 239 240
 * Variadic arguments:
 * - "x": an argument of type unsigned long*; on return gets filled with the
 *        identifier of the tile currently pointed to.
 * Returns 0 if successful; an error code otherwise.
 */
Kamil Iskra's avatar
Kamil Iskra committed
241
int aml_tiling_iterator_get(const struct aml_tiling_iterator *iterator, ...);
242

243 244
/* Tiling types passed to the tiling create()/init()/vinit() routines.  */
/* Regular, linear tiling with uniform tile sizes.  */
245
#define AML_TILING_TYPE_1D 0
246 247
#define AML_TILING_TYPE_2D_ROWMAJOR 1
#define AML_TILING_TYPE_2D_COLMAJOR 2
248

249 250 251 252 253 254 255 256 257 258
/*
 * Allocates and initializes a new tiling.
 * "tiling": an address where the pointer to the newly allocated tiling
 *           structure will be stored.
 * "type": see AML_TILING_TYPE_*.
 * Variadic arguments:
 * - if "type" equals AML_TILING_TYPE_1D, two additional arguments are needed:
 *   - "tilesize": an argument of type size_t; provides the size of each tile.
 *   - "totalsize": an argument of type size_t; provides the size of the
 *                  complete user data structure to be tiled.
259 260
 * - if "type" equals AML_TILING_TYPE_2D, four additional arguments are needed:
 *   - "tilesize": an argument of type size_t; provides the size of a tile.
261 262
 *   - "totalsize": an argument of type size_t; provides the size of the
 *                  complete user data structure to be tiled.
263 264
 *   - "rowsize": an argument of type size_t; the number of tiles in a row
 *   - "colsize": an argument of type size_t; the number of tiles in a column
265 266
 * Returns 0 if successful; an error code otherwise.
 */
267
int aml_tiling_create(struct aml_tiling **tiling, int type, ...);
268 269 270 271 272 273 274 275
/*
 * Initializes a tiling.  This is a varargs-variant of the aml_tiling_vinit()
 * routine.
 * "tiling": an allocated tiling structure.
 * "type": see aml_tiling_create().
 * Variadic arguments: see aml_tiling_create().
 * Returns 0 if successful; an error code otherwise.
 */
276
int aml_tiling_init(struct aml_tiling *tiling, int type, ...);
277 278 279 280 281 282 283
/*
 * Initializes a tiling.
 * "tiling": an allocated tiling structure.
 * "type": see aml_tiling_create().
 * "args": see the variadic arguments of aml_tiling_create().
 * Returns 0 if successful; an error code otherwise.
 */
284
int aml_tiling_vinit(struct aml_tiling *tiling, int type, va_list args);
285 286 287 288 289 290
/*
 * Tears down an initialized tiling.
 * "tiling": an initialized tiling structure.
 * "type": see aml_tiling_create().
 * Returns 0 if successful; an error code otherwise.
 */
291
int aml_tiling_destroy(struct aml_tiling *tiling, int type);
292 293 294 295 296 297 298 299 300 301 302 303 304 305 306 307 308 309 310 311 312 313 314 315 316 317 318 319 320 321 322 323 324 325 326 327 328 329 330

/*******************************************************************************
 * Tiling 1D:
 ******************************************************************************/

extern struct aml_tiling_ops aml_tiling_1d_ops;
extern struct aml_tiling_iterator_ops aml_tiling_iterator_1d_ops;

struct aml_tiling_1d_data {
	size_t blocksize;
	size_t totalsize;
};

struct aml_tiling_iterator_1d_data {
	size_t i;
	struct aml_tiling_1d_data *tiling;
};

#define AML_TILING_1D_DECL(name) \
	struct aml_tiling_1d_data __ ##name## _inner_data; \
	struct aml_tiling name = { \
		&aml_tiling_1d_ops, \
		(struct aml_tiling_data *)&__ ## name ## _inner_data, \
	};

#define AML_TILING_ITERATOR_1D_DECL(name) \
	struct aml_tiling_iterator_1d_data __ ##name## _inner_data; \
	struct aml_tiling_iterator name = { \
		&aml_tiling_iterator_1d_ops, \
		(struct aml_tiling_iterator_data *)&__ ## name ## _inner_data, \
	};

#define AML_TILING_1D_ALLOCSIZE (sizeof(struct aml_tiling_1d_data) + \
				 sizeof(struct aml_tiling))

#define AML_TILING_ITERATOR_1D_ALLOCSIZE \
	(sizeof(struct aml_tiling_iterator_1d_data) + \
	 sizeof(struct aml_tiling_iterator))

331 332
/*******************************************************************************
 * Tiling 2D:
333
 * a contiguous memory area composed of contiguous tiles arranged in 2D grid.
334 335
 ******************************************************************************/

336 337
extern struct aml_tiling_ops aml_tiling_2d_rowmajor_ops;
extern struct aml_tiling_ops aml_tiling_2d_colmajor_ops;
338 339 340 341 342
extern struct aml_tiling_iterator_ops aml_tiling_iterator_2d_ops;

struct aml_tiling_2d_data {
	size_t blocksize;
	size_t totalsize;
343
	size_t ndims[2]; /* # number of rows, # number of cols (in tiles) */
344 345 346 347 348 349 350
};

struct aml_tiling_iterator_2d_data {
	size_t i;
	struct aml_tiling_2d_data *tiling;
};

351 352 353 354 355 356 357 358
#define AML_TILING_2D_ROWMAJOR_DECL(name) \
	struct aml_tiling_2d_data __ ##name## _inner_data; \
	struct aml_tiling name = { \
		&aml_tiling_2d_rowmajor_ops, \
		(struct aml_tiling_data *)&__ ## name ## _inner_data, \
	};

#define AML_TILING_2D_COLMAJOR_DECL(name) \
359 360
	struct aml_tiling_2d_data __ ##name## _inner_data; \
	struct aml_tiling name = { \
361
		&aml_tiling_2d_colmajor_ops, \
362 363 364 365 366 367 368 369 370 371 372 373 374 375 376 377 378
		(struct aml_tiling_data *)&__ ## name ## _inner_data, \
	};

#define AML_TILING_ITERATOR_2D_DECL(name) \
	struct aml_tiling_iterator_2d_data __ ##name## _inner_data; \
	struct aml_tiling_iterator name = { \
		&aml_tiling_iterator_2d_ops, \
		(struct aml_tiling_iterator_data *)&__ ## name ## _inner_data, \
	};

#define AML_TILING_2D_ALLOCSIZE (sizeof(struct aml_tiling_2d_data) + \
				 sizeof(struct aml_tiling))

#define AML_TILING_ITERATOR_2D_ALLOCSIZE \
	(sizeof(struct aml_tiling_iterator_2d_data) + \
	 sizeof(struct aml_tiling_iterator))

379 380 381 382 383
/*******************************************************************************
 * DMA:
 * Management of low-level movement of memory.
 ******************************************************************************/

384 385
/* Internal macros used for tracking DMA request types.  */
/* Invalid request type.  Used for marking inactive requests in the vector.  */
386
#define AML_DMA_REQUEST_TYPE_INVALID -1
387
/* Copy request type.  Uses memcpy() for data migration.  */
388 389
#define AML_DMA_REQUEST_TYPE_COPY 0

390
struct aml_dma_request;
391 392 393
struct aml_dma_data;

struct aml_dma_ops {
394
	int (*create_request)(struct aml_dma_data *dma,
395
			      struct aml_dma_request **req, int type,
396 397 398 399 400
			      va_list args);
	int (*destroy_request)(struct aml_dma_data *dma,
			       struct aml_dma_request *req);
	int (*wait_request)(struct aml_dma_data *dma,
			    struct aml_dma_request *req);
401 402 403 404 405 406 407
};

struct aml_dma {
	struct aml_dma_ops *ops;
	struct aml_dma_data *data;
};

408 409 410 411 412 413 414 415 416 417 418 419 420 421 422 423
/*
 * Requests a synchronous data copy between two different tiles, using
 * memcpy() or equivalent.
 * "dma": an initialized DMA structure.
 * Variadic arguments:
 * - "dt": an argument of type struct aml_tiling*; the destination tiling
 *         structure.
 * - "dptr": an argument of type void*; the start address of the complete
 *           destination user data structure.
 * - "dtid": an argument of type int; the destination tile identifier.
 * - "st": an argument of type struct aml_tiling*; the source tiling structure.
 * - "sptr": an argument of type void*; the start address of the complete
 *           source user data structure.
 * - "stid": an argument of type int; the source tile identifier.
 * Returns 0 if successful; an error code otherwise.
 */
424
int aml_dma_copy(struct aml_dma *dma, ...);
425 426 427 428 429 430 431 432 433
/*
 * Requests a data copy between two different tiles.  This is an asynchronous
 * version of aml_dma_copy().
 * "dma": an initialized DMA structure.
 * "req": an address where the pointer to the newly assigned DMA request will be
 *        stored.
 * Variadic arguments: see aml_dma_copy().
 * Returns 0 if successful; an error code otherwise.
 */
434
int aml_dma_async_copy(struct aml_dma *dma, struct aml_dma_request **req, ...);
435 436 437 438 439 440
/*
 * Waits for an asynchronous DMA request to complete.
 * "dma": an initialized DMA structure.
 * "req": a DMA request obtained using aml_dma_async_*() calls.
 * Returns 0 if successful; an error code otherwise.
 */
441
int aml_dma_wait(struct aml_dma *dma, struct aml_dma_request *req);
442 443 444 445 446 447
/*
 * Tears down an asynchronous DMA request before it completes.
 * "dma": an initialized DMA structure.
 * "req": a DMA request obtained using aml_dma_async_*() calls.
 * Returns 0 if successful; an error code otherwise.
 */
448
int aml_dma_cancel(struct aml_dma *dma, struct aml_dma_request *req);
449

450 451
/*******************************************************************************
 * Scratchpad:
452
 * Use an area to stage data from another area in and out.
453 454 455 456 457 458
 * A dma handles the movement itself.
 ******************************************************************************/

struct aml_scratch_request;
struct aml_scratch_data;

459 460
/* Internal macros used for tracking scratchpad request types.  */
/* Invalid request type.  Used for marking inactive requests in the vector.  */
461
#define AML_SCRATCH_REQUEST_TYPE_INVALID -1
462
/* Push from the scratchpad to regular memory.  */
463
#define AML_SCRATCH_REQUEST_TYPE_PUSH 0
464
/* Pull from regular memory to the scratchpad.  */
465
#define AML_SCRATCH_REQUEST_TYPE_PULL 1
466 467
/* No-op/empty request */
#define AML_SCRATCH_REQUEST_TYPE_NOOP 2
468 469 470 471 472 473 474 475 476

struct aml_scratch_ops {
	int (*create_request)(struct aml_scratch_data *scratch,
			      struct aml_scratch_request **req, int type,
			      va_list args);
	int (*destroy_request)(struct aml_scratch_data *scratch,
			       struct aml_scratch_request *req);
	int (*wait_request)(struct aml_scratch_data *scratch,
			    struct aml_scratch_request *req);
477
	void *(*baseptr)(const struct aml_scratch_data *scratch);
478
	int (*release)(struct aml_scratch_data *scratch, int scratchid);
479 480 481 482 483 484 485
};

struct aml_scratch {
	struct aml_scratch_ops *ops;
	struct aml_scratch_data *data;
};

486 487 488 489 490 491 492 493 494 495 496 497 498
/*
 * Requests a synchronous pull from regular memory to the scratchpad.
 * "scratch": an initialized scratchpad structure.
 * Variadic arguments:
 * - "scratchptr": an argument of type void*; the scratchpad base pointer (see
 *                 aml_scratch_baseptr()).
 * - "scratchid": an argument of type int*; gets filled with the scratch tile
 *                identifier where the data will be pulled into.
 * - "srcptr": an argument of type void*; the start address of the complete
 *             source user data structure.
 * - "srcid": an argument of type int; the source tile identifier.
 * Returns 0 if successful; an error code otherwise.
 */
499
int aml_scratch_pull(struct aml_scratch *scratch, ...);
500 501 502 503 504 505 506 507 508
/*
 * Requests a pull from regular memory to the scratchpad.  This is an
 * asynchronous version of aml_scratch_pull().
 * "scratch": an initialized scratchpad structure.
 * "req": an address where the pointer to the newly assigned scratch request
 *        will be stored.
 * Variadic arguments: see aml_scratch_pull().
 * Returns 0 if successful; an error code otherwise.
 */
509 510
int aml_scratch_async_pull(struct aml_scratch *scratch,
			   struct aml_scratch_request **req, ...);
511 512 513 514
/*
 * Requests a synchronous push from the scratchpad to regular memory.
 * "scratch": an initialized scratchpad structure.
 * Variadic arguments:
515 516 517
 * - "dstptr": an argument of type void*; the start address of the complete
 *             destination user data structure.
 * - "dstid": an argument of type int*; gets filled with the destination tile
518 519 520 521 522 523 524
 *            identifier where the data will be pushed into (and where it was
 *            pulled from in the first place).
 * - "scratchptr": an argument of type void*; the scratchpad base pointer (see
 *                 aml_scratch_baseptr()).
 * - "scratchid": an argument of type int; the scratchpad tile identifier.
 * Returns 0 if successful; an error code otherwise.
 */
525
int aml_scratch_push(struct aml_scratch *scratch, ...);
526 527 528 529 530 531 532 533 534
/*
 * Requests a push from the scratchpad to regular memory.  This is an
 * asynchronous version of aml_scratch_push().
 * "scratch": an initialized scratchpad structure.
 * "req": an address where the pointer to the newly assigned scratch request
 *        will be stored.
 * Variadic arguments: see aml_scratch_push().
 * Returns 0 if successful; an error code otherwise.
 */
535 536
int aml_scratch_async_push(struct aml_scratch *scratch,
			   struct aml_scratch_request **req, ...);
537 538 539 540 541 542
/*
 * Waits for an asynchronous scratch request to complete.
 * "scratch": an initialized scratchpad structure.
 * "req": a scratch request obtained using aml_scratch_async_*() calls.
 * Returns 0 if successful; an error code otherwise.
 */
543 544
int aml_scratch_wait(struct aml_scratch *scratch,
		     struct aml_scratch_request *req);
545

546 547
/*
 * Tears down an asynchronous scratch request before it completes.
548
 * "scratch": an initialized scratchpad structure.
549 550 551
 * "req": a scratch request obtained using aml_scratch_async_*() calls.
 * Returns 0 if successful; an error code otherwise.
 */
552 553
int aml_scratch_cancel(struct aml_scratch *scratch,
		       struct aml_scratch_request *req);
554 555
/*
 * Provides the location of the scratchpad.
556
 * "scratch": an initialized scratchpad structure.
557 558
 * Returns a base pointer to the scratchpad memory buffer.
 */
559
void* aml_scratch_baseptr(const struct aml_scratch *scratch);
560

561 562 563
/*
 * Release a scratch tile for immediate reuse.
 * "scratch": an initialized scratchpad structure.
564 565
 * "scratchid": a scratchpad tile identifier.
 * Returns 0 if successful; an error code otherwise.
566 567 568
 */
int aml_scratch_release(struct aml_scratch *scratch, int scratchid);

569
#endif