Commit 1fad4d65 authored by Nicolas Denoyelle's avatar Nicolas Denoyelle Committed by Swann Perarnau

[doc] Add doxygen documentation to the project

parent 10a2b644
......@@ -3,6 +3,7 @@
# unignore files with extensions
!*.*
# unignore special files
!AUTHORS
!LICENSE
......@@ -50,7 +51,7 @@ stamp-h1
/install-sh
/missing
/stamp-h1
version.h
/version.h
/m4/libtool.m4
/m4/ltmain.sh
/m4/ltoptions.m4
......
......@@ -4,6 +4,7 @@ variables:
stages:
- style
- build
- release
make:generic:
stage: build
......@@ -53,3 +54,12 @@ checkpatch:
stage: style
script:
- nix run -f "$ARGOPKGS" checkpatch --command checkpatch.pl
readthedocs:
stage: release
when: on_success
only:
- master
- /v\.[0-9]+\.[0-9]+\.[0-9]+/
script:
- nix run nixpkgs.curl -c curl -X POST -d "branch=$CI_COMMIT_REF_NAME" -d "token=$READTHEDOCS_TOKEN" https://readthedocs.org/api/v2/webhook/argo-aml/83161/
This diff is collapsed.
......@@ -5,7 +5,12 @@ if ADD_BENCHMARKS
SUBDIRS += benchmarks
endif
if AML_DOXYGEN
SUBDIRS += doc
endif
pkgconfigdir = $(libdir)/pkgconfig
pkgconfig_DATA = aml.pc
EXTRA_DIST = autogen.sh aml.pc README.markdown
......@@ -7,6 +7,30 @@ memory management policies for allocation and placement of data across devices.
This library is still in the prototyping phase. APIs might break often.
## Documentation
The latest documentation can be found
[online](https://argo-aml.readthedocs.io/en/latest/),
can be installed when installing a release or generated
from git clone before installing with
```
make -C doc doxygen
```
## Requirements:
* autoconf
* automake
* libnuma
## Installation
```
sh autogen.sh
./configure
make -j install
```
# General Architecture
The architecture of the library relies on two principles: type-specific
......
......@@ -69,6 +69,12 @@ AC_CHECK_HEADERS([numa.h],,[AC_MSG_ERROR([AML requires libnuma headers.])])
AC_CHECK_HEADERS([numaif.h],,[AC_MSG_ERROR([AML requires libnuma headers.])])
AC_CHECK_LIB(numa, mbind,,[AC_MSG_ERROR([AML requires libnuma.])])
# Doxygen
# See m4/ax_doxygen.m4
AC_CHECK_PROG(doxygen_happy, [doxygen], [yes], [no])
AM_CONDITIONAL([AML_DOXYGEN],[ test "x$doxygen_happy" = "xyes" ])
AC_CONFIG_HEADERS([include/config.h])
AC_CONFIG_FILES([Makefile
......@@ -80,6 +86,7 @@ AC_CONFIG_FILES([Makefile
src/utils/Makefile
include/Makefile
tests/Makefile
doc/Makefile
benchmarks/Makefile
aml.pc
include/aml/utils/version.h])
......
DATA_INSTALL_DIR=$(datadir)/aml
DOXYGEN_BUILD_DIR=./build-doxygen
SPHINX_BUILD_DIR=./build-sphinx
doxygen:
doxygen aml.doxy
create-doc-dir:
mkdir -p $(DATA_INSTALL_DIR)
install-doc-html:
if [ -d $(DOXYGEN_BUILD_DIR)/html ]; then \
cp -r $(DOXYGEN_BUILD_DIR)/html $(DATA_INSTALL_DIR); \
fi
install-data-local: create-doc-dir install-doc-html
uninstall-local:
rm -rf $(DATA_INSTALL_DIR)
clean-local:
rm -rf $(DOXYGEN_BUILD_DIR) $(SPHINX_BUILD_DIR)
dist-hook: doxygen
cp -r $(DOXYGEN_BUILD_DIR) $(distdir)
This source diff could not be displayed because it is too large. You can view the blob instead.
################################################################################
# Copyright 2019 UChicago Argonne, LLC.
# (c.f. AUTHORS, LICENSE)
#
# This file is part of the AML project.
# For more info, see https://xgitlab.cels.anl.gov/argo/aml
#
# SPDX-License-Identifier: BSD-3-Clause
################################################################################
# Configuration file for the Sphinx documentation builder.
#
# This file only contains a selection of the most common options. For a full
# list see the documentation:
# http://www.sphinx-doc.org/en/master/config
# -- Path setup --------------------------------------------------------------
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
import os
import sys
import subprocess
# Generate documentation
subprocess.call('doxygen aml.doxy', shell=True)
# -- General configuration ---------------------------------------------------
# The language for content autogenerated by Sphinx.
language = 'C'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = []
# Add any paths that contain templates here, relative to this directory.
templates_path = []
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This pattern also affects html_static_path and html_extra_path.
exclude_patterns = []
# -- Project information -----------------------------------------------------
project = 'AML'
copyright = 'Copyright 2019 UChicago Argonne, LLC'
author = 'Swann Perarnau, Kamil Iskra, Brian Suchy, Valentin Reis, Nicolas Denoyelle'
release = 'X.X.X'
# -- Options for HTML output -------------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = []
#Overwrite sphinx documentation with doxygen documentation.
html_extra_path = ['build-doxygen/html']
This is an empty file required by shpinx when
compiling documentation on readthedocs.org.
This is an empty landing page overwritten with doxygen documentation.
Welcome to AML Documentation Page {#index}
============
AML is a memory management library designed for highly optimized
codes to delegate complex memory operations.
## Overview
AML goals are the following:
1. to provide a convenient interface for addressing all heterogeneous
addressable memories and moving data around.
2. to provide an interface for expressing and translating data
structures between several representations and physical memories.
Toward achieving these goals, AML implements the following abstractions:
\image html building-blocks-diagram.png width=400cm
* [Areas](@ref aml_area), a set of addressable physical memories,
* [Tilings](@ref aml_tiling), a description of tiling data structures,
* [DMA](@ref aml_dma), an engine to asynchronously move data structures between areas,
* [Scratchpad](@ref aml_scratch), a stage-in, stage-out abstraction for prefetching.
Each of these abstractions have several implementations. For instance,
areas may refer to usual DDR of a subset of them, GPU memory or non-volatile memory.
Tilings are implemented to reflect either 1D or 2D structures etc.
## Problems Solved with AML
AML library is designed for highly optimized codes to delegate complex memory
operations. AML is currently used in a
[proof of concept](https://xgitlab.cels.anl.gov/argo/nauts/tree/master/papers/mchpc18)
of efficient matrix multiplication on architectures with software managed memory side
cache. It is also currently being extended toward specialized memory allocations
for latency sensitive applications.
## Quick Start Guide
### Download
* [Github version](https://xgitlab.cels.anl.gov/argo/aml):
```
git clone git@xgitlab.cels.anl.gov:argo/aml.git
```
* Release versions:
TODO after release.
### Requirements:
* autoconf
* automake
* libnuma
### Installation
```
sh autogen.sh
./configure
make -j install
```
### Workflow
* Include aml header:
```
#include <aml.h>
...
int main(int argc, char **argv){
```
* Check AML version:
```
if(aml_version_major != AML_VERSION_MAJOR){
printf("AML ABI mismatch!");
return 1;
}
```
* Initialize and Cleanup AML:
```
if(aml_init(&argc, &argv) != 0){
printf("AML library init failure!");
return 1;
}
...
aml_finalize();
```
* Link your program with `-laml`
See above building blocks specific pages for further examples and information on
library features.
## Support
AML interface with users is mainly done through
[gitlab issue interface](https://xgitlab.cels.anl.gov/argo/aml/issues) and
direct contact with developers/maintainers:
* Swann Perarnau (swann@anl.gov)
* Nicolas Denoyelle (ndenoyelle@anl.gov)
AML is looking forward to new users. If you think your problem can be solved with AML
eventually with some additional non-existing features which are relevant to the library,
please let us know. Tell us about your topic of interests and your scientific/technical
problems. We are also pleased to improve the software, thus any suggestion, bug
declaration, contribution is welcomed.
## Contributing
AML is looking forward to new collaborations and contributions.
If you wish to contribute to AML project rendez-vous [here](@ref contributing_page)
This diff is collapsed.
......@@ -15,81 +15,86 @@
#include <numa.h>
#include <numaif.h>
/* allowed binding flags */
/**
* @defgroup aml_area_linux "AML Linux Areas"
* @brief Linux Implementation of Areas.
*
* Linux implementation of AML areas.
* This building block relies on libnuma implementation and
* linux mmap/munmap to provide mmap/munmap on NUMA host
* host processor memory. New areas may be created
* to allocate a specific subset of memories.
* This building block also include a static declaration of
* a default initialized area that can be used out of the box with
* abstract area API.
*
* #include <aml/area/linux.h>
* @{
**/
/**
* Allowed binding flag for area creation.
* This flag will apply strict binding to the selected bitmask.
* If subsequent allocation will failt if they cannot enforce binding
* on bitmask.
**/
#define AML_AREA_LINUX_BINDING_FLAG_BIND (MPOL_BIND)
/**
* Allowed binding flag for area creation.
* This flag will make subsequent allocations to interleave
* pages on memories of the bitmask.
**/
#define AML_AREA_LINUX_BINDING_FLAG_INTERLEAVE (MPOL_INTERLEAVE)
/**
* Allowed binding flag for area creation.
* This flag will make subsequent allocations to bound to the
* nodes of bitmask if possible, else to some other node.
**/
#define AML_AREA_LINUX_BINDING_FLAG_PREFERRED (MPOL_PREFERRED)
/* allowed mmap flags to pass */
/**
* Allowed mapping flag for area creation.
* This flag will make subsequent allocations to be private
* to the process making them.
**/
#define AML_AREA_LINUX_MMAP_FLAG_PRIVATE (MAP_PRIVATE | MAP_ANONYMOUS)
#define AML_AREA_LINUX_MMAP_FLAG_SHARED (MAP_SHARED | MAP_ANONYMOUS)
extern struct aml_area_ops aml_area_linux_ops;
/* User data stored inside area */
struct aml_area_linux_data {
/** numanodes to use **/
struct bitmask *nodeset;
/** numaif.h mbind policy or AML_AREA_LINUX_FLAG_* **/
int binding_flags;
/** mmap flags **/
int mmap_flags;
};
/* Default linux area with private mapping and no binding. */
extern struct aml_area aml_area_linux;
/*******************************************************************************
* Linux operators
*******************************************************************************/
/**
* Bind memory of size "size" pointed by "ptr" to binding set in "bind".
* If mbind call was not successfull, i.e AML_FAILURE is returned, then errno
* should be inspected for further error checking.
* Allowed mapping flag for area creation.
* This flag will make subsequent allocations to be visible to
* other processes of the system.
**/
int
aml_area_linux_mbind(struct aml_area_linux_data *bind,
void *ptr,
size_t size);
#define AML_AREA_LINUX_MMAP_FLAG_SHARED (MAP_SHARED | MAP_ANONYMOUS)
/**
* Function to check whether binding of a ptr obtained with
* aml_area_linux_mmap() then aml_area_linux_mbind() match area settings.
* Returns 1 if mapped memory binding in ptr match area_data binding settings,
* else 0.
* This contains area operations implementation
* for linux area.
**/
int
aml_area_linux_check_binding(struct aml_area_linux_data *area_data,
void *ptr,
size_t size);
extern struct aml_area_ops aml_area_linux_ops;
/**
* mmap hook for aml area.
* Fails with AML_FAILURE. On failure errno should be checked for further
* error investigations.
* Default linux area with private mapping and no binding.
* Can be used out of the box with aml_area_*() functions.
**/
void*
aml_area_linux_mmap(const struct aml_area_data *area_data,
void *ptr,
size_t size);
extern struct aml_area aml_area_linux;
/**
* munmap hook for aml area, to unmap memory mapped with aml_area_linux_mmap().
* Fails with AML_FAILURE. On failure errno should be checked for further
* error investigations.
* Implementation of aml_area_data for linux areas.
**/
int
aml_area_linux_munmap(const struct aml_area_data *area_data,
void *ptr,
const size_t size);
/*******************************************************************************
* create/destroy and others
*******************************************************************************/
struct aml_area_linux_data {
/** numanodes to use when allocating data **/
struct bitmask *nodeset;
/** binding policy **/
int binding_flags;
/** mmap flags **/
int mmap_flags;
};
/**
* Static declaration of an aml area with linux ops.
**/
#define AML_AREA_LINUX_DECL(name) \
struct aml_area_linux_data __ ##name## _inner_data; \
struct aml_area name = { \
......@@ -97,12 +102,16 @@ aml_area_linux_munmap(const struct aml_area_data *area_data,
(struct aml_area_data *)&__ ## name ## _inner_data, \
}
/**
* Static declaration of the size of a linux aml area.
**/
#define AML_AREA_LINUX_ALLOCSIZE \
(sizeof(struct aml_area_linux_data) + \
sizeof(struct aml_area))
/**
* \brief Linux area creation.
*
* Allocate and initialize a struct aml_area implemented by aml_area_linux
* operations.
* @param[out] area pointer to an uninitialized struct aml_area pointer to
......@@ -121,24 +130,94 @@ int aml_area_linux_create(struct aml_area **area, const int mmap_flags,
const struct aml_bitmap *nodemask,
const int binding_flags);
/**
* \brief Linux area destruction.
*
* Destroy (finalize and free resources) a struct aml_area created by
* aml_area_linux_create().
*
* @param area is NULL after this call.
**/
void aml_area_linux_destroy(struct aml_area **area);
/**
* Initialize a struct aml_area declared using the AML_AREA_LINUX_DECL macro.
* See aml_area_linux_create for details on arguments.
* @see aml_area_linux_create() for details on arguments.
*/
int aml_area_linux_init(struct aml_area *area, const int mmap_flags,
const struct aml_bitmap *nodemask,
const int binding_flags);
/**
* Finalize a struct aml_area initialized with aml_area_linux_init.
*/
void aml_area_linux_fini(struct aml_area *area);
/**
* Destroy (finalize and free resources) a struct aml_area created by
* aml_area_linux_create.
* Bind memory of size "size" pointed by "ptr" to binding set in "bind".
* If mbind call was not successfull, i.e AML_FAILURE is returned, then errno
* should be inspected for further error checking.
* @param bind: The binding settings. mmap_flags is actually unused.
* @param ptr: The data to bind.
* @param size: The size of the data pointed by ptr.
* @return an AML error code.
**/
int
aml_area_linux_mbind(struct aml_area_linux_data *bind,
void *ptr,
size_t size);
/**
* Function to check whether binding of a ptr obtained with
* aml_area_linux_mmap() then aml_area_linux_mbind() match area settings.
* @param area_data: The expected binding settings.
* @param ptr: The data supposely bound.
* @param size: The data size.
* @return 1 if mapped memory binding in ptr match area_data binding settings,
* else 0.
**/
int
aml_area_linux_check_binding(struct aml_area_linux_data *area_data,
void *ptr,
size_t size);
/**
* \brief mmap block for aml area.
*
* @param area is NULL after this call.
* This function is a wrapper on mmap function using arguments set in
* mmap_flags of area_data.
* This function does not perform binding, unlike it is done in areas created
* with aml_area_linux_create().
* @param area_data: The structure containing mmap_flags for mmap call.
* nodemask and bind_flags fields are ignored.
* @param ptr: A hint provided to mmap function.
* @param size: The size to allocate.
* @return NULL on failure, else a valid pointer to memory.
* Upon failure, errno should be checked for further error investigations.
**/
void*
aml_area_linux_mmap(const struct aml_area_data *area_data,
void *ptr,
size_t size);
/**
* \brief munmap hook for aml area.
*
* unmap memory mapped with aml_area_linux_mmap().
* @param area_data: unused
* @param ptr: The virtual memory to unmap.
* @param size: The size of virtual memory to unmap.
* @return AML_FAILURE on error, AML_SUCCESS.
* Upon failure errno should be checked for further error investigations.
**/
int
aml_area_linux_munmap(const struct aml_area_data *area_data,
void *ptr,
const size_t size);
/**
* @}
**/
void aml_area_linux_destroy(struct aml_area **area);
#endif //AML_AREA_LINUX_NUMA_H
......@@ -11,46 +11,79 @@
#ifndef AML_DMA_LINUX_PAR_H
#define AML_DMA_LINUX_PAR_H 1
/*******************************************************************************
* Linux Parallel DMA API:
* DMA logic implemented based on general linux API, with the caller thread
* used as the only execution thread.
******************************************************************************/
/**
* @defgroup aml_dma_par "AML Parallel DMA"
* @brief Parallel DMA implementation.
*
* DMA logic implemented based on general linux API, asynchronous execution
* threads. This DMA implementation moves between pointers allocated with an
* aml_area_linux.
* @{
**/
/**
* Default table of dma request operations for linux
* parallel dma.
**/
extern struct aml_dma_ops aml_dma_linux_par_ops;
/** Thread data embeded inside an asynchronous dma request. **/
struct aml_dma_linux_par_thread_data {
/**
* A logical identifier of the thread in charge for
* the request progress.
**/
int tid;
/** The actual thread in charge for the request progress**/
pthread_t thread;
/** The dma containing sequential operations **/
struct aml_dma_linux_par *dma;
/** The request handled by this thread **/
struct aml_dma_request_linux_par *req;
};
/** Inside of a parallel request for linux movement. **/
struct aml_dma_request_linux_par {
/**
* The type of dma request
* @see <aml.h>
**/
int type;
/** The destination pointer of the data movement **/
void *dest;
/** The source pointer of the data movement **/
void *src;
/** The size of data to move **/
size_t size;
/** The thread data in charge of the request progress **/
struct aml_dma_linux_par_thread_data *thread_data;
};
/** Inside of a parallel request for linux movement. **/
struct aml_dma_linux_par_data {
size_t nbthreads;
struct aml_vector requests;
pthread_mutex_t lock;
};
/** Declaration of linux parallel dma operations **/
struct aml_dma_linux_par_ops {
void *(*do_thread)(void *thread_data);
int (*do_copy)(struct aml_dma_linux_par_data *data,
struct aml_dma_request_linux_par *request, int tid);
};
/**
* aml_dma structure for linux based, parallel dma movement
* Needs to be initialized with aml_dma_linux_par_init().
* Can be passed to generic aml_dma_*() functions.
**/
struct aml_dma_linux_par {
struct aml_dma_linux_par_ops ops;
struct aml_dma_linux_par_data data;
};
/** Static declaration of aml_dma_linux_par structure. **/
#define AML_DMA_LINUX_PAR_DECL(name) \
struct aml_dma_linux_par __ ##name## _inner_data; \
struct aml_dma name = { \
......@@ -58,6 +91,7 @@ struct aml_dma_linux_par {
(struct aml_dma_data *)&__ ## name ## _inner_data, \
}
/** Static declaration of aml_dma_linux_par structure size. **/
#define AML_DMA_LINUX_PAR_ALLOCSIZE \
(sizeof(struct aml_dma_linux_par) + \
sizeof(struct aml_dma))
......@@ -97,7 +131,10 @@ void aml_dma_linux_par_fini(struct aml_dma *dma);
/**
* Tears down a parallel DMA created with aml_dma_linux_par_create.
* @param dma the address of a pointer to a parallel dma. Will be NULL after.
*/
**/
void aml_dma_linux_par_destroy(struct aml_dma **dma);
/**
* @}
**/
#endif // AML_LINUX_DMA_LINUX_PAR_H
......@@ -11,36 +11,72 @@
#ifndef AML_DMA_LINUX_SEQ_H
#define AML_DMA_LINUX_SEQ_H 1
/*******************************************************************************
* Linux Sequential DMA API:
/**
* @defgroup aml_dma_seq "AML Sequential DMA"
* @brief Sequential DMA implementation.
*
* DMA logic implemented based on general linux API, with the caller thread
* used as the only execution thread.
******************************************************************************/
* used as the only execution thread. This DMA implementation moves between
* pointers allocated with an aml_area_linux.
* @{
**/
/**
* Default table of dma request operations for linux
* sequential dma.
**/
extern struct aml_dma_ops aml_dma_linux_seq_ops;
/** Inside of a sequential request for linux movement. **/
struct aml_dma_request_linux_seq {
/**
* The type of dma request
* @see <aml.h>
**/
int type;
/** The destination pointer of the data movement **/
void *dest;
/** The source pointer of the data movement **/
void *src;
/** The size of data to move **/
size_t size;
};
/** Inner data of sequential linux aml_dma implementation **/
struct aml_dma_linux_seq_data {
/**
* Queue of submitted requests.
* Requests may be submitted concurrently but will all
* be performed by a single thread.
**/
struct aml_vector requests;
/** Lock for queuing requests concurrently **/
pthread_mutex_t lock;
};
/** Declaration of available linux sequential dma operations **/
struct aml_dma_linux_seq_ops {
/**
* Perform a sequential copy between source and destination
* pointers allocated with an aml_area_linux.
* @see aml_area
**/
int (*do_copy)(struct aml_dma_linux_seq_data *dma,
struct aml_dma_request_linux_seq *req);
};
/**
* aml_dma structure for linux based, sequential dma movement.
* Needs to be initialized with aml_dma_linux_seq_init().
* Can be passed to generic aml_dma_*() functions.
**/
struct aml_dma_linux_seq {
struct aml_dma_linux_seq_ops ops;