[doc] Add doxygen documentation to the project

.gitignore

@@ -3,6 +3,7 @@
 
 # unignore files with extensions
 !*.*
+
 # unignore special files
 !AUTHORS
 !LICENSE
@@ -50,7 +51,7 @@ stamp-h1
 /install-sh
 /missing
 /stamp-h1
-version.h
+/version.h
 /m4/libtool.m4
 /m4/ltmain.sh
 /m4/ltoptions.m4

.gitlab-ci.yml

@@ -4,6 +4,7 @@ variables:
 stages:
   - style
   - build
+  - release
 
 make:generic:
   stage: build
@@ -53,3 +54,12 @@ checkpatch:
   stage: style
   script:
     - nix run -f "$ARGOPKGS" checkpatch --command checkpatch.pl
+
+readthedocs:
+  stage: release
+  when: on_success
+  only:
+    - master
+    - /v\.[0-9]+\.[0-9]+\.[0-9]+/
+  script:
+    - nix run nixpkgs.curl -c curl -X POST -d "branch=$CI_COMMIT_REF_NAME" -d "token=$READTHEDOCS_TOKEN" https://readthedocs.org/api/v2/webhook/argo-aml/83161/

Makefile.am

@@ -5,7 +5,12 @@ if ADD_BENCHMARKS
 SUBDIRS += benchmarks
 endif
 
+if AML_DOXYGEN
+SUBDIRS += doc
+endif
+
 pkgconfigdir = $(libdir)/pkgconfig
 pkgconfig_DATA = aml.pc
 
 EXTRA_DIST = autogen.sh aml.pc README.markdown
+

README.markdown

@@ -7,6 +7,30 @@ memory management policies for allocation and placement of data across devices.
 
 This library is still in the prototyping phase. APIs might break often.
 
+## Documentation
+
+The latest documentation can be found
+[online](https://argo-aml.readthedocs.io/en/latest/),
+can be installed when installing a release or generated
+from git clone before installing with
+```
+make -C doc doxygen
+```
+
+## Requirements:
+
+* autoconf
+* automake
+* libnuma
+
+## Installation
+
+```
+sh autogen.sh
+./configure
+make -j install
+```
+
 # General Architecture
 
 The architecture of the library relies on two principles: type-specific

configure.ac

@@ -69,6 +69,12 @@ AC_CHECK_HEADERS([numa.h],,[AC_MSG_ERROR([AML requires libnuma headers.])])
 AC_CHECK_HEADERS([numaif.h],,[AC_MSG_ERROR([AML requires libnuma headers.])])
 AC_CHECK_LIB(numa, mbind,,[AC_MSG_ERROR([AML requires libnuma.])])
 
+# Doxygen
+# See m4/ax_doxygen.m4
+AC_CHECK_PROG(doxygen_happy, [doxygen], [yes], [no])
+
+AM_CONDITIONAL([AML_DOXYGEN],[ test "x$doxygen_happy" = "xyes" ])
+
 AC_CONFIG_HEADERS([include/config.h])
 
 AC_CONFIG_FILES([Makefile
@@ -80,6 +86,7 @@ AC_CONFIG_FILES([Makefile
 		 src/utils/Makefile
 		 include/Makefile
 		 tests/Makefile
+		 doc/Makefile
 		 benchmarks/Makefile
 		 aml.pc
 		 include/aml/utils/version.h])

doc/Makefile.am

+DATA_INSTALL_DIR=$(datadir)/aml
+DOXYGEN_BUILD_DIR=./build-doxygen
+SPHINX_BUILD_DIR=./build-sphinx
+
+doxygen:
+	doxygen aml.doxy
+
+create-doc-dir:
+	mkdir -p $(DATA_INSTALL_DIR)
+
+install-doc-html:
+	if [ -d $(DOXYGEN_BUILD_DIR)/html ]; then \
+	cp -r $(DOXYGEN_BUILD_DIR)/html $(DATA_INSTALL_DIR); \
+	fi
+
+install-data-local: create-doc-dir install-doc-html
+
+uninstall-local:
+	rm -rf $(DATA_INSTALL_DIR)
+
+clean-local:
+	rm -rf $(DOXYGEN_BUILD_DIR) $(SPHINX_BUILD_DIR)
+
+dist-hook: doxygen
+	cp -r $(DOXYGEN_BUILD_DIR) $(distdir)
+
+

doc/conf.py

+################################################################################
+# Copyright 2019 UChicago Argonne, LLC.
+# (c.f. AUTHORS, LICENSE)
+# 
+# This file is part of the AML project.
+# For more info, see https://xgitlab.cels.anl.gov/argo/aml
+# 
+# SPDX-License-Identifier: BSD-3-Clause
+################################################################################
+
+# Configuration file for the Sphinx documentation builder.
+#
+# This file only contains a selection of the most common options. For a full
+# list see the documentation:
+# http://www.sphinx-doc.org/en/master/config
+
+# -- Path setup --------------------------------------------------------------
+
+# If extensions (or modules to document with autodoc) are in another directory,
+# add these directories to sys.path here. If the directory is relative to the
+# documentation root, use os.path.abspath to make it absolute, like shown here.
+#
+import os
+import sys
+import subprocess
+
+# Generate documentation
+subprocess.call('doxygen aml.doxy', shell=True)
+ 
+# -- General configuration ---------------------------------------------------
+
+# The language for content autogenerated by Sphinx.
+language = 'C'
+
+# Add any Sphinx extension module names here, as strings. They can be
+# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
+# ones.
+extensions = []
+
+# Add any paths that contain templates here, relative to this directory.
+templates_path = []
+
+# List of patterns, relative to source directory, that match files and
+# directories to ignore when looking for source files.
+# This pattern also affects html_static_path and html_extra_path.
+exclude_patterns = []
+
+# -- Project information -----------------------------------------------------
+
+project   = 'AML'
+copyright = 'Copyright 2019 UChicago Argonne, LLC'
+author    = 'Swann Perarnau, Kamil Iskra, Brian Suchy, Valentin Reis, Nicolas Denoyelle'
+release   = 'X.X.X'
+
+# -- Options for HTML output -------------------------------------------------
+
+# The theme to use for HTML and HTML Help pages.  See the documentation for
+# a list of builtin themes.
+#
+html_theme = 'alabaster'
+
+# Add any paths that contain custom static files (such as style sheets) here,
+# relative to this directory. They are copied after the builtin static files,
+# so a file named "default.css" will overwrite the builtin "default.css".
+html_static_path = []
+
+#Overwrite sphinx documentation with doxygen documentation.
+html_extra_path = ['build-doxygen/html']
+

doc/contents.rst

+This is an empty file required by shpinx when
+compiling documentation on readthedocs.org.

doc/index.rst

+This is an empty landing page overwritten with doxygen documentation.
+

doc/md/main.md

+Welcome to AML Documentation Page {#index}
+============
+
+AML is a memory management library designed for highly optimized
+codes to delegate complex memory operations.
+
+## Overview
+
+AML goals are the following:
+1. to provide a convenient interface for addressing all heterogeneous
+addressable memories and moving data around.
+2. to provide an interface for expressing and translating data
+structures between several representations and physical memories.
+
+Toward achieving these goals, AML implements the following abstractions:
+
+\image html building-blocks-diagram.png width=400cm
+
+* [Areas](@ref aml_area), a set of addressable physical memories,
+* [Tilings](@ref aml_tiling), a description of tiling data structures,
+* [DMA](@ref aml_dma), an engine to asynchronously move data structures between areas,
+* [Scratchpad](@ref aml_scratch), a stage-in, stage-out abstraction for prefetching.
+
+Each of these abstractions have several implementations. For instance,
+areas may refer to usual DDR of a subset of them, GPU memory or non-volatile memory.
+Tilings are implemented to reflect either 1D or 2D structures etc.
+
+## Problems Solved with AML
+
+AML library is designed for highly optimized codes to delegate complex memory
+operations. AML is currently used in a
+[proof of concept](https://xgitlab.cels.anl.gov/argo/nauts/tree/master/papers/mchpc18)
+of efficient matrix multiplication on architectures with software managed memory side
+cache. It is also currently being extended toward specialized memory allocations
+for latency sensitive applications.
+
+## Quick Start Guide
+
+### Download
+
+* [Github version](https://xgitlab.cels.anl.gov/argo/aml):
+```
+git clone git@xgitlab.cels.anl.gov:argo/aml.git
+```
+* Release versions: 
+TODO after release.
+
+### Requirements:
+
+* autoconf
+* automake
+* libnuma
+
+### Installation
+
+```
+sh autogen.sh
+./configure
+make -j install
+```
+
+### Workflow
+
+* Include aml header:
+```
+#include <aml.h>
+...
+int main(int argc, char **argv){
+```
+
+* Check AML version:
+```
+if(aml_version_major != AML_VERSION_MAJOR){
+    printf("AML ABI mismatch!");
+    return 1;
+}
+```
+
+* Initialize and Cleanup AML:
+```
+if(aml_init(&argc, &argv) != 0){
+    printf("AML library init failure!");
+    return 1;
+}
+...
+aml_finalize();
+```
+
+* Link your program with `-laml`
+
+See above building blocks specific pages for further examples and information on
+library features.
+
+## Support
+
+AML interface with users is mainly done through
+[gitlab issue interface](https://xgitlab.cels.anl.gov/argo/aml/issues) and
+direct contact with developers/maintainers:
+* Swann Perarnau (swann@anl.gov)
+* Nicolas Denoyelle (ndenoyelle@anl.gov)
+
+AML is looking forward to new users. If you think your problem can be solved with AML
+eventually with some additional non-existing features which are relevant to the library,
+please let us know. Tell us about your topic of interests and your scientific/technical
+problems. We are also pleased to improve the software, thus any suggestion, bug
+declaration, contribution is welcomed.
+
+## Contributing
+
+AML is looking forward to new collaborations and contributions.
+If you wish to contribute to AML project rendez-vous [here](@ref contributing_page)
+

include/aml/area/linux.h

@@ -15,81 +15,86 @@
 #include <numa.h>
 #include <numaif.h>
 
-/* allowed binding flags */
+/**
+ * @defgroup aml_area_linux "AML Linux Areas"
+ * @brief Linux Implementation of Areas.
+ *
+ * Linux implementation of AML areas.
+ * This building block relies on libnuma implementation and
+ * linux mmap/munmap to provide mmap/munmap on NUMA host
+ * host processor memory. New areas may be created
+ * to allocate a specific subset of memories.
+ * This building block also include a static declaration of
+ * a default initialized area that can be used out of the box with
+ * abstract area API.
+ *
+ * #include <aml/area/linux.h>
+ * @{
+ **/
+
+/**
+ * Allowed binding flag for area creation.
+ * This flag will apply strict binding to the selected bitmask.
+ * If subsequent allocation will failt if they cannot enforce binding
+ * on bitmask.
+ **/
 #define AML_AREA_LINUX_BINDING_FLAG_BIND       (MPOL_BIND)
+
+/**
+ * Allowed binding flag for area creation.
+ * This flag will make subsequent allocations to interleave
+ * pages on memories of the bitmask.
+ **/
 #define AML_AREA_LINUX_BINDING_FLAG_INTERLEAVE (MPOL_INTERLEAVE)
+
+/**
+ * Allowed binding flag for area creation.
+ * This flag will make subsequent allocations to bound to the
+ * nodes of bitmask if possible, else to some other node.
+ **/
 #define AML_AREA_LINUX_BINDING_FLAG_PREFERRED  (MPOL_PREFERRED)
 
-/* allowed mmap flags to pass */
+/**
+ * Allowed mapping flag for area creation.
+ * This flag will make subsequent allocations to be private
+ * to the process making them.
+ **/
 #define AML_AREA_LINUX_MMAP_FLAG_PRIVATE (MAP_PRIVATE | MAP_ANONYMOUS)
-#define AML_AREA_LINUX_MMAP_FLAG_SHARED  (MAP_SHARED | MAP_ANONYMOUS)
-
-extern struct aml_area_ops aml_area_linux_ops;
-
-/* User data stored inside area */
-struct aml_area_linux_data {
-	/** numanodes to use **/
-	struct bitmask *nodeset;
-	/** numaif.h mbind policy or AML_AREA_LINUX_FLAG_* **/
-	int             binding_flags;
-	/** mmap flags **/
-	int             mmap_flags;
-};
-
-/* Default linux area with private mapping and no binding. */
-extern struct aml_area aml_area_linux;
-
-
-/*******************************************************************************
- * Linux operators
-*******************************************************************************/
 
 /**
- * Bind memory of size "size" pointed by "ptr" to binding set in "bind".
- * If mbind call was not successfull, i.e AML_FAILURE is returned, then errno
- * should be inspected for further error checking.
+ * Allowed mapping flag for area creation.
+ * This flag will make subsequent allocations to be visible to
+ * other processes of the system.
  **/
-int
-aml_area_linux_mbind(struct aml_area_linux_data    *bind,
-		     void                          *ptr,
-		     size_t                         size);
+#define AML_AREA_LINUX_MMAP_FLAG_SHARED  (MAP_SHARED | MAP_ANONYMOUS)
 
 /**
- * Function to check whether binding of a ptr obtained with
- * aml_area_linux_mmap() then aml_area_linux_mbind() match area settings.
- * Returns 1 if mapped memory binding in ptr match area_data binding settings,
- * else 0.
+ * This contains area operations implementation
+ * for linux area.
  **/
-int
-aml_area_linux_check_binding(struct aml_area_linux_data *area_data,
-			     void                       *ptr,
-			     size_t                      size);
+extern struct aml_area_ops aml_area_linux_ops;
 
 /**
- * mmap hook for aml area.
- * Fails with AML_FAILURE. On failure errno should be checked for further
- * error investigations.
+ * Default linux area with private mapping and no binding.
+ * Can be used out of the box with aml_area_*() functions.
  **/
-void*
-aml_area_linux_mmap(const struct aml_area_data  *area_data,
-		    void                        *ptr,
-		    size_t                       size);
+extern struct aml_area aml_area_linux;
 
 /**
- * munmap hook for aml area, to unmap memory mapped with aml_area_linux_mmap().
- * Fails with AML_FAILURE. On failure errno should be checked for further
- * error investigations.
+ * Implementation of aml_area_data for linux areas.
  **/
-int
-aml_area_linux_munmap(const struct aml_area_data *area_data,
-		      void *ptr,
-		      const size_t size);
-
-/*******************************************************************************
- * create/destroy and others
-*******************************************************************************/
-
+struct aml_area_linux_data {
+	/** numanodes to use when allocating data **/
+	struct bitmask *nodeset;
+	/** binding policy **/
+	int             binding_flags;
+	/** mmap flags **/
+	int             mmap_flags;
+};
 
+/**
+ * Static declaration of an aml area with linux ops.
+ **/
 #define AML_AREA_LINUX_DECL(name) \
 	struct aml_area_linux_data __ ##name## _inner_data; \
 	struct aml_area name = { \
@@ -97,12 +102,16 @@ aml_area_linux_munmap(const struct aml_area_data *area_data,
 		(struct aml_area_data *)&__ ## name ## _inner_data, \
 	}
 
+/**
+ * Static declaration of the size of a linux aml area.
+ **/
 #define AML_AREA_LINUX_ALLOCSIZE \
 	(sizeof(struct aml_area_linux_data) + \
 	 sizeof(struct aml_area))
 
-
 /**
+ * \brief Linux area creation.
+ *
  * Allocate and initialize a struct aml_area implemented by aml_area_linux
  * operations.
  * @param[out] area pointer to an uninitialized struct aml_area pointer to
@@ -121,24 +130,94 @@ int aml_area_linux_create(struct aml_area **area, const int mmap_flags,
 			  const struct aml_bitmap *nodemask,
 			  const int binding_flags);
 
+
+/**
+ * \brief Linux area destruction.
+ *
+ * Destroy (finalize and free resources) a struct aml_area created by
+ * aml_area_linux_create().
+ *
+ * @param area is NULL after this call.
+ **/
+void aml_area_linux_destroy(struct aml_area **area);
+
 /**
  * Initialize a struct aml_area declared using the AML_AREA_LINUX_DECL macro.
- * See aml_area_linux_create for details on arguments.
+ * @see aml_area_linux_create() for details on arguments.
  */
 int aml_area_linux_init(struct aml_area *area, const int mmap_flags,
 			const struct aml_bitmap *nodemask,
 			const int binding_flags);
+
 /**
  * Finalize a struct aml_area initialized with aml_area_linux_init.
  */
 void aml_area_linux_fini(struct aml_area *area);
 
 /**
- * Destroy (finalize and free resources) a struct aml_area created by
- * aml_area_linux_create.
+ * Bind memory of size "size" pointed by "ptr" to binding set in "bind".
+ * If mbind call was not successfull, i.e AML_FAILURE is returned, then errno
+ * should be inspected for further error checking.
+ * @param bind: The binding settings. mmap_flags is actually unused.
+ * @param ptr: The data to bind.
+ * @param size: The size of the data pointed by ptr.
+ * @return an AML error code.
+ **/
+int
+aml_area_linux_mbind(struct aml_area_linux_data    *bind,
+		     void                          *ptr,
+		     size_t                         size);
+
+/**
+ * Function to check whether binding of a ptr obtained with
+ * aml_area_linux_mmap() then aml_area_linux_mbind() match area settings.
+ * @param area_data: The expected binding settings.
+ * @param ptr: The data supposely bound.
+ * @param size: The data size.
+ * @return 1 if mapped memory binding in ptr match area_data binding settings,
+ * else 0.
+ **/
+int
+aml_area_linux_check_binding(struct aml_area_linux_data *area_data,
+			     void                       *ptr,
+			     size_t                      size);
+
+/**
+ * \brief mmap block for aml area.
  *
- * @param area is NULL after this call.
+ * This function is a wrapper on mmap function using arguments set in
+ * mmap_flags of area_data.
+ * This function does not perform binding, unlike it is done in areas created
+ * with aml_area_linux_create().
+ * @param area_data: The structure containing mmap_flags for mmap call.
+ *        nodemask and bind_flags fields are ignored.
+ * @param ptr: A hint provided to mmap function.
+ * @param size: The size to allocate.
+ * @return NULL on failure, else a valid pointer to memory.
+ * Upon failure, errno should be checked for further error investigations.
+ **/
+void*
+aml_area_linux_mmap(const struct aml_area_data  *area_data,
+		    void                        *ptr,
+		    size_t                       size);
+
+/**
+ * \brief munmap hook for aml area.
+ *
+ * unmap memory mapped with aml_area_linux_mmap().
+ * @param area_data: unused
+ * @param ptr: The virtual memory to unmap.
+ * @param size: The size of virtual memory to unmap.
+ * @return AML_FAILURE on error, AML_SUCCESS.
+ * Upon failure errno should be checked for further error investigations.
+ **/
+int
+aml_area_linux_munmap(const struct aml_area_data *area_data,
+		      void *ptr,
+		      const size_t size);
+
+/**
+ * @}
  **/
-void aml_area_linux_destroy(struct aml_area **area);
 
 #endif //AML_AREA_LINUX_NUMA_H

include/aml/dma/linux-par.h

@@ -11,46 +11,79 @@
 #ifndef AML_DMA_LINUX_PAR_H
 #define AML_DMA_LINUX_PAR_H 1
 
-/*******************************************************************************
- * Linux Parallel DMA API:
- * DMA logic implemented based on general linux API, with the caller thread
- * used as the only execution thread.
- ******************************************************************************/
+/**
+ * @defgroup aml_dma_par "AML Parallel DMA"
+ * @brief Parallel DMA implementation.
+ *
+ * DMA logic implemented based on general linux API, asynchronous execution
+ * threads. This DMA implementation moves between pointers allocated with an
+ * aml_area_linux.
+ * @{
+ **/
 
+/**
+ * Default table of dma request operations for linux
+ * parallel dma.
+ **/
 extern struct aml_dma_ops aml_dma_linux_par_ops;
 
+/** Thread data embeded inside an asynchronous dma request. **/
 struct aml_dma_linux_par_thread_data {
+	/**
+	 * A logical identifier of the thread in charge for
+	 * the request progress.
+	 **/
 	int tid;
+	/** The actual thread in charge for the request progress**/
 	pthread_t thread;
+	/** The dma containing sequential operations **/
 	struct aml_dma_linux_par *dma;
+	/** The request handled by this thread **/
 	struct aml_dma_request_linux_par *req;
 };
 
+/** Inside of a parallel request for linux movement. **/
 struct aml_dma_request_linux_par {
+	/**
+	 * The type of dma request
+	 * @see <aml.h>
+	 **/
 	int type;
+	/** The destination pointer of the data movement **/
 	void *dest;
+	/** The source pointer of the data movement **/
 	void *src;
+	/** The size of data to move **/
 	size_t size;
+	/** The thread data in charge of the request progress **/
 	struct aml_dma_linux_par_thread_data *thread_data;
 };
 
+/** Inside of a parallel request for linux movement. **/
 struct aml_dma_linux_par_data {
 	size_t nbthreads;
 	struct aml_vector requests;
 	pthread_mutex_t lock;
 };
 
+/** Declaration of linux parallel dma operations **/
 struct aml_dma_linux_par_ops {
 	void *(*do_thread)(void *thread_data);
 	int (*do_copy)(struct aml_dma_linux_par_data *data,
 		       struct aml_dma_request_linux_par *request, int tid);
 };
 
+/**
+ * aml_dma structure for linux based, parallel dma movement
+ * Needs to be initialized with aml_dma_linux_par_init().
+ * Can be passed to generic aml_dma_*() functions.
+ **/
 struct aml_dma_linux_par {
 	struct aml_dma_linux_par_ops ops;
 	struct aml_dma_linux_par_data data;
 };
 
+/** Static declaration of aml_dma_linux_par structure. **/
 #define AML_DMA_LINUX_PAR_DECL(name) \
 	struct aml_dma_linux_par __ ##name## _inner_data; \
 	struct aml_dma name = { \
@@ -58,6 +91,7 @@ struct aml_dma_linux_par {
 		(struct aml_dma_data *)&__ ## name ## _inner_data, \
 	}
 
+/** Static declaration of aml_dma_linux_par structure size. **/
 #define AML_DMA_LINUX_PAR_ALLOCSIZE \
 	(sizeof(struct aml_dma_linux_par) + \
 	 sizeof(struct aml_dma))
@@ -97,7 +131,10 @@ void aml_dma_linux_par_fini(struct aml_dma *dma);
 /**
  * Tears down a parallel DMA created with aml_dma_linux_par_create.
  * @param dma the address of a pointer to a parallel dma. Will be NULL after.
- */
+ **/
 void aml_dma_linux_par_destroy(struct aml_dma **dma);
 
+/**
+ * @}
+ **/
 #endif // AML_LINUX_DMA_LINUX_PAR_H

include/aml/dma/linux-seq.h

@@ -11,36 +11,72 @@
 #ifndef AML_DMA_LINUX_SEQ_H
 #define AML_DMA_LINUX_SEQ_H 1
 
-/*******************************************************************************
- * Linux Sequential DMA API:
+/**
+ * @defgroup aml_dma_seq "AML Sequential DMA"
+ * @brief Sequential DMA implementation.
+ *
  * DMA logic implemented based on general linux API, with the caller thread
- * used as the only execution thread.
- ******************************************************************************/
+ * used as the only execution thread. This DMA implementation moves between
+ * pointers allocated with an aml_area_linux.
+ * @{
+ **/
 
+/**
+ * Default table of dma request operations for linux
+ * sequential dma.
+ **/
 extern struct aml_dma_ops aml_dma_linux_seq_ops;
 
+/** Inside of a sequential request for linux movement. **/
 struct aml_dma_request_linux_seq {
+	/**
+	 * The type of dma request
+	 * @see <aml.h>
+	 **/
 	int type;
+	/** The destination pointer of the data movement **/
 	void *dest;
+	/** The source pointer of the data movement **/
 	void *src;
+	/** The size of data to move **/
 	size_t size;
 };
 
+/** Inner data of sequential linux aml_dma implementation **/
 struct aml_dma_linux_seq_data {
+	/**
+	 * Queue of submitted requests.
+	 * Requests may be submitted concurrently but will all
+	 * be performed by a single thread.
+	 **/
 	struct aml_vector requests;
+	/** Lock for queuing requests concurrently **/
 	pthread_mutex_t lock;
 };
 
+/** Declaration of available linux sequential dma operations **/
 struct aml_dma_linux_seq_ops {