Added pre-finalize callbacks

configure.ac

@@ -2,7 +2,7 @@
 # Process this file with autoconf to produce a configure script.
 
 AC_PREREQ([2.63])
-AC_INIT([margo], [0.5], [],[],[]) 
+AC_INIT([margo], [0.5.3], [],[],[]) 
 AC_CONFIG_MACRO_DIR([m4])
 LT_INIT
 

include/margo.h

@@ -159,6 +159,73 @@ void margo_wait_for_finalize(
 hg_bool_t margo_is_listening(
     margo_instance_id mid);
 
+/**
+ * Installs a callback to be called before the margo instance is finalize,
+ * and before the Mercury progress loop is terminated.
+ * Callbacks installed will be called in reverse ordered than they have been
+ * pushed, and with the user-provider pointer as argument.
+ *
+ * Note that callbacks may not be called within margo_finalize. They are called
+ * when the margo instance is cleaned up, which may happen in margo_wait_for_finalize.
+ * 
+ * Callbacks installed using this function may call RPCs or margo_thread_sleep,
+ * but do not guarantee that the process isn't itself going to receive RPCs in
+ * the mean time.
+ *
+ * @param mid The margo instance
+ * @param cb Callback to install
+ * @param uargs User-provider argument to pass to the callback when called
+ */
+void margo_push_prefinalize_callback(
+    margo_instance_id mid,
+    void(*cb)(void*), 
+    void* uargs);
+
+/**
+ * Removes the last pre-finalize callback that was pushed into the margo instance
+ * without calling it. If a callback was removed, this function returns 1, otherwise
+ * it returns 0.
+ *
+ * @param mid Margo instance.
+ */
+int margo_pop_prefinalize_callback(
+    margo_instance_id mid);
+
+/**
+ * @brief Installs a callback to be called before the margo instance is finalized,
+ * and before the Mercury progress loop is terminated.
+ * The owner pointer allows to identify callbacks installed by particular providers.
+ * Note that one can install multiple callbacks with the same owner. If popped, they
+ * will be popped in reverse order of installation. If they are not popped, they will
+ * be called in reverse order of installation by the margo cleanup procedure.
+ *
+ * Callbacks installed using this function may call RPCs or margo_thread_sleep,
+ * but do not guarantee that the process isn't itself going to receive RPCs in
+ * the mean time.
+ *
+ * @param mid The margo instance
+ * @param owner Owner of the callback (to be used when popping callbacks)
+ * @param cb Callback to install
+ * @param uargs User-provider argument to pass to the callback when called
+ */
+void margo_provider_push_prefinalize_callback(
+    margo_instance_id mid,
+    void* owner,
+    void(*cb)(void*),
+    void* uargs);
+
+/**
+ * Removes the last prefinalize callback that was pushed into the margo instance
+ * by the specified owner. If a callback was removed, this function returns 1, otherwise
+ * it returns 0.
+ *
+ * @param mid Margo instance.
+ * @param owner Owner of the callback.
+ */
+int margo_provider_pop_prefinalize_callback(
+    margo_instance_id mid,
+    void* owner);
+
 /**
  * Installs a callback to be called before the margo instance is finalize.
  * Callbacks installed will be called in reverse ordered than they have been
@@ -167,6 +234,10 @@ hg_bool_t margo_is_listening(
  * Note that callbacks may not be called within margo_finalize. They are called
  * when the margo instance is cleaned up, which may happen in margo_wait_for_finalize.
  *
+ * Important: callbacks cannot make RPC calls nor use margo_thread_sleep. If you
+ * need to be able to make RPC calls or use margo_thread_sleep, you should use
+ * margo_push_prefinalize_callback instead.
+ *
  * @param mid The margo instance
  * @param cb Callback to install
  * @param uargs User-provider argument to pass to the callback when called
@@ -193,6 +264,10 @@ int margo_pop_finalize_callback(
  * will be popped in reverse order of installation. If they are not popped, they will
  * be called in reverse order of installation by the margo cleanup procedure.
  *
+ * Important: callbacks cannot make RPC calls nor use margo_thread_sleep. If you
+ * need to be able to make RPC calls or use margo_thread_sleep, you should use
+ * margo_provider_push_prefinalize_callback instead.
+ *
  * @param mid The margo instance
  * @param owner Owner of the callback (to be used when popping callbacks)
  * @param cb Callback to install

src/margo.c

@@ -128,6 +128,7 @@ struct margo_instance
     ABT_mutex finalize_mutex;
     ABT_cond finalize_cond;
     struct margo_finalize_cb* finalize_cb;
+    struct margo_finalize_cb* prefinalize_cb;
 
     /* control logic to prevent margo_finalize from destroying
        the instance when some operations are pending */
@@ -477,6 +478,7 @@ margo_instance_id margo_init_pool(ABT_pool progress_pool, ABT_pool handler_pool,
 
     mid->refcount = 1;
     mid->finalize_cb = NULL;
+    mid->prefinalize_cb = NULL;
     mid->enable_remote_shutdown = 0;
 
     mid->pending_operations = 0;
@@ -625,6 +627,16 @@ void margo_finalize(margo_instance_id mid)
         return;
     }
 
+    /* before exiting the progress loop, pre-finalize callbacks need to be called */
+    struct margo_finalize_cb* fcb = mid->prefinalize_cb;
+    while(fcb) {
+        mid->prefinalize_cb = fcb->next;
+        (fcb->callback)(fcb->uargs);
+        struct margo_finalize_cb* tmp = fcb;
+        fcb = mid->prefinalize_cb;
+        free(tmp);
+    }
+
     /* tell progress thread to wrap things up */
     mid->hg_progress_shutdown_flag = 1;
 
@@ -688,6 +700,63 @@ hg_bool_t margo_is_listening(
     return HG_Class_is_listening(mid->hg_class);
 }
 
+void margo_push_prefinalize_callback(
+            margo_instance_id mid,
+            void(*cb)(void*),
+            void* uargs)
+{
+    margo_provider_push_prefinalize_callback(
+            mid,
+            NULL,
+            cb,
+            uargs);
+}
+
+int margo_pop_prefinalize_callback(
+                    margo_instance_id mid)
+{   
+    return margo_provider_pop_prefinalize_callback(mid, NULL);
+}
+
+void margo_provider_push_prefinalize_callback(
+            margo_instance_id mid,
+            void* owner,
+            void(*cb)(void*),                  
+            void* uargs)
+{
+    if(cb == NULL) return;
+
+    struct margo_finalize_cb* fcb = 
+        (struct margo_finalize_cb*)malloc(sizeof(*fcb));
+    fcb->owner    = owner;
+    fcb->callback = cb;
+    fcb->uargs    = uargs;
+
+    struct margo_finalize_cb* next = mid->prefinalize_cb;
+    fcb->next = next;
+    mid->prefinalize_cb = fcb;
+}
+
+int margo_provider_pop_prefinalize_callback(
+            margo_instance_id mid,
+            void* owner)
+{
+    struct margo_finalize_cb* prev = NULL;
+    struct margo_finalize_cb* fcb  =  mid->prefinalize_cb;
+    while(fcb != NULL && fcb->owner != owner) {
+        prev = fcb;
+        fcb = fcb->next;
+    }
+    if(fcb == NULL) return 0;
+    if(prev == NULL) {
+        mid->prefinalize_cb = fcb->next;
+    } else {
+        prev->next = fcb->next;
+    }
+    free(fcb);
+    return 1;
+}
+
 void margo_push_finalize_callback(
             margo_instance_id mid,
             void(*cb)(void*),