HEPnOSService.cpp 4.22 KB
Newer Older
Matthieu Dorier's avatar
Matthieu Dorier committed
1
2
3
4
5
6
7
8
9
/*
 * (C) 2018 The University of Chicago
 * 
 * See COPYRIGHT in top-level directory.
 */
#include <string>
#include <iostream>
#include <fstream>
#include <vector>
10
#include <cstring>
Matthieu Dorier's avatar
Matthieu Dorier committed
11
12
#include <unistd.h>
#include <mpi.h>
13
#include <thallium.hpp>
14
#include <sdskv-server.hpp>
15
#include "ServiceConfig.hpp"
Matthieu Dorier's avatar
Matthieu Dorier committed
16
#include "../ItemDescriptor.hpp"
Matthieu Dorier's avatar
Matthieu Dorier committed
17
#include "ConnectionInfoGenerator.hpp"
18
#include "hepnos-service.h"
Matthieu Dorier's avatar
Matthieu Dorier committed
19

20
21
namespace tl = thallium;

Matthieu Dorier's avatar
Matthieu Dorier committed
22
23
#define ASSERT(__cond, __msg, ...) { if(!(__cond)) { fprintf(stderr, "[%s:%d] " __msg, __FILE__, __LINE__, __VA_ARGS__); exit(-1); } }

Matthieu Dorier's avatar
Matthieu Dorier committed
24
25
static void createProviderAndDatabases(tl::engine& engine, hepnos::ProviderConfig& provider_config, sdskv_compare_fn comp = nullptr);
static int hepnosItemDescriptorCompare(const void* id1, hg_size_t size1, const void* id2, hg_size_t size2);
26

27
void hepnos_run_service(MPI_Comm comm, const char* config_file, const char* connection_file)
Matthieu Dorier's avatar
Matthieu Dorier committed
28
29
30
{
    int ret;
    int rank;
31
    int size;
32
33

    MPI_Comm_rank(comm, &rank);
34
    MPI_Comm_size(comm, &size);
Matthieu Dorier's avatar
Matthieu Dorier committed
35

36
37
38
    /* load configuration */
    std::unique_ptr<hepnos::ServiceConfig> config;
    try {
39
        config = std::make_unique<hepnos::ServiceConfig>(config_file, rank, size);
40
41
42
43
44
45
46
47
    } catch(const std::exception& e) {
        std::cerr << "Error: when reading configuration:" << std::endl;
        std::cerr << "  " << e.what() << std::endl;
        std::cerr << "Aborting." << std::endl;
        MPI_Abort(MPI_COMM_WORLD, -1);
        return;
    }

Matthieu Dorier's avatar
Matthieu Dorier committed
48
    tl::engine engine;
49
    try {
Matthieu Dorier's avatar
Matthieu Dorier committed
50
51
52
53
54
        hg_init_info hg_opt;
        memset(&hg_opt, 0, sizeof(hg_opt));
        if(config->busySpin)
            hg_opt.na_init_info.progress_mode = NA_NO_BLOCK;
        engine = tl::engine(
55
                    config->address,
56
                    THALLIUM_SERVER_MODE,
Matthieu Dorier's avatar
Matthieu Dorier committed
57
58
                    false, config->numThreads-1,
                    &hg_opt);
59
60
61
62

    } catch(std::exception& ex) {
        std::cerr << "Error: unable to initialize thallium" << std::endl;
        std::cerr << "Exception: " << ex.what() << std::endl;
63
        std::cerr << "Aborting." << std::endl;
64
65
        MPI_Abort(MPI_COMM_WORLD, -1);
        return;
Matthieu Dorier's avatar
Matthieu Dorier committed
66
    }
Matthieu Dorier's avatar
Matthieu Dorier committed
67
68
    engine.enable_remote_shutdown();
    auto self_addr_str  = static_cast<std::string>(engine.self());
Matthieu Dorier's avatar
Matthieu Dorier committed
69

70
71
    /* SDSKV providers initialization */
    for(auto& provider_config : config->datasetProviders)
Matthieu Dorier's avatar
Matthieu Dorier committed
72
        createProviderAndDatabases(engine, provider_config);
73
    for(auto& provider_config : config->runProviders)
Matthieu Dorier's avatar
Matthieu Dorier committed
74
        createProviderAndDatabases(engine, provider_config, hepnosItemDescriptorCompare);
75
    for(auto& provider_config : config->subrunProviders)
Matthieu Dorier's avatar
Matthieu Dorier committed
76
        createProviderAndDatabases(engine, provider_config, hepnosItemDescriptorCompare);
77
    for(auto& provider_config : config->eventProviders)
Matthieu Dorier's avatar
Matthieu Dorier committed
78
        createProviderAndDatabases(engine, provider_config, hepnosItemDescriptorCompare);
79
    for(auto& provider_config : config->productProviders)
Matthieu Dorier's avatar
Matthieu Dorier committed
80
        createProviderAndDatabases(engine, provider_config);
Matthieu Dorier's avatar
Matthieu Dorier committed
81

82
    hepnos::ConnectionInfoGenerator fileGen(self_addr_str, *config);
Matthieu Dorier's avatar
Matthieu Dorier committed
83
    fileGen.generateFile(MPI_COMM_WORLD, connection_file);
Matthieu Dorier's avatar
Matthieu Dorier committed
84

Matthieu Dorier's avatar
Matthieu Dorier committed
85
    engine.wait_for_finalize();
Matthieu Dorier's avatar
Matthieu Dorier committed
86
}
87

Matthieu Dorier's avatar
Matthieu Dorier committed
88
static void createProviderAndDatabases(tl::engine& engine, hepnos::ProviderConfig& provider_config, sdskv_compare_fn comp) {
89
90
91
92
93

    sdskv::provider* provider = sdskv::provider::create(
            engine.get_margo_instance(), 
            provider_config.provider_id,
            SDSKV_ABT_POOL_DEFAULT);
Matthieu Dorier's avatar
Matthieu Dorier committed
94
    if(comp) provider->add_comparison_function("hepnos-item-descriptor", comp);
95
96
97
98
99
100
101
    for(auto& db_config : provider_config.databases) {
        sdskv_config_t config;
        std::memset(&config, 0, sizeof(config));
        config.db_name = db_config.name.c_str();
        config.db_path = db_config.path.c_str();
        config.db_type = db_config.type;
        config.db_no_overwrite = 1;
Matthieu Dorier's avatar
Matthieu Dorier committed
102
        if(comp) config.db_comp_fn_name = "hepnos-item-descriptor";
103
104
105
        db_config.id = provider->attach_database(config);
    }
}
Matthieu Dorier's avatar
Matthieu Dorier committed
106
107
108
109
110
111
112
113

static int hepnosItemDescriptorCompare(const void* id1_ptr, hg_size_t size1, const void* id2_ptr, hg_size_t size2) {
    const hepnos::ItemDescriptor* id1 = reinterpret_cast<const hepnos::ItemDescriptor*>(id1_ptr);
    const hepnos::ItemDescriptor* id2 = reinterpret_cast<const hepnos::ItemDescriptor*>(id2_ptr);
    if(*id1 == *id2) return 0;
    if(*id1 < *id2) return -1;
    return 1;
}