ad_pe_aggrs.c 6.89 KB
Newer Older
Paul Coffman's avatar
Paul Coffman committed
1
2
3
4
5
6
7
8
9
10
11
12
13
14
15
16
17
18
19
20
/* ---------------------------------------------------------------- */
/* (C)Copyright IBM Corp.  2007, 2008                               */
/* ---------------------------------------------------------------- */
/**
 * \file ad_pe_aggrs.c
 * \brief The externally used function from this file is is declared in ad_pe_aggrs.h
 */

/* -*- Mode: C; c-basic-offset:4 ; -*- */
/*
 *   Copyright (C) 1997-2001 University of Chicago.
 *   See COPYRIGHT notice in top-level directory.
 */

/*#define TRACE_ON */

#include "adio.h"
#include "adio_cb_config_list.h"
#include "../ad_gpfs.h"
#include "ad_pe_aggrs.h"
21
22
#include "mpiimpl.h"

Paul Coffman's avatar
Paul Coffman committed
23
24
25
26
27
28
29
30
31
32
33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
#ifdef AGGREGATION_PROFILE
#include "mpe.h"
#endif

#ifdef USE_DBG_LOGGING
  #define AGG_DEBUG 1
#endif

#ifndef TRACE_ERR
#  define TRACE_ERR(format...)
#endif

/*
 * Compute the aggregator-related parameters that are required in 2-phase collective IO of ADIO.
 * The parameters are
 * 	. the number of aggregators (proxies) : fd->hints->cb_nodes
 *	. the ranks of the aggregators :        fd->hints->ranklist
 * For now set these based on MP_IOTASKLIST
 */
int
ADIOI_PE_gen_agg_ranklist(ADIO_File fd)
{

    int numAggs = 0;
    char *ioTaskList = getenv( "MP_IOTASKLIST" );
48
49
    char *ioAgentCount = getenv("MP_IOAGENT_CNT");
    int i,j;
50
    int inTERcommFlag = 0;
Paul Coffman's avatar
Paul Coffman committed
51

52
53
    int myRank;
    MPI_Comm_rank(fd->comm, &myRank);
Paul Coffman's avatar
Paul Coffman committed
54

55
56
57
58
59
60
61
    MPI_Comm_test_inter(fd->comm, &inTERcommFlag);
    if (inTERcommFlag) {
      FPRINTF(stderr,"inTERcomms are not supported in MPI-IO - aborting....\n");
      perror("ADIOI_PE_gen_agg_ranklist:");
      MPI_Abort(MPI_COMM_WORLD, 1);
    }

62
    if (ioTaskList) {
Paul Coffman's avatar
Paul Coffman committed
63
64
      char tmpBuf[8];   /* Big enough for 1M tasks (7 digits task ID). */
      tmpBuf[7] = '\0';
65
      for (i=0; i<7; i++) {
Paul Coffman's avatar
Paul Coffman committed
66
67
68
69
70
71
72
73
74
         tmpBuf[i] = *ioTaskList++;      /* Maximum is 7 digits for 1 million. */
         if (*ioTaskList == ':') {       /* If the next char is a ':' ends it. */
             tmpBuf[i+1] = '\0';
             break;
         }
      }
      numAggs = atoi(tmpBuf);
      fd->hints->ranklist = (int *) ADIOI_Malloc (numAggs * sizeof(int));

75
      for (j=0; j<numAggs; j++) {
Paul Coffman's avatar
Paul Coffman committed
76
         ioTaskList++;                /* Advance past the ':' */
77
         for (i=0; i<7; i++) {
Paul Coffman's avatar
Paul Coffman committed
78
79
80
81
82
83
84
85
86
87
88
89
90
91
92
93
            tmpBuf[i] = *ioTaskList++;
            if ( (*ioTaskList == ':') || (*ioTaskList == '\0') ) {
                tmpBuf[i+1] = '\0';
                break;
            }
         }
         fd->hints->ranklist[j] = atoi(tmpBuf);

         /* At the end check whether the list is shorter than specified. */
         if (*ioTaskList == '\0') {
            if (j < (numAggs-1)) {
               numAggs = j;
            }
            break;
         }
      }
94
95
96
97
98
99
100
101
102
103
104
    }
    else {
      MPID_Comm *mpidCommData;

      MPID_Comm_get_ptr(fd->comm,mpidCommData);
      int localSize = mpidCommData->local_size;

      // get my node rank
      int myNodeRank = mpidCommData->intranode_table[mpidCommData->rank];

      int *allNodeRanks = (int *) ADIOI_Malloc (localSize * sizeof(int));
Paul Coffman's avatar
Paul Coffman committed
105

106
107
      allNodeRanks[myRank] = myNodeRank;
      MPI_Allgather(MPI_IN_PLACE, 1, MPI_INT, allNodeRanks, 1, MPI_INT, fd->comm);
Paul Coffman's avatar
Paul Coffman committed
108
109

#ifdef AGG_DEBUG
110
111
112
113
114
115
116
117
118
119
120
121
122
123
124
      printf("MPID_Comm data: remote_size is %d local_size is %d\nintranode_table entries:\n",mpidCommData->remote_size,mpidCommData->local_size);
      for (i=0;i<localSize;i++) {
        printf("%d ",mpidCommData->intranode_table[i]);
      }
      printf("\ninternode_table entries:\n");
      for (i=0;i<localSize;i++) {
        printf("%d ",mpidCommData->internode_table[i]);
      }
      printf("\n");

      printf("\allNodeRanks entries:\n");
      for (i=0;i<localSize;i++) {
        printf("%d ",allNodeRanks[i]);
      }
      printf("\n");
Paul Coffman's avatar
Paul Coffman committed
125
126
127

#endif

128
129
130
131
132
133
134
135
136
137
138
139
140
141
142
143
144
145
146
147
148
149
150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
165
166
167
168
169
170
171
172
173
174
175
176
177
178
179
180
181
182
183
184
185
186
187
188
189
190
191
192
193
194
195
196
197
198
199
200
201
202
203
204
205
206
207
208
209
210
211
212
213
214
215
216
217
218
219
220
221
222
223
      if (ioAgentCount) {
        int cntType = -1;

        if ( strcasecmp(ioAgentCount, "ALL") ) {
           if ( (cntType = atoi(ioAgentCount)) <= 0 ) {
              /* Input is other non-digit or less than 1 the  assume */
              /* 1 agent per node.  Note: atoi(-1) reutns -1.        */
              /* No warning message given here -- done earlier.      */
              cntType = -1;
           }
        }
        else {
           /* ALL is specified set agent count to localSize */
           cntType = -2;
        }
        switch(cntType) {
           case -1:
              /* 1 agent/node case */
             {
             int rankListIndex = 0;
             fd->hints->ranklist = (int *) ADIOI_Malloc (localSize * sizeof(int));
             for (i=0;i<localSize;i++) {
               if (allNodeRanks[i] == 0) {
                 fd->hints->ranklist[rankListIndex++] = i;
                 numAggs++;
               }
             }
             }
              break;
           case -2:
              /* ALL tasks case */
             fd->hints->ranklist = (int *) ADIOI_Malloc (localSize * sizeof(int));
             for (i=0;i<localSize;i++) {
               fd->hints->ranklist[i] = i;
               numAggs++;
             }
              break;
           default:
              /* Specific agent count case -- MUST be less than localSize. */
              if (cntType <= localSize) {
                 numAggs = cntType;
                 // Round-robin thru allNodeRanks - pick the 0's, then the 1's, etc
                 int currentNodeRank = 0;  // node rank currently being selected as aggregator
                 int rankListIndex = 0;
                 int currentAllNodeIndex = 0;

                 fd->hints->ranklist = (int *) ADIOI_Malloc (numAggs * sizeof(int));

                 while (rankListIndex < numAggs) {

                   int foundEntry = 0;
                   while (!foundEntry && (currentAllNodeIndex < localSize)) {
                     if (allNodeRanks[currentAllNodeIndex] == currentNodeRank) {
                       fd->hints->ranklist[rankListIndex++] = currentAllNodeIndex;
                       foundEntry = 1;
                     }
                     currentAllNodeIndex++;
                   }
                   if (!foundEntry) {
                     currentNodeRank++;
                     currentAllNodeIndex = 0;
                   }
                } // while
              } // (cntType <= localSize)
              else {
                ADIOI_Assert(1);
              }
              break;
            } // switch(cntType)
        } // if (ioAgentCount)

      else { // default is 1 aggregator per node
        // take the 0 entries from allNodeRanks
          int rankListIndex = 0;
          fd->hints->ranklist = (int *) ADIOI_Malloc (localSize * sizeof(int));
          for (i=0;i<localSize;i++) {
            if (allNodeRanks[i] == 0) {
              fd->hints->ranklist[rankListIndex++] = i;
              numAggs++;
            }
          }
      }
    }

    if ( getenv("MP_I_SHOW_AGENTS") ) {
      if (myRank == 0) {
      printf("Agg rank list of %d generated:\n", numAggs);
      for (i=0;i<numAggs;i++) {
        printf("%d ",fd->hints->ranklist[i]);
      }
      printf("\n");
      }
    }

    fd->hints->cb_nodes = numAggs;

Paul Coffman's avatar
Paul Coffman committed
224
225
226
    return 0;
}