Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Francois Tessier
TAPIOCA
Commits
1d30b3e0
Commit
1d30b3e0
authored
Aug 17, 2017
by
Francois Tessier
Browse files
Rewrite the header files for mem and net abstractions. Add a prototype to implement data uncaching
parent
e1246397
Changes
2
Hide whitespace changes
Inline
Side-by-side
architectures/linux-sles_12-x86_64/tp_memory.hpp
View file @
1d30b3e0
...
...
@@ -3,190 +3,44 @@
#include
<stdio.h>
#include
<stdlib.h>
#include
<string.h>
#include
<mpi.h>
#include
<hbwmalloc.h>
#include
"tp_utils.hpp"
#include
"tp_memory_interface.hpp"
#include
<hbwmalloc.h>
class
Memory
:
public
iMemory
{
public:
Memory
(
);
~
Memory
(
);
/**********************/
/* |-- Allocation */
/**********************/
void
memAlloc
(
int64_t
buffSize
,
mem_t
mem
,
bool
masterRank
,
char
*
fileName
,
MPI_Comm
comm
)
{
int
rank
;
this
->
mem_
=
mem
;
this
->
masterRank_
=
masterRank
;
MPI_Comm_dup
(
comm
,
&
this
->
comm_
);
this
->
buffSize_
=
buffSize
;
strncpy
(
this
->
fileName_
,
fileName
,
sizeof
(
fileName
)
);
MPI_Comm_rank
(
this
->
comm_
,
&
rank
);
switch
(
this
->
mem_
)
{
case
DDR
:
if
(
this
->
masterRank_
)
{
printMsg
(
DEBUG
,
"Allocate memory on DDR (%s:%d)
\n
"
,
__FILE__
,
__LINE__
);
this
->
buffer_
=
malloc
(
this
->
buffSize_
);
MPI_Win_create
(
this
->
buffer_
,
this
->
buffSize_
,
1
,
MPI_INFO_NULL
,
this
->
comm_
,
&
this
->
RMAWin_
);
}
else
MPI_Win_create
(
NULL
,
0
,
1
,
MPI_INFO_NULL
,
this
->
comm_
,
&
this
->
RMAWin_
);
MPI_Win_fence
(
0
,
this
->
RMAWin_
);
break
;
case
HBM
:
if
(
this
->
masterRank_
)
{
printMsg
(
DEBUG
,
"Allocate memory on HBM (%s:%d)
\n
"
,
__FILE__
,
__LINE__
);
this
->
buffer_
=
hbw_malloc
(
this
->
buffSize_
);
MPI_Win_create
(
this
->
buffer_
,
this
->
buffSize_
,
1
,
MPI_INFO_NULL
,
this
->
comm_
,
&
this
->
RMAWin_
);
}
else
MPI_Win_create
(
NULL
,
0
,
1
,
MPI_INFO_NULL
,
this
->
comm_
,
&
this
->
RMAWin_
);
MPI_Win_fence
(
0
,
this
->
RMAWin_
);
break
;
default:
printMsg
(
ERROR
,
"Unable to allocate memory (mem = %s)
\n
"
,
this
->
memName
()
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
}
void
memFree
(
)
{
int
rank
;
MPI_Comm_rank
(
MPI_COMM_WORLD
,
&
rank
);
switch
(
this
->
mem_
)
{
case
DDR
:
MPI_Win_free
(
&
this
->
RMAWin_
);
if
(
this
->
masterRank_
)
{
printMsg
(
DEBUG
,
"Free memory on DDR (%s:%d)
\n
"
,
__FILE__
,
__LINE__
);
free
(
this
->
buffer_
);
}
break
;
case
HBM
:
MPI_Win_free
(
&
this
->
RMAWin_
);
if
(
this
->
masterRank_
)
{
printMsg
(
DEBUG
,
"Free memory on HBM (%s:%d)
\n
"
,
__FILE__
,
__LINE__
);
hbw_free
(
this
->
buffer_
);
}
break
;
default:
printMsg
(
ERROR
,
"Unable to free memory (mem = %s)
\n
"
,
this
->
memName
()
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
}
void
memAlloc
(
int64_t
buffSize
,
mem_t
mem
,
bool
masterRank
,
char
*
fileName
,
MPI_Comm
comm
);
void
memFree
(
);
/**********************/
/* |-- I/O */
/**********************/
int
memWrite
(
void
*
srcBuffer
,
int64_t
srcSize
,
int64_t
offset
,
int
destRank
)
{
int
err
;
MPI_Status
status
;
switch
(
this
->
mem_
)
{
case
DDR
:
case
HBM
:
err
=
MPI_Put
(
srcBuffer
,
srcSize
,
MPI_BYTE
,
destRank
,
offset
,
srcSize
,
MPI_BYTE
,
this
->
RMAWin_
);
break
;
default:
printMsg
(
ERROR
,
"Error while writing data (mem = %s)
\n
"
,
this
->
memName
()
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
return
err
;
}
int
memRead
(
void
*
srcBuffer
,
int64_t
srcSize
,
int64_t
offset
,
int
destRank
)
{
int
err
;
MPI_Status
status
;
switch
(
this
->
mem_
)
{
case
DDR
:
case
HBM
:
err
=
MPI_Get
(
srcBuffer
,
srcSize
,
MPI_BYTE
,
destRank
,
offset
,
srcSize
,
MPI_BYTE
,
this
->
RMAWin_
);
break
;
default:
printMsg
(
ERROR
,
"Error while reading data (mem = %s)
\n
"
,
this
->
memName
()
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
return
err
;
}
int
memFlush
(
)
{
int
err
;
MPI_Status
status
;
switch
(
this
->
mem_
)
{
case
DDR
:
case
HBM
:
MPI_Win_fence
(
0
,
this
->
RMAWin_
);
break
;
default:
printMsg
(
ERROR
,
"Error while flushing data (mem = %s)
\n
"
,
this
->
memName
()
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
return
err
;
}
int
memWrite
(
void
*
srcBuffer
,
int64_t
srcSize
,
int64_t
offset
,
int
destRank
);
int
memRead
(
void
*
srcBuffer
,
int64_t
srcSize
,
int64_t
offset
,
int
destRank
);
int
memFlush
(
);
int
memUncache
(
);
/**********************/
/* |-- Utils */
/**********************/
char
*
memName
(
)
{
switch
(
this
->
mem_
)
{
case
DDR
:
return
"DDR"
;
break
;
case
HBM
:
return
"HBM"
;
break
;
case
SSD
:
return
"SSD"
;
break
;
case
HDD
:
return
"HDD"
;
break
;
default:
printMsg
(
ERROR
,
"Wrong memory type!
\n
"
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
}
mem_t
memType
(
char
*
name
)
{
if
(
!
strcmp
(
"DDR"
,
name
)
)
return
DDR
;
if
(
!
strcmp
(
"HBM"
,
name
)
)
return
HBM
;
if
(
!
strcmp
(
"SSD"
,
name
)
)
return
SSD
;
if
(
!
strcmp
(
"HDD"
,
name
)
)
return
HDD
;
printMsg
(
ERROR
,
"Wrong memory name!
\n
"
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
char
*
memName
(
);
mem_t
memType
(
char
*
name
);
/************************/
/* |-- Characteristics */
/************************/
int64_t
memBandwidth
(
)
{
return
0
;
}
int64_t
memLatency
(
)
{
return
0
;
}
int64_t
memCapacity
(
)
{
return
0
;
}
int64_t
memBandwidth
(
);
int64_t
memLatency
(
);
int64_t
memCapacity
(
);
bool
memPersistency
(
);
};
#endif // TP_MEMORY_H
...
...
architectures/linux-sles_12-x86_64/tp_network.hpp
View file @
1d30b3e0
#ifndef
CRAY_CX40_THETA_LUSTRE
_H
#define
CRAY_CX40_THETA_LUSTRE
_H
#ifndef
TP_NETWORK
_H
#define
TP_NETWORK
_H
#include
<stdio.h>
#include
<stdlib.h>
#include
<sched.h>
#include
<mpi.h>
#include
<pmi.h>
#include
"tp_network_interface.hpp"
#include
<pmi.h>
extern
"C"
{
#include
<lustre/lustreapi.h>
...
...
@@ -20,195 +22,35 @@ public:
/**********************/
/* |-- Nodes */
/**********************/
int
IONodeId
()
{
return
0
;
}
int
BridgeNodeId
()
{
return
0
;
}
int
ComputeNodeId
()
{
return
0
;
}
int
ProcessPerNode
()
{
int
ppn
,
err
;
err
=
PMI_Get_numpes_on_smp
(
&
ppn
);
if
(
err
!=
PMI_SUCCESS
)
{
fprintf
(
stderr
,
"[ERROR] PMI_Get_numpes_on_smp failed!
\n
"
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
return
ppn
;
}
/*
* 7 LNET nodes per OST
*/
int
IONodesPerFile
(
char
*
filename
,
int
*
nodesList
)
{
int
err
,
stripeCount
,
nLnets
,
i
,
idx
,
oid
,
l
;
char
fgrId
[
20
];
int
*
ssuId
,
*
ostId
,
*
lnets
;
struct
find_param
param
=
{
0
};
int
ssu2fgr
[]
=
{
0
,
0
,
0
,
0
,
2
,
3
,
2
,
3
,
2
,
3
,
2
,
3
,
2
,
3
,
2
,
3
,
2
,
3
,
4
,
5
,
4
,
5
,
4
,
5
,
4
,
5
,
4
,
5
,
4
,
5
,
4
,
5
,
6
,
7
,
6
,
7
,
6
,
7
,
6
,
7
,
6
,
7
,
6
,
7
,
6
,
7
,
8
,
9
,
8
,
9
,
8
,
9
,
8
,
9
,
8
,
9
,
8
,
9
,
8
,
9
};
err
=
llapi_getstripe
(
filename
,
&
param
);
if
(
err
)
fprintf
(
stdout
,
"[ERROR] llapi_getstripe
\n
"
);
stripeCount
=
(
&
param
)
->
fp_lmd
->
lmd_lmm
.
lmm_stripe_count
;
nLnets
=
stripeCount
*
LNETS_PER_OST
;
ssuId
=
(
int
*
)
malloc
(
stripeCount
*
sizeof
(
int
)
);
ostId
=
(
int
*
)
malloc
(
stripeCount
*
sizeof
(
int
)
);
// /!\ Hypothesis : OSS id == SNX - 4
for
(
i
=
0
;
i
<
stripeCount
;
i
++
)
{
idx
=
(
&
param
)
->
fp_lmd
->
lmd_lmm
.
lmm_objects
[
i
].
l_ost_idx
;
ssuId
[
i
]
=
idx
+
4
;
lnets
=
(
int
*
)
malloc
(
LNETS_PER_OST
*
sizeof
(
int
)
);
snprintf
(
fgrId
,
20
,
"o2ib100%d"
,
ssu2fgr
[
ssuId
[
i
]
]
);
FgrToLnets
(
fgrId
,
lnets
);
for
(
l
=
0
;
l
<
LNETS_PER_OST
;
l
++
)
nodesList
[
i
*
LNETS_PER_OST
+
l
]
=
lnets
[
l
];
free
(
lnets
);
}
return
nLnets
;
}
int
IONodeId
();
int
BridgeNodeId
();
int
ComputeNodeId
();
int
ProcessPerNode
();
int
IONodesPerFile
(
char
*
filename
,
int
*
nodesList
);
/**********************/
/* |-- Network */
/**********************/
/*
* X : Rank 3 (Optics cables between groups : 12.5 Gbps)
* Y : Rank 2 (Copper cables between 6 backplanes : 14 Gbps)
* Z : Rank 1 (Copper cables between 16 aries routers in a backplane :14 Gbps)
* nid : Node identifier
*
* http://www.nersc.gov/users/computational-systems/edison/configuration/interconnect/
*
*/
int
NetworkDimensions
()
{
return
4
;
}
int
NetworkDimensions
();
/* |---- Coordinates */
void
RankToCoordinates
(
int
rank
,
int
*
coord
)
{
pmi_mesh_coord_t
xyz
;
int
nid
;
/* Hypothesis : PMI_rank == MPI_rank */
PMI_Get_nid
(
rank
,
&
nid
);
PMI_Get_meshcoord
((
pmi_nid_t
)
nid
,
&
xyz
);
coord
[
0
]
=
xyz
.
mesh_x
;
coord
[
1
]
=
xyz
.
mesh_y
;
coord
[
2
]
=
xyz
.
mesh_z
;
coord
[
3
]
=
nid
;
coord
[
4
]
=
sched_getcpu
();
}
void
IONodeCoordinates
(
int
*
coord
)
{
}
void
RankToCoordinates
(
int
rank
,
int
*
coord
);
void
IONodeCoordinates
(
int
*
coord
);
/* |---- Distance */
int
DistanceToIONode
(
int
srcRank
)
{
return
0
;
}
int
DistanceToIONode
(
int
srcRank
);
/* Minimal distance between two ranks considering a dragonfly network */
int
DistanceBetweenRanks
(
int
srcRank
,
int
destRank
)
{
int
dim
=
NetworkDimensions
(),
d
;
int
srcCoord
[
dim
],
destCoord
[
dim
];
int
distance
;
RankToCoordinates
(
srcRank
,
srcCoord
);
RankToCoordinates
(
destRank
,
destCoord
);
distance
=
0
;
for
(
d
=
0
;
d
<
dim
;
d
++
)
{
if
(
srcCoord
[
d
]
!=
destCoord
[
d
]
)
distance
++
;
}
return
distance
;
}
int
DistanceBetweenRanks
(
int
srcRank
,
int
destRank
);
/* |---- Routes */
int
RouteToIONode
(
int
srcRank
,
int
*
path
)
{
return
0
;
}
int
RouteBetweenRanks
(
int
srcRank
,
int
destRank
,
int
*
path
)
{
return
0
;
}
int
RouteToIONode
(
int
srcRank
,
int
*
path
);
int
RouteBetweenRanks
(
int
srcRank
,
int
destRank
,
int
*
path
);
/* |---- Links */
void
LinksList
(
int
*
linksList
)
{
}
void
LinksList
(
int
*
linksList
);
private:
void
FgrToLnets
(
char
*
fgr_id
,
int
*
lnet
)
{
int
count
=
0
;
FILE
*
fp
;
char
fline
[
100
];
char
*
lnet_list
,
*
item
;
fp
=
fopen
(
"/etc/lnet/routes.conf"
,
"r"
);
if
(
fp
==
NULL
)
{
fprintf
(
stdout
,
"[ERROR] Error while opening routes.conf file!
\n
"
);
return
;
}
while
(
fgets
(
fline
,
100
,
fp
)
!=
NULL
)
{
const
char
*
c
=
strstr
(
fline
,
fgr_id
);
if
(
c
!=
NULL
)
{
const
char
*
b1
=
strstr
(
fline
,
"["
)
+
1
;
const
char
*
b2
=
strstr
(
fline
,
"]"
);
lnet_list
=
(
char
*
)
malloc
(
sizeof
(
char
)
*
(
b2
-
b1
+
1
)
);
strncpy
(
lnet_list
,
b1
,
b2
-
b1
);
item
=
strtok
(
lnet_list
,
","
);
while
(
item
)
{
lnet
[
count
]
=
atoi
(
item
);
item
=
strtok
(
0
,
","
);
count
++
;
}
}
count
=
0
;
}
fclose
(
fp
);
return
;
}
void
FgrToLnets
(
char
*
fgr_id
,
int
*
lnet
);
};
#endif //
CRAY_CX40_THETA_LUSTRE
_H
#endif //
TP_NETWORK
_H
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment