Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
Francois Tessier
TAPIOCA
Commits
741c8c8a
Commit
741c8c8a
authored
Aug 31, 2017
by
Francois Tessier
Browse files
Draft of the NVRAM implementation of the abstraction layer for XC40
parent
a67035bf
Changes
2
Hide whitespace changes
Inline
Side-by-side
architectures/linux-sles_12-x86_64/tp_memory.cpp
View file @
741c8c8a
...
...
@@ -3,6 +3,7 @@
Memory
::
Memory
()
{
this
->
request_
=
NULL
;
this
->
mmapAllocatorRank_
=
0
;
}
...
...
@@ -55,6 +56,37 @@ void Memory::memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* file
}
err
=
MPI_File_open
(
this
->
comm_
,
this
->
fileName_
,
MPI_MODE_RDWR
|
MPI_MODE_CREATE
,
MPI_INFO_NULL
,
&
this
->
fileHandle_
);
// Preallocate the file
break
;
case
NVR
:
strcpy
(
this
->
fileName_
,
fileName
);
if
(
this
->
mmapAllocatorRank_
==
rank
)
{
printMsg
(
DEBUG
,
"Map file %s in DRAM (%s:%d)
\n
"
,
this
->
fileName_
,
__FILE__
,
__LINE__
);
// Add test for memory capacity
this
->
fd_
=
open
(
this
->
fileName_
,
O_RDWR
|
O_CREAT
,
S_IRWXU
);
if
(
this
->
fd_
==
-
1
)
{
printMsg
(
ERROR
,
"Error while opening the file %s (%s:%d)
\n
"
,
this
->
fileName_
,
__FILE__
,
__LINE__
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
if
(
ftruncate
(
this
->
fd_
,
this
->
buffSize_
)
==
-
1
)
{
printMsg
(
ERROR
,
"Error while truncating the file %s (%s:%d)
\n
"
,
this
->
fileName_
,
__FILE__
,
__LINE__
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
this
->
ptrMap_
=
mmap
(
0
,
this
->
buffSize_
,
PROT_READ
|
PROT_WRITE
,
MAP_SHARED
,
this
->
fd_
,
0
);
if
(
this
->
ptrMap_
==
MAP_FAILED
)
{
printMsg
(
ERROR
,
"Mmap of file %s has failed (%s:%d)
\n
"
,
this
->
fileName_
,
__FILE__
,
__LINE__
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
MPI_Win_create
(
this
->
ptrMap_
,
this
->
buffSize_
,
1
,
MPI_INFO_NULL
,
this
->
comm_
,
&
this
->
RMAWin_
);
}
else
MPI_Win_create
(
NULL
,
0
,
1
,
MPI_INFO_NULL
,
this
->
comm_
,
&
this
->
RMAWin_
);
MPI_Win_fence
(
0
,
this
->
RMAWin_
);
break
;
default:
printMsg
(
ERROR
,
"Unable to allocate memory (mem = %s)
\n
"
,
this
->
memName
()
);
...
...
@@ -64,7 +96,7 @@ void Memory::memAlloc ( int64_t buffSize, mem_t mem, bool masterRank, char* file
void
Memory
::
memFree
(
)
{
int
rank
;
int
rank
,
err
;
MPI_Comm_rank
(
MPI_COMM_WORLD
,
&
rank
);
switch
(
this
->
mem_
)
...
...
@@ -89,6 +121,20 @@ void Memory::memFree ( ) {
}
MPI_File_close
(
&
this
->
fileHandle_
);
break
;
case
NVR
:
this
->
memFlush
();
MPI_Win_free
(
&
this
->
RMAWin_
);
if
(
this
->
mmapAllocatorRank_
==
rank
)
{
printMsg
(
DEBUG
,
"Free memory on NVRAM and unmap file %s (%s:%d)
\n
"
,
this
->
fileName_
,
__FILE__
,
__LINE__
);
err
=
munmap
(
this
->
ptrMap_
,
this
->
buffSize_
);
if
(
err
==
-
1
)
{
printMsg
(
ERROR
,
"Error while unmaping the file %s (%s:%d)
\n
"
,
this
->
fileName_
,
__FILE__
,
__LINE__
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
close
(
this
->
fd_
);
}
break
;
default:
printMsg
(
ERROR
,
"Unable to free memory (mem = %s)
\n
"
,
this
->
memName
()
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
...
...
@@ -107,6 +153,7 @@ int Memory::memWrite ( void* srcBuffer, int64_t srcSize, int64_t offset, int des
{
case
DDR
:
case
HBM
:
case
NVR
:
err
=
MPI_Put
(
srcBuffer
,
srcSize
,
MPI_BYTE
,
destRank
,
offset
,
srcSize
,
MPI_BYTE
,
this
->
RMAWin_
);
break
;
case
HDD
:
...
...
@@ -129,6 +176,7 @@ int Memory::memRead ( void* srcBuffer, int64_t srcSize, int64_t offset, int des
{
case
DDR
:
case
HBM
:
case
NVR
:
err
=
MPI_Get
(
srcBuffer
,
srcSize
,
MPI_BYTE
,
destRank
,
offset
,
srcSize
,
MPI_BYTE
,
this
->
RMAWin_
);
break
;
case
HDD
:
...
...
@@ -144,9 +192,11 @@ int Memory::memRead ( void* srcBuffer, int64_t srcSize, int64_t offset, int des
int
Memory
::
memFlush
(
)
{
int
err
;
int
err
,
rank
;
MPI_Status
status
;
MPI_Comm_rank
(
MPI_COMM_WORLD
,
&
rank
);
switch
(
this
->
mem_
)
{
case
DDR
:
...
...
@@ -157,6 +207,17 @@ int Memory::memFlush ( ) {
if
(
this
->
request_
!=
NULL
)
MPI_Wait
(
&
this
->
request_
,
&
status
);
break
;
case
NVR
:
if
(
this
->
mmapAllocatorRank_
==
rank
)
{
printMsg
(
DEBUG
,
"Sync memory and file %s (%s:%d)
\n
"
,
this
->
fileName_
,
__FILE__
,
__LINE__
);
err
=
msync
(
this
->
ptrMap_
,
this
->
buffSize_
,
MS_SYNC
);
if
(
err
==
-
1
)
{
printMsg
(
ERROR
,
"Error while syncing memory and file %s (%s:%d)
\n
"
,
this
->
fileName_
,
__FILE__
,
__LINE__
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
}
}
//MPI_Win_fence ( 0, this->RMAWin_ );
break
;
default:
printMsg
(
ERROR
,
"Error while flushing data (mem = %s)
\n
"
,
this
->
memName
()
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
...
...
@@ -189,6 +250,9 @@ char* Memory::memName ( ) {
case
HDD
:
return
"HDD"
;
break
;
case
NVR
:
return
"NVR"
;
break
;
default:
printMsg
(
ERROR
,
"Wrong memory type!
\n
"
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
...
...
@@ -201,6 +265,7 @@ mem_t Memory::memType ( char* name ) {
if
(
!
strcmp
(
"HBM"
,
name
)
)
return
HBM
;
if
(
!
strcmp
(
"SSD"
,
name
)
)
return
SSD
;
if
(
!
strcmp
(
"HDD"
,
name
)
)
return
HDD
;
if
(
!
strcmp
(
"NVR"
,
name
)
)
return
NVR
;
printMsg
(
ERROR
,
"Wrong memory name!
\n
"
);
MPI_Abort
(
MPI_COMM_WORLD
,
-
1
);
...
...
architectures/linux-sles_12-x86_64/tp_memory.hpp
View file @
741c8c8a
...
...
@@ -4,6 +4,11 @@
#include
<stdio.h>
#include
<stdlib.h>
#include
<string.h>
#include
<sys/mman.h>
#include
<unistd.h>
#include
<sys/types.h>
#include
<sys/stat.h>
#include
<fcntl.h>
#include
<mpi.h>
#include
<hbwmalloc.h>
...
...
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment