Skip to content
Projects
Groups
Snippets
Help
Loading...
Help
Support
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
D
darshan
Project overview
Project overview
Details
Activity
Releases
Repository
Repository
Files
Commits
Branches
Tags
Contributors
Graph
Compare
Issues
72
Issues
72
List
Boards
Labels
Milestones
Merge Requests
5
Merge Requests
5
Analytics
Analytics
Repository
Value Stream
Wiki
Wiki
Members
Members
Collapse sidebar
Close sidebar
Activity
Graph
Create a new issue
Commits
Issue Boards
Open sidebar
darshan
darshan
Commits
03053837
Commit
03053837
authored
Jan 30, 2017
by
Shane Snyder
Browse files
Options
Browse Files
Download
Plain Diff
Merge branch 'issue-219-dxt-mem-config' into 'master'
Issue 219 dxt mem config See merge request !9
parents
a4a77ada
915885cf
Changes
5
Hide whitespace changes
Inline
Side-by-side
Showing
5 changed files
with
58 additions
and
17 deletions
+58
-17
darshan-runtime/darshan-core.h
darshan-runtime/darshan-core.h
+0
-3
darshan-runtime/lib/darshan-core.c
darshan-runtime/lib/darshan-core.c
+7
-6
darshan-runtime/lib/darshan-dxt.c
darshan-runtime/lib/darshan-dxt.c
+49
-6
darshan-runtime/lib/darshan-mpiio.c
darshan-runtime/lib/darshan-mpiio.c
+1
-1
darshan-runtime/lib/darshan-posix.c
darshan-runtime/lib/darshan-posix.c
+1
-1
No files found.
darshan-runtime/darshan-core.h
View file @
03053837
...
@@ -48,9 +48,6 @@
...
@@ -48,9 +48,6 @@
/* default name record buf can store 2048 records of size 100 bytes */
/* default name record buf can store 2048 records of size 100 bytes */
#define DARSHAN_NAME_RECORD_BUF_SIZE (2048 * 100)
#define DARSHAN_NAME_RECORD_BUF_SIZE (2048 * 100)
/* Default runtime compression buffer size */
#define DARSHAN_COMP_BUF_SIZE DARSHAN_MOD_MEM_MAX
/* structure to track registered modules */
/* structure to track registered modules */
struct
darshan_core_module
struct
darshan_core_module
{
{
...
...
darshan-runtime/lib/darshan-core.c
View file @
03053837
...
@@ -156,8 +156,9 @@ void darshan_core_initialize(int argc, char **argv)
...
@@ -156,8 +156,9 @@ void darshan_core_initialize(int argc, char **argv)
char
*
jobid_str
;
char
*
jobid_str
;
int
jobid
;
int
jobid
;
int
ret
;
int
ret
;
int
tmpval
;
int
i
;
int
i
;
int
tmpval
;
double
tmpfloat
;
DARSHAN_MPI_CALL
(
PMPI_Comm_size
)(
MPI_COMM_WORLD
,
&
nprocs
);
DARSHAN_MPI_CALL
(
PMPI_Comm_size
)(
MPI_COMM_WORLD
,
&
nprocs
);
DARSHAN_MPI_CALL
(
PMPI_Comm_rank
)(
MPI_COMM_WORLD
,
&
my_rank
);
DARSHAN_MPI_CALL
(
PMPI_Comm_rank
)(
MPI_COMM_WORLD
,
&
my_rank
);
...
@@ -219,11 +220,11 @@ void darshan_core_initialize(int argc, char **argv)
...
@@ -219,11 +220,11 @@ void darshan_core_initialize(int argc, char **argv)
envstr
=
getenv
(
DARSHAN_MOD_MEM_OVERRIDE
);
envstr
=
getenv
(
DARSHAN_MOD_MEM_OVERRIDE
);
if
(
envstr
)
if
(
envstr
)
{
{
ret
=
sscanf
(
envstr
,
"%
d"
,
&
tmpval
);
ret
=
sscanf
(
envstr
,
"%
lf"
,
&
tmpfloat
);
/* silently ignore if the env variable is set poorly */
/* silently ignore if the env variable is set poorly */
if
(
ret
==
1
&&
tmp
val
>
0
)
if
(
ret
==
1
&&
tmp
float
>
0
)
{
{
darshan_mod_mem_quota
=
tmp
val
*
1024
*
1024
;
/* convert from MiB */
darshan_mod_mem_quota
=
tmp
float
*
1024
*
1024
;
/* convert from MiB */
}
}
}
}
...
@@ -404,7 +405,7 @@ void darshan_core_shutdown()
...
@@ -404,7 +405,7 @@ void darshan_core_shutdown()
final_core
->
log_job_p
->
end_time
=
last_end_time
;
final_core
->
log_job_p
->
end_time
=
last_end_time
;
}
}
final_core
->
comp_buf
=
malloc
(
DARSHAN_COMP_BUF_SIZE
);
final_core
->
comp_buf
=
malloc
(
darshan_mod_mem_quota
);
if
(
!
(
final_core
->
comp_buf
))
if
(
!
(
final_core
->
comp_buf
))
{
{
darshan_core_cleanup
(
final_core
);
darshan_core_cleanup
(
final_core
);
...
@@ -1557,7 +1558,7 @@ static int darshan_deflate_buffer(void **pointers, int *lengths, int count,
...
@@ -1557,7 +1558,7 @@ static int darshan_deflate_buffer(void **pointers, int *lengths, int count,
}
}
tmp_stream
.
next_out
=
(
unsigned
char
*
)
comp_buf
;
tmp_stream
.
next_out
=
(
unsigned
char
*
)
comp_buf
;
tmp_stream
.
avail_out
=
DARSHAN_COMP_BUF_SIZE
;
tmp_stream
.
avail_out
=
darshan_mod_mem_quota
;
/* loop over the input pointers */
/* loop over the input pointers */
for
(
i
=
0
;
i
<
count
;
i
++
)
for
(
i
=
0
;
i
<
count
;
i
++
)
...
...
darshan-runtime/lib/darshan-dxt.c
View file @
03053837
...
@@ -35,7 +35,11 @@ typedef int64_t off64_t;
...
@@ -35,7 +35,11 @@ typedef int64_t off64_t;
#endif
#endif
/* maximum amount of memory to use for storing DXT records */
/* maximum amount of memory to use for storing DXT records */
#define DXT_IO_TRACE_MEM_MAX (4 * 1024 * 1024)
/* 4 MiB */
#ifdef __DARSHAN_MOD_MEM_MAX
#define DXT_IO_TRACE_MEM_MAX (__DARSHAN_MOD_MEM_MAX * 1024 * 1024)
#else
#define DXT_IO_TRACE_MEM_MAX (4 * 1024 * 1024)
/* 4 MiB default */
#endif
/* initial size of read/write trace buffer (in number of segments) */
/* initial size of read/write trace buffer (in number of segments) */
/* NOTE: when this size is exceeded, the buffer size is doubled */
/* NOTE: when this size is exceeded, the buffer size is doubled */
...
@@ -137,7 +141,8 @@ static pthread_mutex_t dxt_runtime_mutex =
...
@@ -137,7 +141,8 @@ static pthread_mutex_t dxt_runtime_mutex =
PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
;
PTHREAD_RECURSIVE_MUTEX_INITIALIZER_NP
;
static
int
dxt_my_rank
=
-
1
;
static
int
dxt_my_rank
=
-
1
;
static
int
dxt_mem_remaining
=
DXT_IO_TRACE_MEM_MAX
;
static
int
dxt_total_mem
=
DXT_IO_TRACE_MEM_MAX
;
static
int
dxt_mem_remaining
=
0
;
#define DXT_LOCK() pthread_mutex_lock(&dxt_runtime_mutex)
#define DXT_LOCK() pthread_mutex_lock(&dxt_runtime_mutex)
#define DXT_UNLOCK() pthread_mutex_unlock(&dxt_runtime_mutex)
#define DXT_UNLOCK() pthread_mutex_unlock(&dxt_runtime_mutex)
...
@@ -375,6 +380,9 @@ static void dxt_posix_runtime_initialize()
...
@@ -375,6 +380,9 @@ static void dxt_posix_runtime_initialize()
* over realloc'ing module memory as needed.
* over realloc'ing module memory as needed.
*/
*/
int
dxt_psx_buf_size
=
0
;
int
dxt_psx_buf_size
=
0
;
int
ret
;
double
tmpfloat
;
char
*
envstr
;
/* register the DXT module with darshan core */
/* register the DXT module with darshan core */
darshan_core_register_module
(
darshan_core_register_module
(
...
@@ -391,14 +399,30 @@ static void dxt_posix_runtime_initialize()
...
@@ -391,14 +399,30 @@ static void dxt_posix_runtime_initialize()
return
;
return
;
}
}
DXT_LOCK
();
dxt_posix_runtime
=
malloc
(
sizeof
(
*
dxt_posix_runtime
));
dxt_posix_runtime
=
malloc
(
sizeof
(
*
dxt_posix_runtime
));
if
(
!
dxt_posix_runtime
)
if
(
!
dxt_posix_runtime
)
{
{
darshan_core_unregister_module
(
DXT_POSIX_MOD
);
darshan_core_unregister_module
(
DXT_POSIX_MOD
);
DXT_UNLOCK
();
return
;
return
;
}
}
memset
(
dxt_posix_runtime
,
0
,
sizeof
(
*
dxt_posix_runtime
));
memset
(
dxt_posix_runtime
,
0
,
sizeof
(
*
dxt_posix_runtime
));
/* set the memory quota for DXT, if it has not been initialized */
envstr
=
getenv
(
"ENABLE_DXT_IO_TRACE_MEM"
);
if
(
envstr
&&
dxt_mpiio_runtime
==
NULL
)
{
ret
=
sscanf
(
envstr
,
"%lf"
,
&
tmpfloat
);
/* silently ignore if the env variable is set poorly */
if
(
ret
==
1
&&
tmpfloat
>
0
)
{
dxt_total_mem
=
tmpfloat
*
1024
*
1024
;
/* convert from MiB */
}
}
dxt_mem_remaining
=
dxt_total_mem
;
DXT_UNLOCK
();
return
;
return
;
}
}
...
@@ -409,6 +433,9 @@ void dxt_mpiio_runtime_initialize()
...
@@ -409,6 +433,9 @@ void dxt_mpiio_runtime_initialize()
* over realloc'ing module memory as needed.
* over realloc'ing module memory as needed.
*/
*/
int
dxt_mpiio_buf_size
=
0
;
int
dxt_mpiio_buf_size
=
0
;
int
ret
;
double
tmpfloat
;
char
*
envstr
;
/* register the DXT module with darshan core */
/* register the DXT module with darshan core */
darshan_core_register_module
(
darshan_core_register_module
(
...
@@ -425,14 +452,30 @@ void dxt_mpiio_runtime_initialize()
...
@@ -425,14 +452,30 @@ void dxt_mpiio_runtime_initialize()
return
;
return
;
}
}
DXT_LOCK
();
dxt_mpiio_runtime
=
malloc
(
sizeof
(
*
dxt_mpiio_runtime
));
dxt_mpiio_runtime
=
malloc
(
sizeof
(
*
dxt_mpiio_runtime
));
if
(
!
dxt_mpiio_runtime
)
if
(
!
dxt_mpiio_runtime
)
{
{
darshan_core_unregister_module
(
DXT_MPIIO_MOD
);
darshan_core_unregister_module
(
DXT_MPIIO_MOD
);
DXT_UNLOCK
();
return
;
return
;
}
}
memset
(
dxt_mpiio_runtime
,
0
,
sizeof
(
*
dxt_mpiio_runtime
));
memset
(
dxt_mpiio_runtime
,
0
,
sizeof
(
*
dxt_mpiio_runtime
));
/* set the memory quota for DXT, if it has not been initialized */
envstr
=
getenv
(
"ENABLE_DXT_IO_TRACE_MEM"
);
if
(
envstr
&&
dxt_posix_runtime
==
NULL
)
{
ret
=
sscanf
(
envstr
,
"%lf"
,
&
tmpfloat
);
/* silently ignore if the env variable is set poorly */
if
(
ret
==
1
&&
tmpfloat
>
0
)
{
dxt_total_mem
=
tmpfloat
*
1024
*
1024
;
/* convert from MiB */
}
}
dxt_mem_remaining
=
dxt_total_mem
;
DXT_UNLOCK
();
return
;
return
;
}
}
...
@@ -682,10 +725,10 @@ static void dxt_posix_shutdown(
...
@@ -682,10 +725,10 @@ static void dxt_posix_shutdown(
*
dxt_posix_buf_sz
=
0
;
*
dxt_posix_buf_sz
=
0
;
dxt_posix_runtime
->
record_buf
=
malloc
(
DXT_IO_TRACE_MEM_MAX
);
dxt_posix_runtime
->
record_buf
=
malloc
(
dxt_total_mem
);
if
(
!
(
dxt_posix_runtime
->
record_buf
))
if
(
!
(
dxt_posix_runtime
->
record_buf
))
return
;
return
;
memset
(
dxt_posix_runtime
->
record_buf
,
0
,
DXT_IO_TRACE_MEM_MAX
);
memset
(
dxt_posix_runtime
->
record_buf
,
0
,
dxt_total_mem
);
dxt_posix_runtime
->
record_buf_size
=
0
;
dxt_posix_runtime
->
record_buf_size
=
0
;
/* iterate all dxt posix records and serialize them to the output buffer */
/* iterate all dxt posix records and serialize them to the output buffer */
...
@@ -792,10 +835,10 @@ static void dxt_mpiio_shutdown(
...
@@ -792,10 +835,10 @@ static void dxt_mpiio_shutdown(
*
dxt_mpiio_buf_sz
=
0
;
*
dxt_mpiio_buf_sz
=
0
;
dxt_mpiio_runtime
->
record_buf
=
malloc
(
DXT_IO_TRACE_MEM_MAX
);
dxt_mpiio_runtime
->
record_buf
=
malloc
(
dxt_total_mem
);
if
(
!
(
dxt_mpiio_runtime
->
record_buf
))
if
(
!
(
dxt_mpiio_runtime
->
record_buf
))
return
;
return
;
memset
(
dxt_mpiio_runtime
->
record_buf
,
0
,
DXT_IO_TRACE_MEM_MAX
);
memset
(
dxt_mpiio_runtime
->
record_buf
,
0
,
dxt_total_mem
);
dxt_mpiio_runtime
->
record_buf_size
=
0
;
dxt_mpiio_runtime
->
record_buf_size
=
0
;
/* iterate all dxt posix records and serialize them to the output buffer */
/* iterate all dxt posix records and serialize them to the output buffer */
...
...
darshan-runtime/lib/darshan-mpiio.c
View file @
03053837
...
@@ -862,7 +862,7 @@ static void mpiio_runtime_initialize()
...
@@ -862,7 +862,7 @@ static void mpiio_runtime_initialize()
memset
(
mpiio_runtime
,
0
,
sizeof
(
*
mpiio_runtime
));
memset
(
mpiio_runtime
,
0
,
sizeof
(
*
mpiio_runtime
));
/* check if DXT (Darshan extended tracing) should be enabled */
/* check if DXT (Darshan extended tracing) should be enabled */
if
(
getenv
(
"ENABLE_DXT_IO_TRACE"
))
{
if
(
getenv
(
"ENABLE_DXT_IO_TRACE
_MEM
"
))
{
enable_dxt_io_trace
=
1
;
enable_dxt_io_trace
=
1
;
}
}
...
...
darshan-runtime/lib/darshan-posix.c
View file @
03053837
...
@@ -1291,7 +1291,7 @@ static void posix_runtime_initialize()
...
@@ -1291,7 +1291,7 @@ static void posix_runtime_initialize()
memset
(
posix_runtime
,
0
,
sizeof
(
*
posix_runtime
));
memset
(
posix_runtime
,
0
,
sizeof
(
*
posix_runtime
));
/* check if DXT (Darshan extended tracing) should be enabled */
/* check if DXT (Darshan extended tracing) should be enabled */
if
(
getenv
(
"ENABLE_DXT_IO_TRACE"
))
{
if
(
getenv
(
"ENABLE_DXT_IO_TRACE
_MEM
"
))
{
enable_dxt_io_trace
=
1
;
enable_dxt_io_trace
=
1
;
}
}
...
...
Write
Preview
Markdown
is supported
0%
Try again
or
attach a new file
Attach a file
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment