cffi_backend.py 13.8 KB
Newer Older
1
# -*- coding: utf-8 -*-
Kevin Harms's avatar
Kevin Harms committed
2
3
4
5
"""The cfii_backend package will read a darshan log
using the functions defined in libdarshan-util.so
and is interfaced via the python CFFI module.
"""
6
7

import cffi
8
9
import ctypes

10
11
12
import numpy as np
import pandas as pd

13
14
15
16
import logging
logger = logging.getLogger(__name__)


Kevin Harms's avatar
Kevin Harms committed
17
from darshan.backend.api_def_c import load_darshan_header
18
19
from darshan.discover_darshan import find_utils
from darshan.discover_darshan import check_version
20
21
22
23
24
25

API_def_c = load_darshan_header()

ffi = cffi.FFI()
ffi.cdef(API_def_c)

26
27
28
libdutil = None
libdutil = find_utils(ffi, libdutil)

29
check_version(ffi, libdutil)
30
31
32



33
34
35
36
37
38
39
40
41
42
43
44
45
46
47
_structdefs = {
    "BG/Q": "struct darshan_bgq_record **",
    "DXT_MPIIO": "struct dxt_file_record **",
    "DXT_POSIX": "struct dxt_file_record **",
    "H5F": "struct darshan_hdf5_file **",
    "H5D": "struct darshan_hdf5_dataset **",
    "LUSTRE": "struct darshan_lustre_record **",
    "MPI-IO": "struct darshan_mpiio_file **",
    "PNETCDF": "struct darshan_pnetcdf_file **",
    "POSIX": "struct darshan_posix_file **",
    "STDIO": "struct darshan_stdio_file **",
}



48
49
50
51
52
53
54
55
56
57
58
59
60
61
def get_lib_version():
    """
    Return the version information hardcoded into the shared library.
    
    Args:
        None
        
    Return:
        version (str): library version number
    """
    ver = ffi.new("char **")
    ver = libdutil.darshan_log_get_lib_version()
    version = ffi.string(ver).decode("utf-8")
    return version
62

63

64
65
66
67
68
69
70
71
72
73
74
def log_open(filename):
    """
    Opens a darshan logfile.

    Args:
        filename (str): Path to a darshan log file

    Return:
        log handle
    """
    b_fname = filename.encode()
75
76
    handle = libdutil.darshan_log_open(b_fname)
    log = {"handle": handle, 'modules': None, 'name_records': None}
77
78
79
80
81
82
83
84

    return log


def log_close(log):
    """
    Closes the logfile and releases allocated memory.
    """
85
    libdutil.darshan_log_close(log['handle'])
Jakob Luettgau's avatar
Jakob Luettgau committed
86
    #modules = {}
87
88
89
90
91
92
93
    return


def log_get_job(log):
    """
    Returns a dictionary with information about the current job.
    """
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
94

95
    job = {}
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
96

97
    jobrec = ffi.new("struct darshan_job *")
98
    libdutil.darshan_log_get_job(log['handle'], jobrec)
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
99

100
101
102
    job['uid'] = jobrec[0].uid
    job['start_time'] = jobrec[0].start_time
    job['end_time'] = jobrec[0].end_time
103
104
    job['nprocs'] = jobrec[0].nprocs
    job['jobid'] = jobrec[0].jobid
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
105

106
107
    mstr = ffi.string(jobrec[0].metadata).decode("utf-8")
    md = {}
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
108

109
110
111
    for kv in mstr.split('\n')[:-1]:
        k,v = kv.split('=', maxsplit=1)
        md[k] = v
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
112

113
114
115
116
117
118
119
120
121
122
123
124
125
126
127
128
    job['metadata'] = md
    return job


def log_get_exe(log):
    """
    Get details about the executable (path and arguments)

    Args:
        log: handle returned by darshan.open

    Return:
        string: executeable path and arguments
    """

    exestr = ffi.new("char[]", 4096)
129
    libdutil.darshan_log_get_exe(log['handle'], exestr)
130
131
132
133
134
135
136
137
138
139
    return ffi.string(exestr).decode("utf-8")


def log_get_mounts(log):
    """
    Returns a list of available mounts recorded for the log.

    Args:
        log: handle returned by darshan.open
    """
140

141
142
143
    mntlst = []
    mnts = ffi.new("struct darshan_mnt_info **")
    cnt = ffi.new("int *")
144
    libdutil.darshan_log_get_mounts(log['handle'], mnts, cnt)
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
145

146
147
148
    for i in range(0, cnt[0]):
        mntlst.append((ffi.string(mnts[0][i].mnt_path).decode("utf-8"),
            ffi.string(mnts[0][i].mnt_type).decode("utf-8")))
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
149

150
151
152
153
154
155
156
157
158
159
160
161
162
163
164
    return mntlst


def log_get_modules(log):
    """
    Return a dictionary containing available modules including information 
    about the contents available for each module in the current log.

    Args:
        log: handle returned by darshan.open

    Return:
        dict: Modules with additional info for current log.

    """
165

Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
166
    # use cached module index if already present
167
168
169
    if log['modules'] != None:
        return log['modules']

Jakob Luettgau's avatar
Jakob Luettgau committed
170
171
    modules = {}

172
173
    mods = ffi.new("struct darshan_mod_info **")
    cnt    = ffi.new("int *")
174
    libdutil.darshan_log_get_modules(log['handle'], mods, cnt)
175
176
177
    for i in range(0, cnt[0]):
        modules[ffi.string(mods[0][i].name).decode("utf-8")] = \
                {'len': mods[0][i].len, 'ver': mods[0][i].ver, 'idx': mods[0][i].idx}
178
179
180
181

    # add to cache
    log['modules'] = modules

182
183
184
    return modules


185
186
187
188
189
190
191
192
193
194
195
def log_get_name_records(log):
    """
    Return a dictionary resovling hash to string (typically a filepath).

    Args:
        log: handle returned by darshan.open
        hash: hash-value (a number)

    Return:
        dict: the name records
    """
196
197
198
199
200
201

    # used cached name_records if already present
    if log['name_records'] != None:
        return log['name_records']


202
203
204
205
    name_records = {}

    nrecs = ffi.new("struct darshan_name_record **")
    cnt = ffi.new("int *")
206
    libdutil.darshan_log_get_name_records(log['handle'], nrecs, cnt)
207
208
209
210

    for i in range(0, cnt[0]):
        name_records[nrecs[0][i].id] = ffi.string(nrecs[0][i].name).decode("utf-8")

211
212
213
214

    # add to cache
    log['name_records'] = name_records

215
216
217
218
    return name_records



219
220
221
222
223
224
225
226
227
228
229
230
231
232
233
234
235
236
237
238
239
240
241
242
243
244
245
246
247
248
249
250
251
252
253
def log_lookup_name_records(log, ids=[]):
    """
    Resolve a single hash to it's name record string (typically a filepath).

    Args:
        log: handle returned by darshan.open
        hash: hash-value (a number)

    Return:
        dict: the name records
    """

    name_records = {}

    #cids = ffi.new("darshan_record_id *") * len(ids)
    whitelist = (ctypes.c_ulonglong * len(ids))(*ids)
    whitelist_cnt = len(ids)

    whitelistp = ffi.from_buffer(whitelist)

    nrecs = ffi.new("struct darshan_name_record **")
    cnt = ffi.new("int *")
    libdutil.darshan_log_get_filtered_name_records(log['handle'], nrecs, cnt, ffi.cast("darshan_record_id *", whitelistp), whitelist_cnt)

    for i in range(0, cnt[0]):
        name_records[nrecs[0][i].id] = ffi.string(nrecs[0][i].name).decode("utf-8")

    # add to cache
    log['name_records'] = name_records

    return name_records




254

255
def log_get_record(log, mod, dtype='numpy'):
256
    """
257
    Standard entry point fetch records via mod string.
258
259
260
261
262
263

    Args:
        log: Handle returned by darshan.open
        mod_name (str): Name of the Darshan module

    Return:
264
265
        log record of type dtype
    
266
267
    """

268
269
270
    if mod in ['LUSTRE']:
        rec = _log_get_lustre_record(log, dtype=dtype)
    elif mod in ['DXT_POSIX', 'DXT_MPIIO']:
Kevin Harms's avatar
Kevin Harms committed
271
        rec = log_get_dxt_record(log, mod, dtype=dtype)
272
    else:
Kevin Harms's avatar
Kevin Harms committed
273
        rec = log_get_generic_record(log, mod, dtype=dtype)
274

275
    return rec
276
277


278

Kevin Harms's avatar
Kevin Harms committed
279
def log_get_generic_record(log, mod_name, dtype='numpy'):
280
281
    """
    Returns a dictionary holding a generic darshan log record.
282

283
284
285
286
    Args:
        log: Handle returned by darshan.open
        mod_name (str): Name of the Darshan module

287
288
    Return:
        dict: generic log record
289

290
    Example:
291

292
293
294
295
    The typical darshan log record provides two arrays, on for integer counters
    and one for floating point counters:

    >>> darshan.log_get_generic_record(log, "POSIX", "struct darshan_posix_file **")
296
    {'counters': array([...], dtype=int64), 'fcounters': array([...])}
297

298
    """
Jakob Luettgau's avatar
Jakob Luettgau committed
299
    modules = log_get_modules(log)
Kevin Harms's avatar
Kevin Harms committed
300
    mod_type = _structdefs[mod_name]
Jakob Luettgau's avatar
Jakob Luettgau committed
301

302
303
    rec = {}
    buf = ffi.new("void **")
304
    r = libdutil.darshan_log_get_record(log['handle'], modules[mod_name]['idx'], buf)
305
306
307
308
    if r < 1:
        return None
    rbuf = ffi.cast(mod_type, buf)

309
310
    rec['id'] = rbuf[0].base_rec.id
    rec['rank'] = rbuf[0].base_rec.rank
311
312
    if mod_name == 'H5D':
        rec['file_rec_id'] = rbuf[0].file_rec_id
313

314
    clst = []
315
316
    for i in range(0, len(rbuf[0].counters)):
        clst.append(rbuf[0].counters[i])
317
318
    rec['counters'] = np.array(clst, dtype=np.int64)
    cdict = dict(zip(counter_names(mod_name), rec['counters']))
319

320
    flst = []
321
322
    for i in range(0, len(rbuf[0].fcounters)):
        flst.append(rbuf[0].fcounters[i])
323
324
    rec['fcounters'] = np.array(flst, dtype=np.float64)
    fcdict = dict(zip(fcounter_names(mod_name), rec['fcounters']))
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
325

326
    if dtype == "dict":
327
328
329
330
        rec.update({
            'counters': cdict, 
            'fcounters': fcdict
            })
331

332
    if dtype == "pandas":
333
334
335
        df_c = pd.DataFrame(cdict, index=[0])
        df_fc = pd.DataFrame(fcdict, index=[0])

336
        # flip column order (to prepend id and rank)
337
338
339
        df_c = df_c[df_c.columns[::-1]]
        df_fc = df_fc[df_fc.columns[::-1]]

340
        # attach id and rank to counters and fcounters
341
342
343
344
345
346
347
348
349
350
        df_c['id'] = rec['id']
        df_c['rank'] = rec['rank']

        df_fc['id'] = rec['id']
        df_fc['rank'] = rec['rank']

        # flip column order
        df_c = df_c[df_c.columns[::-1]]
        df_fc = df_fc[df_fc.columns[::-1]]

351
        rec.update({
352
353
            'counters': df_c,
            'fcounters': df_fc
354
            })
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
355

356
357
358
359
360
361
362
363
364
365
366
367
368
369
370
371
372
    return rec


def counter_names(mod_name, fcnts=False):
    """
    Returns a list of available counter names for the module.
    By default only integer counter names are listed, unless fcnts is set to
    true in which case only the floating point counter names are listed.

    Args:
        mod_name (str): Name of the module to return counter names.
        fcnts (bool): Switch to request floating point counters instead of integer. (Default: False)

    Return:
        list: Counter names as strings.

    """
373
374
375
376

    if mod_name == 'MPI-IO':
        mod_name = 'MPIIO'

377
378
    names = []
    i = 0
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
379

380
381
382
383
    if fcnts:
        F = "f_"
    else:
        F = ""
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
384

385
386
    end = "{0}_{1}NUM_INDICES".format(mod_name.upper(), F.upper())
    var_name = "{0}_{1}counter_names".format(mod_name.lower(), F.lower())
Jakob Luettgau's avatar
PEP8.    
Jakob Luettgau committed
387

388
389
390
391
392
393
394
395
396
397
398
399
400
401
402
403
404
405
406
407
408
409
410
411
412
413
414
415
416
    while True: 
        try:
            var = getattr(libdutil, var_name)
        except:
            var = None
        if not var:
            return None
        name = ffi.string(var[i]).decode("utf-8")
        if name == end:
            break
        names.append(name)
        i += 1
    return names


def fcounter_names(mod_name):
    """
    Returns a list of available floating point counter names for the module.

    Args:
        mod_name (str): Name of the module to return counter names.

    Return:
        list: Available floiting point counter names as strings.

    """
    return counter_names(mod_name, fcnts=True)


417
def _log_get_lustre_record(log, dtype='numpy'):
418
419
420
421
422
423
424
425
426
427
428
429
430
431
432
433
434
435
436
437
438
439
    """
    Returns a darshan log record for Lustre.

    Args:
        log: handle returned by darshan.open
    """
    modules = log_get_modules(log)

    rec = {}
    buf = ffi.new("void **")
    r = libdutil.darshan_log_get_record(log['handle'], modules['LUSTRE']['idx'], buf)
    if r < 1:
        return None
    rbuf = ffi.cast("struct darshan_lustre_record **", buf)

    rec['id'] = rbuf[0].base_rec.id
    rec['rank'] = rbuf[0].base_rec.rank

    clst = []
    for i in range(0, len(rbuf[0].counters)):
        clst.append(rbuf[0].counters[i])
    rec['counters'] = np.array(clst, dtype=np.int64)
440
441
   
    # counters
442
443
    cdict = dict(zip(counter_names('LUSTRE'), rec['counters']))

444
445
446
447
448
449
450
    # ost_ids 
    sizeof_64 = ffi.sizeof("int64_t")
    sizeof_base = ffi.sizeof("struct darshan_base_record")
    offset = sizeof_base + sizeof_64 * len(rbuf[0].counters)
    offset = int(offset/sizeof_64)

    ost_ids = ffi.cast("int64_t *", rbuf[0])
451
    ostlst = []
452
453
    for i in range(offset, cdict['LUSTRE_STRIPE_WIDTH']+offset):
        ostlst.append(ost_ids[i])
454
455
456
    rec['ost_ids'] = np.array(ostlst, dtype=np.int64)


457
    # dtype conversion
458
459
460
    if dtype == "dict":
        rec.update({
            'counters': cdict, 
461
            'ost_ids': ostlst
462
            })
463

464
    if dtype == "pandas":
465
466
467
468
469
470
471
472
        df_c = pd.DataFrame(cdict, index=[0])

        # prepend id and rank
        df_c = df_c[df_c.columns[::-1]] # flip colum order
        df_c['id'] = rec['id']
        df_c['rank'] = rec['rank']
        df_c = df_c[df_c.columns[::-1]] # flip back

473
        rec.update({
474
475
            'counters': df_c,
            'ost_ids': pd.DataFrame(rec['ost_ids'], columns=['ost_ids']),
476
            })
477
478

    return rec
479

480
481


Kevin Harms's avatar
Kevin Harms committed
482
def log_get_dxt_record(log, mod_name, reads=True, writes=True, dtype='dict'):
483
    """
484
    Returns a dictionary holding a dxt darshan log record.
485
486

    Args:
487
488
489
        log: Handle returned by darshan.open
        mod_name (str): Name of the Darshan module
        mod_type (str): String containing the C type
490

491
492
    Return:
        dict: generic log record
493

494
    Example:
495

496
497
498
499
500
    The typical darshan log record provides two arrays, on for integer counters
    and one for floating point counters:

    >>> darshan.log_get_dxt_record(log, "DXT_POSIX", "struct dxt_file_record **")
    {'rank': 0, 'read_count': 11, 'read_segments': array([...]), ...}
501
502
503
504


    """

505
    modules = log_get_modules(log)
Kevin Harms's avatar
Kevin Harms committed
506
    mod_type = _structdefs[mod_name]
507
    #name_records = log_get_name_records(log)
508

509
510
511
512
513
514
515
    rec = {}
    buf = ffi.new("void **")
    r = libdutil.darshan_log_get_record(log['handle'], modules[mod_name]['idx'], buf)
    if r < 1:
        return None
    filerec = ffi.cast(mod_type, buf)
    clst = []
516

517
518
519
520
    rec['id'] = filerec[0].base_rec.id
    rec['rank'] = filerec[0].base_rec.rank
    rec['hostname'] = ffi.string(filerec[0].hostname).decode("utf-8")
    #rec['filename'] = name_records[rec['id']]
521

522
523
    wcnt = filerec[0].write_count
    rcnt = filerec[0].read_count
524

525
526
527
528
529
    rec['write_count'] = wcnt
    rec['read_count'] = rcnt
 
    rec['write_segments'] = []
    rec['read_segments'] = []
530

531

532
533
534
535
536
537
538
539
540
541
542
543
544
545
546
547
548
549
550
551
552
553
554
555
556
557
558
559
560
561
562
    size_of = ffi.sizeof("struct dxt_file_record")
    segments = ffi.cast("struct segment_info *", buf[0] + size_of  )


    for i in range(wcnt):
        seg = {
            "offset": segments[i].offset,
            "length": segments[i].length,
            "start_time": segments[i].start_time,
            "end_time": segments[i].end_time
        }
        rec['write_segments'].append(seg)


    for i in range(rcnt):
        i = i + wcnt
        seg = {
            "offset": segments[i].offset,
            "length": segments[i].length,
            "start_time": segments[i].start_time,
            "end_time": segments[i].end_time
        }
        rec['read_segments'].append(seg)


    if dtype == "pandas":
        rec['read_segments'] = pd.DataFrame(rec['read_segments'])
        rec['write_segments'] = pd.DataFrame(rec['write_segments'])

    return rec

563
564


565