Skip to content
GitLab
Projects
Groups
Snippets
/
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
sds
py-bake
Commits
3fb096ac
Commit
3fb096ac
authored
Feb 22, 2021
by
Matthieu Dorier
Browse files
adapted to API of bake 0.6
parent
d935ded0
Changes
4
Hide whitespace changes
Inline
Side-by-side
pybake/server.py
View file @
3fb096ac
...
...
@@ -5,9 +5,6 @@ import _pybakeserver
import
pymargo
from
pybake.target
import
BakeTargetID
def
make_pool
(
name
,
size
,
mode
):
_pybakeserver
.
make_pool
(
name
,
size
,
mode
)
class
BakeProvider
(
pymargo
.
Provider
):
"""
The BakeProvide class wraps a C-level bake_provider_t object.
...
...
@@ -20,52 +17,47 @@ class BakeProvider(pymargo.Provider):
super
(
BakeProvider
,
self
).
__init__
(
engine
,
provider_id
)
self
.
_provider
=
_pybakeserver
.
register
(
engine
.
_mid
,
provider_id
)
def
add_storage_target
(
self
,
path
):
def
create_target
(
self
,
path
,
size
):
"""
Create a storage target and attach it to the provider.
Returns a BakeTargetID instance that can be used to access the storage target.
"""
tid
=
_pybakeserver
.
create_target
(
self
.
_provider
,
path
,
size
)
return
BakeTargetID
(
tid
)
def
attach_target
(
self
,
path
):
"""
Adds a storage target to the provider.
Returns a BakeTargetID instance that can be used to access the storage target.
"""
tid
=
_pybakeserver
.
a
dd_storage
_target
(
self
.
_provider
,
path
)
tid
=
_pybakeserver
.
a
ttach
_target
(
self
.
_provider
,
path
)
return
BakeTargetID
(
tid
)
def
remove_storage
_target
(
self
,
target
):
def
detach
_target
(
self
,
target
):
"""
Removes a storage target from the provider. This does not delete the underlying file.
The target argument must be a BakeTargetID object.
"""
_pybakeserver
.
remove_storage
_target
(
self
.
_provider
,
target
.
_tid
)
_pybakeserver
.
detach
_target
(
self
.
_provider
,
target
.
_tid
)
def
remove_all_storage
_targets
(
self
):
def
detach_all
_targets
(
self
):
"""
Removes all the storage targets managed by this provider.
"""
_pybakeserver
.
remove_all_storage
_targets
(
self
.
_provider
)
_pybakeserver
.
detach_all
_targets
(
self
.
_provider
)
def
count_
storage_
targets
(
self
):
def
count_targets
(
self
):
"""
Returns the number of storage targets that this provider manages.
"""
return
_pybakeserver
.
count_
storage_
targets
(
self
.
_provider
)
return
_pybakeserver
.
count_targets
(
self
.
_provider
)
def
list_
storage_
targets
(
self
):
def
list_targets
(
self
):
"""
Returns the list of storage targets (BakeTargetIDs) that this provider manages.
"""
l
=
_pybakeserver
.
list_
storage_
targets
(
self
.
_provider
)
l
=
_pybakeserver
.
list_targets
(
self
.
_provider
)
if
(
l
is
None
):
return
[]
else
:
return
[
BakeTargetID
(
tid
)
for
tid
in
l
]
# def set_target_xfer_buffer(self, target, count, size):
# """
# Sets the number and size of intermediate buffers that can be used to
# execute transfers to a specific target.
# """
# _pybakeserver.set_target_xfer_buffer(self._provider, target._tid, count, size)
# def set_target_xfer_concurrency(self, target, num_threads):
# """
# Sets the number of ULTs that can be used to execute transfers concurrently.
# """
# _pybakeserver.set_target_xfer_concurrency(self._provider, target._tid, num_threads)
pybake/src/server.cpp
View file @
3fb096ac
/*
* (C) 2018 The University of Chicago
*
*
* See COPYRIGHT in top-level directory.
*/
#include
<pybind11/pybind11.h>
...
...
@@ -32,98 +32,82 @@ typedef py11::capsule pybake_provider_t;
#define ADDR2CAPSULE(__addr) py11::capsule((void*)(__addr), "hg_addr_t", nullptr)
#define BAKEPR2CAPSULE(__bpr) py11::capsule((void*)(__bpr), "bake_provider_t", nullptr)
static
pybake_provider_t
pybake_provider_register
(
pymargo_instance_id
mid
,
uint8_t
provider_id
)
{
static
pybake_provider_t
pybake_provider_register
(
pymargo_instance_id
mid
,
uint8_t
provider_id
)
{
bake_provider_t
provider
;
int
ret
=
bake_provider_register
(
mid
,
provider_id
,
BAKE_ABT_POOL_DEFAULT
,
&
provider
);
int
ret
=
bake_provider_register
(
mid
,
provider_id
,
NULL
,
&
provider
);
HANDLE_ERROR
(
bake_provider_register
,
ret
);
return
BAKEPR2CAPSULE
(
provider
);
}
static
py11
::
object
pybake_provider_a
dd_storage
_target
(
static
py11
::
object
pybake_provider_a
ttach
_target
(
pybake_provider_t
provider
,
const
std
::
string
&
target_name
)
{
bake_target_id_t
target_id
;
std
::
memset
(
&
target_id
,
0
,
sizeof
(
target_id
));
int
ret
=
bake_provider_a
dd_storage
_target
(
int
ret
=
bake_provider_a
ttach
_target
(
provider
,
target_name
.
c_str
(),
&
target_id
);
HANDLE_ERROR
(
bake_provider_a
dd_storage
_target
,
ret
);
HANDLE_ERROR
(
bake_provider_a
ttach
_target
,
ret
);
return
py11
::
cast
(
target_id
);
}
static
void
pybake_provider_remove_storage_target
(
static
py11
::
object
pybake_provider_create_target
(
pybake_provider_t
provider
,
const
std
::
string
&
target_name
,
size_t
target_size
)
{
bake_target_id_t
target_id
;
std
::
memset
(
&
target_id
,
0
,
sizeof
(
target_id
));
int
ret
=
bake_provider_create_target
(
provider
,
target_name
.
c_str
(),
target_size
,
&
target_id
);
HANDLE_ERROR
(
bake_provider_destroy_target
,
ret
);
return
py11
::
cast
(
target_id
);
}
static
void
pybake_provider_detach_target
(
pybake_provider_t
provider
,
bake_target_id_t
target_id
)
{
int
ret
=
bake_provider_
remove_storage
_target
(
provider
,
target_id
);
HANDLE_ERROR
(
bake_provider_
remove_storage
_target
,
ret
);
int
ret
=
bake_provider_
detach
_target
(
provider
,
target_id
);
HANDLE_ERROR
(
bake_provider_
detach
_target
,
ret
);
}
static
void
pybake_provider_
remove_all_storage
_targets
(
static
void
pybake_provider_
detach_all
_targets
(
pybake_provider_t
provider
)
{
int
ret
=
bake_provider_
remove_all_storage
_targets
(
provider
);
HANDLE_ERROR
(
bake_provider_
remove_all_storage
_targets
,
ret
);
int
ret
=
bake_provider_
detach_all
_targets
(
provider
);
HANDLE_ERROR
(
bake_provider_
detach_all
_targets
,
ret
);
}
static
uint64_t
pybake_provider_count_
storage_
targets
(
static
uint64_t
pybake_provider_count_targets
(
pybake_provider_t
provider
)
{
uint64_t
n
=
0
;
int
ret
=
bake_provider_count_
storage_
targets
(
provider
,
&
n
);
HANDLE_ERROR
(
bake_provider_count_
storage_
targets
,
ret
);
int
ret
=
bake_provider_count_targets
(
provider
,
&
n
);
HANDLE_ERROR
(
bake_provider_count_targets
,
ret
);
return
n
;
}
static
py11
::
list
pybake_provider_list_
storage_
targets
(
static
py11
::
list
pybake_provider_list_targets
(
pybake_provider_t
provider
)
{
std
::
vector
<
bake_target_id_t
>
result
;
uint64_t
n
=
pybake_provider_count_
storage_
targets
(
provider
);
uint64_t
n
=
pybake_provider_count_targets
(
provider
);
if
(
n
==
0
)
return
py11
::
list
();
result
.
resize
(
n
);
bake_provider_list_
storage_
targets
(
provider
,
result
.
data
());
bake_provider_list_targets
(
provider
,
result
.
data
());
py11
::
list
list_result
;
for
(
const
auto
&
t
:
result
)
list_result
.
append
(
t
);
return
list_result
;
}
static
void
pybake_make_pool
(
const
std
::
string
&
pool_name
,
size_t
pool_size
,
mode_t
mode
)
{
int
ret
=
bake_makepool
(
pool_name
.
c_str
(),
pool_size
,
mode
);
HANDLE_ERROR
(
bake_makepool
,
ret
);
}
#if 0
static void pybake_provider_set_target_xfer_buffer(
pybake_provider_t provider,
bake_target_id_t target_id,
size_t count,
size_t size) {
int ret = bake_provider_set_target_xfer_buffer(provider, target_id, count, size);
HANDLE_ERROR(bake_provider_set_target_xfer_buffer, ret);
}
static void pybake_provider_set_target_xfer_concurrency(
pybake_provider_t provider,
bake_target_id_t target_id,
uint32_t num_threads) {
int ret = bake_provider_set_target_xfer_concurrency(provider, target_id, num_threads);
HANDLE_ERROR(bake_provider_set_target_xfer_concurrency, ret);
}
#endif
PYBIND11_MODULE
(
_pybakeserver
,
m
)
{
py11
::
module
::
import
(
"_pybaketarget"
);
m
.
def
(
"register"
,
&
pybake_provider_register
);
m
.
def
(
"add_storage_target"
,
&
pybake_provider_add_storage_target
);
m
.
def
(
"remove_storage_target"
,
&
pybake_provider_remove_storage_target
);
m
.
def
(
"remove_all_storage_targets"
,
&
pybake_provider_remove_all_storage_targets
);
m
.
def
(
"count_storage_targets"
,
&
pybake_provider_count_storage_targets
);
m
.
def
(
"list_storage_targets"
,
&
pybake_provider_list_storage_targets
);
m
.
def
(
"make_pool"
,
&
pybake_make_pool
);
#if 0
m.def("set_target_xfer_buffer", &pybake_provider_set_target_xfer_buffer);
m.def("set_target_xfer_concurrency", &pybake_provider_set_target_xfer_concurrency);
#endif
m
.
def
(
"create_target"
,
&
pybake_provider_create_target
);
m
.
def
(
"attach_target"
,
&
pybake_provider_attach_target
);
m
.
def
(
"detach_target"
,
&
pybake_provider_detach_target
);
m
.
def
(
"detach_all_targets"
,
&
pybake_provider_detach_all_targets
);
m
.
def
(
"count_targets"
,
&
pybake_provider_count_targets
);
m
.
def
(
"list_targets"
,
&
pybake_provider_list_targets
);
}
spack.yaml
0 → 100644
View file @
3fb096ac
spack
:
specs
:
-
mochi-bake
-
py-mochi-margo
-
py-pkgconfig
-
py-pybind11
concretization
:
together
test/server.py
View file @
3fb096ac
...
...
@@ -10,25 +10,20 @@ from pybake.server import BakeProvider
mid
=
MargoInstance
(
'tcp'
)
mid
.
enable_remote_shutdown
()
mplex
_id
=
42
print
(
"Server running at address "
+
str
(
mid
.
addr
())
+
" with
mplex
_id="
+
str
(
mplex
_id
))
provider
_id
=
42
print
(
"Server running at address "
+
str
(
mid
.
addr
())
+
" with
provider
_id="
+
str
(
provider
_id
))
provider
=
BakeProvider
(
mid
,
mplex_id
)
#target = provider.add_storage_target("/dev/shm/baketarget")
#print("target id is "+str(target))
#print("number of targets: "+str(provider.count_storage_targets()))
#pybake.server.make_pool("/dev/shm/baketarget2", 2*8388608, 0o664)
target
=
provider
.
add_storage_target
(
"/dev/shm/baketarget2"
)
provider
=
BakeProvider
(
mid
,
provider_id
)
target
=
provider
.
create_target
(
"/dev/shm/baketarget"
,
10
*
1024
*
1024
)
print
(
"target id is "
+
str
(
target
))
print
(
"number of targets: "
+
str
(
provider
.
count_storage_targets
()))
print
(
"storage targets: "
)
targets
=
provider
.
list_
storage_
targets
()
targets
=
provider
.
list_targets
()
for
t
in
targets
:
print
(
str
(
t
))
#
provider.
remove_all_storage
_targets()
#
print "number of targets: "+str(provider.count_
storage_
targets())
provider
.
detach_all
_targets
()
print
"number of targets: "
+
str
(
provider
.
count_targets
())
mid
.
wait_for_finalize
()
Write
Preview
Supports
Markdown
0%
Try again
or
attach a new file
.
Cancel
You are about to add
0
people
to the discussion. Proceed with caution.
Finish editing this message first!
Cancel
Please
register
or
sign in
to comment