Skip to content
GitLab
Menu
Projects
Groups
Snippets
Loading...
Help
Help
Support
Community forum
Keyboard shortcuts
?
Submit feedback
Contribute to GitLab
Sign in
Toggle navigation
Menu
Open sidebar
sds
HEP
HEPnOS
Commits
651208a7
Commit
651208a7
authored
Apr 10, 2018
by
Matthieu Dorier
Browse files
started implementation of real mochi-based storage system
parent
98e7aee4
Changes
20
Hide whitespace changes
Inline
Side-by-side
CMakeLists.txt
View file @
651208a7
...
...
@@ -34,9 +34,16 @@ include_directories(${CMAKE_CURRENT_SOURCE_DIR}/include)
# packages we depend on
include
(
xpkg-import
)
find_package
(
mercury CONFIG REQUIRED
)
find_package
(
Boost REQUIRED COMPONENTS serialization filesystem
)
find_package
(
Boost REQUIRED COMPONENTS serialization
system
filesystem
)
include_directories
(
${
Boost_INCLUDE_DIRS
}
)
xpkg_import_module
(
margo REQUIRED margo
)
xpkg_import_module
(
sdskv-client REQUIRED sdskv-client
)
xpkg_import_module
(
bake-client REQUIRED bake-client
)
xpkg_import_module
(
sdskv-server REQUIRED sdskv-server
)
xpkg_import_module
(
bake-server REQUIRED bake-server
)
xpkg_import_module
(
ch-placement REQUIRED ch-placement
)
find_package
(
yaml-cpp REQUIRED
)
add_subdirectory
(
src
)
add_subdirectory
(
test
)
#add_subdirectory (bin)
include/hepnos/DataSet.hpp
0 → 100644
View file @
651208a7
#ifndef __HEPNOS_DATA_SET_H
#define __HEPNOS_DATA_SET_H
#include <hepnos/DataStore.hpp>
namespace
hepnos
{
class
DataSet
{
friend
class
DataStore
;
private:
DataSet
(
DataStore
&
ds
,
uint8_t
level
,
const
std
::
string
&
name
);
DataStore
&
m_datastore
;
uint8_t
m_level
;
std
::
string
m_name
;
};
}
#endif
include/hepnos/DataStore.hpp
0 → 100644
View file @
651208a7
#ifndef __HEPNOS_DATA_STORE_H
#define __HEPNOS_DATA_STORE_H
#include <string>
#include <memory>
namespace
hepnos
{
class
DataSet
;
class
DataStore
{
public:
DataStore
(
const
std
::
string
&
configFile
);
DataStore
(
const
DataStore
&
)
=
delete
;
DataStore
(
DataStore
&&
);
DataStore
&
operator
=
(
const
DataStore
&
)
=
delete
;
DataStore
&
operator
=
(
DataStore
&&
);
~
DataStore
();
class
iterator
;
class
const_iterator
;
iterator
find
(
const
std
::
string
&
datasetName
);
inline
const_iterator
find
(
const
std
::
string
&
datasetName
)
const
;
iterator
begin
();
iterator
end
();
const_iterator
cbegin
()
const
;
const_iterator
cend
()
const
;
iterator
lower_bound
(
const
std
::
string
&
lb
);
const_iterator
lower_bound
(
const
std
::
string
&
lb
)
const
;
iterator
upper_bound
(
const
std
::
string
&
ub
);
const_iterator
upper_bound
(
const
std
::
string
&
ub
)
const
;
DataSet
createDataSet
(
const
std
::
string
&
name
);
private:
class
Impl
;
std
::
unique_ptr
<
Impl
>
m_impl
;
};
class
DataStore
::
const_iterator
{
friend
class
DataStore
::
Impl
;
friend
class
DataStore
;
protected:
DataStore
*
m_datastore
;
const_iterator
(
DataStore
&
ds
);
public:
typedef
const_iterator
self_type
;
typedef
DataSet
value_type
;
typedef
DataSet
&
reference
;
typedef
DataSet
*
pointer
;
typedef
int
difference_type
;
typedef
std
::
forward_iterator_tag
iterator_category
;
virtual
~
const_iterator
();
const_iterator
(
const
const_iterator
&
);
const_iterator
(
const_iterator
&&
);
const_iterator
&
operator
=
(
const
const_iterator
&
);
const_iterator
&
operator
=
(
const_iterator
&&
);
self_type
operator
++
();
self_type
operator
++
(
int
);
const
reference
operator
*
();
const
pointer
operator
->
();
bool
operator
==
(
const
self_type
&
rhs
)
const
;
bool
operator
!=
(
const
self_type
&
rhs
)
const
;
};
class
DataStore
::
iterator
:
public
DataStore
::
const_iterator
{
friend
class
DataStore
::
Impl
;
friend
class
DataStore
;
private:
iterator
(
DataStore
&
ds
);
public:
typedef
iterator
self_type
;
typedef
DataSet
value_type
;
typedef
DataSet
&
reference
;
typedef
DataSet
*
pointer
;
typedef
int
difference_type
;
typedef
std
::
forward_iterator_tag
iterator_category
;
~
iterator
();
iterator
(
const
iterator
&
);
iterator
(
iterator
&&
);
iterator
&
operator
=
(
const
iterator
&
);
iterator
&
operator
=
(
iterator
&&
);
reference
operator
*
();
pointer
operator
->
();
};
}
#endif
include/hepnos/Exception.hpp
0 → 100644
View file @
651208a7
#ifndef __HEPNOS_EXCEPTION_H
#define __HEPNOS_EXCEPTION_H
#include <string>
#include <exception>
namespace
hepnos
{
class
Exception
:
public
std
::
exception
{
std
::
string
m_msg
;
public:
Exception
(
const
std
::
string
&
msg
)
:
m_msg
(
msg
){}
virtual
const
char
*
what
()
const
noexcept
override
{
return
m_msg
.
c_str
();
}
};
}
#endif
include/hepnos/FileDataStore.hpp
deleted
100644 → 0
View file @
98e7aee4
#ifndef __HEPNOS_FILE_DATASTORE_H
#define __HEPNOS_FILE_DATASTORE_H
#include <string>
#include <iterator>
#include <boost/filesystem.hpp>
#include <hepnos/FileNamespace.hpp>
namespace
hepnos
{
namespace
fs
=
boost
::
filesystem
;
class
FileDataStore
{
public:
typedef
FileNamespace
namespace_type
;
typedef
FileRun
run_type
;
typedef
FileSubRun
subrun_type
;
typedef
FileEvent
event_type
;
FileDataStore
(
const
std
::
string
path
)
:
_path
(
path
)
{
if
(
_path
.
empty
())
_path
=
"./"
;
if
(
_path
.
back
()
!=
'/'
)
_path
+=
std
::
string
(
"/"
);
fs
::
create_directories
(
_path
);
fs
::
create_directory
(
_path
+
std
::
string
(
".ref"
));
}
namespace_type
createNamespace
(
const
std
::
string
&
name
)
{
std
::
string
dir
=
_path
+
name
;
if
(
fs
::
is_directory
(
dir
))
return
FileNamespace
(
name
,
dir
);
fs
::
create_directory
(
dir
);
return
FileNamespace
(
name
,
dir
);
}
namespace_type
openNamespace
(
const
std
::
string
&
name
)
{
std
::
string
dir
=
_path
+
name
;
if
(
fs
::
is_directory
(
dir
))
return
FileNamespace
(
name
,
dir
);
else
return
FileNamespace
();
}
event_type
getEventByID
(
std
::
uint64_t
id
)
{
std
::
stringstream
ss
;
ss
<<
_path
<<
".ref/"
<<
std
::
setfill
(
'0'
)
<<
std
::
setw
(
20
)
<<
id
;
std
::
string
link
=
ss
.
str
();
if
(
!
fs
::
exists
(
link
))
return
FileEvent
();
std
::
string
p
=
fs
::
canonical
(
link
).
string
();
return
FileEvent
(
p
);
}
private:
std
::
string
_path
;
};
}
#endif
include/hepnos/FileEvent.hpp
deleted
100644 → 0
View file @
98e7aee4
#ifndef __HEPNOS_FILE_EVENT_H
#define __HEPNOS_FILE_EVENT_H
#include <string>
#include <limits>
#include <boost/filesystem.hpp>
#include <hepnos/FileProductAccessorBackend.hpp>
#include <hepnos/FileObjectIterator.hpp>
#include <hepnos/ProductAccessor.hpp>
namespace
hepnos
{
namespace
fs
=
boost
::
filesystem
;
class
FileSubRun
;
class
FileDataStore
;
class
FileEvent
:
public
ProductAccessor
<
FileProductAccessorBackend
>
{
private:
friend
class
FileDataStore
;
friend
class
FileSubRun
;
friend
class
FileObjectIterator
<
FileEvent
>
;
void
createRefAndSetID
()
{
std
::
size_t
h
=
std
::
hash
<
std
::
string
>
()(
_path
);
std
::
string
linkName
;
std
::
stringstream
ss
;
do
{
ss
.
str
(
""
);
ss
<<
_path
<<
"../../../../.ref/"
;
ss
<<
std
::
setfill
(
'0'
)
<<
std
::
setw
(
20
)
<<
h
;
h
+=
1
;
}
while
(
fs
::
exists
(
ss
.
str
()));
_eventID
=
h
-
1
;
fs
::
create_symlink
(
_path
,
ss
.
str
());
}
FileEvent
(
std
::
uint64_t
eventNumber
,
const
std
::
string
&
dir
)
:
ProductAccessor
<
FileProductAccessorBackend
>
(
dir
)
,
_eventNumber
(
eventNumber
)
,
_path
(
dir
)
{
createRefAndSetID
();
}
FileEvent
(
const
std
::
string
&
dir
)
:
ProductAccessor
<
FileProductAccessorBackend
>
(
dir
)
,
_eventNumber
(
0
)
,
_path
(
dir
)
{
if
(
_path
.
back
()
!=
'/'
)
_path
+=
std
::
string
(
"/"
);
std
::
size_t
i
,
j
;
j
=
_path
.
size
()
-
1
;
if
(
_path
[
j
]
==
'/'
)
j
--
;
i
=
j
;
while
(
_path
[
i
]
!=
'/'
)
i
--
;
i
+=
1
;
while
(
_path
[
i
]
==
'0'
)
i
++
;
j
+=
1
;
std
::
string
eventDir
(
&
dir
[
i
],
j
-
i
);
if
(
eventDir
.
size
()
>
0
)
_eventNumber
=
std
::
stoi
(
eventDir
);
}
FileEvent
()
:
ProductAccessor
<
FileProductAccessorBackend
>
(
""
)
,
_eventNumber
(
std
::
numeric_limits
<
std
::
uint64_t
>::
max
())
,
_path
(
""
)
{}
std
::
uint64_t
_eventNumber
;
std
::
string
_path
;
std
::
size_t
_eventID
;
public:
std
::
uint64_t
getEventNumber
()
const
{
return
_eventNumber
;
}
std
::
size_t
getEventID
()
const
{
return
_eventID
;
}
bool
isValid
()
const
{
return
_eventNumber
!=
std
::
numeric_limits
<
std
::
uint64_t
>::
max
();
}
};
}
#endif
include/hepnos/FileNamespace.hpp
deleted
100644 → 0
View file @
98e7aee4
#ifndef __HEPNOS_FILE_NAMESPACE_H
#define __HEPNOS_FILE_NAMESPACE_H
#include <string>
#include <iomanip>
#include <boost/filesystem.hpp>
#include <hepnos/FileObjectIterator.hpp>
#include <hepnos/FileRun.hpp>
namespace
hepnos
{
namespace
fs
=
boost
::
filesystem
;
class
FileDataStore
;
class
FileNamespace
{
friend
class
FileDataStore
;
private:
std
::
string
_name
;
std
::
string
_path
;
FileNamespace
(
const
std
::
string
&
name
,
const
std
::
string
&
dir
)
:
_name
(
name
),
_path
(
dir
+
std
::
string
(
"/"
))
{}
FileNamespace
()
{}
public:
typedef
FileObjectIterator
<
FileRun
>
iterator
;
const
std
::
string
&
getName
()
const
{
return
_name
;
}
bool
isValid
()
const
{
return
!
_name
.
empty
();
}
FileRun
createRun
()
{
fs
::
directory_iterator
begin
(
_path
),
end
;
size_t
runNumber
=
std
::
count_if
(
begin
,
end
,
[](
const
fs
::
directory_entry
&
d
)
{
return
fs
::
is_directory
(
d
.
path
());
});
std
::
stringstream
ss
;
ss
<<
_path
<<
std
::
setfill
(
'0'
)
<<
std
::
setw
(
12
)
<<
runNumber
<<
"/"
;
std
::
string
dir
=
ss
.
str
();
fs
::
create_directory
(
dir
);
return
FileRun
(
runNumber
,
dir
);
}
FileRun
openRun
(
std
::
uint64_t
runNumber
)
{
std
::
stringstream
ss
;
ss
<<
_path
<<
std
::
setfill
(
'0'
)
<<
std
::
setw
(
12
)
<<
runNumber
<<
"/"
;
std
::
string
dir
=
ss
.
str
();
if
(
fs
::
is_directory
(
dir
))
return
FileRun
(
runNumber
,
dir
);
else
return
FileRun
();
}
std
::
size_t
numRuns
()
const
{
fs
::
directory_iterator
begin
(
_path
),
end
;
return
std
::
count_if
(
begin
,
end
,
[](
const
fs
::
directory_entry
&
d
)
{
return
fs
::
is_directory
(
d
.
path
());
});
}
iterator
begin
()
{
return
iterator
(
_path
,
0
,
numRuns
());
}
iterator
end
()
{
auto
n
=
numRuns
();
return
iterator
(
_path
,
n
,
n
);
}
};
}
#endif
include/hepnos/FileObjectIterator.hpp
deleted
100644 → 0
View file @
98e7aee4
#ifndef __HEPNOS_FILE_OBJECT_ITERATOR_H
#define __HEPNOS_FILE_OBJECT_ITERATOR_H
#include <boost/filesystem.hpp>
namespace
hepnos
{
namespace
fs
=
boost
::
filesystem
;
template
<
typename
T
>
class
FileObjectIterator
{
private:
std
::
string
_path
;
std
::
uint64_t
_numObjects
;
std
::
uint64_t
_currentIdx
;
T
_current
;
public:
FileObjectIterator
(
const
std
::
string
&
path
,
std
::
uint64_t
idx
,
std
::
uint64_t
nobj
)
:
_path
(
path
),
_numObjects
(
nobj
),
_currentIdx
(
idx
)
{
if
(
_path
.
back
()
!=
'/'
)
_path
+=
std
::
string
(
"/"
);
if
(
_currentIdx
!=
_numObjects
)
{
std
::
stringstream
ss
;
ss
<<
_path
<<
std
::
setfill
(
'0'
)
<<
std
::
setw
(
12
)
<<
_currentIdx
<<
"/"
;
_current
=
T
(
_currentIdx
,
ss
.
str
());
}
}
bool
operator
==
(
const
FileObjectIterator
&
other
)
const
{
return
_path
==
other
.
_path
&&
_currentIdx
==
other
.
_currentIdx
&&
_numObjects
==
other
.
_numObjects
;
}
bool
operator
!=
(
const
FileObjectIterator
&
other
)
const
{
return
!
(
*
this
==
other
);
}
const
T
&
operator
*
()
const
{
return
_current
;
}
const
T
*
operator
->
()
const
{
return
&
_current
;
}
T
*
operator
->
()
{
return
&
_current
;
}
FileObjectIterator
operator
++
()
{
FileObjectIterator
old
=
*
this
;
_currentIdx
+=
1
;
if
(
_currentIdx
!=
_numObjects
)
{
std
::
stringstream
ss
;
ss
<<
_path
<<
std
::
setfill
(
'0'
)
<<
std
::
setw
(
12
)
<<
_currentIdx
<<
"/"
;
_current
=
T
(
_currentIdx
,
ss
.
str
());
}
return
old
;
}
FileObjectIterator
&
operator
++
(
int
)
{
_currentIdx
+=
1
;
if
(
_currentIdx
!=
_numObjects
)
{
std
::
stringstream
ss
;
ss
<<
_path
<<
std
::
setfill
(
'0'
)
<<
std
::
setw
(
12
)
<<
_currentIdx
<<
"/"
;
_current
=
T
(
_currentIdx
,
ss
.
str
());
}
return
*
this
;
}
};
}
#endif
include/hepnos/FileProductAccessorBackend.hpp
deleted
100644 → 0
View file @
98e7aee4
#ifndef __HEPNOS_FILE_PRODUCT_ACCESSOR_BACKEND_H
#define __HEPNOS_FILE_PRODUCT_ACCESSOR_BACKEND_H
#include <vector>
#include <string>
#include <sstream>
#include <fstream>
#include <hepnos/InputTag.hpp>
namespace
hepnos
{
class
FileProductAccessorBackend
{
private:
std
::
string
makeFileNameFor
(
const
std
::
string
&
objName
,
const
InputTag
&
tag
)
{
std
::
string
dot
(
"."
);
std
::
string
fileName
=
objName
+
dot
+
tag
.
moduleLabel
+
dot
+
tag
.
instanceName
+
dot
+
tag
.
processName
;
if
(
_path
.
empty
())
return
fileName
;
else
if
(
_path
.
back
()
==
'/'
)
return
_path
+
fileName
;
else
return
_path
+
std
::
string
(
"/"
)
+
fileName
;
}
std
::
string
_path
;
public:
FileProductAccessorBackend
(
const
std
::
string
&
path
)
:
_path
(
path
)
{}
void
store
(
const
std
::
string
&
objName
,
const
InputTag
&
tag
,
const
std
::
vector
<
char
>&
data
)
{
std
::
string
fileName
=
makeFileNameFor
(
objName
,
tag
);
std
::
ofstream
outfile
;
outfile
.
open
(
fileName
,
std
::
ofstream
::
out
|
std
::
ofstream
::
trunc
);
outfile
.
write
(
data
.
data
(),
data
.
size
());
outfile
.
close
();
}
bool
load
(
const
std
::
string
&
objName
,
const
InputTag
&
tag
,
std
::
vector
<
char
>&
data
)
{
std
::
string
fileName
=
makeFileNameFor
(
objName
,
tag
);
std
::
ifstream
infile
;
infile
.
open
(
fileName
,
std
::
ifstream
::
ate
|
std
::
ifstream
::
binary
);
if
(
!
infile
.
good
())
return
false
;
std
::
size_t
size
=
infile
.
tellg
();
infile
.
seekg
(
0
,
infile
.
beg
);
data
.
resize
(
size
);
infile
.
read
(
data
.
data
(),
size
);
infile
.
close
();
return
true
;
}
~
FileProductAccessorBackend
()
{}
};
}
#endif
include/hepnos/FileRun.hpp
deleted
100644 → 0
View file @
98e7aee4
#ifndef __HEPNOS_FILE_RUN_H
#define __HEPNOS_FILE_RUN_H
#include <string>
#include <limits>
#include <boost/filesystem.hpp>
#include <hepnos/FileProductAccessorBackend.hpp>
#include <hepnos/ProductAccessor.hpp>
#include <hepnos/FileSubRun.hpp>
#include <hepnos/FileObjectIterator.hpp>
namespace
hepnos
{
namespace
fs
=
boost
::
filesystem
;
class
FileNamespace
;
template
<
typename
T
>
class
FileObjectIterator
;
class
FileRun
:
public
ProductAccessor
<
FileProductAccessorBackend
>
{
private:
friend
class
FileNamespace
;
friend
class
FileObjectIterator
<
FileRun
>
;
FileRun
(
std
::
uint64_t
runNumber
,
const
std
::
string
&
dir
)
:
ProductAccessor
<
FileProductAccessorBackend
>
(
dir
)
,
_runNumber
(
runNumber
)
,
_path
(
dir
)
{}
FileRun
(
const
std
::
string
&
dir
)
:
ProductAccessor
<
FileProductAccessorBackend
>
(
dir
)
,
_runNumber
(
0
)
,
_path
(
dir
)
{
std
::
size_t
i
,
j
;
j
=
dir
.
size
()
-
1
;
if
(
dir
[
j
]
==
'/'
)
j
--
;