Commit ab05fbd8 authored by Michael Salim's avatar Michael Salim
Browse files

major reorganization; added setuptools and sphinx docs folder

parent 75d8ac3b
import os,sys,logging
import os
import pathlib
import sys
import socket
from django.core.management import call_command
import logging
logger = logging.getLogger('console')
try:
INSTALL_PATH = os.environ['ARGOBALSAM_INSTALL_PATH']
LOGGING_DIRECTORY = os.path.join(INSTALL_PATH, 'log') # where to store log files
except KeyError as e:
logger.error('Environment not setup: ' + str(e))
raise
here = os.path.abspath(os.path.dirname(__file__))
INSTALL_PATH = pathlib.Path(here).parent
LOGGING_DIRECTORY = os.path.join(INSTALL_PATH, 'log') # where to store log files
#------------------------------
......@@ -56,6 +57,7 @@ BALSAM_SCHEDULER_SUBMIT_EXE = '/usr/bin/qsub'
BALSAM_SCHEDULER_STATUS_EXE = '/usr/bin/qstat'
BALSAM_SCHEDULER_HISTORY_EXE = '/usr/bin/'
BALSAM_SERVICE_PERIOD = 1 # seconds between service loop execution
BALSAM_RUNNER_CREATION_PERIOD_SEC = 5
BALSAM_MAX_QUEUED = 20 # the maximum number of jobs allowed on the local queue
BALSAM_SUBMIT_JOBS = True # submit jobs to queue, turn off when testing
BALSAM_DEFAULT_QUEUE = 'debug-cache-quad' # default local queue name
......@@ -192,23 +194,23 @@ LOGGING = {
'level': 'DEBUG',
'propagate': True,
},
'argo': {
'balsam.argo': {
'handlers': ['default'],
'level': 'DEBUG',
},
'common': {
'balsam.common': {
'handlers': ['default'],
'level': 'DEBUG',
},
'console': {
'balsam.console': {
'handlers': ['console'],
'level': 'INFO',
},
'balsam': {
'balsam.service': {
'handlers': ['default'],
'level': 'DEBUG',
},
'balsamlauncher': {
'balsam.launcher': {
'handlers': ['default'],
'level': 'DEBUG',
},
......@@ -223,11 +225,11 @@ LOGGING = {
}
}
if 'argoservice' in ' '.join(sys.argv):
logger = logging.getLogger('argo')
logger = logging.getLogger('balsam.argo')
elif 'service' in sys.argv:
logger = logging.getLogger('balsam')
logger = logging.getLogger('balsam.service')
elif 'launcher' in ' '.join(sys.argv):
logger = logging.getLogger('balsamlauncher')
logger = logging.getLogger('balsam.launcher')
else:
logger = logging.getLogger('console')
......
#!/bin/bash
PYTHON="$(which python)"
$PYTHON $ARGOBALSAM_INSTALL_PATH/bin/cli.py $*
# Minimal makefile for Sphinx documentation
#
# You can set these variables from the command line.
SPHINXOPTS =
SPHINXBUILD = sphinx-build
SPHINXPROJ = balsam
SOURCEDIR = .
BUILDDIR = _build
# Put it first so that "make" without argument is like "make help".
help:
@$(SPHINXBUILD) -M help "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
.PHONY: help Makefile
# Catch-all target: route all unknown targets to Sphinx using the new
# "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS).
%: Makefile
@$(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O)
\ No newline at end of file
#!/usr/bin/env python3
# -*- coding: utf-8 -*-
#
# balsam documentation build configuration file, created by
# sphinx-quickstart on Fri Dec 15 13:22:13 2017.
#
# This file is execfile()d with the current directory set to its
# containing dir.
#
# Note that not all possible configuration values are present in this
# autogenerated file.
#
# All configuration values have a default; values that are commented out
# serve to show the default.
# If extensions (or modules to document with autodoc) are in another directory,
# add these directories to sys.path here. If the directory is relative to the
# documentation root, use os.path.abspath to make it absolute, like shown here.
#
# import os
# import sys
# sys.path.insert(0, os.path.abspath('.'))
# -- General configuration ------------------------------------------------
# If your documentation needs a minimal Sphinx version, state it here.
#
# needs_sphinx = '1.0'
# Add any Sphinx extension module names here, as strings. They can be
# extensions coming with Sphinx (named 'sphinx.ext.*') or your custom
# ones.
extensions = ['sphinx.ext.autodoc']
# Add any paths that contain templates here, relative to this directory.
templates_path = ['_templates']
# The suffix(es) of source filenames.
# You can specify multiple suffix as a list of string:
#
# source_suffix = ['.rst', '.md']
source_suffix = '.rst'
# The master toctree document.
master_doc = 'index'
# General information about the project.
project = 'balsam'
copyright = '2017, Misha Salim, Tom Uram, J. Taylor Childers'
author = 'Misha Salim, Tom Uram, J. Taylor Childers'
# The version info for the project you're documenting, acts as replacement for
# |version| and |release|, also used in various other places throughout the
# built documents.
#
# The short X.Y version.
version = '0.1'
# The full version, including alpha/beta/rc tags.
release = '0.1'
# The language for content autogenerated by Sphinx. Refer to documentation
# for a list of supported languages.
#
# This is also used if you do content translation via gettext catalogs.
# Usually you set "language" from the command line for these cases.
language = None
# List of patterns, relative to source directory, that match files and
# directories to ignore when looking for source files.
# This patterns also effect to html_static_path and html_extra_path
exclude_patterns = ['_build', 'Thumbs.db', '.DS_Store']
# The name of the Pygments (syntax highlighting) style to use.
pygments_style = 'sphinx'
# If true, `todo` and `todoList` produce output, else they produce nothing.
todo_include_todos = False
# -- Options for HTML output ----------------------------------------------
# The theme to use for HTML and HTML Help pages. See the documentation for
# a list of builtin themes.
#
html_theme = 'alabaster'
# Theme options are theme-specific and customize the look and feel of a theme
# further. For a list of options available for each theme, see the
# documentation.
#
# html_theme_options = {}
# Add any paths that contain custom static files (such as style sheets) here,
# relative to this directory. They are copied after the builtin static files,
# so a file named "default.css" will overwrite the builtin "default.css".
html_static_path = ['_static']
# Custom sidebar templates, must be a dictionary that maps document names
# to template names.
#
# This is required for the alabaster theme
# refs: http://alabaster.readthedocs.io/en/latest/installation.html#sidebars
html_sidebars = {
'**': [
'relations.html', # needs 'show_related': True theme option to display
'searchbox.html',
]
}
# -- Options for HTMLHelp output ------------------------------------------
# Output file base name for HTML help builder.
htmlhelp_basename = 'balsamdoc'
# -- Options for LaTeX output ---------------------------------------------
latex_elements = {
# The paper size ('letterpaper' or 'a4paper').
#
# 'papersize': 'letterpaper',
# The font size ('10pt', '11pt' or '12pt').
#
# 'pointsize': '10pt',
# Additional stuff for the LaTeX preamble.
#
# 'preamble': '',
# Latex figure (float) alignment
#
# 'figure_align': 'htbp',
}
# Grouping the document tree into LaTeX files. List of tuples
# (source start file, target name, title,
# author, documentclass [howto, manual, or own class]).
latex_documents = [
(master_doc, 'balsam.tex', 'balsam Documentation',
'Misha Salim, Tom Uram, J. Taylor Childers', 'manual'),
]
# -- Options for manual page output ---------------------------------------
# One entry per manual page. List of tuples
# (source start file, name, description, authors, manual section).
man_pages = [
(master_doc, 'balsam', 'balsam Documentation',
[author], 1)
]
# -- Options for Texinfo output -------------------------------------------
# Grouping the document tree into Texinfo files. List of tuples
# (source start file, target name, title, author,
# dir menu entry, description, category)
texinfo_documents = [
(master_doc, 'balsam', 'balsam Documentation',
author, 'balsam', 'One line description of project.',
'Miscellaneous'),
]
.. balsam documentation master file, created by
sphinx-quickstart on Fri Dec 15 13:22:13 2017.
You can adapt this file completely to your liking, but it should at least
contain the root `toctree` directive.
Welcome to balsam's documentation!
==================================
.. toctree::
:maxdepth: 2
:caption: Contents:
Indices and tables
==================
* :ref:`genindex`
* :ref:`modindex`
* :ref:`search`
@ECHO OFF
pushd %~dp0
REM Command file for Sphinx documentation
if "%SPHINXBUILD%" == "" (
set SPHINXBUILD=sphinx-build
)
set SOURCEDIR=.
set BUILDDIR=_build
set SPHINXPROJ=balsam
if "%1" == "" goto help
%SPHINXBUILD% >NUL 2>NUL
if errorlevel 9009 (
echo.
echo.The 'sphinx-build' command was not found. Make sure you have Sphinx
echo.installed, then set the SPHINXBUILD environment variable to point
echo.to the full path of the 'sphinx-build' executable. Alternatively you
echo.may add the Sphinx directory to PATH.
echo.
echo.If you don't have Sphinx installed, grab it from
echo.http://sphinx-doc.org/
exit /b 1
)
%SPHINXBUILD% -M %1 %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
goto end
:help
%SPHINXBUILD% -M help %SOURCEDIR% %BUILDDIR% %SPHINXOPTS%
:end
popd
#!/usr/bin/python
import sys
import django
django.setup()
from django.conf import settings
from balsam import models
num=1
if len(sys.argv)>1:
num=int(sys.argv[1])
for i in range(num):
job = models.BalsamJob()
job.job_id = models.BalsamJob.generate_job_id()
job.working_directory = models.BalsamJob.create_working_path(job.job_id)
#job.site = 'cooley'
#job.job_name = 'helloworld'
#job.job_description = 'helloworld'
#job.origin_id = options['origin_id']
#job.queue = 'default'
#job.project = 'visualization'
job.wall_time_minutes = 30
job.num_nodes = 1
#job.processes_per_node = 1
job.application = 'helloworld'
job.config_file = '/home/turam/helloworld.sh'
#job.input_url = options['input_url']
#job.output_url = options['output_url']
job.save()
print 'Created job', job.id
#!/usr/bin/env bash
SECURITY_PATH=FIXME
ARGOBALSAM_PATH=$SECURITY_PATH/argobalsam/dev
GRIDCERT_PATH=$SECURITY_PATH/argobalsam/gridsecurity
export X509_USER_CERT=$GRIDCERT_PATH/$USER/xrootdsrv-cert.pem
export X509_USER_KEY=$GRIDCERT_PATH/$USER/xrootdsrv-key.pem
export X509_CACERTS=$GRIDCERT_PATH/$USER/cacerts.pem
export X509_CERT_DIR=/etc/grid-security/certificates
......@@ -13,12 +13,12 @@ def set_permissions(top):
for file in (os.path.join(root, f) for f in files):
os.chmod(file, 0o644)
if __name__ == "__main__":
def main():
tempdir = tempfile.TemporaryDirectory(dir=os.getcwd(), prefix="testdata_")
os.environ['BALSAM_TEST_DIRECTORY'] = tempdir.name
os.environ['BALSAM_TEST']='1'
os.environ['DJANGO_SETTINGS_MODULE'] = 'argobalsam.settings'
os.environ['DJANGO_SETTINGS_MODULE'] = 'balsam.django_config.settings'
django.setup()
set_permissions(tempdir.name)
......@@ -30,3 +30,6 @@ if __name__ == "__main__":
else:
suite = loader.discover('tests')
unittest.TextTestRunner(verbosity=2).run(suite)
if __name__ == "__main__":
main()
#!/usr/bin/env bash
ROOT_PATH=/users/hpcusers
ARGOBALSAM_PATH=$ROOT_PATH/argobalsam/dev
GRIDCERT_PATH=$ROOT_PATH/argobalsam/gridsecurity
#source $ROOT_PATH/scripts/setupPython-2.7.sh
#export X509_USER_CERT=/tmp/x509up_u30168
#export X509_USER_KEY=/tmp/x509up_u30168
export X509_USER_CERT=$GRIDCERT_PATH/$USER/xrootdsrv-cert.pem
export X509_USER_KEY=$GRIDCERT_PATH/$USER/xrootdsrv-key.pem
#export X509_CERT_DIR=$GRIDCERT_PATH/certificates
export X509_CACERTS=$GRIDCERT_PATH/$USER/cacerts.pem
export X509_CERT_DIR=/etc/grid-security/certificates
#export X509_USER_CERT=/users/hpcusers/argobalsam/production/argobalsam/keycert.txt
#export X509_USER_KEY=/users/hpcusers/argobalsam/production/argobalsam/keycert.txt
PID_FILE=pids.txt
start() {
# kill old processes first
while read pid; do
kill -0 $pid > /dev/null 2>&1
if [[ $? == '0' ]]; then
echo killing old instance before starting: PID = $pid, cmd:
echo $(cat /proc/$pid/cmdline | strings -1)
kill -- -$pid
fi
done <$PID_FILE
python manage.py argo_service > ./log/argo_service.out 2>&1 &
echo $! > $PID_FILE
python manage.py balsam_service > ./log/balsam_service.out 2>&1 &
echo $! >> $PID_FILE
python manage.py runserver 8001 > ./log/runserver.log 2>&1 &
echo $! >> $PID_FILE
}
stop() {
while read pid; do
kill -0 $pid > /dev/null 2>&1
if [[ $? == '0' ]]; then
echo killing instance: PID = $pid, cmd:
echo $(cat /proc/$pid/cmdline | strings -1)
kill -- -$pid
fi
done <$PID_FILE
}
case "$1" in
start)
start
;;
stop)
stop
;;
reload|restart|force-reload)
stop
start
;;
**)
echo "Usage: $0 {start|stop|reload}" 1>&2
;;
esac
'''A setuptools based setup module.
https://packaging.python.org/en/latest/distributing.html
'''
from setuptools import setup, find_packages
from codecs import open
from os import path
here = path.abspath(path.dirname(__file__))
with open(path.join(here, 'README.md'), encoding='utf-8') as f:
long_description = f.read()
setup(
name='balsam',
version='0.1',
description='HPC Edge service and workflow management',
long_description=long_description,
url='', # Home page
author='',
author_email='',
classifiers = [],
keywords='',
packages=find_packages(exclude=['docs','__pycache__','data','experiments','log','tests']),
install_requires=['django', 'django-concurrency'],
include_package_data=True,
# Register command-line tools here
entry_points={
'console_scripts': [
'balsam = balsam.scripts.cli:main',
'balsam-test = run_tests:main'
],
'gui_scripts': [],
},
)
#!/usr/bin/env bash
#source env/bin/activate
export ARGOBALSAM_INSTALL_PATH=$(pwd)
export PYTHONPATH=$ARGOBALSAM_INSTALL_PATH:$PYTHONPATH
export PATH=$ARGOBALSAM_INSTALL_PATH/bin:$PATH
export ARGOBALSAM_DATA_PATH=$ARGOBALSAM_INSTALL_PATH/data
export ARGOBALSAM_EXE_PATH=$ARGOBALSAM_INSTALL_PATH/exe
......@@ -7,7 +7,7 @@ from django.core.management import call_command
from django import db
from django.conf import settings
from balsam.models import BalsamJob, ApplicationDefinition
from balsam.service.models import BalsamJob, ApplicationDefinition
class BalsamTestCase(unittest.TestCase):
@classmethod
......
......@@ -6,7 +6,7 @@ import time
import sys
parser = argparse.ArgumentParser()
parser.add_argument('--sleep', type=int, default=0)
parser.add_argument('--sleep', type=float, default=0)
parser.add_argument('--retcode', type=int, default=0)
args = parser.parse_args()
......
#!/Users/misha/anaconda3/envs/testmpi/bin/python
import balsamlauncher.dag as dag
import balsam.launcher.dag as dag
import glob
import sys
import os
......
Markdown is supported
0% or .
You are about to add 0 people to the discussion. Proceed with caution.
Finish editing this message first!
Please register or to comment