[feature] Add clouds.yaml and lease management

Using pipenv as the environment manager, implement as clouds.yaml
workflow and a script to manage leases on chameleon.

Note that the clouds.yaml only support chi-tacc for now. We will need to
add another region for UC later.

.gitignore

+**/secure.yaml

Pipfile

+[[source]]
+url = "https://pypi.org/simple"
+verify_ssl = true
+name = "pypi"
+
+[packages]
+ansible = "*"
+python-openstackclient = "*"
+shade = "*"
+python-blazarclient = "*"
+os-client-config = "*"
+
+[dev-packages]
+ipython = "*"
+
+[requires]
+python_version = "3.6"

README.markdown

+Software-management of the Argo test infrastructure
+---------------------------------------------------
+
+This repository contains scripts and configuration files to manage an Argo
+testbed. Most of the contents are aimed at the Chameleoncloud infrastructure in
+particular.
+
+# Requirements
+
+All the necessary requirements are managed by the python `Pipfile`. You will
+need `pipenv` to use it. See `https://github.com/pypa/pipenv` for how to
+install it.
+
+# Environment Setup
+
+Enter the pipenv environment:
+
+> pipenv shell
+
+Create the required secure.yaml file with your Chameleon credentials
+
+> sed -e 's/USERNAME/myuser/g' -e 's/PASSWORD/mypass/g' secure.yaml.templ > secure.yaml
+
+**DO NOT COMMIT THIS FILE** git will ignore it by default, but remember that
+this repository is public and these are private credentials to Chameleon.
+
+# Lease Management
+
+Chameleon requires that a lease is created before we can boot nodes. You can
+create a lease on the Chameleon web interface, or use the `chi-lease.py` script
+in this repo.
+
+*Remenber that we have limited node-hours on Chameleon, so try to keep lease
+allocations to the minimal amount of hours. You can always extend a lease if
+required*
+
+> ./chi-lease.py list
+> ./chi-lease.py create
+> ./chi-lease.py show
+> ./chi-lease.py delete

chi-lease.py

+#!/usr/bin/env python
+
+import argparse
+import datetime
+import json
+import logging
+import os
+
+import os_client_config
+from blazarclient import client as blazar_client
+import keystoneauth1
+
+
+def get_client(cloud=None):
+    """Retrieve a client to blazar based on clouds.yaml config."""
+    config = os_client_config.OpenStackConfig()
+    cloud_config = config.get_one(cloud)
+    session = cloud_config.get_session()
+
+    # blazar acces
+    return blazar_client.Client(1, session=session, service_type='reservation')
+
+
+def do_create(argv):
+    """Create a lease using <name>, with <size> nodes, starting now."""
+    client = get_client(argv.cloud)
+    name = argv.name
+    node_count = int(argv.size)
+
+    # start the lease now
+    start_time = datetime.datetime.utcnow()
+    end_time = start_time + datetime.timedelta(hours=argv.length)
+
+    # resource properties
+    props = ['=', '$node_type', 'compute_haswell']
+    resources = {'min': node_count,
+                 'max': node_count,
+                 'resource_type': 'physical:host',
+                 'hypervisor_properties': "",
+                 'resource_properties': props,
+                 }
+    resources = {k: v if isinstance(v, str) else json.dumps(v)
+                 for k, v in resources.items()}
+
+    # put everything into blazar arguments
+    args = {'name': name,
+            'start': start_time.isoformat(' ', 'minutes'),
+            'end': end_time.isoformat(' ', 'minutes'),
+            'events': [],
+            'reservations': [resources],
+            }
+    try:
+        resp = client.lease.create(**args)
+        print(json.dumps(resp, indent=4))
+    except keystoneauth1.exceptions.http.InternalServerError as e:
+        print(e.message)
+
+
+def do_delete(argv):
+    """Delete all leases with <name> as name."""
+    client = get_client(argv.cloud)
+    # leases are listed by lease-id, so we cannot use lease.delete()
+    lease_list = client.lease.list()
+    leases = [l for l in lease_list if l['name'] == argv.name]
+    for lease in leases:
+        client.lease.delete(lease['id'])
+    if not leases:
+        print("No lease named {}".format(argv.name))
+
+
+def do_show(argv):
+    """Show all leases with <name> as name."""
+    client = get_client(argv.cloud)
+    # leases are listed by lease-id, so we cannot use lease.get()
+    lease_list = client.lease.list()
+    # make sure to print every lease with that name.
+    leases = [l for l in lease_list if l['name'] == argv.name]
+    for lease in leases:
+        print(json.dumps(lease, indent=4))
+    if not leases:
+        print("No lease named {}".format(argv.name))
+
+
+def do_list(argv):
+    """List all leases."""
+    client = get_client(argv.cloud)
+    lease_list = client.lease.list()
+    if lease_list:
+        print(json.dumps(lease_list, indent=4))
+    else:
+        print("No leases.")
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Chameleon Lease Helper')
+    parser.add_argument('--cloud', default=os.environ.get('OS_CLOUD'),
+                        help='Cloud name')
+    parser.add_argument('--debug', help="Print debugging output",
+                        action='store_true')
+    subparsers = parser.add_subparsers(title='Commands', dest='command')
+    subparsers.required = True
+
+    # create a lease
+    parser_create = subparsers.add_parser("create", help="Create a lease")
+    parser_create.add_argument("name", help="Lease name")
+    parser_create.add_argument("size", help="Number of hosts to lease",
+                               type=int)
+    parser_create.add_argument("length", help="How long to lease for (hours)",
+                               type=int)
+    parser_create.set_defaults(func=do_create)
+
+    parser_delete = subparsers.add_parser("delete", help="Delete a lease")
+    parser_delete.add_argument("name", help="Lease name")
+    parser_delete.set_defaults(func=do_delete)
+
+    parser_show = subparsers.add_parser("show", help="Show a lease")
+    parser_show.add_argument("name", help="Lease name")
+    parser_show.set_defaults(func=do_show)
+
+    parser_list = subparsers.add_parser("list", help="List all leases")
+    parser_list.set_defaults(func=do_list)
+
+    args = parser.parse_args()
+    if args.debug:
+        logger = logging.getLogger('keystoneauth')
+        logger.addHandler(logging.StreamHandler())
+        logger.setLevel(logging.DEBUG)
+    args.func(args)
+
+
+if __name__ == '__main__':
+    main()

clouds.yaml

+# clouds information to connect to openstack
+clouds:
+  chi-tacc:
+    auth:
+      project_name: 'CH-817005'
+      project_id: '615f0f3f9fb74922b9b1bdb22acc3acc'
+      auth_url: 'https://chi.tacc.chameleoncloud.org:5000/v3'
+      user_domain_name: "Default"
+    region_name: 'CHI@TACC'
+    interface: 'public'
+    identity_api_version: 3

secure.yaml.templ

+# clouds information to connect to openstack
+clouds:
+  chi-tacc:
+    auth:
+      username: 'USERNAME'
+      password: 'PASSWORD'