Merge branch 'region-support' into 'master'

Region support

See merge request !1

.gitignore

 **/secure.yaml
 *.retry
+ansible/inventory.yaml

README.markdown

@@ -15,19 +15,48 @@ install it.
 
 Enter the pipenv environment:
 
-> pipenv shell
+```
+pipenv shell
+```
 
 Install required packages
 
-> pipenv install
+```
+pipenv install
+```
 
 Create the required secure.yaml file with your Chameleon credentials
 
-> sed -e 's/USERNAME/myuser/g' -e 's/PASSWORD/mypass/g' secure.yaml.templ > secure.yaml
+```
+sed -e 's/USERNAME/myuser/g' -e 's/PASSWORD/mypass/g' secure.yaml.templ > secure.yaml
+```
 
 **DO NOT COMMIT THIS FILE** git will ignore it by default, but remember that
 this repository is public and these are private credentials to Chameleon.
 
+Note that the default clouds.yaml contains two regions, and they need to be
+specified as the `--region` option for all operations in a specific site. If
+you don't want to repeat this option everytime, set the `OS_REGION_NAME`
+environment variable.
+
+# General Help
+
+The `chi-mngmt.py` script contains a few commands to help you figure out if
+your configuration is working properly, or access specific information about a
+region.
+
+```
+./chi-mngmt.py list-clouds
+./chi-mngmt.py list-keys
+```
+
+In general the scripts included in this repo are here to simplify access to the
+openstack tools. But if you need to do something more complex, you can always
+rely on the openstack GUI or CLI directly. The `pipenv` setup will give you
+access to the `openstack` command in particular. You can have this tool use the
+same `clouds.yaml` configuration by using the `--os-cloud` and `--os-region`
+options.
+
 # Lease Management
 
 Chameleon requires that a lease is created before we can boot nodes. You can
@@ -38,20 +67,29 @@ in this repo.
 allocations to the minimal amount of hours. You can always extend a lease if
 required*
 
-> ./chi-lease.py list
-> ./chi-lease.py create
-> ./chi-lease.py show
-> ./chi-lease.py delete
+```
+./chi-lease.py list
+./chi-lease.py create
+./chi-lease.py show
+./chi-lease.py delete
+```
 
 # Appliance Management
 
 Once a lease is active, you can manage an appliance on it.
 
-> ./chi-appliance list
-> ./chi-appliance create
-> ./chi-appliance show
-> ./chi-appliance configure
-> ./chi-appliance delete
+```
+./chi-appliance.py list
+./chi-appliance.py create
+./chi-appliance.py show
+./chi-appliance.py configure
+./chi-appliance.py delete
+```
+
+Note that `create` requires as a last argument a JSON-formatted dictionary of
+additional template parameters. These parameters can be found in the
+`chameleon-appliances` directory. The command will fill out `reservation_id`
+and `node_count` by itself.
 
 In particular, `configure` will use ansible to reconfigure the appliance nodes.
 The playbook used for this configuration is available in the `ansible`

chi-appliance.py

@@ -13,20 +13,20 @@ from blazarclient import client as blazar_client
 import shade
 
 
-def get_cloud_config(cloud=None):
+def get_cloud_config(region=None):
     """Retrieve the config in clouds.yaml."""
     config = os_client_config.OpenStackConfig()
-    return config.get_one(cloud)
+    return config.get_one('chameleon', region_name=region)
 
 
-def get_shade_client(cloud=None):
-    cloud = get_cloud_config(cloud)
+def get_shade_client(region=None):
+    cloud = get_cloud_config(region)
     return shade.OpenStackCloud(cloud_config=cloud)
 
 
-def get_blazar_client(cloud=None):
+def get_blazar_client(region=None):
     """Retrieve a client to blazar based on clouds.yaml config."""
-    cloud_config = get_cloud_config(cloud)
+    cloud_config = get_cloud_config(region)
     session = cloud_config.get_session()
 
     # blazar acces
@@ -35,8 +35,8 @@ def get_blazar_client(cloud=None):
 
 def do_create(argv):
     """Create an appliance inside a lease, based on template."""
-    shade_client = get_shade_client(argv.cloud)
-    blazar_client = get_blazar_client(argv.cloud)
+    shade_client = get_shade_client(argv.region)
+    blazar_client = get_blazar_client(argv.region)
     # build common parameters
     leases = blazar_client.lease.list()
     leases = [l for l in leases if l['name'] == argv.lease]
@@ -59,7 +59,7 @@ def do_create(argv):
 
 def do_delete(argv):
     """Delete an appliance with <name>."""
-    shade_client = get_shade_client(argv.cloud)
+    shade_client = get_shade_client(argv.region)
     ret = shade_client.delete_stack(argv.name)
     if ret:
         print("Appliance successfully deleted.")
@@ -69,7 +69,7 @@ def do_delete(argv):
 
 def do_show(argv):
     """Show appliance with <name>."""
-    shade_client = get_shade_client(argv.cloud)
+    shade_client = get_shade_client(argv.region)
     app = shade_client.get_stack(argv.name)
     if app:
         print(json.dumps(app, indent=4))
@@ -79,7 +79,7 @@ def do_show(argv):
 
 def do_list(argv):
     """List all appliances."""
-    shade_client = get_shade_client(argv.cloud)
+    shade_client = get_shade_client(argv.region)
     app_list = shade_client.list_stacks()
     if app_list:
         print(json.dumps(app_list, indent=4))
@@ -97,8 +97,8 @@ def do_configure(argv):
       - one playbook to copy the ansible config over to the frontend and launch
         ansible inside.
     """
-    shade_client = get_shade_client(argv.cloud)
-    blazar_client = get_blazar_client(argv.cloud)
+    shade_client = get_shade_client(argv.region)
+    blazar_client = get_blazar_client(argv.region)
     # basic info
     appliance = shade_client.get_stack(argv.name)
     if not appliance:
@@ -207,8 +207,8 @@ def do_configure(argv):
 
 def main():
     parser = argparse.ArgumentParser(description='Chameleon Appliance Helper')
-    parser.add_argument('--cloud', default=os.environ.get('OS_CLOUD'),
-                        help='Cloud name')
+    parser.add_argument('--region', default=os.environ.get('OS_REGION_NAME'),
+                        help='Region name (in clouds.yaml)')
     parser.add_argument('--debug', help="Print debugging output",
                         action='store_true')
     subparsers = parser.add_subparsers(title='Commands', dest='command')

chi-lease.py

@@ -11,16 +11,15 @@ from blazarclient import client as blazar_client
 import keystoneauth1
 
 
-def get_cloud_config(cloud=None):
+def get_cloud_config(region=None):
     """Retrieve the config in clouds.yaml."""
     config = os_client_config.OpenStackConfig()
-    return config.get_one(cloud)
+    return config.get_one('chameleon', region_name=region)
 
 
-def get_client(cloud=None):
+def get_client(region=None):
     """Retrieve a client to blazar based on clouds.yaml config."""
-    config = os_client_config.OpenStackConfig()
-    cloud_config = config.get_one(cloud)
+    cloud_config = get_cloud_config(region)
     session = cloud_config.get_session()
 
     # blazar acces
@@ -29,7 +28,7 @@ def get_client(cloud=None):
 
 def do_create(argv):
     """Create a lease using <name>, with <size> nodes, starting now."""
-    client = get_client(argv.cloud)
+    client = get_client(argv.region)
     name = argv.name
     node_count = int(argv.size)
 
@@ -67,7 +66,7 @@ def do_create(argv):
 
 def do_delete(argv):
     """Delete all leases with <name> as name."""
-    client = get_client(argv.cloud)
+    client = get_client(argv.region)
     # leases are listed by lease-id, so we cannot use lease.delete()
     lease_list = client.lease.list()
     leases = [l for l in lease_list if l['name'] == argv.name]
@@ -79,7 +78,7 @@ def do_delete(argv):
 
 def do_show(argv):
     """Show all leases with <name> as name."""
-    client = get_client(argv.cloud)
+    client = get_client(argv.region)
     # leases are listed by lease-id, so we cannot use lease.get()
     lease_list = client.lease.list()
     # make sure to print every lease with that name.
@@ -92,7 +91,7 @@ def do_show(argv):
 
 def do_list(argv):
     """List all leases."""
-    client = get_client(argv.cloud)
+    client = get_client(argv.region)
     lease_list = client.lease.list()
     if lease_list:
         print(json.dumps(lease_list, indent=4))
@@ -102,8 +101,8 @@ def do_list(argv):
 
 def main():
     parser = argparse.ArgumentParser(description='Chameleon Lease Helper')
-    parser.add_argument('--cloud', default=os.environ.get('OS_CLOUD'),
-                        help='Cloud name')
+    parser.add_argument('--region', default=os.environ.get('OS_REGION_NAME'),
+                        help='Region name (in clouds.yaml)')
     parser.add_argument('--debug', help="Print debugging output",
                         action='store_true')
     subparsers = parser.add_subparsers(title='Commands', dest='command')

chi-mngmt.py

+#!/usr/bin/env python
+
+import argparse
+import json
+import logging
+import os
+
+import os_client_config
+from blazarclient import client as blazar_client
+import shade
+
+
+def get_cloud_config(region=None):
+    """Retrieve the config in clouds.yaml."""
+    config = os_client_config.OpenStackConfig()
+    return config.get_one('chameleon', region_name=region)
+
+
+def get_shade_client(region=None):
+    cloud = get_cloud_config(region)
+    return shade.OpenStackCloud(cloud_config=cloud)
+
+
+def get_blazar_client(region=None):
+    """Retrieve a client to blazar based on clouds.yaml config."""
+    cloud_config = get_cloud_config(region)
+    session = cloud_config.get_session()
+
+    # blazar acces
+    return blazar_client.Client(1, session=session, service_type='reservation')
+
+
+def do_listclouds(argv):
+    """List available clouds."""
+    config = os_client_config.OpenStackConfig()
+    clouds = config.get_all()
+    if not clouds:
+        print("No cloud in the clouds.yaml definition.")
+    else:
+        for cloud in clouds:
+            print(json.dumps(cloud.config, indent=4))
+
+
+def do_listkeys(argv):
+    """List keys in a cloud."""
+    shade_client = get_shade_client(argv.region)
+    ret = shade_client.list_keypairs()
+    if ret:
+        for k in ret:
+            print(json.dumps(k, indent=4))
+    else:
+        print("No keypair found.")
+
+
+def main():
+    parser = argparse.ArgumentParser(description='Chameleon General Helper')
+    parser.add_argument('--region', default=os.environ.get('OS_REGION_NAME'),
+                        help='Region name (in clouds.yaml)')
+    parser.add_argument('--debug', help="Print debugging output",
+                        action='store_true')
+    subparsers = parser.add_subparsers(title='Commands', dest='command')
+    subparsers.required = True
+
+    # create a lease
+    parser_listclouds = subparsers.add_parser("list-clouds",
+                                              help="List available clouds")
+    parser_listclouds.set_defaults(func=do_listclouds)
+
+    parser_listkeys = subparsers.add_parser("list-keys",
+                                            help="List available keypairs")
+    parser_listkeys.set_defaults(func=do_listkeys)
+
+    args = parser.parse_args()
+    if args.debug:
+        logger = logging.getLogger('keystoneauth')
+        logger.addHandler(logging.StreamHandler())
+        logger.setLevel(logging.DEBUG)
+    args.func(args)
+
+
+if __name__ == '__main__':
+    main()

clouds.yaml

 # clouds information to connect to openstack
 clouds:
-  chi-tacc:
+  chameleon:
+    interface: 'public'
+    identity_api_version: 3
     auth:
       project_name: 'CH-817005'
-      project_id: '615f0f3f9fb74922b9b1bdb22acc3acc'
-      auth_url: 'https://chi.tacc.chameleoncloud.org:5000/v3'
       user_domain_name: "Default"
-    region_name: 'CHI@TACC'
-    interface: 'public'
-    identity_api_version: 3
+      auth_url: 'https://chameleoncloud.org/'
+    regions:
+      - name: 'CHI@TACC'
+        values:
+          auth:
+            project_name: 'CH-817005'
+            project_id: '615f0f3f9fb74922b9b1bdb22acc3acc'
+            auth_url: 'https://chi.tacc.chameleoncloud.org:5000/v3'
+      - name: 'CHI@UC'
+        values:
+          auth:
+            project_name: 'CH-817005'
+            project_id: '615f0f3f9fb74922b9b1bdb22acc3acc'
+            auth_url: 'https://chi.uc.chameleoncloud.org:5000/v3'

secure.yaml.templ

 # clouds information to connect to openstack
 clouds:
-  chi-tacc:
+  chameleon:
     auth:
       username: 'USERNAME'
       password: 'PASSWORD'