Minimal/2023.1 kvm gpu res

site-config/globals.yml

@@ -34,7 +34,13 @@ virtualenv: /opt/kolla/venv
 kolla_base_distro: "ubuntu"
 kolla_install_type: "source"
 
+# Where to get container images from
+docker_registry: "ghcr.io"
+docker_namespace: "chameleoncloud/kolla"
+openstack_tag: "2023.1-ubuntu-jammy"
 
+horizon_tag: "2023.1-kvm-res"
+blazar_tag: "2023.1-kvm-res"
 
 # default superadmin user and project
 keystone_admin_user: "admin"
@@ -90,9 +96,64 @@ identity_provider_url: "{{ keystone_identity_providers[0].identifier }}"
 keystone_oidc_provider_metadata_url: "{{ identity_provider_url }}/.well-known/openid-configuration"
 keystone_federation_oidc_jwks_uri: "{{ identity_provider_url }}/protocol/openid-connect/certs"
 
-# keystone must support mapping multiple projects or keycloak federation will fail
-keystone_image_full: ghcr.io/chameleoncloud/kolla/keystone:2023.1-ubuntu-jammy
-keystone_fernet_image_full: ghcr.io/chameleoncloud/kolla/keystone-fernet:2023.1-ubuntu-jammy
-keystone_ssh_image_full: ghcr.io/chameleoncloud/kolla/keystone-ssh:2023.1-ubuntu-jammy
-horizon_image_full: ghcr.io/chameleoncloud/kolla/horizon:2023.1-ubuntu-jammy
-fluentd_image_full: ghcr.io/chameleoncloud/kolla/fluentd:2023.1-ubuntu-jammy
+############
+# GPU config
+############
+
+# nova_pci_device_spec = 
+# nova_pci_alias =
+
+####################
+# Reservation Config
+####################
+enable_blazar: true
+
+horizon_image_full: ghcr.io/chameleoncloud/kolla/horizon:2023.1-kvm-res
+blazar_api_image_full: ghcr.io/chameleoncloud/kolla/blazar-api:2023.1-kvm-res
+blazar_manager_image_full: ghcr.io/chameleoncloud/kolla/blazar-manager:2023.1-kvm-res
+
+# reservation type flags
+blazar_enable_flavor_reservation: true
+# The trait which permits flavor reservation
+blazar_flavor_reservation_trait: custom_blazar_flavor_reservation
+
+blazar_enable_host_reservation: false
+blazar_physical_polling_monitor: "{{ blazar_enable_host_reservation }}"
+blazar_physical_polling_monitor_dry_run: false
+
+blazar_enable_floatingip_reservation: false
+blazar_fip_polling_monitor: "{{ blazar_enable_floatingip_reservation }}"
+blazar_fip_polling_monitor_dry_run: false
+
+blazar_enable_network_reservation: false
+blazar_network_polling_monitor: "{{ blazar_enable_network_reservation }}"
+blazar_network_polling_monitor_dry_run: false
+
+blazar_enable_device_reservation: false
+blazar_zun_polling_monitor: "{{ blazar_enable_device_reservation }}"
+
+enable_blazar_allocation_enforcement: false
+blazar_randomize_hosts: false
+
+blazar_project_enforcement_id: charge_code
+# When to send lease notification email
+blazar_minutes_before_end_lease: 2880
+blazar_default_max_lease_duration: "{{ 60 * 60 * 24 * 7 }}"
+blazar_default_reservation_extension_window: 172800
+blazar_usage_default_allocated: 20000.0
+# Redis currently binds on public IP, use public FQDN
+blazar_usage_db_host: "{{ kolla_external_fqdn }}"
+blazar_usage_project_exemptions: []
+blazar_external_service_check_create_endpoint: "{{ chameleon_portal_url }}/api/balance_service/v2/check-create/"
+blazar_external_service_check_update_endpoint: "{{ chameleon_portal_url }}/api/balance_service/v2/check-update/"
+blazar_external_service_on_end_endpoint: "{{ chameleon_portal_url }}/api/balance_service/v2/on-end/"
+blazar_floatingip_billrate: 0.0
+blazar_email_relay: "127.0.0.1"
+blazar_api_allocation_extras: user_name
+blazar_host_default_resource_properties: '["=", "\$node_type", "compute_skylake"]'
+blazar_host_retry_without_default_resources: yes
+blazar_enable_plugin_network: yes
+blazar_network_default_resource_properties: '["=", "\$stitch_provider", "none"]'
+blazar_network_retry_without_default_resources: no
+blazar_floatingip_reservation_network_regex: "[pP]ublic"
+blazar_host_url_format: "https://chameleoncloud.org/hardware/node/sites/{{ chameleon_site_name }}/clusters/chameleon/nodes/{hypervisor_hostname}/"

site-config/node_custom_config/blazar.conf

@@ -0,0 +1,96 @@
+[DEFAULT]
+os_region_name = {{ openstack_region_name }}
+
+[enforcement]
+{# note: important to preserve newline after this #}
+exempt_projects = {% for project_id in blazar_usage_project_exemptions %}{{ project_id }}{% if not loop.last %},{% endif %}{% endfor %}
+
+{% set filters = [] %}
+{% if enable_blazar_max_lease_enforcement | bool %}
+{%   set _ = filters.append("MaxLeaseDurationFilter") %}
+{% endif %}
+{% if enable_blazar_allocation_enforcement | bool %}
+{%   set _ = filters.append("ExternalServiceFilter") %}
+{% endif %}
+enabled_filters = {{ filters | join(", ") }}
+
+{% if blazar_external_service_endpoint is defined %}
+external_service_base_endpoint = {{ blazar_external_service_endpoint }}
+{% endif %}
+{% if blazar_external_service_check_create_endpoint is defined %}
+external_service_check_create_endpoint = {{ blazar_external_service_check_create_endpoint }}
+{% endif %}
+{% if blazar_external_service_check_update_endpoint is defined %}
+external_service_check_update_endpoint = {{ blazar_external_service_check_update_endpoint }}
+{% endif %}
+{% if blazar_external_service_on_end_endpoint is defined %}
+external_service_on_end_endpoint = {{ blazar_external_service_on_end_endpoint }}
+{% endif %}
+max_lease_duration = {{ blazar_default_max_lease_duration }}
+reservation_extension_window = {{ blazar_default_reservation_extension_window }}
+
+[keystone_authtoken]
+region_name = {{ openstack_region_name }}
+
+[manager]
+{# note: important to preserve newline after this #}
+plugins = network.plugin,virtual.floatingip.plugin{% if enable_nova | bool %},physical.host.plugin{% endif %}{% if enable_zun | bool %},device.plugin{% endif %}{% if blazar_enable_flavor_reservation | bool %},flavor.instance.plugin{% endif %}
+
+minutes_before_end_lease = {{ blazar_minutes_before_end_lease }}
+
+[oslo_messaging_notifications]
+# Experiment Precis requires 2.0 message format, i.e. set driver to messagingv2
+driver = messagingv2
+
+[oslo_messaging_rabbit]
+# Currently needed (Xena) to avoid eventlet deadlock issue
+heartbeat_in_pthread = false
+
+[physical:host]
+before_end = email
+email_relay = {{ blazar_email_relay }}
+{% if blazar_email_port is defined %}
+email_port = {{ blazar_email_port }}
+{% endif %}
+{% if blazar_email_ssl is defined %}
+email_ssl = {{ blazar_email_ssl }}
+{% endif %}
+{% if blazar_email_user is defined %}
+email_user = {{ blazar_email_user }}
+{% endif %}
+{% if blazar_email_password is defined %}
+email_password = {{ blazar_email_password }}
+{% endif %}
+{% if enable_ironic | bool %}
+enable_polling_monitor = {{ blazar_physical_polling_monitor}}
+enable_polling_monitor_dry_run = {{ blazar_physical_polling_monitor_dry_run }}
+retry_allocation_without_defaults = {{ blazar_host_retry_without_default_resources | bool }}
+default_resource_properties = {{ blazar_host_default_resource_properties }}
+allow_reservation = {{ blazar_enable_host_reservation | bool }}
+randomize_host_selection = {{ blazar_randomize_hosts | bool }}
+{% endif %}
+
+{% if enable_zun | bool %}
+[device]
+enable_polling_monitor = {{ blazar_zun_polling_monitor}}
+{% endif %}
+
+[virtual:floatingip]
+billrate = {{ blazar_floatingip_billrate }}
+enable_polling_monitor = {{ blazar_fip_polling_monitor}}
+enable_polling_monitor_dry_run = {{ blazar_fip_polling_monitor_dry_run }}
+
+[network]
+retry_allocation_without_defaults = {{ blazar_network_retry_without_default_resources | bool }}
+default_resource_properties = {{ blazar_network_default_resource_properties }}
+enable_polling_monitor = {{ blazar_network_polling_monitor}}
+enable_polling_monitor_dry_run = {{ blazar_network_polling_monitor_dry_run }}
+
+[flavor:instance]
+randomize_host_selection = {{ blazar_randomize_hosts | bool }}
+# Note this configures if email is sent. Email relay uses host config.
+before_end = email
+placement_reservation_permitted_trait = {{ blazar_flavor_reservation_trait }}
+
+[api]
+allocation_extras = {{ blazar_api_allocation_extras }}

site-config/node_custom_config/blazar/policy.yaml

@@ -0,0 +1,104 @@
+# Default rule for most Admin APIs.
+#"admin": "is_admin:True or role:admin"
+
+# Default rule for most non-Admin APIs.
+"admin_or_owner": "rule:admin or project_id:%(project_id)s"
+
+# Policy rule for List/Show Lease(s) API.
+# GET  /{api_version}/leases
+# GET  /{api_version}/leases/{lease_id}
+#"blazar:leases:get": "rule:admin_or_owner"
+
+# Policy rule for Delete Lease API.
+# DELETE  /{api_version}/leases/{lease_id}
+#"blazar:leases:delete": "rule:admin_or_owner"
+
+# Policy rule for List/Show Host(s) API.
+# GET  /{api_version}/os-hosts
+# GET  /{api_version}/os-hosts/{host_id}
+"blazar:oshosts:get": "rule:admin_or_owner"
+
+# Policy rule for Delete Host API.
+# DELETE  /{api_version}/os-hosts/{host_id}
+"blazar:oshosts:delete": "rule:admin_api"
+
+# Policy rule for List/Get Host(s) Allocations API.
+# GET  /{api_version}/os-hosts/allocations
+# GET  /{api_version}/os-hosts/{host_id}/allocation
+"blazar:oshosts:get_allocations": "rule:admin_or_owner"
+
+# Policy rule for Reallocate Host API.
+# PUT  /{api_version}/os-hosts/{host_id}/allocation
+"blazar:oshosts:reallocate": "rule:admin_api"
+
+# Policy rule for Resource Properties API.
+# GET  /{api_version}/os-hosts/resource_properties
+"blazar:oshosts:get_resource_properties": "@"
+
+# Policy rule for Resource Properties API.
+# PATCH  /{api_version}/os-hosts/resource_properties/{property_name}
+"blazar:oshosts:patch_resource_properties": "rule:admin_api"
+
+# Policy rule for List/Show Network(s) API.
+# GET  /{api_version}/networks
+# GET  /{api_version}/networks/{network_id}
+"blazar:networks:get": "rule:admin_or_owner"
+
+# Policy rule for Delete Network API.
+# DELETE  /{api_version}/networks/{network_id}
+"blazar:networks:delete": "rule:admin_api"
+
+# Policy rule for List/Get Network(s) Allocations API.
+# GET  /{api_version}/networks/allocations
+# GET  /{api_version}/networks/{network_id}/allocation
+"blazar:networks:get_allocations": "rule:admin_or_owner"
+
+# Policy rule for Resource Properties API.
+# GET  /{api_version}/networks/resource_properties
+"blazar:networks:get_resource_properties": "@"
+
+# Policy rule for Resource Properties API.
+# PATCH  /{api_version}/networks/resource_properties/{property_name}
+"blazar:networks:patch_resource_properties": "rule:admin_api"
+
+# Policy rule for List/Show Device(s) API.
+# GET  /{api_version}/devices
+# GET  /{api_version}/devices/{device_id}
+"blazar:devices:get": "rule:admin_or_owner"
+
+# Policy rule for Update Host API.
+# PUT  /{api_version}/devices/{device_id}
+"blazar:devices:put": "rule:admin_api"
+
+# Policy rule for Delete Device API.
+# DELETE  /{api_version}/devices/{device_id}
+"blazar:devices:delete": "rule:admin_api"
+
+# Policy rule for List/Get Device(s) Allocations API.
+# GET  /{api_version}/devices/allocations
+# GET  /{api_version}/devices/{device_id}/allocation
+"blazar:devices:get_allocations": "rule:admin_or_owner"
+
+# Policy rule for Resource Properties API.
+# GET  /{api_version}/devices/resource_properties
+"blazar:devices:get_resource_properties": "@"
+
+# Policy rule for Resource Properties API.
+# PATCH  /{api_version}/devices/resource_properties/{property_name}
+"blazar:devices:patch_resource_properties": "rule:admin_api"
+
+"default": "!"
+"admin_api": "role:admin"
+"blazar:leases:create": "rule:admin_or_owner"
+"blazar:leases:update": "rule:admin_or_owner"
+"blazar:plugins:get": "@"
+"blazar:oshosts:create": "rule:admin_api"
+"blazar:oshosts:update": "rule:admin_api"
+"blazar:oshosts:list_allocations": "rule:admin_or_owner"
+"blazar:networks:create": "rule:admin_api"
+"blazar:networks:update": "rule:admin_api"
+"blazar:networks:put": "rule:admin_or_owner"
+"blazar:networks:list_allocations": "rule:admin_or_owner"
+"blazar:devices:create": "rule:admin_api"
+"blazar:devices:update": "rule:admin_api"
+"blazar:devices:list_allocations": "rule:admin_or_owner"

site-config/node_custom_config/glance/policy.yaml

@@ -0,0 +1,18 @@
+---
+member_or_reader: role:member or role:reader
+
+get_image: role:admin or (rule:member_or_reader and (project_id:%(project_id)s or project_id:%(member_id)s or 'community':%(visibility)s or 'public':%(visibility)s or 'shared':%(visibility)s))
+get_images: role:admin or (rule:member_or_reader and project_id:%(project_id)s)
+get_image_location: role:admin or (rule:member_or_reader and project_id:%(project_id)s)
+get_member: role:admin or rule:member_or_reader and (project_id:%(project_id)s or project_id:%(member_id)s)
+get_members: role:admin or rule:member_or_reader and (project_id:%(project_id)s or project_id:%(member_id)s)
+get_metadef_namespace: role:admin or (rule:member_or_reader and (project_id:%(project_id)s or 'public':%(visibility)s))
+get_metadef_namespaces: role:admin or (rule:member_or_reader and project_id:%(project_id)s)
+get_metadef_object: role:admin or (rule:member_or_reader and (project_id:%(project_id)s or 'public':%(visibility)s)) 
+get_metadef_objects: role:admin or (rule:member_or_reader and (project_id:%(project_id)s or 'public':%(visibility)s))
+list_metadef_resource_types: role:admin or (rule:member_or_reader and (project_id:%(project_id)s or 'public':%(visibility)s))
+get_metadef_resource_type: role:admin or (rule:member_or_reader and (project_id:%(project_id)s or 'public':%(visibility)s))
+get_metadef_property: role:admin or (rule:member_or_reader and (project_id:%(project_id)s or 'public':%(visibility)s))
+get_metadef_properties: role:admin or (rule:member_or_reader and (project_id:%(project_id)s or 'public':%(visibility)s))
+get_metadef_tag: role:admin or (rule:member_or_reader and (project_id:%(project_id)s or 'public':%(visibility)s))
+get_metadef_tags: role:admin or (rule:member_or_reader and (project_id:%(project_id)s or 'public':%(visibility)s))

site-config/node_custom_config/horizon/custom_local_settings

@@ -147,3 +147,25 @@ X_FRAME_OPTIONS = 'SAMEORIGIN'
 
 # disable usage report on overview page
 OPENSTACK_USE_SIMPLE_TENANT_USAGE = False
+
+{% if enable_blazar | bool %}
+OPENSTACK_BLAZAR_HOST_RESERVATION = {
+  'enabled': {{ blazar_enable_host_reservation | bool }},
+  'url_format': '{{ blazar_host_url_format }}',
+}
+OPENSTACK_BLAZAR_FLAVOR_RESERVATION = {
+  "enabled": {{ blazar_enable_flavor_reservation | bool}},
+  "blazar_flavor_reservation_trait": '{{ blazar_flavor_reservation_trait }}',
+}
+OPENSTACK_BLAZAR_FLOATINGIP_RESERVATION = {
+  # Allow reserving floating IPs on this network
+  'enabled': {{ blazar_enable_floatingip_reservation | bool }},
+  'network_name_regex': '{{ blazar_floatingip_reservation_network_regex }}'
+}
+OPENSTACK_BLAZAR_DEVICE_RESERVATION = {
+  'enabled': {{ blazar_enable_device_reservation | bool }},
+}
+OPENSTACK_BLAZAR_NETWORK_RESERVATION = {
+  'enabled': {{ blazar_enable_network_reservation | bool }},
+}
+{% endif %}

site-config/node_custom_config/nova.conf

@@ -0,0 +1,16 @@
+[filter_scheduler]
+available_filters = nova.scheduler.filters.all_filters
+available_filters = blazarnova.scheduler.filters.blazar_filter.BlazarFilter
+enabled_filters = ComputeFilter, ComputeCapabilitiesFilter, ImagePropertiesFilter, ServerGroupAntiAffinityFilter, ServerGroupAffinityFilter, PciPassthroughFilter, BlazarFilter
+
+[pci]
+
+report_in_placement = True
+
+{% if nova_pci_device_spec is defined %}
+device_spec = {{ nova_pci_device_spec | to_json }}
+{% endif %}
+
+{% for alias in nova_pci_alias | default('[]') %}
+alias = {{ alias | to_json }}
+{% endfor %}

site-config/node_custom_config/nova/policy.yaml

@@ -0,0 +1,3 @@
+# Original: #"project_reader_api": "role:reader and project_id:%(project_id)s"
+# replace due to keystone app credential not honoring implicit roles
+"project_reader_api": "(role:reader or role:member) and project_id:%(project_id)s"