CI: Use precompiled Director release

ci/ops_files/deployment-configuration.yml

@@ -1,30 +1,15 @@
 ---
-- type: replace
-  path: /releases/name=bosh
-  value:
-    name: bosh
-    url: file://bosh-release.tgz
-
 - type: replace
   path: /releases/name=bosh-openstack-cpi
   value:
     name: bosh-openstack-cpi
     url: file://bosh-openstack-cpi.tgz
 
 - type: replace
-  path: /resource_pools/name=vms
+  path: /resource_pools/name=vms/cloud_properties?
   value:
-    name: vms
-    network: default
-    stemcell:
-      url: file://stemcell.tgz
-    cloud_properties:
-      instance_type: ((openstack_flavor))
-      availability_zone: ((availability_zone))
-    env:
-      bosh:
-        mbus:
-          cert: ((mbus_bootstrap_ssl))
+    instance_type: ((openstack_flavor))
+    availability_zone: ((availability_zone))
 
 - type: replace
   path: /instance_groups/name=bosh/properties/hm/resurrector_enabled

ci/ops_files/use-golang-cpi.yml

@@ -0,0 +1,18 @@
+---
+# Deploy the director with the Go CPI job instead of the Ruby one (bosh-deployment/openstack/cpi.yml
+# wires openstack_cpi).
+- type: replace
+  path: /instance_groups/name=bosh/jobs/name=openstack_cpi
+  value:
+    name: openstack_cpi_golang
+    release: bosh-openstack-cpi
+
+- type: replace
+  path: /instance_groups/name=bosh/properties/director/cpi_job
+  value: openstack_cpi_golang
+
+- type: replace
+  path: /cloud_provider/template
+  value:
+    name: openstack_cpi_golang
+    release: bosh-openstack-cpi

ci/pipeline.yml

@@ -40,15 +40,9 @@ jobs:
         timeout: *timeouts-long
       - get: terraform-image
       - get: openstack-cpi-release-docker-image
-      - get: stemcell-director
-        resource: openstack-ubuntu-jammy-stemcell
-        timeout: *timeouts-long
-        trigger: true
       - get: stemcell
         resource: openstack-ubuntu-jammy-stemcell
         timeout: *timeouts-long
-      - get: bosh-release
-        timeout: *timeouts-long
       - get: bats
         timeout: *timeouts-long
       - get: bosh-deployment
@@ -154,7 +148,6 @@ jobs:
           openstack_username: ((openstack_ci_user.username))
           openstack_api_key: ((openstack_ci_user.password))
           openstack_domain: Default
-          distro: "ubuntu"
           availability_zone: nova
           DEBUG_BATS: *debug_bats
 
@@ -371,13 +364,6 @@ resource_types:
     repository: ljfranklin/terraform-resource
     username: ((dockerhub_username))
     password: ((dockerhub_password))
-- name: bosh-release
-  type: registry-image
-  check_every: 168h
-  source:
-    repository: dpb587/bosh-release-resource
-    username: ((dockerhub_username))
-    password: ((dockerhub_password))
 - name: gcs
   type: registry-image
   check_every: 168h
@@ -459,11 +445,6 @@ resources:
     uri: https://github.com/cloudfoundry/bosh-acceptance-tests.git
     branch: master
 
-- name: bosh-release
-  type: bosh-io-release
-  source:
-    repository: cloudfoundry/bosh
-
 - name: openstack-ubuntu-jammy-stemcell
   type: bosh-io-stemcell
   source:

ci/tasks/deploy-manual-networking.sh

@@ -16,7 +16,6 @@ source bosh-openstack-cpi-release/ci/tasks/utils.sh
 : "${openstack_api_key:?}"
 : "${openstack_domain:?}"
 : "${DEBUG_BATS:?}"
-: "${distro:?}"
 openstack_ca_file_path="${openstack_ca_file_path:-}"
 optional_value availability_zone
 
@@ -56,13 +55,6 @@ pushd bosh-openstack-cpi-release
 
 popd
 cp ./bosh-openstack-cpi-dev.tgz "${deployment_dir}/${cpi_release_name}.tgz"
-cp ./stemcell-director/*.tgz "${deployment_dir}/stemcell.tgz"
-prepare_bosh_release "${distro}"
-
-echo "Calculating MD5 of original stemcell:"
-md5sum stemcell-director/*.tgz
-echo "Calculating MD5 of copied stemcell:"
-md5sum "${deployment_dir}/stemcell.tgz"
 
 cd "${deployment_dir}"
 
@@ -77,9 +69,9 @@ bosh-go int ../bosh-deployment/bosh.yml \
     -o ../bosh-deployment/openstack/cpi.yml \
     "${maybe_use_custom_ca_ops_file[@]}" \
     -o ../bosh-deployment/external-ip-not-recommended.yml \
-    -o ../bosh-deployment/misc/source-releases/bosh.yml \
     -o ../bosh-deployment/jumpbox-user.yml \
     -o ../bosh-openstack-cpi-release/ci/ops_files/deployment-configuration.yml \
+    -o ../bosh-openstack-cpi-release/ci/ops_files/use-golang-cpi.yml \
     -o ../bosh-openstack-cpi-release/ci/ops_files/custom-manual-networking.yml \
     -o ../bosh-openstack-cpi-release/ci/ops_files/timeouts.yml \
     -o ../bosh-openstack-cpi-release/ci/ops_files/remove-registry.yml \

ci/tasks/deploy-manual-networking.yml

@@ -6,8 +6,6 @@ image_resource:
     repository: boshcpi/openstack-cpi-release
 inputs:
   - name: bosh-openstack-cpi-release
-  - name: bosh-release
-  - name: stemcell-director
   - name: terraform-cpi
   - name: bosh-deployment
 outputs:
@@ -25,6 +23,5 @@ params:
   openstack_api_key:                   ""
   openstack_domain:                    ""
   DEBUG_BATS:                          ""
-  distro:                              ""
   openstack_file_path:                 ""
   availability_zone:                   replace-me

ci/tasks/run-golang-lifecycle-specs.sh

@@ -0,0 +1,43 @@
+#!/usr/bin/env bash
+
+set -euo pipefail
+
+source bosh-openstack-cpi-release/ci/tasks/utils.sh
+
+: "${BOSH_OPENSTACK_DOMAIN:?}"
+: "${BOSH_OPENSTACK_AUTH_URL_V3:?}"
+: "${BOSH_OPENSTACK_USERNAME_V3:?}"
+: "${BOSH_OPENSTACK_API_KEY_V3:?}"
+: "${BOSH_OPENSTACK_PROJECT:?}"
+: "${BOSH_OPENSTACK_FLAVOR_WITH_NO_ROOT_DISK:?}"
+: "${BOSH_OPENSTACK_VOLUME_TYPE:?}"
+
+optional_value BOSH_OPENSTACK_AVAILABILITY_ZONE
+
+# Infrastructure identifiers come from the lifecycle terraform output.
+metadata=terraform-cpi/metadata
+
+BOSH_OPENSTACK_MANUAL_IP=$(jq --raw-output ".manual_ip" "${metadata}")
+BOSH_OPENSTACK_ALLOWED_ADDRESS_PAIRS=$(jq --raw-output ".allowed_address_pairs" "${metadata}")
+BOSH_OPENSTACK_NO_DHCP_MANUAL_IP_1=$(jq --raw-output ".no_dhcp_manual_ip_1" "${metadata}")
+BOSH_OPENSTACK_NO_DHCP_MANUAL_IP_2=$(jq --raw-output ".no_dhcp_manual_ip_2" "${metadata}")
+BOSH_OPENSTACK_NET_ID=$(jq --raw-output ".net_id" "${metadata}")
+BOSH_OPENSTACK_NET_ID_NO_DHCP_1=$(jq --raw-output ".net_id_no_dhcp_1" "${metadata}")
+BOSH_OPENSTACK_NET_ID_NO_DHCP_2=$(jq --raw-output ".net_id_no_dhcp_2" "${metadata}")
+BOSH_OPENSTACK_DEFAULT_KEY_NAME=$(jq --raw-output ".default_key_name" "${metadata}")
+BOSH_OPENSTACK_FLOATING_IP=$(jq --raw-output ".floating_ip" "${metadata}")
+BOSH_OPENSTACK_SECURITY_GROUP_NAME=$(jq --raw-output ".security_group_name" "${metadata}")
+BOSH_OPENSTACK_SECURITY_GROUP_ID=$(jq --raw-output ".security_group_id" "${metadata}")
+export BOSH_OPENSTACK_MANUAL_IP BOSH_OPENSTACK_ALLOWED_ADDRESS_PAIRS \
+  BOSH_OPENSTACK_NO_DHCP_MANUAL_IP_1 BOSH_OPENSTACK_NO_DHCP_MANUAL_IP_2 \
+  BOSH_OPENSTACK_NET_ID BOSH_OPENSTACK_NET_ID_NO_DHCP_1 BOSH_OPENSTACK_NET_ID_NO_DHCP_2 \
+  BOSH_OPENSTACK_DEFAULT_KEY_NAME BOSH_OPENSTACK_FLOATING_IP \
+  BOSH_OPENSTACK_SECURITY_GROUP_NAME BOSH_OPENSTACK_SECURITY_GROUP_ID
+
+# Extract the stemcell; the Go harness reads <dir>/stemcell.MF and <dir>/image.
+mkdir -p "${PWD}/openstack-lifecycle-stemcell/stemcell"
+tar -C "${PWD}/openstack-lifecycle-stemcell/stemcell" -xzf "${PWD}/openstack-lifecycle-stemcell/stemcell.tgz"
+export BOSH_OPENSTACK_STEMCELL_PATH="${PWD}/openstack-lifecycle-stemcell/stemcell"
+
+cd bosh-openstack-cpi-release/src/openstack_cpi_golang
+go run github.com/onsi/ginkgo/v2/ginkgo --tags lifecycle -r -v lifecycle

ci/tasks/run-golang-lifecycle-specs.yml

@@ -0,0 +1,25 @@
+---
+platform: linux
+
+image_resource:
+  type: registry-image
+  source:
+    repository: ghcr.io/cloudfoundry/bosh/golang-release
+
+inputs:
+- name: bosh-openstack-cpi-release
+- name: openstack-lifecycle-stemcell
+- name: terraform-cpi
+
+run:
+  path: bosh-openstack-cpi-release/ci/tasks/run-golang-lifecycle-specs.sh
+
+params:
+  BOSH_OPENSTACK_DOMAIN:                   ""
+  BOSH_OPENSTACK_AUTH_URL_V3:              ""
+  BOSH_OPENSTACK_USERNAME_V3:              ""
+  BOSH_OPENSTACK_API_KEY_V3:               ""
+  BOSH_OPENSTACK_PROJECT:                  ""
+  BOSH_OPENSTACK_VOLUME_TYPE:              ""
+  BOSH_OPENSTACK_FLAVOR_WITH_NO_ROOT_DISK: ""
+  BOSH_OPENSTACK_AVAILABILITY_ZONE:        replace-me

ci/tasks/run-manual-networking-bats.sh

@@ -1,12 +1,16 @@
 #!/usr/bin/env bash
 
-set -e -x
+# Network and credential variables are exported at runtime by the utils.sh
+# helpers (export_terraform_variable/optional_value), which shellcheck can't follow.
+# shellcheck disable=SC1091,SC2154
+
+set -eo pipefail
 
 source bosh-openstack-cpi-release/ci/tasks/utils.sh
 
-: ${stemcell_name:?}
-: ${openstack_flavor_with_ephemeral_disk:?}
-: ${openstack_flavor_with_no_ephemeral_disk:?}
+: "${stemcell_name:?}"
+: "${openstack_flavor_with_ephemeral_disk:?}"
+: "${openstack_flavor_with_no_ephemeral_disk:?}"
 
 optional_value availability_zone
 optional_value bats_rspec_tags
@@ -32,25 +36,27 @@ export_terraform_variable terraform-cpi/metadata "security_group"
 working_dir=$PWD
 # checked by BATs environment helper (bosh-acceptance-tests.git/lib/bat/env.rb)
 export BAT_STEMCELL="${working_dir}/stemcell/stemcell.tgz"
-export BAT_DIRECTOR=${director_public_ip}
+export BAT_DIRECTOR="${director_public_ip}"
 export BAT_INFRASTRUCTURE='openstack'
 export BAT_BOSH_CLI='bosh-go'
 
 export BOSH_ENVIRONMENT="${director_public_ip}"
 export BOSH_CLIENT="admin"
-export BOSH_CLIENT_SECRET="$( creds_path /admin_password )"
-export BOSH_CA_CERT="$( creds_path /director_ssl/ca )"
+BOSH_CLIENT_SECRET="$(creds_path /admin_password)"
+export BOSH_CLIENT_SECRET
+BOSH_CA_CERT="$(creds_path /director_ssl/ca)"
+export BOSH_CA_CERT
 
 ssh_private_key=$( creds_path /jumpbox_ssh/private_key | sed 's/$/\\n/' | tr -d '\n' )
 private_key_path=$(mktemp)
-echo -e "${ssh_private_key}" > ${private_key_path}
+echo -e "${ssh_private_key}" > "${private_key_path}"
 export BOSH_ALL_PROXY="ssh+socks5://jumpbox@${director_public_ip}:22?private-key=${private_key_path}"
 
 echo "using bosh CLI version..."
 bosh-go --version
 
 export BAT_DEPLOYMENT_SPEC="${working_dir}/bats-config.yml"
-cat > $BAT_DEPLOYMENT_SPEC <<EOF
+cat > "${BAT_DEPLOYMENT_SPEC}" <<EOF
 ---
 cpi: openstack
 properties:
@@ -95,4 +101,6 @@ EOF
 
 cd bats
 bundle install -j4
-bundle exec rspec --tag ~raw_ephemeral_storage --tag ~multiple_manual_networks ${bats_rspec_tags} spec
+# bats_rspec_tags may hold multiple space-separated flags; split into an array.
+read -ra rspec_tags <<< "${bats_rspec_tags:-}"
+bundle exec rspec --tag ~raw_ephemeral_storage --tag ~multiple_manual_networks "${rspec_tags[@]}" spec

ci/tasks/utils.sh

@@ -45,41 +45,6 @@ init_openstack_cli_env(){
 
 }
 
-prepare_bosh_release() {
-    local distribution=${1}
-    local bosh_release_version=${2}
-    local stemcell_version=${3}
-
-    use_compiled_release=true
-
-    local s3_path_to_bosh_release
-    s3_path_to_bosh_release=$(find_bosh_compiled_release "${distribution}" "${bosh_release_version}" "${stemcell_version}")
-
-    if [ -n "${s3_path_to_bosh_release}" ]; then
-        echo "Using compiled BOSH release: s3://bosh-compiled-release-tarballs/$s3_path_to_bosh_release"
-        # shellcheck disable=SC2154
-        aws --no-sign-request s3 cp "s3://bosh-compiled-release-tarballs/${s3_path_to_bosh_release}" "${deployment_dir}/bosh-release.tgz"
-    else
-        use_compiled_release=false
-    fi
-
-    if [ "${use_compiled_release}" = "false" ]; then
-       echo "Using BOSH release from sources"
-       # shellcheck disable=SC2154
-       cp ./bosh-release/*.tgz "${deployment_dir}/bosh-release.tgz"
-    fi
-}
-
-find_bosh_compiled_release(){
-    local distribution=$1
-    local bosh_release_version=${2:-$(cat ./bosh-release/version)}
-    local stemcell_version=${3:-$(cat ./stemcell-director/version)}
-
-    local s3_path_to_bosh_release
-    s3_path_to_bosh_release=$(aws --no-sign-request s3 ls s3://bosh-compiled-release-tarballs | grep -oE "[^ ](\w|-)*$bosh_release_version.+$distribution.+$stemcell_version.*\.tgz" | sort -r | head -1)
-    echo "${s3_path_to_bosh_release}"
-}
-
 export_terraform_variable() {
     local metadata=$1
     local variable_name=$2

src/openstack_cpi_golang/cpi/config/config.go

@@ -31,7 +31,7 @@
 }
 
 type OpenstackConfig struct {
 	AuthURL                      string   `json:"auth_url"`
 	Username                     string   `json:"username"`
 	APIKey                       string   `json:"api_key"`
 	ApplicationCredentialID      string   `json:"application_credential_id"`
@@ -48,7 +48,7 @@
 	IgnoreServerAvailabilityZone bool     `json:"ignore_server_availability_zone"`
 	HumanReadableVMNames         bool     `json:"human_readable_vm_names"`
 	UseNovaNetworking            bool     `json:"use_nova_networking"`
-	ConnectionOptions            string   `json:"connection_options"`
+	ConnectionOptions            map[string]interface{} `json:"connection_options"`
 	DomainName                   string   `json:"domain"`
 	ProjectName                  string   `json:"project"`
 	Tenant                       string   `json:"tenant"`

src/openstack_cpi_golang/cpi/methods/create_vm.go

@@ -83,9 +83,12 @@ func (m CreateVMMethod) CreateVMV2(
 		return apiv1.VMCID{}, apiv1.Networks{}, fmt.Errorf("failed to create image service: %w", err)
 	}
 
-	loadbalancerService, err := m.loadbalancerServiceBuilder.Build()
-	if err != nil {
-		return apiv1.VMCID{}, apiv1.Networks{}, fmt.Errorf("failed to create loadbalancer service: %w", err)
+	var loadbalancerService loadbalancer.LoadbalancerService
+	if len(cloudProps.LoadbalancerPools) > 0 {
+		loadbalancerService, err = m.loadbalancerServiceBuilder.Build()
+		if err != nil {
+			return apiv1.VMCID{}, apiv1.Networks{}, fmt.Errorf("failed to create loadbalancer service: %w", err)
+		}
 	}
 
 	_, err = imageService.GetImage(stemcellCID.AsString())