diff --git a/collection/stages/roles/day2ops/defaults/main.yml b/collection/stages/roles/day2ops/defaults/main.yml index ce63b11a..ddc03f6d 100644 --- a/collection/stages/roles/day2ops/defaults/main.yml +++ b/collection/stages/roles/day2ops/defaults/main.yml @@ -2,3 +2,6 @@ # defaults file for day2ops day2ops_steps: [] day2ops_report_filename: shiftstack-qa-day2ops-results.xml + +# Application Credentials rotation +app_credential_name: "AppCreds-{{ user_cloud }}" diff --git a/collection/stages/roles/day2ops/tasks/procedures/rotate_app_creds.yml b/collection/stages/roles/day2ops/tasks/procedures/rotate_app_creds.yml new file mode 100644 index 00000000..b07fcd88 --- /dev/null +++ b/collection/stages/roles/day2ops/tasks/procedures/rotate_app_creds.yml @@ -0,0 +1,45 @@ +--- +# Source: https://github.com/shiftstack/installer/blob/master/docs/user/openstack/README.md#openstack-credentials-update +- name: Restore user/password auth to ensure full permissions for app credential creation + ansible.builtin.include_role: + name: shiftstack.stages.prepare + tasks_from: clouds.yml + +- name: Create application credentials and update clouds.yaml + ansible.builtin.include_role: + name: shiftstack.stages.prepare + tasks_from: app_creds.yml + +- name: Rotate OpenShift Cloud Credentials + ansible.builtin.shell: | + set -o pipefail && \ + cat {{ clouds_yaml_file_path }} | sed 's/{{ user_cloud }}:/openstack:/' | \ + oc set data -n kube-system secret/openstack-credentials clouds.yaml=- + environment: + KUBECONFIG: "{{ kubeconfig }}" + changed_when: true + +- name: Get OpenStack Credentials from OCP cluster + ansible.builtin.shell: | + set -o pipefail && \ + oc get secret -n kube-system openstack-credentials -o json | jq -r '.data."clouds.yaml"' | base64 -d + environment: + KUBECONFIG: "{{ kubeconfig }}" + register: ocp_creds_output + changed_when: false + +- name: Parse OCP credentials + ansible.builtin.set_fact: + ocp_creds: "{{ ocp_creds_output.stdout | from_yaml }}" + +- name: Verify credentials rotated to application credentials + ansible.builtin.assert: + that: + - ocp_creds.clouds.openstack.auth_type == 'v3applicationcredential' + fail_msg: "Credential rotation failed — auth_type is not v3applicationcredential" + success_msg: "Credential rotation verified — auth_type is v3applicationcredential" + +- name: Wait until the Cluster Operators are healthy + ansible.builtin.include_role: + name: tools_cluster_checks + tasks_from: wait_until_cluster_operators_ready.yml diff --git a/collection/stages/roles/prepare/defaults/main.yml b/collection/stages/roles/prepare/defaults/main.yml index 6dc3cdba..0e31dc8d 100644 --- a/collection/stages/roles/prepare/defaults/main.yml +++ b/collection/stages/roles/prepare/defaults/main.yml @@ -21,6 +21,10 @@ project: load_balancers: 1000 load_balancer_listeners: 5000 load_balancer_pools: 5000 +# Application Credentials +use_application_credentials: false +app_credential_name: "AppCreds-{{ user_cloud }}" + ocp_api_description: "API {{ ocp_cluster_name }}.{{ ocp_base_domain }}" ocp_apps_description: "APPS {{ ocp_cluster_name }}.{{ ocp_base_domain }}" ocp_bootstrap_fip_description: "Bootstrap {{ ocp_cluster_name }}.{{ ocp_base_domain }}" diff --git a/collection/stages/roles/prepare/tasks/app_creds.yml b/collection/stages/roles/prepare/tasks/app_creds.yml new file mode 100644 index 00000000..a6ad035f --- /dev/null +++ b/collection/stages/roles/prepare/tasks/app_creds.yml @@ -0,0 +1,76 @@ +--- +- name: Delete stale temporary Application Credential from interrupted previous run + ansible.builtin.shell: | + openstack application credential delete {{ app_credential_name }}-new + environment: + OS_CLOUD: "{{ user_cloud }}" + failed_when: false + changed_when: false + +- name: Create new Application Credential + ansible.builtin.shell: | + openstack application credential create \ + --description "App Creds - All roles" \ + "{{ app_credential_name }}-new" -f yaml + environment: + OS_CLOUD: "{{ user_cloud }}" + register: app_cred_output + changed_when: true + +- name: Parse Application Credential output + ansible.builtin.set_fact: + app_cred_info: "{{ app_cred_output.stdout | from_yaml }}" + +- name: Read current clouds.yaml + ansible.builtin.slurp: + src: "{{ clouds_yaml_file_path }}" + register: clouds_yaml_file + +- name: Set clouds.yaml fact + ansible.builtin.set_fact: + clouds_yaml_params: "{{ clouds_yaml_file.content | b64decode | from_yaml }}" + +- name: Build updated cloud entry with application credentials + ansible.builtin.set_fact: + updated_cloud_entry: + auth: + auth_url: "{{ clouds_yaml_params.clouds[user_cloud].auth.auth_url }}" + application_credential_id: "{{ app_cred_info.id }}" + application_credential_secret: "{{ app_cred_info.secret }}" + auth_type: v3applicationcredential + identity_api_version: "{{ clouds_yaml_params.clouds[user_cloud].identity_api_version | default('3') }}" + region_name: "{{ clouds_yaml_params.clouds[user_cloud].region_name | default(omit) }}" + +- name: Add cacert to updated cloud entry + ansible.builtin.set_fact: + updated_cloud_entry: "{{ updated_cloud_entry | combine({'cacert': clouds_yaml_params.clouds[user_cloud].cacert}, recursive=True) }}" + when: clouds_yaml_params.clouds[user_cloud].cacert is defined + +- name: Update clouds.yaml with application credentials + ansible.builtin.set_fact: + clouds_yaml_params: "{{ {'clouds': (clouds_yaml_params.clouds | combine({user_cloud: updated_cloud_entry}))} }}" + +- name: Write updated clouds.yaml + ansible.builtin.copy: + content: "{{ clouds_yaml_params | to_nice_yaml(indent=4) }}" + dest: "{{ clouds_yaml_file_path }}" + mode: u=rw,g=rw,o=r + +- name: Update clouds.yaml copy in osp_config_dir + ansible.builtin.copy: + content: "{{ clouds_yaml_params | to_nice_yaml(indent=4) }}" + dest: "{{ osp_config_dir }}/clouds.yaml" + mode: u=rw,g=rw,o=r + when: osp_config_dir is defined + +- name: Validate new application credentials work + openstack.cloud.auth: + cloud: "{{ user_cloud }}" + +- name: Delete previous Application Credential + ansible.builtin.shell: | + openstack application credential delete {{ app_credential_name }} + environment: + OS_CLOUD: "{{ user_cloud }}" + failed_when: false + changed_when: false diff --git a/collection/stages/roles/prepare/tasks/main.yml b/collection/stages/roles/prepare/tasks/main.yml index 3bac350f..afedf7f2 100644 --- a/collection/stages/roles/prepare/tasks/main.yml +++ b/collection/stages/roles/prepare/tasks/main.yml @@ -32,6 +32,10 @@ - name: Update clouds.yml file with new Project ansible.builtin.include_tasks: clouds.yml +- name: Configure Application Credentials for OpenStack authentication + ansible.builtin.include_tasks: app_creds.yml + when: use_application_credentials | default(false) + - name: Restricted Network Preparations ansible.builtin.include_tasks: restricted_network.yml when: diff --git a/collection/tools/roles/tools_get_openshift_release/defaults/main.yml b/collection/tools/roles/tools_get_openshift_release/defaults/main.yml index f54042ac..64f604c6 100644 --- a/collection/tools/roles/tools_get_openshift_release/defaults/main.yml +++ b/collection/tools/roles/tools_get_openshift_release/defaults/main.yml @@ -2,6 +2,5 @@ # defaults file for tools_get_openshift_release openshift_releasestream_url: "https://openshift-release.apps.ci.l2s4.p1.openshiftapps.com/api/v1/releasestream" release_name: "{{ openshift_release_build_name | default('') }}" -openshift_download_url: "{{ 'https://openshift-release-artifacts.apps.ci.l2s4.p1.openshiftapps.com' + '/' + release_name }}" openshift_mirror_url: "https://mirror.openshift.com/pub/openshift-v4/x86_64/clients/ocp" ocp_build_info_file: "{{ controller_home_dir }}/latest_build.json" diff --git a/collection/tools/roles/tools_get_openshift_release/tasks/get_openshift_release_binaries.yml b/collection/tools/roles/tools_get_openshift_release/tasks/get_openshift_release_binaries.yml index a2515771..a53995ae 100644 --- a/collection/tools/roles/tools_get_openshift_release/tasks/get_openshift_release_binaries.yml +++ b/collection/tools/roles/tools_get_openshift_release/tasks/get_openshift_release_binaries.yml @@ -1,25 +1,124 @@ --- +# Extract OCP installer and/or client binaries directly from the release image +# using `oc adm release extract --tools` instead of the release-controller's +# file-cache (openshift-release-artifacts), which has no SLA and can get stuck +# indefinitely during tool extraction. +# +# The pull secret is extracted from the host cluster via the Kubernetes API +# using the kubeconfig's client certificate. If `oc` is not already present +# in the pod (cold-start), a stable client is bootstrapped from +# mirror.openshift.com before running `oc adm release extract --tools`. - name: Get the OCP installer and/or client binaries vars: - installer_url: "{{ openshift_download_url }}/openshift-install-linux-{{ release_name }}.tar.gz" - client_url: "{{ openshift_download_url }}/openshift-client-linux-{{ release_name }}.tar.gz" + installer_tarball: "openshift-install-linux-{{ release_name }}.tar.gz" + client_tarball: "openshift-client-linux-{{ release_name }}.tar.gz" + pull_secret_file: "{{ home_dir }}/pull-secret.json" + bootstrap_oc_dir: "{{ home_dir }}/bootstrap-oc" + bootstrap_oc_url: "{{ openshift_mirror_url }}/stable/openshift-client-linux.tar.gz" block: - name: Fail if release_name var is not defined ansible.builtin.fail: msg: "'release_name' variable must be defined and cannot be empty" when: release_name == '' - - name: Wait for content to come up on {{ openshift_download_url }} - ansible.builtin.uri: - url: "{{ openshift_download_url }}" - method: GET - return_content: yes - status_code: 200 - body_format: json - register: result - until: result.content.find("openshift-install-linux") != -1 - retries: 20 - delay: 60 + - name: Fail if openshift_release_pull_spec is not defined + ansible.builtin.fail: + msg: "'openshift_release_pull_spec' must be set by get_openshift_release_build_name.yml" + when: openshift_release_pull_spec is not defined or openshift_release_pull_spec == '' + + - name: Extract pull secret from host cluster via Kubernetes API + ansible.builtin.shell: | + python3 << 'PYEOF' + import yaml, json, base64, subprocess, os, sys, tempfile + + kubeconfig_path = "{{ rhoso_kubeconfig }}" + output_path = "{{ pull_secret_file }}" + + with open(kubeconfig_path) as f: + kc = yaml.safe_load(f) + + server = kc['clusters'][0]['cluster']['server'] + user = kc['users'][0]['user'] + + try: + cert_data = user['client-certificate-data'] + key_data = user['client-key-data'] + except KeyError: + print(f"rhoso_kubeconfig must use client-certificate auth, " + f"found auth keys: {list(user.keys())}", file=sys.stderr) + sys.exit(1) + + with tempfile.TemporaryDirectory() as tmpdir: + ca_path = os.path.join(tmpdir, 'ca.crt') + cert_path = os.path.join(tmpdir, 'client.crt') + key_path = os.path.join(tmpdir, 'client.key') + + with open(ca_path, 'wb') as f: + f.write(base64.b64decode(kc['clusters'][0]['cluster']['certificate-authority-data'])) + with open(cert_path, 'wb') as f: + f.write(base64.b64decode(cert_data)) + with open(key_path, 'wb') as f: + f.write(base64.b64decode(key_data)) + + result = subprocess.run([ + 'curl', '-s', '--fail', + '--cacert', ca_path, + '--cert', cert_path, + '--key', key_path, + f'{server}/api/v1/namespaces/openshift-config/secrets/pull-secret' + ], capture_output=True, text=True) + + if result.returncode != 0: + print(f"Failed to fetch pull secret from {server}: {result.stderr}", file=sys.stderr) + sys.exit(1) + + data = json.loads(result.stdout) + decoded = base64.b64decode(data['data']['.dockerconfigjson']).decode() + auths = json.loads(decoded) + + with open(output_path, 'w') as f: + f.write(decoded) + + print(f"Pull secret extracted: {len(auths.get('auths', {}))} registries") + PYEOF + register: _pull_secret_result + no_log: true + + - name: Verify pull secret file is valid + ansible.builtin.shell: >- + python3 -c "import json; d=json.load(open('{{ pull_secret_file }}')); + print(len(d.get('auths',{})), 'registries found')" + register: _pull_secret_verify + changed_when: false + + - name: Check if oc is already available + ansible.builtin.command: which oc + register: _oc_available + ignore_errors: true + changed_when: false + + - name: Bootstrap oc client from {{ bootstrap_oc_url }} + when: _oc_available is failed + block: + - name: Create bootstrap directory + ansible.builtin.file: + path: "{{ bootstrap_oc_dir }}" + state: directory + mode: u=rwx,g=rw,o=r + + - name: Download stable oc client from mirror + ansible.builtin.unarchive: + src: "{{ bootstrap_oc_url }}" + dest: "{{ bootstrap_oc_dir }}" + remote_src: yes + register: _bootstrap_download + until: _bootstrap_download is not failed + retries: 3 + delay: 10 + + - name: Set oc binary path + ansible.builtin.set_fact: + _oc_bin: "{{ (bootstrap_oc_dir + '/oc') if _oc_available is failed else 'oc' }}" - name: Create the installer directory ansible.builtin.file: @@ -27,18 +126,28 @@ state: directory mode: u=rwx,g=rw,o=r + - name: Extract OCP tools from release image {{ openshift_release_pull_spec }} + ansible.builtin.command: + cmd: >- + timeout 900 + {{ _oc_bin }} adm release extract + --tools + --registry-config={{ pull_secret_file }} + --to={{ home_dir }}/{{ release_name }} + {{ openshift_release_pull_spec }} + register: extract_result + until: extract_result is not failed + retries: 3 + delay: 30 + - name: Get the installer binary and create a symlink when: "'installer' in binaries" block: - - name: Download and unarchive the installer from {{ installer_url }} + - name: Unarchive the installer from {{ installer_tarball }} ansible.builtin.unarchive: - src: "{{ installer_url }}" + src: "{{ home_dir }}/{{ release_name }}/{{ installer_tarball }}" dest: "{{ home_dir }}/{{ release_name }}" remote_src: yes - register: result - until: result is not failed - retries: 3 - delay: 10 - name: Create a symlink to the openshift-install binary from /usr/local/bin ansible.builtin.file: @@ -47,18 +156,14 @@ state: link become: true - - name: Get the installer binary and create symlinks + - name: Get the client binary and create symlinks when: "'client' in binaries" block: - - name: Download and unarchive the client from {{ client_url }} + - name: Unarchive the client from {{ client_tarball }} ansible.builtin.unarchive: - src: "{{ client_url }}" + src: "{{ home_dir }}/{{ release_name }}/{{ client_tarball }}" dest: "{{ home_dir }}/{{ release_name }}" remote_src: yes - register: result - until: result is not failed - retries: 3 - delay: 10 - name: Create a symlink to the oc binary from /usr/local/bin ansible.builtin.file: @@ -73,3 +178,14 @@ dest: /usr/bin/kubectl state: link become: true + + always: + - name: Remove pull secret file + ansible.builtin.file: + path: "{{ pull_secret_file }}" + state: absent + + - name: Remove bootstrap oc directory + ansible.builtin.file: + path: "{{ bootstrap_oc_dir }}" + state: absent diff --git a/collection/tools/roles/tools_get_openshift_release/tasks/get_openshift_release_build_name.yml b/collection/tools/roles/tools_get_openshift_release/tasks/get_openshift_release_build_name.yml index 78c07a79..7461eb58 100644 --- a/collection/tools/roles/tools_get_openshift_release/tasks/get_openshift_release_build_name.yml +++ b/collection/tools/roles/tools_get_openshift_release/tasks/get_openshift_release_build_name.yml @@ -39,12 +39,25 @@ ansible.builtin.set_fact: openshift_release_build_name: "{{ latest_build_info.name }}" -- name: Set openshift_release_build_name when a specific build is given - ansible.builtin.set_fact: - openshift_release_build_name: "{{ build_name }}" + - name: Set openshift_release_pull_spec from release stream API response + ansible.builtin.set_fact: + openshift_release_pull_spec: "{{ latest_build_info.pullSpec }}" + +- name: Set build name and pull spec when a specific build is given when: - release is not match("4-stable") - build_name not in ['','candidate','fast','stable','eus'] + block: + - name: Set openshift_release_build_name for specific build + ansible.builtin.set_fact: + openshift_release_build_name: "{{ build_name }}" + + - name: Construct openshift_release_pull_spec for specific build + ansible.builtin.set_fact: + openshift_release_pull_spec: >- + {{ 'registry.ci.openshift.org/ocp/release:' + build_name + if build_name is search('nightly') + else 'quay.io/openshift-release-dev/ocp-release:' + build_name + '-x86_64' }} - name: Discover the release build name for the z-stream promoted to upgrade channel on {{ release }} # Ref: https://docs.openshift.com/container-platform/4.9/updating/understanding-upgrade-channels-release.html @@ -68,3 +81,12 @@ - name: Set openshift_release_build_name when openshift.build is set to a channel ansible.builtin.set_fact: openshift_release_build_name: "{{ result.stdout }}" + + - name: Parse openshift_release_pull_spec from Pull From field in release.txt + ansible.builtin.shell: set -o pipefail && grep '^Pull From:' {{ home_dir }}/release.txt | awk '{print $3}' + changed_when: false + register: pull_from_result + + - name: Set openshift_release_pull_spec from channel release.txt + ansible.builtin.set_fact: + openshift_release_pull_spec: "{{ pull_from_result.stdout }}" diff --git a/jobs_definitions/osp_verification.yaml b/jobs_definitions/osp_verification.yaml index 98d341b6..77059273 100644 --- a/jobs_definitions/osp_verification.yaml +++ b/jobs_definitions/osp_verification.yaml @@ -19,9 +19,13 @@ stages: - install - post - verification + - day2ops - openstack_test - lb_tests +day2ops_procedures: + - rotate_app_creds + ocp_deployment_topology: network_type: OVNKubernetes primary_ip_protocol: ipv4 # ipv4 or ipv6