diff --git a/.ci/ansible/Containerfile.j2 b/.ci/ansible/Containerfile.j2 index fb9ed4a1..a446f339 100644 --- a/.ci/ansible/Containerfile.j2 +++ b/.ci/ansible/Containerfile.j2 @@ -44,4 +44,8 @@ RUN PULP_STATIC_ROOT=/var/lib/operator/static/ PULP_CONTENT_ORIGIN=localhost \ RUN mkdir /var/lib/pulp/.config USER root:root +# Lots of plugins try to use this path, and throw warnings if they cannot access it. +RUN mkdir /.pytest_cache +RUN chown pulp:pulp /.pytest_cache + ENTRYPOINT ["/init"] diff --git a/.ci/ansible/build_container.yaml b/.ci/ansible/build_container.yaml index 0a188cba..0ffd21d4 100644 --- a/.ci/ansible/build_container.yaml +++ b/.ci/ansible/build_container.yaml @@ -1,6 +1,7 @@ # Ansible playbook to create the pulp service containers image --- -- hosts: "localhost" +- name: "Build CI Container Image" + hosts: "localhost" gather_facts: false vars_files: - "vars/main.yaml" @@ -9,6 +10,7 @@ ansible.builtin.template: src: "Containerfile.j2" dest: "Containerfile" + - name: "Build pulp image" # We build from the ../.. (parent dir of pulpcore git repo) Docker build # "context" so that repos like pulp-smash are accessible to Docker @@ -18,9 +20,20 @@ # 1-off-builds and CI purposes (which has no cache across CI runs.) # Run build.yaml with -e cache=false if your builds are using outdated # layers. - command: "docker build --network host --no-cache={{ not cache | default(true) | bool }} -t {{ image.name }}:{{ image.tag }} -f {{ playbook_dir }}/Containerfile ../../.." + ansible.builtin.command: + argv: + - "docker" + - "build" + - "--network" + - "host" + - "--no-cache={{ not cache | default(true) | bool }}" + - "-t" + - "{{ image.name }}:{{ image.tag }}" + - "-f" + - "{{ playbook_dir }}/Containerfile" + - "../../.." - name: "Clean image cache" - docker_prune: - images : true + community.docker.docker_prune: + images: true ... diff --git a/.ci/ansible/settings.py.j2 b/.ci/ansible/settings.py.j2 index dfe2851d..ab8ebb3d 100644 --- a/.ci/ansible/settings.py.j2 +++ b/.ci/ansible/settings.py.j2 @@ -10,10 +10,6 @@ REDIS_HOST = "localhost" REDIS_PORT = 6379 ANALYTICS = False -{% if api_root is defined %} -API_ROOT = {{ api_root | repr }} -{% endif %} - {% if pulp_settings %} {% for key, value in pulp_settings.items() %} {{ key | upper }} = {{ value | repr }} diff --git a/.ci/ansible/start_container.yaml b/.ci/ansible/start_container.yaml index acdc22ad..e0891b7a 100644 --- a/.ci/ansible/start_container.yaml +++ b/.ci/ansible/start_container.yaml @@ -1,6 +1,7 @@ # Ansible playbook to start the pulp service container and its supporting services --- -- hosts: "localhost" +- name: "Start CI Containers" + hosts: "localhost" gather_facts: false vars_files: - "vars/main.yaml" @@ -14,16 +15,16 @@ - "settings" - name: "Generate Pulp Settings" - template: + ansible.builtin.template: src: "settings.py.j2" dest: "settings/settings.py" - name: "Setup docker networking" - docker_network: + community.docker.docker_network: name: "pulp_ci_bridge" - name: "Start Service Containers" - docker_container: + community.docker.docker_container: name: "{{ item.name }}" image: "{{ item.image }}" auto_remove: true @@ -39,12 +40,12 @@ loop: "{{ services | default([]) }}" - name: "Retrieve Docker Network Info" - docker_network_info: + community.docker.docker_network_info: name: "pulp_ci_bridge" register: "pulp_ci_bridge_info" - name: "Update /etc/hosts" - lineinfile: + ansible.builtin.lineinfile: path: "/etc/hosts" regexp: "\\s{{ item.value.Name }}\\s*$" line: "{{ item.value.IPv4Address | ansible.utils.ipaddr('address') }}\t{{ item.value.Name }}" @@ -55,16 +56,27 @@ amazon.aws.s3_bucket: aws_access_key: "{{ minio_access_key }}" aws_secret_key: "{{ minio_secret_key }}" - s3_url: "http://minio:9000" + endpoint_url: "http://minio:9000" region: "eu-central-1" name: "pulp3" state: "present" when: "s3_test | default(false)" - - block: + - name: "Wait on Services" + block: + - name: "Wait for azurite" + ansible.builtin.uri: + url: "http://ci-azurite:10000/" + status_code: + - 200 + - 400 + when: "azure_test | default(false)" + retries: 2 + delay: 5 + - name: "Wait for Pulp" - uri: - url: "http://pulp{{ lookup('env', 'PULP_API_ROOT') | default('\/pulp\/', True) }}api/v3/status/" + ansible.builtin.uri: + url: "http://pulp{{ pulp_scenario_settings.api_root | default(pulp_settings.api_root | default('\/pulp\/', True), True) }}api/v3/status/" follow_redirects: "all" validate_certs: "no" register: "result" @@ -73,11 +85,12 @@ delay: 5 rescue: - name: "Output pulp container log" - command: "docker logs pulp" + ansible.builtin.command: + cmd: "docker logs pulp" failed_when: true - name: "Check version of component being tested" - assert: + ansible.builtin.assert: that: - "(result.json.versions | items2dict(key_name='component', value_name='version'))[item.app_label] | canonical_semver == (component_version | canonical_semver)" fail_msg: | @@ -86,14 +99,15 @@ loop: "{{ 'plugins' | ansible.builtin.extract(lookup('ansible.builtin.file', '../../template_config.yml') | from_yaml) }}" - name: "Set pulp password in .netrc" - copy: + ansible.builtin.copy: dest: "~/.netrc" content: | machine pulp login admin password password -- hosts: "pulp" +- name: "Prepare Pulp Application Container" + hosts: "pulp" gather_facts: false tasks: - name: "Create directory for pulp-smash config" @@ -108,6 +122,6 @@ dest: "/var/lib/pulp/.config/pulp_smash/settings.json" - name: "Set pulp admin password" - command: + ansible.builtin.command: cmd: "pulpcore-manager reset-admin-password --password password" ... diff --git a/.github/workflows/scripts/before_install.sh b/.github/workflows/scripts/before_install.sh index b33df1d5..2f78d393 100755 --- a/.github/workflows/scripts/before_install.sh +++ b/.github/workflows/scripts/before_install.sh @@ -29,7 +29,7 @@ if [ "$TEST" = "s3" ]; then COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[s3] git+https://github.com/gerrod3/botocore.git@fix-100-continue" fi if [ "$TEST" = "azure" ]; then - COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure]" + COMPONENT_SOURCE="${COMPONENT_SOURCE} pulpcore[azure,uvloop]" fi if [[ "$TEST" = "pulp" ]]; then @@ -39,10 +39,6 @@ if [[ "$TEST" = "lowerbounds" ]]; then python3 .ci/scripts/calc_constraints.py requirements.txt > lowerbounds_constraints.txt fi -export PULP_API_ROOT=$(test "${TEST}" = "s3" && echo "/rerouted/djnd/" || echo "/pulp/") - -echo "PULP_API_ROOT=${PULP_API_ROOT}" >> "$GITHUB_ENV" - # Compose the scenario definition. mkdir -p .ci/ansible/vars @@ -54,9 +50,8 @@ legacy_component_name: "pulp_python" component_name: "python" component_version: "${COMPONENT_VERSION}" pulp_env: {} -pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} +pulp_settings: {"allowed_export_paths": "/tmp", "allowed_import_paths": "/tmp", "api_root": "/pulp/", "orphan_protection_time": 0, "pypi_api_hostname": "https://pulp:443"} pulp_scheme: "https" -api_root: "${PULP_API_ROOT}" image: name: "pulp" tag: "ci_build" @@ -93,7 +88,7 @@ if [ "$TEST" = "s3" ]; then s3_test: true minio_access_key: "${MINIO_ACCESS_KEY}" minio_secret_key: "${MINIO_SECRET_KEY}" -pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "domain_enabled": true} +pulp_scenario_settings: {"MEDIA_ROOT": "", "STORAGES": {"default": {"BACKEND": "storages.backends.s3boto3.S3Boto3Storage", "OPTIONS": {"access_key": "AKIAIT2Z5TDYPX3ARJBA", "addressing_style": "path", "bucket_name": "pulp3", "default_acl": "@none", "endpoint_url": "http://minio:9000", "region_name": "eu-central-1", "secret_key": "fqRvjWaPU5o0fCqQuUWbj9Fainj2pVZtBCiDiieS", "signature_version": "s3v4"}}, "staticfiles": {"BACKEND": "django.contrib.staticfiles.storage.StaticFilesStorage"}}, "api_root": "/rerouted/djnd/", "domain_enabled": true} pulp_scenario_env: {} VARSYAML fi diff --git a/.github/workflows/scripts/install.sh b/.github/workflows/scripts/install.sh index 9df47ccd..bfdcc8d9 100755 --- a/.github/workflows/scripts/install.sh +++ b/.github/workflows/scripts/install.sh @@ -15,7 +15,7 @@ REPO_ROOT="$PWD" source .github/workflows/scripts/utils.sh -PIP_REQUIREMENTS=("pulp-cli") +PIP_REQUIREMENTS=("pulp-cli" "yq") # This must be the **only** call to "pip install" on the test runner. pip install ${PIP_REQUIREMENTS[*]} @@ -23,7 +23,7 @@ pip install ${PIP_REQUIREMENTS[*]} if [[ "$TEST" = "s3" ]]; then for i in {1..3} do - ansible-galaxy collection install "amazon.aws:8.1.0" && s=0 && break || s=$? && sleep 3 + ansible-galaxy collection install "amazon.aws:11.1.0" && s=0 && break || s=$? && sleep 3 done if [[ $s -gt 0 ]] then @@ -36,18 +36,15 @@ fi PULP_CLI_VERSION="$(pip freeze | sed -n -e 's/pulp-cli==//p')" git clone --depth 1 --branch "$PULP_CLI_VERSION" https://github.com/pulp/pulp-cli.git ../pulp-cli -cd .ci/ansible/ +PULP_API_ROOT="$(yq -r '.pulp_scenario_settings.api_root // .pulp_settings.api_root // "/pulp/"' < .ci/ansible/vars/main.yaml)" pulp config create --base-url https://pulp --api-root "${PULP_API_ROOT}" --username "admin" --password "password" -cp ~/.config/pulp/cli.toml "${REPO_ROOT}/../pulp-cli/tests/cli.toml" +cp ~/.config/pulp/cli.toml "../pulp-cli/tests/cli.toml" + +cd .ci/ansible/ ansible-playbook build_container.yaml ansible-playbook start_container.yaml - -# Plugins often write to ~/.config/pulp/cli.toml from the host -chmod 777 ~/.config/pulp -chmod 666 ~/.config/pulp/cli.toml -sudo chown -R 700:700 ~/.config echo ::group::SSL # Copy pulp CA sudo docker cp pulp:/etc/pulp/certs/pulp_webserver.crt /usr/local/share/ca-certificates/pulp_webserver.crt @@ -72,7 +69,3 @@ fi # Needed for some functional tests cmd_prefix bash -c "echo '%wheel ALL=(ALL) NOPASSWD: ALL' > /etc/sudoers.d/nopasswd" cmd_prefix bash -c "usermod -a -G wheel pulp" - -# Lots of plugins try to use this path, and throw warnings if they cannot access it. -cmd_prefix mkdir /.pytest_cache -cmd_prefix chown pulp:pulp /.pytest_cache diff --git a/.github/workflows/test.yml b/.github/workflows/test.yml index 8f7e14ca..00d2c3ec 100644 --- a/.github/workflows/test.yml +++ b/.github/workflows/test.yml @@ -134,7 +134,7 @@ jobs: if: always() run: | echo "Need to debug? Please check: https://github.com/marketplace/actions/debugging-with-tmate" - http --timeout 30 --check-status --pretty format --print hb "https://pulp${PULP_API_ROOT}api/v3/status/" || true + pulp status || true docker images || true docker ps -a || true docker logs pulp || true diff --git a/template_config.yml b/template_config.yml index 84ae8e66..4d8adf5e 100644 --- a/template_config.yml +++ b/template_config.yml @@ -6,7 +6,6 @@ # After editing this file please always reapply the plugin template before committing any changes. --- -api_root: "/pulp/" black: false check_commit_message: true check_gettext: true @@ -45,6 +44,7 @@ pulp_scheme: "https" pulp_settings: allowed_export_paths: "/tmp" allowed_import_paths: "/tmp" + api_root: "/pulp/" orphan_protection_time: 0 pypi_api_hostname: "https://pulp:443" pulp_settings_azure: @@ -80,6 +80,7 @@ pulp_settings_s3: signature_version: "s3v4" staticfiles: BACKEND: "django.contrib.staticfiles.storage.StaticFilesStorage" + api_root: "/rerouted/djnd/" domain_enabled: true pydocstyle: true release_email: "pulp-infra@redhat.com"