Files
opendesk/.gitlab-ci.yml
2024-02-21 12:04:33 +00:00

643 lines
19 KiB
YAML

# SPDX-FileCopyrightText: 2023 Bundesministerium des Innern und für Heimat, PG ZenDiS "Projektgruppe für Aufbau ZenDiS"
# SPDX-License-Identifier: Apache-2.0
---
include:
- project: "${PROJECT_PATH_GITLAB_CONFIG_TOOLING}"
ref: "v2.3.2"
file:
- "ci/common/automr.yml"
- "ci/common/lint.yml"
- "ci/release-automation/semantic-release.yml"
- local: "/.gitlab/generate/generate-docs.yml"
- project: "${PROJECT_PATH_CUSTOM_ENVIRONMENT_CONFIG}"
file: "gitlab/environments.yaml"
- local: "/.gitlab/lint/lint-opendesk.yml"
rules:
- if: "$JOB_OPENDESK_LINTER_ENABLED == 'false' || $CI_PIPELINE_SOURCE =~ 'tags|merge_request_event'"
when: "never"
- when: "always"
- local: "/.gitlab/lint/lint-kyverno.yml"
rules:
- if: "$JOB_KYVERNO_LINTER_ENABLED == 'false' || $CI_PIPELINE_SOURCE =~ 'tags|merge_request_event'"
when: "never"
- when: "always"
stages:
- ".pre"
- "scan"
- "automr"
- "lint"
- "env-cleanup"
- "env"
- "basic-services-deploy"
- "component-deploy-stage-1"
- "component-deploy-stage-2"
- "tests"
- "env-stop"
- "generate-release-assets"
- ".post"
variables:
NAMESPACE:
description: "The name of namespaces to deploy to."
value: ""
CLUSTER:
description: "Define which cluster to use. Cluster must be defined in gitlab/environments.yaml of
sovereign-workplace-env included above."
value: "dev"
MASTER_PASSWORD_WEB_VAR:
description: "Optional: Provide a passphrase to be used for password generation."
value: ""
ENV_STOP_BEFORE:
description: "Stop environment/delete namespace for the deployment"
value: "no"
options:
- "yes"
- "no"
DEPLOY_ALL_COMPONENTS:
description: "Enable all component deployment (overwrites 'no' setting on component level)."
value: "no"
options:
- "yes"
- "no"
DEPLOY_SERVICES:
description: "Enable Service deployment."
value: "no"
options:
- "yes"
- "no"
DEPLOY_UMS:
description: "Enable Univention Management Stack deployment."
value: "no"
options:
- "yes"
- "no"
DEPLOY_PROVISIONING:
description: "Enable Provisioning Components."
value: "no"
options:
- "yes"
- "no"
DEPLOY_COLLABORA:
description: "Enable Collabora deployment."
value: "no"
options:
- "yes"
- "no"
DEPLOY_CRYPTPAD:
description: "Enable CryptPad deployment."
value: "no"
options:
- "yes"
- "no"
DEPLOY_ELEMENT:
description: "Enable Element deployment."
value: "no"
options:
- "yes"
- "no"
DEPLOY_OX:
description: "Enable OX AppSuite8 deployment."
value: "no"
options:
- "yes"
- "no"
DEPLOY_ICS:
description: "Enable ICS deployment."
value: "no"
options:
- "yes"
- "no"
DEPLOY_XWIKI:
description: "Enable XWiki deployment."
value: "no"
options:
- "yes"
- "no"
DEPLOY_NEXTCLOUD:
description: "Enable Nextcloud deployment."
value: "no"
options:
- "yes"
- "no"
DEPLOY_OPENPROJECT:
description: "Enable OpenProject deployment."
value: "no"
options:
- "yes"
- "no"
DEPLOY_JITSI:
description: "Enable Jitsi deployment."
value: "no"
options:
- "yes"
- "no"
RUN_TESTS:
description: "Triggers execution of E2E-tests."
value: "yes"
options:
- "yes"
- "no"
TESTS_BRANCH:
description: "Branch of E2E-tests on which the test pipeline is triggered"
value: "main"
.deploy-common:
cache: {}
dependencies: []
extends: ".environments"
image: "registry.opencode.de/bmi/opendesk/components/platform-development/images/helm:1.0.1\
@sha256:d38f41b88374e055332860018f2936db8807b763caf6089735db0484cbb2842a"
script:
- "cd ${CI_PROJECT_DIR}/helmfile/apps/${COMPONENT}"
# MASTER_PASSWORD_WEB_VAR as precedence for MASTER_PASSWORD
- |
if ! [ -z "${MASTER_PASSWORD_WEB_VAR}" ]; then
export MASTER_PASSWORD="${MASTER_PASSWORD_WEB_VAR}"
fi;
- >
echo "Installing ${COMPONENT} into ${NAMESPACE} namespace as ${HELMFILE_ENVIRONMENT} environment on ${CLUSTER}"
- "helmfile --namespace ${NAMESPACE} apply --suppress-diff"
tags:
- "docker"
- "kubernetes"
- "${CLUSTER}"
variables:
HELMFILE_ENVIRONMENT: "dev"
env-cleanup:
extends: ".deploy-common"
environment:
name: "${NAMESPACE}"
action: "stop"
needs: []
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
$ENV_STOP_BEFORE != "no"
when: "on_success"
script:
- |
if [ "${OPENDESK_SLEDGEHAMMER_DESTROY_ENABLED}" = "yes" ]; then
for OPENDESK_RELEASE in $(helm ls -n ${NAMESPACE} -aq); do
helm uninstall -n ${NAMESPACE} ${OPENDESK_RELEASE};
done
kubectl delete pvc --all --namespace ${NAMESPACE};
kubectl delete jobs --all --namespace ${NAMESPACE};
else
helmfile destroy --namespace ${NAMESPACE};
fi
stage: "env-cleanup"
env-start:
environment:
name: "${NAMESPACE}"
on_stop: "env-stop"
extends: ".deploy-common"
image: "${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/alpine/k8s:1.25.6"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/
when: "on_success"
script:
- "echo \"Deploying to Environment ${NAMESPACE} in ${CLUSTER} Cluster\""
- "kubectl create namespace ${NAMESPACE} --dry-run=client -o yaml | kubectl apply -f -"
- >
kubectl create secret
--namespace "${NAMESPACE}"
docker-registry external-registry
--docker-server "${EXTERNAL_REGISTRY}"
--docker-username "${EXTERNAL_REGISTRY_USERNAME}"
--docker-password "${EXTERNAL_REGISTRY_PASSWORD}"
--dry-run=client -o yaml | kubectl apply -f -
stage: "env"
services-deploy:
stage: "basic-services-deploy"
extends: ".deploy-common"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || $DEPLOY_SERVICES != "no")
when: "on_success"
variables:
COMPONENT: "services"
provisioning-deploy:
stage: "component-deploy-stage-2"
extends: ".deploy-common"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || $DEPLOY_UMS != "no" || $DEPLOY_PROVISIONING != "no")
when: "on_success"
variables:
COMPONENT: "provisioning"
ums-deploy:
stage: "component-deploy-stage-1"
extends: ".deploy-common"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || $DEPLOY_UMS != "no")
when: "on_success"
variables:
COMPONENT: "univention-management-stack"
ox-deploy:
stage: "component-deploy-stage-1"
extends: ".deploy-common"
timeout: "30m"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || $DEPLOY_OX != "no")
when: "on_success"
variables:
COMPONENT: "open-xchange"
ics-deploy:
stage: "component-deploy-stage-1"
extends: ".deploy-common"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || $DEPLOY_ICS != "no")
when: "on_success"
variables:
COMPONENT: "intercom-service"
xwiki-deploy:
stage: "component-deploy-stage-1"
extends: ".deploy-common"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || $DEPLOY_XWIKI != "no")
when: "on_success"
variables:
COMPONENT: "xwiki"
collabora-deploy:
stage: "component-deploy-stage-1"
extends: ".deploy-common"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || $DEPLOY_NEXTCLOUD != "no" || $DEPLOY_COLLABORA != "no")
when: "on_success"
variables:
COMPONENT: "collabora"
cryptpad-deploy:
stage: "component-deploy-stage-1"
extends: ".deploy-common"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || $DEPLOY_NEXTCLOUD != "no" || $DEPLOY_CRYPTPAD != "no")
when: "on_success"
variables:
COMPONENT: "cryptpad"
nextcloud-deploy:
stage: "component-deploy-stage-1"
extends: ".deploy-common"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || $DEPLOY_NEXTCLOUD != "no")
when: "on_success"
variables:
COMPONENT: "nextcloud"
openproject-deploy:
stage: "component-deploy-stage-1"
extends: ".deploy-common"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || $DEPLOY_OPENPROJECT != "no")
when: "on_success"
variables:
COMPONENT: "openproject"
openproject-bootstrap-deploy:
stage: "component-deploy-stage-2"
extends: ".deploy-common"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || ($DEPLOY_OPENPROJECT != "no" && $DEPLOY_NEXTCLOUD != "no"))
when: "on_success"
variables:
COMPONENT: "openproject-bootstrap"
jitsi-deploy:
stage: "component-deploy-stage-1"
extends: ".deploy-common"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || $DEPLOY_JITSI != "no")
when: "on_success"
variables:
COMPONENT: "jitsi"
element-deploy:
stage: "component-deploy-stage-1"
extends: ".deploy-common"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" &&
$NAMESPACE =~ /.+/ &&
($DEPLOY_ALL_COMPONENTS != "no" || $DEPLOY_ELEMENT != "no")
when: "on_success"
variables:
COMPONENT: "element"
env-stop:
extends: ".deploy-common"
environment:
name: "${NAMESPACE}"
action: "stop"
image: "${CI_DEPENDENCY_PROXY_DIRECT_GROUP_IMAGE_PREFIX}/alpine/k8s:1.25.6"
needs: []
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" && $NAMESPACE =~ /.+/
when: "manual"
script:
- "echo 'We do not stop the env (delete the namespace) at the moment in this stage, as deleting a branches also
triggers this env-stop stage and we do not want this to happen.'"
# - kubectl delete namespace "${NAMESPACE}"
stage: "env-stop"
variables:
GIT_STRATEGY: "none"
.ums-default-password: &ums-default-password
- |
UMS_PASSWORDS=$( \
kubectl -n ${NAMESPACE} get cm ums-stack-data-swp-data -o jsonpath='{.data.dev-test-users\.yaml}' \
| yq '.properties.password' > passwords.txt \
)
DEFAULT_USER_PASSWORD=$( \
awk 'NR==1{print $1}' passwords.txt \
)
DEFAULT_ADMIN_PASSWORD=$(
awk 'NR==3{print $1}' passwords.txt \
)
run-tests:
extends: ".deploy-common"
environment:
name: "${NAMESPACE}"
stage: "tests"
rules:
- if: >
$CI_PIPELINE_SOURCE =~ "web|schedules|triggers" && $NAMESPACE =~ /.+/ && $RUN_TESTS == "yes"
when: "on_success"
script:
- *ums-default-password
- |
curl --request POST \
--header "Content-Type: application/json" \
--data "{ \
\"ref\": \"${TESTS_BRANCH}\", \
\"token\": \"${CI_JOB_TOKEN}\", \
\"variables\": { \
\"url\": \"https://portal.${DOMAIN}\", \
\"user_name\": \"${DEFAULT_USER_NAME}\", \
\"user_password\": \"${DEFAULT_USER_PASSWORD}\", \
\"admin_name\": \"${DEFAULT_ADMIN_NAME}\", \
\"admin_password\": \"${DEFAULT_ADMIN_PASSWORD}\", \
\"DEPLOY_ALL_COMPONENTS\": \"${DEPLOY_ALL_COMPONENTS}\", \
\"DEPLOY_COLLABORA\": \"${DEPLOY_COLLABORA}\", \
\"DEPLOY_ELEMENT\": \"${DEPLOY_ELEMENT}\", \
\"DEPLOY_ICS\": \"${DEPLOY_ICS}\", \
\"DEPLOY_JITSI\": \"${DEPLOY_JITSI}\", \
\"DEPLOY_KEYCLOAK\": \"${DEPLOY_UMS}\", \
\"DEPLOY_NEXTCLOUD\": \"${DEPLOY_NEXTCLOUD}\", \
\"DEPLOY_OPENPROJECT\": \"${DEPLOY_OPENPROJECT}\", \
\"DEPLOY_OX\": \"${DEPLOY_OX}\", \
\"DEPLOY_SERVICES\": \"${DEPLOY_SERVICES}\", \
\"DEPLOY_UCS\": \"${DEPLOY_UMS}\", \
\"DEPLOY_XWIKI\": \"${DEPLOY_XWIKI}\", \
\"DEPLOY_PROVISIONING\": \"${DEPLOY_PROVISIONING}\" \
} \
}" \
"https://${TESTS_PROJECT_URL}/trigger/pipeline"
avscan-prepare:
stage: ".pre"
rules:
- if: >
$JOB_AVSCAN_ENABLED != 'false' &&
$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH &&
$CI_PIPELINE_SOURCE =~ "push|merge_request_event"
when: "always"
- when: "never"
image: "external-registry.souvap-univention.de/docker-remote/mikefarah/yq"
script:
- |
cat << 'EOF' > dynamic-scans.yml
---
stages:
- "scan"
.container-clamav:
stage: "scan"
image: "registry.opencode.de/bmi/opendesk/components/platform-development/images/clamav-imagescan:1.0.0"
before_script:
- "sed -i \"/^DatabaseMirror .*$/c DatabaseMirror ${DATABASE_MIRROR}\" /etc/clamav/freshclam.conf"
- "freshclam"
- "mkdir /scan"
script:
- "export IMAGE=${EXTERNAL_REGISTRY:-${CONTAINER_REGISTRY}}/${CONTAINER_IMAGE}:${CONTAINER_TAG}"
- "echo Pulling and scanning $IMAGE..."
- "crane pull $IMAGE /scan/image.tar"
- "clamscan /scan"
variables:
CONTAINER_IMAGE: ""
CONTAINER_REGISTRY: ""
CONTAINER_TAG: ""
DATABASE_MIRROR: "https://nexus.souvap-univention.de/repository/ClamAV"
EOF
- >
yq '.images
| with_entries(.key |= "scan-" + .)
| .[].extends=".container-clamav"
| with(.[]; .variables.CONTAINER_IMAGE = .repository
| .variables.CONTAINER_TAG = .tag | .variables.CONTAINER_REGISTRY = .registry)
| del(.[].repository)
| del(.[].tag)
| del(.[].registry)'
helmfile/environments/default/images.yaml
>> dynamic-scans.yml
artifacts:
paths:
- "dynamic-scans.yml"
avscan-start:
stage: "scan"
rules:
- if: >
$JOB_AVSCAN_ENABLED != 'false' &&
$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH &&
$CI_PIPELINE_SOURCE =~ "push|merge_request_event"
when: "always"
- when: "never"
trigger:
include:
- artifact: "dynamic-scans.yml"
job: "avscan-prepare"
strategy: "depend"
generate-release-assets:
stage: "generate-release-assets"
image: "registry.souvap-univention.de/souvap/tooling/images/ansible:4.10.0"
rules:
- if: >
$JOB_AVSCAN_ENABLED != 'false' &&
$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH &&
$CI_PIPELINE_SOURCE =~ "push|merge_request_event"
when: "on_success"
- when: "never"
script:
- |
git clone https://gitlab-ci-token:${CI_JOB_TOKEN}@${CI_SERVER_HOST}/${ASSET_GENERATOR_REPO_PATH}
cd opendesk-asset-generator
export OPENDESK_DEPLOYMENT_AUTOMATION_PATH=${CI_PROJECT_DIR}
./opendesk_asset_generator.py
mv ./build_artefacts ${CI_PROJECT_DIR}
cd ..
rm -rf opendesk-asset-generator
ls -l ./build_artefacts
artifacts:
paths:
- "./build_artefacts/chart-index.json"
- "./build_artefacts/image-index.json"
tags: []
variables:
ASSET_GENERATOR_REPO_PATH: "bmi/opendesk/tooling/opendesk-asset-generator"
# Declare .environments which is in environments repository. In case it is not available
# 'cache' is used because job must contain at least one key, so cache is just a dummy key.
.environments:
cache: {}
# Overwrite shared settings
.common-semantic-release:
image: "registry.souvap-univention.de/souvap/tooling/images/semantic-release-patched:latest"
tags: []
conventional-commits-linter:
rules:
- if: "$JOB_CONVENTIONAL_COMMITS_LINTER_ENABLED == 'false' || $CI_PIPELINE_SOURCE =~ 'tags|merge_request_event'"
when: "never"
- when: "always"
common-yaml-linter:
rules:
- if: "$JOB_COMMON_YAML_LINTER_ENABLED == 'false' || $CI_PIPELINE_SOURCE =~ 'tags|triggers|web|merge_request_event'"
when: "never"
- when: "always"
reuse-linter:
allow_failure: false
rules:
- if: "$JOB_REUSE_LINTER_ENABLED == 'false' || $CI_PIPELINE_SOURCE =~ 'tags|triggers|web|merge_request_event'"
when: "never"
- when: "always"
generate-release-version:
rules:
- if: >
$JOB_RELEASE_ENABLED != 'false' &&
$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH &&
$CI_PIPELINE_SOURCE =~ "push|merge_request_event"
when: "on_success"
release:
dependencies:
- "generate-release-assets"
rules:
- if: >
$JOB_AVSCAN_ENABLED != 'false' &&
$CI_COMMIT_BRANCH == $CI_DEFAULT_BRANCH &&
$CI_PIPELINE_SOURCE =~ "push|merge_request_event"
when: "on_success"
script:
- >
export RELEASE_VERSION=$(semantic-release --dry-run --branches $CI_COMMIT_REF_NAME --plugins
"@semantic-release/gitlab" | grep -oP "Published release [0-9]+\.[0-9]+\.[0-9]+ on" |
grep -oP "[0-9]+\.[0-9]+\.[0-9]+")
- |
if [ -z "${RELEASE_VERSION}" ]; then
echo "RELEASE_VERSION=$(git describe --tags --abbrev=0 | sed s@^v@@g )"
else
echo "RELEASE_VERSION=${RELEASE_VERSION}"
fi
- |
echo -e "\n[INFO] Writing data to helm value file..."
cat <<EOF >helmfile/environments/default/global.generated.yaml
# SPDX-FileCopyrightText: 2024 Bundesministerium des Innern und für Heimat, PG ZenDiS "Projektgruppe für Aufbau ZenDiS"
# SPDX-License-Identifier: Apache-2.0
---
global:
systemInformation:
releaseVersion: "v$(echo -E "$RELEASE_VERSION")"
...
EOF
- |
cat << 'EOF' > ${CI_PROJECT_DIR}/.releaserc
{
"branches": ["main"],
"plugins": [
["@semantic-release/gitlab",
{
"assets": [
{ "path": "./build_artefacts/chart-index.json",
"label": "Chart Index JSON" },
{ "path": "./build_artefacts/image-index.json",
"label": "Image Index JSON" },
]
}
],
"@semantic-release/release-notes-generator",
"@semantic-release/changelog",
["@semantic-release/git", {
"assets": [
"charts/**/Chart.yaml",
"CHANGELOG.md",
"charts/**/README.md",
"helmfile/environments/default/global.generated.yaml",
".kyverno/kyverno-test.yaml",
"docs"
],
"message": "chore(release): ${nextRelease.version} [skip ci]\n\n${nextRelease.notes}"
}]
]
}
EOF
- "semantic-release"
needs:
- "generate-release-assets"
- "generate-docs"
...