fix(helmfile): Streamline file extensions in /helmfile/environments/default to

`*.yaml.gotmpl`., please read migrations.md for more details
This commit is contained in:
Thorsten Roßner
2024-12-10 10:47:40 +01:00
parent 627b9c1e84
commit 0e3b661565
45 changed files with 75 additions and 68 deletions

View File

@@ -148,7 +148,8 @@ variables:
- "yes"
- "no"
CREATE_DEFAULT_ACCOUNTS:
description: "Creates `default` and `default-admin` in the instance using the password defined as CI variable `DEFAULT_ACCOUNTS_PASSWORD`."
description: "Creates `default` and `default-admin` in the instance using the password defined as CI variable
`DEFAULT_ACCOUNTS_PASSWORD`."
value: "no"
options:
- "yes"
@@ -172,13 +173,14 @@ variables:
description: "Project url for e2e-tests (`<domain of gitlab>/api/v4/projects/<id>`)"
value: "gitlab.opencode.de/api/v4/projects/1506"
TESTS_TESTSET:
description: "Selects testset for E2E-tests"
description: "Selects test set for E2E-tests"
value: "Smoke"
options:
- "Regression"
- "Smoke"
TESTS_GRACE_PERIOD:
description: "A new deployment sometimes needs a few minutes to sort itself. If tested too early tests may fail. GRACE_PERIOD is the period in seconds that should be waited before running the tests."
description: "A new deployment sometimes needs a few minutes to sort itself. If tested too early tests may fail.
GRACE_PERIOD is the period in seconds that should be waited before running the tests."
value: "0"
.deploy-common:
@@ -462,13 +464,13 @@ fetch-administrator-credentials:
)" >> .env
artifacts:
reports:
dotenv: .env
dotenv: ".env"
import-default-accounts:
stage: "post-execute"
extends: ".environments"
dependencies:
- fetch-administrator-credentials
- "fetch-administrator-credentials"
environment:
name: "${NAMESPACE}"
rules:
@@ -477,8 +479,8 @@ import-default-accounts:
when: "on_success"
image: "registry.opencode.de/bmi/opendesk/components/platform-development/images/user-import:3.0.0"
script:
- echo "Starting default account import for ${DOMAIN}"
- cd /app
- "echo \"Starting default account import for ${DOMAIN}\""
- "cd /app"
- |
./user_import_udm_rest_api.py \
--import_domain ${DOMAIN} \
@@ -494,7 +496,7 @@ run-tests:
stage: "post-execute"
extends: ".deploy-common"
dependencies:
- fetch-administrator-credentials
- "fetch-administrator-credentials"
environment:
name: "${NAMESPACE}"
rules:
@@ -658,7 +660,7 @@ release:
fi
- |
echo -e "\n[INFO] Writing data to helm value file..."
cat <<EOF >helmfile/environments/default/global.generated.yaml
cat <<EOF >helmfile/environments/default/global.generated.yaml.gotmpl
# SPDX-FileCopyrightText: 2024 Zentrum für Digitale Souveränität der Öffentlichen Verwaltung (ZenDiS) GmbH
# SPDX-License-Identifier: Apache-2.0
---
@@ -680,7 +682,7 @@ release:
"charts/**/Chart.yaml",
"CHANGELOG.md",
"charts/**/README.md",
"helmfile/environments/default/global.generated.yaml",
"helmfile/environments/default/global.generated.yaml.gotmpl",
".kyverno/kyverno-test.yaml",
"docs"
],

View File

@@ -2,10 +2,10 @@
# SPDX-License-Identifier: Apache-2.0
---
variables:
OPENDESK_CI_CLI_IMAGE: "registry.opencode.de/bmi/opendesk/tooling/opendesk-ci-cli:2.5.2\
@sha256:8445978facabf1526cdb4434e7da8b9c8b9b930337c247ea811b1a53765817e3"
OPENDESK_LINT_IMAGE: "registry.opencode.de/bmi/opendesk/components/platform-development/images/ci-lint:1.0.7\
@sha256:632ea46d4ff70d39a627404576ea3b14a1cf96d04ea0701cbea6b771765d1769"
OPENDESK_CI_CLI_IMAGE: "registry.opencode.de/bmi/opendesk/tooling/opendesk-ci-cli:2.5.3\
@sha256:1296e8590b59f02311881307bb14c58b72bafc92a58e7e7e7212508abf902b00"
OPENDESK_LINT_IMAGE: "registry.opencode.de/bmi/opendesk/components/platform-development/images/ci-lint:1.0.8\
@sha256:59e714cad38e873cf8a9a132af76ad868b46a51ed12d0bd45477f328c0136991"
.common:
cache: {}

View File

@@ -35,12 +35,12 @@ repositories:
dependencyDashboard: true
# Include package files only within these defined paths
includePaths:
- "helmfile/environments/default/images.yaml"
- "helmfile/environments/default/charts.yaml"
- "helmfile/environments/default/images.yaml.gotmpl"
- "helmfile/environments/default/charts.yaml.gotmpl"
customManagers:
- customType: "regex"
fileMatch:
- "helmfile/environments/default/images.yaml"
- "helmfile/environments/default/images.yaml.gotmpl"
datasourceTemplate: "docker"
matchStrings:
# yamllint disable rule:line-length rule:quoted-strings
@@ -48,7 +48,7 @@ repositories:
# yamllint enable rule:line-length rule:quoted-strings
- customType: "regex"
fileMatch:
- "helmfile/environments/default/charts.yaml"
- "helmfile/environments/default/charts.yaml.gotmpl"
datasourceTemplate: "docker"
matchStrings:
# yamllint disable rule:line-length rule:quoted-strings

View File

@@ -87,8 +87,8 @@ of this project.
Please find a list of the artifacts related to the release either in the source code archive attached to the release or
in the files from the release's git-tag:
- `./helmfile/environments/default/images.yaml`
- `./helmfile/environments/default/charts.yaml`
- `./helmfile/environments/default/images.yaml.gotmpl`
- `./helmfile/environments/default/charts.yaml.gotmpl`
Find more information in our [Workflow documentation](./docs/workflow.md).

View File

@@ -23,7 +23,7 @@ options = p.parse_args()
script_path = os.path.dirname(os.path.realpath(__file__))
# some static definitions
log_path = script_path+'/../logs'
charts_yaml = script_path+'/../helmfile/environments/default/charts.yaml'
charts_yaml = script_path+'/../helmfile/environments/default/charts.yaml.gotmpl'
base_repo_path = script_path+'/..'
base_helmfile = base_repo_path+'/helmfile_generic.yaml.gotmpl'
helmfile_backup_extension = '.bak'
@@ -43,8 +43,8 @@ consoleHandler.setFormatter(logFormatter)
rootLogger.addHandler(consoleHandler)
logging.debug(f"Working with relative paths from script location: {script_path}")
logging.debug(f"Log directory: {log_path}")
logging.debug(f"charts.yaml : {charts_yaml}")
logging.debug(f"Log directory: {log_path}")
logging.debug(f"charts.yaml.gotmpl: {charts_yaml}")
def create_or_switch_branch_base_repo():

View File

@@ -35,7 +35,7 @@ environments, you should use them thoughtfully and carefully if needed.
# Enable debugging
Check the openDesk [`debug.yaml`](../helmfile/environments/default/debug.yaml) and set for your deployment
Check the openDesk [`debug.yaml.gotmpl`](../helmfile/environments/default/debug.yaml.gotmpl) and set for your deployment
```
debug:
enable: true

View File

@@ -12,7 +12,7 @@ However, contributions are possible using the [CLA](https://gitlab.opencode.de/b
<!-- TOC -->
* [Overview](#overview)
* [Default branch, `develop` and other branches](#default-branch-develop-and-other-branches)
* [External artifacts - `charts.yaml` and `images.yaml`](#external-artifacts---chartsyaml-and-imagesyaml)
* [External artifacts - `charts.yaml.gotmpl` and `images.yaml.gotmpl`](#external-artifacts---chartsyamlgotmpl-and-imagesyamlgotmpl)
* [Linting](#linting)
* [Disable linting selectively](#disable-linting-selectively)
* [Renovate](#renovate)
@@ -29,29 +29,29 @@ developing the openDesk platform.
```mermaid
flowchart TD
J[helmfile.yaml\nor a helmfile outside of this repository]-->A
J[helmfile.yaml.gotmpl\nor a helmfile outside of this repository]-->A
J-->K[./helmfile/environemnts/*your_environment*/values.yaml.gotmpl\nor any an environment values file]
A[./helmfile_generic.yaml]-->B[./helmfile/apps/*all_configured_apps*/helmfile.yaml\nReferences the relevant app Helm\ncharts using details from 'charts.yaml']
B-->C[./values-*all_configured_components*.yaml.gotmpl\nValues to template the charts\nwith references to the `images.yaml`]
A[./helmfile_generic.yaml.gotmpl]-->B[./helmfile/apps/*all_configured_apps*/helmfile.yaml.gotmpl\nReferences the relevant app Helm\ncharts using details from 'charts.yaml.gotmpl']
B-->C[./values-*all_configured_components*.yaml.gotmpl\nValues to template the charts\nwith references to the `images.yaml.gotmpl`]
A-->D[./helmfile/environments/default/*\nwith just some examples below]
D-->F[charts.yaml]
D-->G[images.yaml]
D-->F[charts.yaml.gotmpl]
D-->G[images.yaml.gotmpl]
D-->H[global.*]
D-->I[secrets.yaml\nreplicas.yaml\nresources.yaml\n...]
D-->I[secrets.yaml.gotmpl\nreplicas.yaml.gotmpl\nresources.yaml.gotmpl\n...]
A-->|overwrite defaults with your\ndeployment/environment specific values|E[./helmfile/environments/default/values.yaml.gotmpl]
```
The `helmfile.yaml` file in the root folder is the foundation
for the entire deployment. It references the `helmfile_generic.yaml`
file, which includes app-specific `helmfile.yaml` files and
The `helmfile.yaml.gotmpl` file in the root folder is the foundation
for the entire deployment. It references the `helmfile_generic.yaml.gotmpl`
file, which includes app-specific `helmfile.yaml.gotmpl` files and
global values files located in `./environments/default`.
`helmfile.yaml` also refers to three predefined environments: `dev`,
`helmfile.yaml.gotmpl` also refers to three predefined environments: `dev`,
`test`, and `prod`.
The `helmfile_generic.yaml` file is designed to be referenced from
The `helmfile_generic.yaml.gotmpl` file is designed to be referenced from
external repositories, where custom environments may be defined. An
example is demonstrated in the `helmfile.yaml` file.
example is demonstrated in the `helmfile.yaml.gotmpl` file.
Before you investigate any app-specific configuration, it is recommended that you review the contents of `./helmfile/environments/default` to understand what configuration details are set there, as the app deployments reference them.
@@ -68,9 +68,9 @@ Of course, the merge request will target the `develop` branch, be in status `dra
If you do not plan to merge from the branch you have pushed, please close the auto-created MR.
# External artifacts - `charts.yaml` and `images.yaml`
# External artifacts - `charts.yaml.gotmpl` and `images.yaml.gotmpl`
The `charts.yaml` and `images.yaml` files are the central place to reference external artifacts used for the deployment.
The `charts.yaml.gotmpl` and `images.yaml.gotmpl` files are the central place to reference external artifacts used for the deployment.
Besides the deployment automation itself, some tools work with the contents of the files:
@@ -142,7 +142,7 @@ The mirror script takes the information on what artifacts to mirror from the ann
- `# upstreamRegistry` *required*: To identify the source registry
- `# upstreamRegistryCredentialId`: *optional*: In case the source registry is not public, the access credentials have to be specified as ENV variables containing the value of this key in their name, so you want to specify that key all uppercase:
  - `MIRROR_CREDENTIALS_SRC_<upstreamRegistryCredentialId>_USERNAME`
  - `MIRROR_CREDENTIALS_SRC_<upstreamRegistryCredentialId>_PASSWORT`
  - `MIRROR_CREDENTIALS_SRC_<upstreamRegistryCredentialId>_PASSWORD`
- `# upstreamRepository` *required*: To identify the source repository
- `# upstreamMirrorTagFilterRegEx` *required*: If this annotation is set, the mirror for the component will be activated. Only tags that match the given regular expression are being mirrored. **Note:** You must use single quotes for this attribute's value if you use backslash leading regex notation like `\d`.
- `# upstreamMirrorStartFrom` *optional*: Array of numeric values in case you want to mirror only artifacts beginning with a specific version. You must use capturing group

View File

@@ -39,7 +39,7 @@ access openDesk.
```
1. Create a Kubernetes secret named `opendesk-certificates-tls` of type `kubernetes.io/tls` containing either a valid
wildcard certificate or a certificate with [all required subdomains](../../helmfile/environments/default/global.yaml)
wildcard certificate or a certificate with [all required subdomains](../../helmfile/environments/default/global.yaml.gotmpl)
set as SANs (Subject Alternative Name).
1. Create a Kubernetes secret with name `opendesk-certificates-ca-tls` of type `kubernetes.io/tls` containing the custom

View File

@@ -448,9 +448,9 @@ docker run --rm registry.opencode.de/bmi/opendesk/components/platform-developmen
## Using from external repository
Referring to `./helmfile_generic.yaml` from an external
directory or repository is possible. The `helmfile.yaml` that refers to
`./helmfile_generic.yaml` may define custom environments. These custom
Referring to `./helmfile_generic.yaml.gotmpl` from an external
directory or repository is possible. The `helmfile.yaml.gotmpl` that refers to
`./helmfile_generic.yaml.gotmpl` may define custom environments. These custom
environments may overwrite specific configuration values. These
configuration values are:

View File

@@ -11,6 +11,7 @@ SPDX-License-Identifier: Apache-2.0
* [Releases upgrade details](#releases-upgrade-details)
* [From v1.0.0](#from-v100)
* [Pre-upgrade: Manual checks/steps](#pre-upgrade-manual-checkssteps)
* [Helmfile Cleanup: Consistent use of `*.yaml.gotmpl`](#helmfile-cleanup-consistent-use-of-yamlgotmpl)
* [New openDesk default: Enforce login](#new-opendesk-default-enforce-login)
* [Changed openDesk default: Jitsi room history enabled](#changed-opendesk-default-jitsi-room-history-enabled)
* [Streamlining `openxchange` and `oxAppSuite` attribute names](#streamlining-openxchange-and-oxappsuite-attribute-names)
@@ -70,6 +71,16 @@ Explanation of the table's columns:
### Pre-upgrade: Manual checks/steps
#### Helmfile Cleanup: Consistent use of `*.yaml.gotmpl`
In v1.0.0 the files in [`/helmfile/environments/default`](../helmfile/environments/default/) had mixed extensions,
we have streamlined them to consistently use `*.yaml.gotmpl`.
This change requires manual action likely in two situations:
1. You are referencing our upstream files from the aforementioned directory, e.g. in your Argo CD deployment. Please update your references to use the filenames with the new extension.
2. You have custom files containing configuration information that are named just `*.yaml`: Please rename them to `*.yaml.gotmpl`.
#### New openDesk default: Enforce login
Users accessing the openDesk portal are now automatically redirected to the login screen as a default.
@@ -167,7 +178,7 @@ If you make use of the `customization.release` option, you have to switch to a d
```yaml
customization:
release:
collaboraOnline: "./my_custom_templating.yaml"
collaboraOnline: "./my_custom_templating.yaml.gotmpl"
```
to
@@ -176,7 +187,7 @@ to
customization:
release:
collaboraOnline:
file1: "./my_custom_templating.yaml"
file1: "./my_custom_templating.yaml.gotmpl"
```
You can freely choose the `file1` dictionary key used in the example above, but it should start with a letter.
@@ -434,11 +445,11 @@ The status of the upgrade migrations is tracked in the ConfigMap `migrations-sta
When a new upgrade migration is required, ensure to address the following list:
- Update the generated release version file [`global.generated.yaml`](../helmfile/environments/default/global.generated.yaml) at least on the patch level to test the upgrade in your feature branch and trigger it in the `develop` branch after the feature branch was merged. During the release process, the value is overwritten by the release's version number.
- Update the generated release version file [`global.generated.yaml.gotmpl`](../helmfile/environments/default/global.generated.yaml.gotmpl) at least on the patch level to test the upgrade in your feature branch and trigger it in the `develop` branch after the feature branch was merged. During the release process, the value is overwritten by the release's version number.
- You have to implement the migration logic as a runner script in the [`opendesk-migrations`](https://gitlab.opencode.de/bmi/opendesk/components/platform-development/images/opendesk-migrations) image. Please find more instructions in the linked repository.
- You most likely have to update the [`opendesk-migrations` Helm chart](https://gitlab.opencode.de/bmi/opendesk/components/platform-development/charts/opendesk-migrations) within the `rules` section of the [`role.yaml`](https://gitlab.opencode.de/bmi/opendesk/components/platform-development/charts/opendesk-migrations/-/blob/main/charts/opendesk-migrations/templates/role.yaml) to provide the permissions required for the execution of your migration's logic.
- You must set the runner's ID you want to execute in the [migrations.yaml.gotmpl](../helmfile/shared/migrations.yaml.gotmpl). See also the `migrations.*` section of [the Helm chart's README.md](https://gitlab.opencode.de/bmi/opendesk/components/platform-development/charts/opendesk-migrations/-/blob/main/charts/opendesk-migrations/README.md).
- Update the [`charts.yaml`](../helmfile/environments/default/charts.yaml) and [`images.yaml`](../helmfile/environments/default/images.yaml) to reflect the newer releases of the `opendesk-migrations` Helm chart and container image.
- Update the [`charts.yaml.gotmpl`](../helmfile/environments/default/charts.yaml.gotmpl) and [`images.yaml.gotmpl`](../helmfile/environments/default/images.yaml.gotmpl) to reflect the newer releases of the `opendesk-migrations` Helm chart and container image.
[^1]: We do not follow a brand name's specific spelling when it comes to upper and lower case and only use new word
uppercase when names consist of multiple, space divided words.

View File

@@ -30,7 +30,7 @@ repository or Prometheus operator.
# Defaults
All configurable options and their defaults can be found in
[`monitoring.yaml`](../helmfile/environments/default/monitoring.yaml).
[`monitoring.yaml.gotmpl`](../helmfile/environments/default/monitoring.yaml.gotmpl).
# Metrics

View File

@@ -10,13 +10,13 @@ This document covers the ability to scale applications.
# Horizontal scalability
We are working on generating this document automatically based on the file
[`replicas.yaml`](../helmfile/environments/default/replicas.yaml) that contains necessary annotations.
[`replicas.yaml.gotmpl`](../helmfile/environments/default/replicas.yaml.gotmpl) that contains necessary annotations.
In the meantime, this file can be used to check the components scaling support/capabilities.
# Upstream information
While scaling services horizontally is the ideal solution, information about vertical scaling is helpful
when defining the application's resources, see [`resources.yaml`](../helmfile/environments/default/resources.yaml) for references.
when defining the application's resources, see [`resources.yaml.gotmpl`](../helmfile/environments/default/resources.yaml.gotmpl) for references.
Please find below links to the application's upstream resources about scaling:

View File

@@ -235,7 +235,7 @@ The Standard Quality Gate addresses quality assurance steps that should be execu
1. Linting
- Blocking
- Licensing: [reuse](https://github.com/fsfe/reuse-tool)
- openDesk specific: Especially `images.yaml` and `charts.yaml`, find more details in [development.md](./development.md).
- openDesk specific: Especially `images.yaml.gotmpl` and `charts.yaml.gotmpl`, find more details in [development.md](./development.md).
- Non-Blocking
- Security: [Kyverno policy check](../.kyverno) addressing some IT-Grundschutz requirements
- Formal: Yaml

View File

@@ -22,7 +22,7 @@ helmfiles:
#
# Use this format from a remote repository
#
# - path: "git::https://gitlab.opencode.de/bmi/opendesk/deployment/opendesk.git@helmfile_generic.yaml?ref=v0.7.1"
# - path: "git::https://gitlab.opencode.de/bmi/opendesk/deployment/opendesk.git@helmfile_generic.yaml.gotmpl?ref=v0.7.1"
# values:
# - {{ toYaml .Values | nindent 8 }}
# */}}

View File

@@ -42,7 +42,7 @@ job:
- username: {{ .Values.databases.nextcloud.username | quote }}
password: {{ .Values.secrets.mariadb.nextcloudUser | quote}}
connectionLimit: {{ .Values.databases.nextcloud.connectionLimit | default .Values.databases.defaults.userConnectionLimit }}
# OX and XWiki are using the db's `root` users (see `database.yaml`). So we are statically referencing their dedicated
# OX and XWiki are using the db's `root` users (see `database.yaml.gotmpl`). So we are statically referencing their dedicated
# users for the moment.
- username: "openxchange_user"
# - username: {{ .Values.databases.xwiki.username | quote }}
@@ -55,7 +55,7 @@ job:
databases:
- name: {{ .Values.databases.nextcloud.name | quote }}
user: {{ .Values.databases.nextcloud.username | quote }}
# OX and XWiki are using the db's `root` users (see `database.yaml`). So we are statically referencing their dedicated
# OX and XWiki are using the db's `root` users (see `database.yaml.gotmpl`). So we are statically referencing their dedicated
# users for the moment.
- name: "openxchange"
user: "openxchange_user"

View File

@@ -4,21 +4,17 @@
environments:
default:
values:
- "../../environments/default/*.gotmpl"
- "../../environments/default/*.yaml"
- "../../environments/default/*.yaml.gotmpl"
dev:
values:
- "../../environments/default/*.gotmpl"
- "../../environments/default/*.yaml"
- "../../environments/default/*.yaml.gotmpl"
- "../../environments/dev/*.yaml.gotmpl"
test:
values:
- "../../environments/default/*.gotmpl"
- "../../environments/default/*.yaml"
- "../../environments/default/*.yaml.gotmpl"
- "../../environments/test/*.yaml.gotmpl"
prod:
values:
- "../../environments/default/*.gotmpl"
- "../../environments/default/*.yaml"
- "../../environments/default/*.yaml.gotmpl"
- "../../environments/prod/*.yaml.gotmpl"
...

View File

@@ -13,7 +13,7 @@
# customization:
# release:
# collaboraOnline:
# myCustomConfig: "/path/to/additional/file.yaml"
# myCustomConfig: "/path/to/additional/file.yaml.gotmpl"
customization:
release:
# collabora

View File

@@ -14,6 +14,5 @@ debug:
# should activate debug output in all components and even allow e.g. successfully executed jobs
# to stay available. This is going to be implemented on a case by case basis when we actually
# need debugging in a component.
# Use: `{{ if .Values.debug.enabled }}"DEBUG"{{ else }}"WARN"{{ end }}`
enabled: false
...

View File

@@ -1,6 +1,6 @@
# GPG public keys
Place GPG public keys in binary format into this directory and reference them in the respective `helmfile.yaml`.
Place GPG public keys in binary format into this directory and reference them in the respective `helmfile.yaml.gotmpl`.
In case you have only an ASCII Armored file you can just base64 decode the payload of that file, but remember to
**not include** the 4-letter checksum that is prefixed with an `=` sign at the end of the payload.

View File

@@ -8,8 +8,7 @@ helmfiles:
# Path to the helmfile state file being processed BEFORE releases in this state file
- path: "helmfile/apps/migrations-pre/helmfile-child.yaml.gotmpl"
values: &values
- "helmfile/environments/default/*.yaml"
- "helmfile/environments/default/*.gotmpl"
- "helmfile/environments/default/*.yaml.gotmpl"
- {{ toYaml .Values | nindent 8 }}
- path: "helmfile/apps/services/helmfile-child.yaml.gotmpl"
values: *values