diff --git a/.github/workflows/automatic_doc_checks.yaml b/.github/workflows/automatic_doc_checks.yaml index bcc332b8a..1e7b64086 100644 --- a/.github/workflows/automatic_doc_checks.yaml +++ b/.github/workflows/automatic_doc_checks.yaml @@ -8,8 +8,7 @@ on: - '8.0/edge' pull_request: paths: - - 'kubernetes/docs/**' - - 'machines/docs/**' + - '/docs/**' workflow_dispatch: @@ -19,12 +18,7 @@ concurrency: jobs: documentation-checks: - strategy: - matrix: - path: - - kubernetes - - machines uses: canonical/documentation-workflows/.github/workflows/documentation-checks.yaml@main with: - working-directory: ${{ matrix.path }}/docs + working-directory: "docs" fetch-depth: 0 diff --git a/.github/workflows/check_libs.yaml b/.github/workflows/check_libs.yaml index 698f19fe2..09b1e08fb 100644 --- a/.github/workflows/check_libs.yaml +++ b/.github/workflows/check_libs.yaml @@ -11,15 +11,14 @@ on: paths-ignore: - 'kubernetes/.gitignore' - 'kubernetes/.jujuignore' - - 'kubernetes/docs/**' - 'machines/.gitignore' - 'machines/.jujuignore' - - 'machines/docs/**' - 'LICENSE' - '**.md' - '.github/renovate.json5' - '.github/workflows/*.yaml' - + - 'docs/**' + jobs: lib-check: name: Check libraries diff --git a/.github/workflows/ci.yaml b/.github/workflows/ci.yaml index 19b77a0a6..6d058b696 100644 --- a/.github/workflows/ci.yaml +++ b/.github/workflows/ci.yaml @@ -9,8 +9,7 @@ concurrency: on: pull_request: paths-ignore: - - 'kubernetes/docs/**' - - 'machines/docs/**' + - 'docs/**' - '**.md' - '.github/renovate.json5' diff --git a/.github/workflows/integration_test.yaml b/.github/workflows/integration_test.yaml index 965192343..1c345791f 100644 --- a/.github/workflows/integration_test.yaml +++ b/.github/workflows/integration_test.yaml @@ -28,7 +28,7 @@ jobs: - name: Set up environment run: | sudo snap install go --classic - go install github.com/snapcore/spread/cmd/spread@latest + go install github.com/canonical/spread/cmd/spread@latest pipx install tox pipx install poetry - name: Collect spread jobs @@ -83,7 +83,7 @@ jobs: # Default test results in case the integration tests time out or runner set up fails # (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test) if: ${{ github.event_name == 'schedule' && github.run_attempt == '1' }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: allure-default-results-integration-test-${{ inputs.path-to-charm-directory }} path: ${{ inputs.path-to-charm-directory }}/allure-default-results/ @@ -114,15 +114,19 @@ jobs: - name: Checkout timeout-minutes: 3 uses: actions/checkout@v5 + - name: Install Python for tests + uses: actions/setup-python@v6 + with: + python-version: '3.12' - name: Set up environment timeout-minutes: 5 run: | sudo snap install charmcraft --classic sudo snap install go --classic - go install github.com/snapcore/spread/cmd/spread@latest + go install github.com/canonical/spread/cmd/spread@latest - name: Download packed charm(s) timeout-minutes: 5 - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v8 with: pattern: ${{ inputs.artifact-prefix }}-* merge-multiple: true @@ -145,7 +149,7 @@ jobs: # Allure can only process one result per pytest test ID. If parameterization is done via # spread instead of pytest, there will be overlapping pytest test IDs. if: ${{ (success() || (failure() && steps.spread.outcome == 'failure')) && startsWith(matrix.job.spread_job, 'github-ci:ubuntu-24.04:') && endsWith(matrix.job.spread_job, ':juju36') && github.event_name == 'schedule' && github.run_attempt == '1' }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: allure-results-integration-test-${{ inputs.path-to-charm-directory }}-${{ matrix.job.name_in_artifact }} path: ${{ inputs.path-to-charm-directory }}/artifacts/${{ matrix.job.spread_job }}/allure-results/ @@ -186,7 +190,7 @@ jobs: - name: Upload logs timeout-minutes: 5 if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: logs-integration-test-${{ inputs.path-to-charm-directory }}-${{ matrix.job.name_in_artifact }} path: ~/logs/ diff --git a/.github/workflows/release.yaml b/.github/workflows/release.yaml index 6227008c2..73eff14e6 100644 --- a/.github/workflows/release.yaml +++ b/.github/workflows/release.yaml @@ -7,8 +7,7 @@ on: branches: - '8.0/edge' paths-ignore: - - 'kubernetes/docs/**' - - 'machines/docs/**' + - 'docs/**' - '.github/renovate.json5' - '.github/workflows/*.yaml' @@ -58,7 +57,7 @@ jobs: secrets: inherit release: - name: Release charm | ${{ matrix.charm.path }} + name: Release charm | ${{ matrix.path }} needs: - build - integration-test diff --git a/.github/workflows/release_test.yaml b/.github/workflows/release_test.yaml index 05bb7e03d..581d0ed7a 100644 --- a/.github/workflows/release_test.yaml +++ b/.github/workflows/release_test.yaml @@ -28,7 +28,7 @@ jobs: - name: Set up environment run: | sudo snap install go --classic - go install github.com/snapcore/spread/cmd/spread@latest + go install github.com/canonical/spread/cmd/spread@latest pipx install tox pipx install poetry - name: Collect spread jobs @@ -83,7 +83,7 @@ jobs: # Default test results in case the integration tests time out or runner set up fails # (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test) if: ${{ github.event_name == 'schedule' && github.run_attempt == '1' }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: allure-default-results-release-test-${{ inputs.path-to-charm-directory }} path: ${{ inputs.path-to-charm-directory }}/allure-default-results/ @@ -119,10 +119,10 @@ jobs: run: | sudo snap install charmcraft --classic sudo snap install go --classic - go install github.com/snapcore/spread/cmd/spread@latest + go install github.com/canonical/spread/cmd/spread@latest - name: Download packed charm(s) timeout-minutes: 5 - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v8 with: pattern: ${{ inputs.artifact-prefix }}-* merge-multiple: true @@ -145,7 +145,7 @@ jobs: # Allure can only process one result per pytest test ID. If parameterization is done via # spread instead of pytest, there will be overlapping pytest test IDs. if: ${{ (success() || (failure() && steps.spread.outcome == 'failure')) && startsWith(matrix.job.spread_job, 'github-ci:ubuntu-24.04:') && endsWith(matrix.job.spread_job, ':juju36') && github.event_name == 'schedule' && github.run_attempt == '1' }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: allure-results-release-test-${{ inputs.path-to-charm-directory }}-${{ matrix.job.name_in_artifact }} path: ${{ inputs.path-to-charm-directory }}/artifacts/${{ matrix.job.spread_job }}/allure-results/ @@ -186,7 +186,7 @@ jobs: - name: Upload logs timeout-minutes: 5 if: ${{ !contains(matrix.job.spread_job, 'juju29') && (success() || (failure() && steps.spread.outcome == 'failure')) }} - uses: actions/upload-artifact@v4 + uses: actions/upload-artifact@v7 with: name: logs-release-test-${{ inputs.path-to-charm-directory }}-${{ matrix.job.name_in_artifact }} path: ~/logs/ diff --git a/.github/workflows/schedule.yaml b/.github/workflows/schedule.yaml index 5ca8906f5..100a081e2 100644 --- a/.github/workflows/schedule.yaml +++ b/.github/workflows/schedule.yaml @@ -92,12 +92,12 @@ jobs: - name: Download default test results # Default test results in case the integration tests time out or runner set up fails # (So that Allure report will show "unknown"/"failed" test result, instead of omitting the test) - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v8 with: path: allure-default-results/ name: allure-default-results-integration-test - name: Download test results - uses: actions/download-artifact@v5 + uses: actions/download-artifact@v8 with: path: allure-results/ pattern: allure-results-integration-test-* diff --git a/machines/.readthedocs.yaml b/.readthedocs.yaml similarity index 87% rename from machines/.readthedocs.yaml rename to .readthedocs.yaml index fc5f69a52..920e6a42f 100644 --- a/machines/.readthedocs.yaml +++ b/.readthedocs.yaml @@ -18,7 +18,7 @@ build: # This is a special exit code on Read the Docs that will cancel the build immediately. # https://docs.readthedocs.io/en/stable/build-customization.html#cancel-build-based-on-a-condition - | - if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet origin/8.0/edge -- 'machines/docs/' 'machines/.readthedocs.yaml'; + if [ "$READTHEDOCS_VERSION_TYPE" = "external" ] && git diff --quiet origin/8.0/edge -- 'docs/' '.readthedocs.yaml'; then exit 183; fi @@ -26,7 +26,7 @@ build: # Build documentation in the docs/ directory with Sphinx sphinx: builder: dirhtml - configuration: machines/docs/conf.py + configuration: docs/conf.py fail_on_warning: true # If using Sphinx, optionally build your docs in additional formats such as PDF @@ -36,4 +36,4 @@ formats: # Optionally declare the Python requirements required to build your docs python: install: - - requirements: machines/docs/requirements.txt + - requirements: docs/requirements.txt diff --git a/CONTRIBUTING.md b/CONTRIBUTING.md index 4bfe63f8f..e46584581 100644 --- a/CONTRIBUTING.md +++ b/CONTRIBUTING.md @@ -16,12 +16,13 @@ this operator. - Test coverage - User experience for Juju administrators of this charm. - Please help us out in ensuring easy to review branches by rebasing your pull request branch onto - the `main` branch. This also avoids merge commits and creates a linear Git commit history. + the `8.0/edge` branch. This also avoids merge commits and creates a linear Git commit history. ## Develop -Install `tox`, `poetry`, and `charmcraftcache` +Install `yq`, `tox`, `poetry`, and `charmcraftcache` ```shell +pipx install yq pipx install tox pipx install poetry pipx install charmcraftcache @@ -63,9 +64,12 @@ Build the charm in this git repository using: juju add-model dev juju model-config logging-config="=INFO;unit=DEBUG" +# Extract the K8s image +export MYSQL_IMAGE=$(yq -r '.["resources"]["mysql-image"]["upstream-source"]' kubernetes/metadata.yaml) + # Deploy the K8s or VM charm -(cd kubernetes && juju deploy ./mysql-k8s_ubuntu-22.04-amd64.charm --resource mysql-image=...) -(cd machines && juju deploy ./mysql_ubuntu-22.04-amd64.charm) +(cd kubernetes && juju deploy ./mysql-k8s_ubuntu@22.04-amd64.charm --resource mysql-image=${MYSQL_IMAGE}) +(cd machines && juju deploy ./mysql_ubuntu@22.04-amd64.charm) ``` ## Canonical Contributor Agreement diff --git a/README.md b/README.md index b4320bee5..00c704a2d 100644 --- a/README.md +++ b/README.md @@ -1,6 +1,6 @@ # MySQL operators -[![Charmhub](https://charmhub.io/mysql/badge.svg)](https://charmhub.io/mysql) -[![Charmhub](https://charmhub.io/mysql-k8s/badge.svg)](https://charmhub.io/mysql-k8s) +[![Charmhub](https://charmhub.io/mysql/badge.svg?channel=8.0/edge)](https://charmhub.io/mysql) +[![Charmhub](https://charmhub.io/mysql-k8s/badge.svg?channel=8.0/edge)](https://charmhub.io/mysql-k8s) [![Release](https://github.com/canonical/mysql-operators/actions/workflows/release.yaml/badge.svg?branch=8.0/edge)](https://github.com/canonical/mysql-operators/actions/workflows/release.yaml) [![Tests](https://github.com/canonical/mysql-operators/actions/workflows/ci.yaml/badge.svg?branch=8.0/edge)](https://github.com/canonical/mysql-operators/actions/workflows/ci.yaml) @@ -35,8 +35,7 @@ juju destroy-model mysql --destroy-storage --yes ## Documentation -Please follow the tutorial guide ([K8s](https://canonical-charmed-mysql-k8s.readthedocs-hosted.com/tutorial/) or [VM](https://canonical-charmed-mysql.readthedocs-hosted.com/tutorial/)) -with detailed explanation how to access DB, configure cluster, change credentials and/or enable TLS. +See the [official documentation](https://canonical-charmed-mysql.readthedocs-hosted.com/8.0) for more operational guidance, such as deployment on specific clouds, TLS, monitoring, backups, and troubleshooting. ## Relations diff --git a/kubernetes/docs/.custom_wordlist.txt b/docs/.custom_wordlist.txt similarity index 80% rename from kubernetes/docs/.custom_wordlist.txt rename to docs/.custom_wordlist.txt index fe54bb86d..4563636a3 100644 --- a/kubernetes/docs/.custom_wordlist.txt +++ b/docs/.custom_wordlist.txt @@ -1,16 +1,23 @@ # Leave a blank line at the end of this file to support concatenation airgap +airgapped Artifactory async backend backends backport +binlog Charmcraft cjk +cron cryptographically +CSR +CSRs databag dvipng +failover fonts +fqdn freefont Furo github @@ -24,24 +31,30 @@ Intersphinx io ip Jira +kubectl landscape lang lastmod LaTeX latexmk +logrotate MinIO Multipass mydumper mysql +mysqldump MyST nameserver nameservers Open Graph otf +Paxos PDF Percona plantuml PNG +PPA +PPAs PR PVC PVCs @@ -58,18 +71,27 @@ snap_daemon Sphinx Spread spread_test_example +stdout +storages subproject subprojects SVG +Sysbench tex texlive TOC toctree +tracebacks txt +teardown uncommenting +systemd +systemctl +uncordon URL utils VMs +walkthrough WCAG whitespace whitespaces diff --git a/kubernetes/docs/.gitignore b/docs/.gitignore similarity index 100% rename from kubernetes/docs/.gitignore rename to docs/.gitignore diff --git a/kubernetes/docs/.sphinx/.pre-commit-config.yaml b/docs/.sphinx/.pre-commit-config.yaml similarity index 100% rename from kubernetes/docs/.sphinx/.pre-commit-config.yaml rename to docs/.sphinx/.pre-commit-config.yaml diff --git a/kubernetes/docs/.sphinx/.pymarkdown.json b/docs/.sphinx/.pymarkdown.json similarity index 100% rename from kubernetes/docs/.sphinx/.pymarkdown.json rename to docs/.sphinx/.pymarkdown.json diff --git a/machines/docs/.sphinx/get_vale_conf.py b/docs/.sphinx/get_vale_conf.py old mode 100644 new mode 100755 similarity index 96% rename from machines/docs/.sphinx/get_vale_conf.py rename to docs/.sphinx/get_vale_conf.py index e2a81c088..13e7966f8 --- a/machines/docs/.sphinx/get_vale_conf.py +++ b/docs/.sphinx/get_vale_conf.py @@ -31,12 +31,12 @@ def clone_repo_and_copy_paths(file_source_dest, overwrite=False): """ Clone the repository to a temporary directory and copy required files - + Args: file_source_dest: dictionary of file paths to copy from the repository, and their destination paths overwrite: boolean flag to overwrite existing files in the destination - + Returns: bool: True if all files were copied successfully, False otherwise """ @@ -52,8 +52,8 @@ def clone_repo_and_copy_paths(file_source_dest, overwrite=False): try: result = subprocess.run( - clone_cmd, - capture_output=True, + clone_cmd, + capture_output=True, text=True, check=True ) @@ -73,7 +73,7 @@ def clone_repo_and_copy_paths(file_source_dest, overwrite=False): continue if not copy_files_to_path(source_path, dest, overwrite): - is_copy_success = False + is_copy_success = False logging.error("Failed to copy %s to %s", source_path, dest) # Clean up temporary directory @@ -85,12 +85,12 @@ def clone_repo_and_copy_paths(file_source_dest, overwrite=False): def copy_files_to_path(source_path, dest_path, overwrite=False): """ Copy a file or directory from source to destination - + Args: source_path: Path to the source file or directory dest_path: Path to the destination overwrite: Boolean flag to overwrite existing files in the destination - + Returns: bool: True if copy was successful, False otherwise """ @@ -138,7 +138,7 @@ def main(): # Parse command line arguments, default to overwrite_enabled = True overwrite_enabled = not parse_arguments().no_overwrite - # Download into /tmp through git clone + # Download into /tmp through git clone if not clone_repo_and_copy_paths(vale_files_dict, overwrite=overwrite_enabled): logging.error("Failed to download files from repository") return 1 diff --git a/kubernetes/docs/.sphinx/pa11y.json b/docs/.sphinx/pa11y.json similarity index 100% rename from kubernetes/docs/.sphinx/pa11y.json rename to docs/.sphinx/pa11y.json diff --git a/docs/.sphinx/version b/docs/.sphinx/version new file mode 100644 index 000000000..347f5833e --- /dev/null +++ b/docs/.sphinx/version @@ -0,0 +1 @@ +1.4.1 diff --git a/machines/docs/Makefile b/docs/Makefile similarity index 69% rename from machines/docs/Makefile rename to docs/Makefile index 0b498e12f..edee045f2 100644 --- a/machines/docs/Makefile +++ b/docs/Makefile @@ -7,17 +7,17 @@ SPHINXDIR = .sphinx SPHINXOPTS ?= -c . -d $(SPHINXDIR)/.doctrees -j auto SPHINXBUILD ?= $(VENVDIR)/bin/sphinx-build -SOURCEDIR = . -BUILDDIR = _build -VENVDIR = $(SPHINXDIR)/venv +SOURCEDIR ?= . +BUILDDIR ?= _build +VENVDIR ?= $(SPHINXDIR)/venv PA11Y = $(SPHINXDIR)/node_modules/pa11y/bin/pa11y.js --config $(SPHINXDIR)/pa11y.json VENV = $(VENVDIR)/bin/activate TARGET = * -ALLFILES = *.rst **/*.rst -METRICSDIR = $(SOURCEDIR)/.sphinx/metrics REQPDFPACKS = latexmk fonts-freefont-otf texlive-latex-recommended texlive-latex-extra texlive-fonts-recommended texlive-font-utils texlive-lang-cjk texlive-xetex plantuml xindy tex-gyre dvipng CONFIRM_SUDO ?= N VALE_CONFIG = $(SPHINXDIR)/vale.ini +VALEDIR ?= $(VENVDIR)/lib/python*/site-packages/vale +VOCAB_CANONICAL = $(SPHINXDIR)/styles/config/vocabularies/Canonical SPHINX_HOST ?= 127.0.0.1 SPHINX_PORT ?= 8000 @@ -38,14 +38,13 @@ help: @echo "* check accessibility: make pa11y" @echo "* check style guide compliance: make vale" @echo "* check style guide compliance on target: make vale TARGET=*" - @echo "* check metrics for documentation: make allmetrics" @echo "* other possible targets: make " @echo "-------------------------------------------------------------" @echo -.PHONY: help full‑help html epub pdf linkcheck spelling spellcheck woke \ - vale pa11y run serve install pa11y‑install \ - vale‑install pdf‑prep pdf‑prep‑force clean clean‑doc allmetrics \ +.PHONY: help full-help html epub pdf linkcheck spelling spellcheck woke \ + vale pa11y run serve install pa11y-install \ + vale-install pdf-prep pdf-prep-force clean clean-doc \ update lint-md full-help: $(VENVDIR) @@ -72,8 +71,8 @@ pa11y-install: npm install --prefix $(SPHINXDIR) pa11y; \ } -pymarkdownlnt-install: - @. $(VENV); test -d $(SPHINXDIR)/venv/lib/python*/site-packages/pymarkdown || pip install pymarkdownlnt +pymarkdownlnt-install: install + @. $(VENV); test -d $(VENVDIR)/lib/python*/site-packages/pymarkdown || pip install pymarkdownlnt==0.9.35 install: $(VENVDIR) @@ -82,13 +81,13 @@ run: install # Does not depend on $(BUILDDIR) to rebuild properly at every run. html: install - . $(VENV); $(SPHINXBUILD) -W --keep-going -b dirhtml "$(SOURCEDIR)" "$(BUILDDIR)" -w $(SPHINXDIR)/warnings.txt $(SPHINXOPTS) + . $(VENV); $(SPHINXBUILD) --fail-on-warning --keep-going -b dirhtml "$(SOURCEDIR)" "$(BUILDDIR)" -w $(SPHINXDIR)/warnings.txt $(SPHINXOPTS) epub: install . $(VENV); $(SPHINXBUILD) -b epub "$(SOURCEDIR)" "$(BUILDDIR)" -w $(SPHINXDIR)/warnings.txt $(SPHINXOPTS) serve: html - cd "$(BUILDDIR)"; python3 -m http.server --bind 127.0.0.1 8000 + cd "$(BUILDDIR)"; python3 -m http.server --bind $(SPHINX_HOST) $(SPHINX_PORT) clean: clean-doc @test ! -e "$(VENVDIR)" -o -d "$(VENVDIR)" -a "$(abspath $(VENVDIR))" != "$(VENVDIR)" @@ -109,36 +108,35 @@ pa11y: pa11y-install html find $(BUILDDIR) -name *.html -print0 | xargs -n 1 -0 $(PA11Y) lint-md: pymarkdownlnt-install - @. $(VENV); pymarkdownlnt --config $(SPHINXDIR)/.pymarkdown.json scan --recurse --exclude=./$(SPHINXDIR)/** $(SOURCEDIR) + @. $(VENV); pymarkdownlnt --config $(SPHINXDIR)/.pymarkdown.json scan --recurse --exclude=$(SPHINXDIR)/** $(SOURCEDIR) vale-install: install - @. $(VENV); test -d $(SPHINXDIR)/venv/lib/python*/site-packages/vale || pip install rst2html vale @. $(VENV); test -f $(VALE_CONFIG) || python3 $(SPHINXDIR)/get_vale_conf.py @echo '.Name=="Canonical.400-Enforce-inclusive-terms"' > $(SPHINXDIR)/styles/woke.filter @echo '.Level=="error" and .Name!="Canonical.500-Repeated-words" and .Name!="Canonical.000-US-spellcheck"' > $(SPHINXDIR)/styles/error.filter @echo '.Name=="Canonical.000-US-spellcheck"' > $(SPHINXDIR)/styles/spelling.filter - @. $(VENV); find $(SPHINXDIR)/venv/lib/python*/site-packages/vale/vale_bin -size 195c -exec vale --version \; + @. $(VENV); find $(VALEDIR)/vale_bin -size 195c -exec vale --version \; woke: vale-install - @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt - @cat $(SOURCEDIR)/.custom_wordlist.txt >> $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt + @cat $(VOCAB_CANONICAL)/accept.txt > $(VOCAB_CANONICAL)/accept_backup.txt + @cat $(SOURCEDIR)/.custom_wordlist.txt >> $(VOCAB_CANONICAL)/accept.txt @echo "Running Vale acceptable term check against $(TARGET). To change target set TARGET= with make command" @. $(VENV); vale --config="$(VALE_CONFIG)" --filter='$(SPHINXDIR)/styles/woke.filter' --glob='*.{md,rst}' $(TARGET) - @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt && rm $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt + @cat $(VOCAB_CANONICAL)/accept_backup.txt > $(VOCAB_CANONICAL)/accept.txt && rm $(VOCAB_CANONICAL)/accept_backup.txt vale: vale-install - @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt - @cat $(SOURCEDIR)/.custom_wordlist.txt >> $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt + @cat $(VOCAB_CANONICAL)/accept.txt > $(VOCAB_CANONICAL)/accept_backup.txt + @cat $(SOURCEDIR)/.custom_wordlist.txt >> $(VOCAB_CANONICAL)/accept.txt @echo "Running Vale against $(TARGET). To change target set TARGET= with make command" @. $(VENV); vale --config="$(VALE_CONFIG)" --filter='$(SPHINXDIR)/styles/error.filter' --glob='*.{md,rst}' $(TARGET) - @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt && rm $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt + @cat $(VOCAB_CANONICAL)/accept_backup.txt > $(VOCAB_CANONICAL)/accept.txt && rm $(VOCAB_CANONICAL)/accept_backup.txt spelling: vale-install - @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt - @cat $(SOURCEDIR)/.custom_wordlist.txt >> $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt + @cat $(VOCAB_CANONICAL)/accept.txt > $(VOCAB_CANONICAL)/accept_backup.txt + @cat $(SOURCEDIR)/.custom_wordlist.txt >> $(VOCAB_CANONICAL)/accept.txt @echo "Running Vale against $(TARGET). To change target set TARGET= with make command" @. $(VENV); vale --config="$(VALE_CONFIG)" --filter='$(SPHINXDIR)/styles/spelling.filter' --glob='*.{md,rst}' $(TARGET) - @cat $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt > $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept.txt && rm $(SPHINXDIR)/styles/config/vocabularies/Canonical/accept_backup.txt + @cat $(VOCAB_CANONICAL)/accept_backup.txt > $(VOCAB_CANONICAL)/accept.txt && rm $(VOCAB_CANONICAL)/accept_backup.txt spellcheck: spelling @echo "Please note that the \`make spellcheck\` command is being deprecated in favor of \`make spelling\`" @@ -165,21 +163,11 @@ pdf: pdf-prep @echo "Output can be found in ./$(BUILDDIR)" @echo -allmetrics: html - @echo "Recording documentation metrics..." - @echo "Checking for existence of vale..." - . $(VENV) - @. $(VENV); test -d $(SPHINXDIR)/venv/lib/python*/site-packages/vale || pip install vale - @. $(VENV); test -f $(VALE_CONFIG) || python3 $(SPHINXDIR)/get_vale_conf.py - @. $(VENV); find $(SPHINXDIR)/venv/lib/python*/site-packages/vale/vale_bin -size 195c -exec vale --config "$(VALE_CONFIG)" $(TARGET) > /dev/null \; - @eval '$(METRICSDIR)/source_metrics.sh $(PWD)' - @$(METRICSDIR)/build_metrics.py $(BUILDDIR) - update: install @. $(VENV); .sphinx/update_sp.py # Catch-all target: route all unknown targets to Sphinx using the new # "make mode" option. $(O) is meant as a shortcut for $(SPHINXOPTS). %: - $(MAKE) —no-print-directory install + $(MAKE) --no-print-directory install . $(VENV); $(SPHINXBUILD) -M $@ "$(SOURCEDIR)" "$(BUILDDIR)" $(SPHINXOPTS) $(O) diff --git a/kubernetes/docs/conf.py b/docs/conf.py similarity index 88% rename from kubernetes/docs/conf.py rename to docs/conf.py index d40f4ae38..edc476f94 100644 --- a/kubernetes/docs/conf.py +++ b/docs/conf.py @@ -24,7 +24,7 @@ # # TODO: Update with the official name of your project or product -project = "Charmed MySQL K8s" +project = "Charmed MySQL" author = "Canonical Ltd." @@ -34,8 +34,7 @@ # # TODO: To disable the title, set to an empty string. -html_title = project + " documentation" - +html_title = project + " 8.0" + " documentation" # Copyright string; shown at the bottom of the page # @@ -139,7 +138,7 @@ 'repo_default_branch': '8.0/edge', # Docs location in the repo; used in links for viewing the source files - "repo_folder": "/kubernetes/docs/", + "repo_folder": "/docs/", # TODO: To enable or disable the Previous / Next buttons at the bottom of pages # Valid options: none, prev, next, both @@ -152,6 +151,13 @@ 'github_issues': 'enabled', } +html_extra_path = [] + +# Allow opt-in build of the OpenAPI "Hello" example so docs stay clean by default. +if os.getenv("OPENAPI", ""): + tags.add("openapi") + html_extra_path.append("how-to/assets/openapi.yaml") + # TODO: To enable the edit button on pages, uncomment and change the link to a # public repository on GitHub or Launchpad. Any of the following link domains # are accepted: @@ -174,29 +180,35 @@ # Sitemap configuration: https://sphinx-sitemap.readthedocs.io/ ####################### -# Base URL of RTD hosted project +# Use RTD canonical URL to ensure duplicate pages have a specific canonical URL -html_baseurl = 'https://canonical-charmed-mysql-k8s.readthedocs-hosted.com/' +html_baseurl = os.environ.get("READTHEDOCS_CANONICAL_URL", "/") -# URL scheme. Add language and version scheme elements. -# When configured with RTD variables, check for RTD environment so manual runs succeed: +# sphinx-sitemap uses html_baseurl to generate the full URL for each page: -if 'READTHEDOCS_VERSION' in os.environ: - version = os.environ["READTHEDOCS_VERSION"] - sitemap_url_scheme = '{version}{link}' -else: - sitemap_url_scheme = 'MANUAL/{link}' +sitemap_url_scheme = '{link}' # Include `lastmod` dates in the sitemap: sitemap_show_lastmod = True +# Exclude generated pages from the sitemap: + +sitemap_excludes = [ + '404/', + 'genindex/', + 'search/', +] + +# TODO: Add more pages to sitemap_excludes if needed. Wildcards are supported. +# For example, to exclude module pages generated by autodoc, add '_modules/*'. + ####################### # Template and asset locations ####################### -#html_static_path = ["_static"] -#templates_path = ["_templates"] +# html_static_path = [".sphinx/_static"] +# templates_path = [".sphinx/_templates"] ############# @@ -277,12 +289,27 @@ extensions = [ "canonical_sphinx", + "notfound.extension", + "sphinx_design", + "sphinx_reredirects", + "sphinx_tabs.tabs", + "sphinxcontrib.jquery", + "sphinxext.opengraph", + "sphinx_config_options", + "sphinx_contributor_listing", + "sphinx_filtered_toctree", + "sphinx_related_links", + "sphinx_roles", + "sphinx_terminal", + "sphinx_ubuntu_images", + "sphinx_youtube_links", "sphinxcontrib.cairosvgconverter", "sphinx_last_updated_by_git", "sphinx.ext.intersphinx", "sphinx_sitemap", "sphinxcontrib.mermaid", - "sphinxext.rediraffe" + "sphinxext.rediraffe", + "sphinx_new_tab_link" ] # Excludes files or directories from processing @@ -292,7 +319,7 @@ # Adds custom CSS files, located under 'html_static_path' -# html_css_files = [] +html_css_files = [] # Adds custom JavaScript files, located under 'html_static_path' diff --git a/docs/explanation/architecture.md b/docs/explanation/architecture.md new file mode 100644 index 000000000..09b5db548 --- /dev/null +++ b/docs/explanation/architecture.md @@ -0,0 +1,240 @@ +--- +myst: + html_meta: + description: "Understand the architecture of Charmed MySQL for VM and Kubernetes deployments, built on the charmed-mysql snap and MySQL InnoDB ClusterSet." +--- + +(architecture)= +# Architecture + +[MySQL](https://www.mysql.com/) is the world’s most popular open source database. Charmed MySQL is a Juju-based operator to deploy and support MySQL from [day 0 to day 2](https://codilime.com/blog/day-0-day-1-day-2-the-software-lifecycle-in-the-cloud-age/). It is based on the [MySQL Community Edition](https://www.mysql.com/products/community/) using the built-in cluster functionality: [MySQL InnoDB ClusterSet](https://dev.mysql.com/doc/mysql-shell/8.0/en/innodb-clusterset.html). + +## High-level design + +Charmed MySQL is developed for deployment on machine clouds or Kubernetes. Although both versions are extremely similar in functionality, there are some key differences in their architecture. + +(machine-charm)= +### Machine charm + +[Charmed MySQL VM](https://charmhub.io/mysql) leverages the [charmed-mysql snap](https://snapcraft.io/charmed-mysql) which is deployed by Juju on the specified VM/MAAS/bare-metal machine based on Ubuntu Jammy/22.04. snap allows to run MySQL service(s) in a secure and isolated environment ([strict confinement](https://snapcraft.io/blog/demystifying-snap-confinement)). + +The installed snap: + +```shell +$ juju ssh mysql/0 +$ snap list charmed-mysql +Name Version Rev Tracking Publisher Notes +charmed-mysql 8.0.34 69 latest/stable dataplatformbot held +``` + +The snap ships the following components: + +* MySQL Community Edition (based on Ubuntu APT package "[mysql-server-8.0](https://packages.ubuntu.com/jammy/mysql-server-8.0)") +* MySQL Router (based on Ubuntu APT package "[mysql-router](https://packages.ubuntu.com/jammy/mysql-router)") +* MySQL Shell (based on Canonical [backport](https://launchpad.net/~data-platform/+archive/ubuntu/mysql-shell)) +* Percona XtraBackup (based on Canonical [backport](https://launchpad.net/~data-platform/+archive/ubuntu/xtrabackup)) +* Prometheus MySQLd Exporter (based on Canonical [backport](https://launchpad.net/~data-platform/+archive/ubuntu/mysqld-exporter)) +* Prometheus MySQL Router Exporter (based on Canonical [backport](https://launchpad.net/~data-platform/+archive/ubuntu/mysqlrouter-exporter)) +* Prometheus Grafana dashboards and Loki alert rules are part of the charm revision and missing in snap. + +Versions of all the components above are carefully chosen to fit functionality of each other. + +The Charmed MySQL unit consisting of a several services which are enabled/activated accordingly to the setup: + +```shell +$ snap services charmed-mysql +Service Startup Current Notes +charmed-mysql.mysqld enabled active - +charmed-mysql.mysqld-exporter disabled inactive - +charmed-mysql.mysqlrouter-service disabled inactive - +charmed-mysql.mysqlrouterd-exporter disabled inactive - +``` + +The `mysqld` snap service is a main MySQL instance which is normally up and running right after the charm deployment. + +The `mysql-router` snap service used in [Charmed MySQL Router](https://charmhub.io/mysql-router?channel=dpe/edge) only and should be stopped on [Charmed MySQL](https://charmhub.io/mysql) deployments. + +All `exporter` services are activated only after relating with {ref}`COS `. + +```{caution} +* It is possible to start, stop, and restart snap services manually but it is NOT recommended to avoid a split brain with a charm state machine! Do it with a caution!!! +* All snap resources must be executed under the special user `snapd_daemon` only! +``` + +The snap "charmed-mysql" also ships list of tools used by charm: +* `charmed-mysql.mysql` (alias `mysql`) - mysql client to connect `mysqld`. +* `charmed-mysql.mysqlsh` - new [mysql-shell](https://dev.mysql.com/doc/mysql-shell/8.0/en/) client to configure MySQL cluster. +* `charmed-mysql.xbcloud` - a tool to download and upload full or part of xbstream archive from/to the cloud. +* `charmed-mysql.xbstream` - a tool to support simultaneous compression and streaming. +* `charmed-mysql.xtrabackup` - a tool to backup/restore MySQL DB. + +The `mysql` and `mysqlsh` are well known and popular tools to manage MySQL. +The `xtrabackup (xbcloud+xbstream)` used for [MySQL Backups](/how-to/back-up-and-restore/create-a-backup) only to store backups on S3 compatible storage. + +### Kubernetes charm + +[Charmed MySQL K8s](https://charmhub.io/mysql-k8s) leverages the [sidecar](https://kubernetes.io/blog/2015/06/the-distributed-system-toolkit-patterns/#example-1-sidecar-containers) pattern to allow multiple containers in each pod with [Pebble](https://juju.is/docs/sdk/pebble) running as the workload container’s entrypoint. + +Pebble is a lightweight, API-driven process supervisor that is responsible for configuring processes to run in a container and controlling those processes throughout the workload lifecycle. + +Pebble `services` are configured through [layers](https://github.com/canonical/pebble#layer-specification), and the following containers represent each one a layer forming the effective Pebble configuration, or `pebble plan`: + +1. a charm container runs Juju operator code: `juju ssh mysql-k8s/0 bash` +1. a [mysql](https://www.mysql.com/) (workload) container runs the MySQL application along with other services (like monitoring metrics exporters, etc): `juju ssh --container mysql mysql-k8s/0 bash` + +As a result, if you run a `kubectl get pods` on a namespace named for the Juju model you’ve deployed the "Charmed MySQL K8s" charm into, you’ll see something like the following: + +```shell +NAME READY STATUS RESTARTS AGE +mysql-k8s-0 2/2 Running 0 65m +``` + +This shows there are 2 containers in the pod: `charm` and `workload` mentioned above. + +And if you run `kubectl describe pod mysql-k8s-0`, all the containers will have as Command `/charm/bin/pebble`. That’s because Pebble is responsible for the processes startup as explained above. + +The Charmed MySQL K8s (`workload` container) based on the `mysql-image` resource defined in the [charm metadata.yaml](https://github.com/canonical/mysql-operators/blob/8.0/edge/kubernetes/metadata.yaml). It is an official Canonical "[charmed-mysql](https://github.com/canonical/charmed-mysql-rock)" [OCI/ROCK](https://documentation.ubuntu.com/server/explanation/virtualisation/about-rock-images/) image, which is recursively based on Canonical SNAP “[charmed-mysql](https://snapcraft.io/charmed-mysql)” (read more about the snap details in {ref}`machine-charm`). + +[Charmcraft](https://juju.is/docs/sdk/install-charmcraft) uploads an image as a [charm resource](https://charmhub.io/mysql-k8s/resources/mysql-image) to [Charmhub](https://charmhub.io/mysql-k8s) during the [publishing](https://github.com/canonical/mysql-k8s-operator/blob/main/.github/workflows/release.yaml#L40-L53), as described in the [Juju SDK How-to guides](https://juju.is/docs/sdk/publishing). + +The charm supports Juju deployment to all Kubernetes environments: [MicroK8s](https://microk8s.io/), [Charmed Kubernetes](https://ubuntu.com/kubernetes/charmed-k8s), [GKE](https://charmhub.io/mysql-k8s/docs/h-deploy-gke), [Amazon EKS](https://aws.amazon.com/eks/), ... + +The OCI/ROCK ships the following components based on the [`charmed-mysql` snap](https://canonical-charmed-mysql.readthedocs-hosted.com/explanation/architecture): + +* MySQL Community Edition +* MySQL Router +* MySQL Shell +* Percona XtraBackup +* Prometheus MySQLd Exporter +* Prometheus MySQL Router Exporter + +**Prometheus Grafana dashboards and Loki alert rules** are part of the charm revision, but missing in the snap. + +SNAP-based ROCK images guaranties the same components versions and functionality between VM and K8s charm flavors. + +Pebble runs layers of all the currently enabled services, e.g. monitoring, backups, etc: + +```shell +> juju ssh --container mysql mysql-k8s/0 /charm/bin/pebble plan +services: + mysqld_exporter: + summary: mysqld exporter + startup: disabled <= COS Monitoring disabled + override: replace + command: /start-mysqld-exporter.sh + environment: + DATA_SOURCE_NAME: user:password@unix(/var/run/mysqld/mysqld.sock)/ + user: mysql + group: mysql + mysqld_safe: + summary: mysqld safe + startup: enabled <= MySQL is up and running + override: replace + command: mysqld_safe + user: mysql + group: mysql + kill-delay: 24h0m0s +``` + +The `mysqld_safe` is a main MySQL wrapper which is normally up and running right after the charm deployment. + +The `mysql-router` used in [Charmed MySQL Router K8s](https://charmhub.io/mysql-router-k8s?channel=8.0/edge) only and should be stopped on [Charmed MySQL K8s](https://charmhub.io/mysql-k8s) deployments. + +All `exporter` services are activated only after relating with {ref}`COS `. + +```{caution} +* It is possible to start, stop, and restart pebble services manually but it is NOT recommended to avoid a split brain with a charm state machine! Do it with a caution!!! +* All pebble resources must be executed under the proper user (defined in user:group options of pebble layer)! +``` + +The ROCK "charmed-mysql" also ships list of tools used by charm: + +* `mysql` - mysql client to connect `mysqld`. +* `mysqlsh` - new [mysql-shell](https://dev.mysql.com/doc/mysql-shell/8.0/en/) client to configure MySQL cluster. +* `xbcloud` - a tool to download and upload full or part of xbstream archive from/to the cloud. +* `xbstream` - a tool to support simultaneous compression and streaming. +* `xtrabackup` - a tool to backup/restore MySQL DB. + +The `mysql` and `mysqlsh` are well known and popular tools to manage MySQL. + +The `xtrabackup (xbcloud+xbstream)` is used only to store {ref}`backups ` on S3 compatible storage. + +## Integrations + +### MySQL Router + +[MySQL Router](https://dev.mysql.com/doc/mysql-router/8.0/en/) is part of MySQL InnoDB Cluster, and is lightweight middle-ware that provides transparent routing between your application and back-end MySQL Servers. The MySQL Router charm ([VM](https://charmhub.io/mysql-router) | [K8s](https://charmhub.io/mysql-router-k8s)) is an independent charm that can be related with MySQL. + +### TLS Certificates Operator + +The [TLS Certificates](https://charmhub.io/tls-certificates-operator) charm is responsible for distributing certificates through relationship. Certificates are provided by the operator through Juju configs. For playground deployments, the [self-signed operator](https://charmhub.io/self-signed-certificates) is available as well. + +### S3 Integrator + +[S3 Integrator](https://charmhub.io/s3-integrator) is an integrator charm for providing S3 credentials to Charmed MySQL which seek to access shared S3 data. Store the credentials centrally in the integrator charm and relate consumer charms as needed. + +### Data Integrator + +The [Data Integrator](https://charmhub.io/data-integrator) charm is a solution to request DB credentials for non-native Juju applications. Not all applications implement a data_interfaces relation but allow setting credentials through config options. Also, some of the applications are run outside of juju. This integrator charm allows receiving credentials which can be passed into application config directly without implementing juju-native relation. + +### MySQL Test App + +The charm [MySQL Test App](https://charmhub.io/mysql-test-app) is a Canonical test application to validate the charm installation / functionality and perform the basic performance tests. + +### Grafana + +Grafana is an open-source visualization tools that allows to query, visualize, alert on, and visualize metrics from mixed data sources in configurable dashboards for observability. This charms is shipped with its own Grafana dashboard and supports integration with the [Grafana Operator](https://charmhub.io/grafana-k8s) to simplify observability. See: {ref}`enable-monitoring`. + +### Loki + +Loki is an open-source fully-featured logging system. This charms is shipped with support for the [Loki Operator](https://charmhub.io/loki-k8s) to collect the generated logs. See: {ref}`enable-monitoring`. + +### Prometheus + +Prometheus is an open-source systems monitoring and alerting toolkit with a dimensional data model, flexible query language, efficient time series database and modern alerting approach. This charm is shipped with a Prometheus exporters, alerts and support for integrating with the [Prometheus Operator](https://charmhub.io/prometheus-k8s) to automatically scrape the targets. See: {ref}`enable-monitoring`. + +## Low-level design + +See the charm state machines displayed in {ref}`flowcharts`. The low-level logic is mostly common for both VM and K8s charms. + + + +### Juju events + +Accordingly to the [Juju SDK](https://juju.is/docs/sdk/event): “an event is a data structure that encapsulates part of the execution context of a charm”. + +For this charm, the following events are observed: + +1. [`on_install`](https://documentation.ubuntu.com/juju/3.6/reference/hook/#install): install the snap "charmed-mysql" and perform basic preparations to bootstrap the cluster on the first leader (or join the already configured cluster). +2. [`leader-elected`](https://documentation.ubuntu.com/juju/3.6/reference/hook/#leader-elected): generate all the secrets to bootstrap the cluster. +3. [`leader-settings-changed`](https://documentation.ubuntu.com/juju/3.6/reference/hook/#leader-settings-changed): Handle the leader settings changed event. +4. [`start`](https://documentation.ubuntu.com/juju/3.6/reference/hook/#start): Init/setting up the cluster node. +5. [`config_changed`](https://documentation.ubuntu.com/juju/3.6/reference/hook/#config-changed): usually fired in response to a configuration change using the GUI or CLI. Create and set default cluster and cluster-set names in the peer relation databag (on the leader only). +6. [`update-status`](https://documentation.ubuntu.com/juju/3.6/reference/hook/#update-status): Takes care of workload health checks. + + +### Charm code overview + +The code for both VM and K8s charms is located in the same repository, [`mysql-operators`](https://github.com/canonical/mysql-operators/tree/8.0/edge) under the `machines` and `kubernetes` directories respectively. + +For each substrate, `src/charm.py` is the default entry point for a charm and has the `MySQLCharmBase` Python class which inherits from `CharmBase`. + +`CharmBase` is the base class from which all Charms are formed, defined by [Ops](https://ops.readthedocs.io/en/latest/) (Python framework for developing charms). See more information in the [Ops documentation for `CharmBase`](https://ops.readthedocs.io/en/latest/reference/ops.html#ops.CharmBase). + +The `__init__` method guarantees that the charm observes all events relevant to its operation and handles them. + + \ No newline at end of file diff --git a/kubernetes/docs/explanation/flowcharts.md b/docs/explanation/flowcharts.md similarity index 86% rename from kubernetes/docs/explanation/flowcharts.md rename to docs/explanation/flowcharts.md index c23d45057..abb67b949 100644 --- a/kubernetes/docs/explanation/flowcharts.md +++ b/docs/explanation/flowcharts.md @@ -1,3 +1,10 @@ +--- +myst: + html_meta: + description: "Explore Mermaid flowchart diagrams illustrating the Charmed MySQL charm lifecycle, including leader election and pebble ready hook events." +--- + +(flowcharts)= # Charm lifecycle flowcharts ```{mermaid} diff --git a/kubernetes/docs/explanation/index.md b/docs/explanation/index.md similarity index 76% rename from kubernetes/docs/explanation/index.md rename to docs/explanation/index.md index 88ac0992d..701d9086b 100644 --- a/kubernetes/docs/explanation/index.md +++ b/docs/explanation/index.md @@ -1,3 +1,10 @@ +--- +myst: + html_meta: + description: "Conceptual documentation for Charmed MySQL covering architecture, Juju integration, users, roles, logs, security, and key interfaces." +--- + +(explanation)= # Explanation Additional context about key concepts behind the MySQL charm. @@ -44,4 +51,3 @@ Mermaid diagrams of charm events and hooks. Charm flowcharts ``` - diff --git a/docs/explanation/interfaces-and-endpoints.md b/docs/explanation/interfaces-and-endpoints.md new file mode 100644 index 000000000..dcb260a63 --- /dev/null +++ b/docs/explanation/interfaces-and-endpoints.md @@ -0,0 +1,158 @@ +--- +myst: + html_meta: + description: "Reference for Charmed MySQL interfaces and endpoints, including mysql_client and legacy mysql, mysql-shared, and mysql-router interfaces." +--- + +(interfaces-and-endpoints)= +# Interfaces and endpoints + +Charmed MySQL supports modern `mysql_client` and legacy `mysql`, `mysql-shared`, `mysql-router` interfaces (in a backward compatible mode). + +| | Interface | Endpoints | VM charm | K8s charm | +|--------|----------------|-----------------------|----------|-----------| +| modern | `mysql_client` | `database` | ![check] | ![check] | +| legacy | `mysql` | `mysql` | ![check] | ![check] | +| | | `mysql_root` | | ![check] | +| legacy | `mysql-router` | `db-router` | ![check] | | +| legacy | `mysql-shared` | `shared-db` | ![check] | | + +```{caution} +Do **not** relate both modern and legacy interfaces simultaneously. +``` + +## Modern relations + +This charm provides the modern [`mysql_client`](https://github.com/canonical/charm-relation-interfaces)interface. Applications can easily connect MySQL using [`data_interfaces`](https://charmhub.io/data-platform-libs/libraries/data_interfaces) library from [`data-platform-libs`](https://github.com/canonical/data-platform-libs/). + +### `mysql_client` interface, `database` endpoint + +Adding a [Juju relation](https://documentation.ubuntu.com/juju/3.6/reference/relation/) is accomplished with `juju relate` via endpoint `database`. + +Example: + +````{tab-set} +```{tab-item} VM +:sync: vm + + # Deploy Charmed MySQL cluster with 3 nodes + juju deploy mysql -n 3 --channel 8.0 + + # Deploy the relevant charms, e.g. mysql-test-app + juju deploy mysql-test-app + + # Relate MySQL with your application + juju relate mysql:database mysql-test-app:database + + # Check established relation (using mysql_client interface): + juju status --relations + + # Example of the properly established relation: + # > Relation provider Requirer Interface Type + # > mysql:database mysql-test-app:database mysql_client regular +``` + +```{tab-item} K8s +:sync: k8s + + # Deploy Charmed MySQL cluster with 3 nodes + juju deploy mysql-k8s -n 3 --trust --channel 8.0 + + # Deploy the relevant charms, e.g. mysql-test-app + juju deploy mysql-test-app + + # Relate MySQL with your application + juju relate mysql-k8s:database mysql-test-app:database + + # Check established relation (using mysql_client interface): + juju status --relations + + # Example of the properly established relation: + # > Relation provider Requirer Interface Type + # > mysql-k8s:database mysql-test-app:database mysql_client regular +``` +```` + +See details about database user roles in {ref}`users`. + +```{note} +In order to integrate with this charm, every table created by the integrated application must have a primary key. This is required by the [group replication plugin](https://dev.mysql.com/doc/refman/8.0/en/group-replication-requirements.html) enabled in this charm. +``` + +## Legacy relations + +**Legacy relations are deprecated and will be discontinued** from Charmed MySQL 8.4 onward. Their usage should be avoided. + +Check the legacy interface implementation limitations in {ref}`legacy-charm`. + +This charm supports several legacy interfaces, e.g. `mysql`, `mysql-shared`, `mysql-router`. They were used in some legacy charms in [cross-model relations](https://documentation.ubuntu.com/juju/3.6/reference/relation/#cross-model-relation). + +### `mysql` interface, `mysql` endpoint + +This was a popular interface used by some legacy charms on both VM and K8s (e.g. [MariaDB](https://charmhub.io/mariadb), [OSM MariaDB](https://charmhub.io/charmed-osm-mariadb-k8s), [Percona Cluster](https://charmhub.io/percona-cluster) and [MySQL Innodb Cluster](https://charmhub.io/mysql-innodb-cluster)), often in [cross-model relations](https://documentation.ubuntu.com/juju/3.6/reference/relation/#cross-model-relation). + +Example usage of this interface: + +````{tab-set} +```{tab-item} VM +:sync: vm + + juju deploy mysql --channel 8.0 + juju config mysql mysql-interface-database=mediawiki mysql-interface-user=mediawiki + juju deploy mediawiki + juju relate mysql:mysql mediawiki:db +``` + +```{tab-item} K8s +:sync: k8s + + juju deploy mysql-k8s --trust --channel 8.0 + juju config mysql-k8s mysql-interface-database=wordpress mysql-interface-user=wordpress + juju deploy wordpress-k8s + juju relate mysql-k8s:mysql wordpress-k8s:db +``` +```` + +#### `mysql_root` endpoint (K8s only) + +The K8s charm additionally supports the endpoint `mysql_root`, which provides the same legacy interface `mysql` with MySQL root-level privileges. + +```{caution} +Usage of `mysql_root` is **not** recommended from security point of view. +``` + +### `mysql-router` interface, `db-router` endpoint (VM only) + +It is a relation that one uses with the [mysql router](https://charmhub.io/mysql-router) charm. + +As an example, the following commands can be executed to deploy and integrate Charmed MySQL VM to the keystone charm: + +```shell +juju deploy mysql --channel 8.0 +juju deploy mysql-router --series focal +juju deploy keystone --series focal +juju relate mysql-router keystone +juju relate mysql:db-router mysql-router:db-router +``` + +```{note} +Make sure to deploy identical [series/base](https://documentation.ubuntu.com/juju/3.6/reference/machine/#machine-base) for `keystone` and `mysql-router` applications. + +This is necessary due to the [subordinate](https://documentation.ubuntu.com/juju/3.6/reference/charm/#subordinate-charm) charm nature of `mysql-router`. +``` + +### `mysql-shared` interface, `shared-db` endpoint (VM only) + +It is a relation that one uses when the application needs to connect directly to the database cluster. It is supported by various legacy charms, e.g. [mysql-innodb-cluster](https://charmhub.io/mysql-innodb-cluster). + +As an example, the following commands can be executed to deploy and integrate Charmed MySQL VM to the keystone charm: + +```shell +juju deploy mysql --channel 8.0 +juju deploy keystone --series focal +juju relate keystone:shared-db mysql:shared-db +``` + + + +[check]: https://img.icons8.com/color/20/checkmark--v1.png \ No newline at end of file diff --git a/machines/docs/explanation/juju.md b/docs/explanation/juju.md similarity index 60% rename from machines/docs/explanation/juju.md rename to docs/explanation/juju.md index 8d25c46ce..5141ee2c0 100644 --- a/machines/docs/explanation/juju.md +++ b/docs/explanation/juju.md @@ -1,8 +1,15 @@ +--- +myst: + html_meta: + description: "Understand how Juju orchestrates Charmed MySQL and learn about command differences between Juju 2.9 and Juju 3.x relevant to this documentation." +--- + +(juju)= # Juju [Juju](https://juju.is/) is an open source orchestration engine for software operators that enables the deployment, integration and lifecycle management of applications at any scale, on any infrastructure using charms. -This [charm](https://charmhub.io/mysql) is an operator - business logic encapsulated in reusable software packages that automate every aspect of an application's life. Charms are shared via [CharmHub](https://charmhub.io/). +This charm is an operator - business logic encapsulated in reusable software packages that automate every aspect of an application's life. Charms are shared via [CharmHub](https://charmhub.io/). See also: @@ -11,7 +18,8 @@ See also: This page aims to provide some context on some of the inner workings of Juju that affect this charm. -## Breaking changes between Juju 2.9.x and 3.x +(breaking-changes-juju)= +## Breaking changes between Juju 2.9 and 3 As this charm documentation is written for Juju 3.x, users of 2.9.x will encounter noteworthy changes when following the instructions. This section explains those changes. @@ -32,18 +40,44 @@ The response is to therefore substitute the documented command with the equivale ### Juju 3.x: -```shell -juju integrate mysql:database mysql-test-app +````{tab-set} +```{tab-item} VM +:sync: vm + + juju integrate mysql:database mysql-test-app + + juju run mysql/leader get-password +``` + +```{tab-item} K8s +:sync: k8s -juju run mysql/leader get-password + juju integrate mysql-k8s:database mysql-test-app + + juju run mysql-k8s/leader get-password ``` +```` + ### Juju 2.9.x: -```shell -juju relate mysql:database mysql-test-app +````{tab-set} +```{tab-item} VM +:sync: vm + + juju relate mysql:database mysql-test-app -juju run-action --wait mysql/leader get-password + juju run-action --wait mysql/leader get-password ``` + +```{tab-item} K8s +:sync: k8s + + juju relate mysql-k8s:database mysql-test-app + + juju run-action --wait mysql-k8s/leader get-password +``` +```` + ```{note} This section is based on the [OpenStack guide.](https://docs.openstack.org/charm-guide/latest/project/support-notes.html#breaking-changes-between-juju-2-9-x-and-3-x) ``` @@ -53,9 +87,9 @@ This section is based on the [OpenStack guide.](https://docs.openstack.org/charm Newly released charm revisions might require a new Juju version. This is usually because the new revision requires new Juju features, e.g. [Juju secrets](https://juju.is/docs/juju/secret). -Information about Juju requirements will be clearly indicated in the charm's [release notes](/reference/releases) and in the repository's [metadata.yaml](https://github.com/canonical/mysql-operator/blob/14c06ff88c4e564cd6d098aa213bd03e78e84b52/metadata.yaml#L72-L80) file. +Information about Juju requirements will be clearly indicated in the charm's {ref}`release notes ` and in the repository's `metadata.yaml` file. -When upgrading your database charm with juju refresh, Juju checks that its version is compatible with the target revision. If not, it stops the upgrade and prevents further changes to keep the installation safe. +When upgrading your database charm with {command}`juju refresh` Juju checks that its version is compatible with the target revision. If not, it stops the upgrade and prevents further changes to keep the installation safe. ```shell ~$ juju refresh mysql @@ -66,5 +100,5 @@ ERROR Charm feature requirements cannot be met: - charm requires feature "juju" (version >= 3.1.5) but model currently supports version 3.1.4 ``` -You must then [upgrade to the required Juju version](/how-to/refresh/upgrade-juju) before proceeding with the charm upgrade. +You must then {ref}`upgrade to the required Juju version ` before proceeding with the charm upgrade. diff --git a/docs/explanation/legacy-charm.md b/docs/explanation/legacy-charm.md new file mode 100644 index 000000000..4362df5f5 --- /dev/null +++ b/docs/explanation/legacy-charm.md @@ -0,0 +1,137 @@ +--- +relatedlinks: "[Charm generations](https://documentation.ubuntu.com/charmcraft/stable/)" +myst: + html_meta: + description: "Learn about legacy MySQL and MariaDB charms, their history, and how to migrate from them to the modern Charmed MySQL operator." +--- + +(legacy-charm)= +# Legacy charm + +Historically, there were several **legacy charms** that provided MySQL/MariaDB functionality: + +| Legacy VM charms | Legacy K8s charms | +|:----------------------|:------------------| +|[MariaDB] | [OSM MariaDB] | +|[Percona Cluster] | | +|[MySQL Innodb Cluster] | | + +These legacy charms provided endpoints `mysql` and `mysql-root` for the interface `mysql`. + +This Charmed MySQL operator is a **modern charm** - i.e. it is based on the [Ops framework](https://documentation.ubuntu.com/ops/latest/) and designed to replace all legacy {ref}`interfaces and endpoints ` of legacy charms. + +The modern charm provides old endpoints as well as the new endpoint `database` for the interface `mysql_client`. + +See all available endpoints/interfaces for Charmed MySQL on Charmhub: +* [Charmed MySQL VM](https://charmhub.io/mysql/integrations) +* [Charmed MySQL K8s](https://charmhub.io/mysql-k8s/integrations) + +## The default track `latest` vs. `8.0` + +The [default track](https://docs.openstack.org/charm-guide/yoga/project/charm-delivery.html) has been switched from the `latest` to `8.0` for both VM and K8s MySQL charms. + +This was done to ensure all new deployments use a modern codebase. For more context, see this [Discourse topic](https://discourse.charmhub.io/t/request-switch-default-track-latest-8-0-for-charms-mysql-and-mysql-k8s/9977). + +We strongly advise against using the `latest` track, as a future charm upgrade may result in a MySQL version incompatible with an integrated application. Track `8.0` guarantees MySQL `8.0` deployment only. + +The track `latest` is closed to avoid confusion. + +## How to migrate from legacy to modern charm + +The modern charm provides temporary support for legacy interfaces. + +**Quick try**: Relate the current application with new charm using endpoint `mysql` (set the channel to `8.0/stable`). No extra changes are necessary: + +`````{tab-set} +````{tab-item} VM +:sync: vm + +```yaml +mysql: +charm: mysql +channel: 8.0/stable +trust: true +``` +```` + +````{tab-item} K8s +:sync: k8s + +```yaml +mysql: +charm: mysql-k8s +channel: 8.0/stable +trust: true +``` + +```{note} +The `trust` option must be enabled if [Role Based Access Control (RBAC)](https://kubernetes.io/docs/concepts/security/rbac-good-practices/) is in use in your Kubernetes. +``` +```` +````` + +**Proper migration**: Migrate the application to the new interface [`mysql_client`](https://github.com/canonical/charm-relation-interfaces). + +The application will connect MySQL using [`data_interfaces`](https://charmhub.io/data-platform-libs/libraries/data_interfaces) library from [`data-platform-libs`](https://github.com/canonical/data-platform-libs/) via the `database` endpoint. + +```{caution} +In-place upgrades from the legacy charm to the modern, Ops-based charm are **not supported**. + +To migrate database data, see the following guides: +* {ref}`migrate-data-mysqldump` +* {ref}`migrate-data-mydumper` +* {ref}`migrate-data-backup-restore` +``` + +## How to deploy a legacy MySQL charm + +`````{tab-set} +````{tab-item} VM +:sync: vm + +```yaml +mariadb: +charm: mariadb +channel: latest/stable + +percona-cluster: +charm: percona-cluster +channel: latest/stable +``` +```` + +````{tab-item} K8s +:sync: k8s + +```yaml +osm-mariadb: +charm: charmed-osm-mariadb-k8s +channel: latest/stable + +mysql: +charm: mysql-innodb-cluster +channel: 8.0/stable +``` +```` +````` + +## Modern charm + +For more information about the modern charm, see: + +* {ref}`system-requirements` +* {ref}`release-notes` + +## Report issues and contact authors + +The modern charms (from `8.0/stable`) for VM and K8s are stored in the [`mysql-operators` GitHub repository](https://github.com/canonical/mysql-operators). + +Bug reports and feature requests can be submitted as GitHub issues. + +See {ref}`contacts` for more information. + + +[MariaDB]: https://charmhub.io/mariadb +[OSM MariaDB]: https://charmhub.io/charmed-osm-mariadb-k8s +[Percona Cluster]: https://charmhub.io/percona-cluster +[MySQL Innodb Cluster]: https://charmhub.io/mysql-innodb-cluster \ No newline at end of file diff --git a/docs/explanation/logs/audit-logs.md b/docs/explanation/logs/audit-logs.md new file mode 100644 index 000000000..8cda2ada7 --- /dev/null +++ b/docs/explanation/logs/audit-logs.md @@ -0,0 +1,101 @@ +--- +myst: + html_meta: + description: "Understand how the Audit Log plugin works in Charmed MySQL, including output samples, storage paths, rotation frequency, and configuration options." +--- + +(audit-logs)= +# Audit logs + +The Audit Log plugin allows fine grained configuration for all login/logout, queries or both records to be stored in a log file. It is enabled in Charmed MySQL by default. + +## Overview + +The following is a sample of the audit logs, in JSON format with only logins records (default configuration): + +```json +{"audit_record":{"name":"Quit","record":"6_2024-09-03T01:53:14","timestamp":"2024-09-03T01:53:33Z","connection_id":"992","status":0,"user":"clusteradmin","priv_user":"clusteradmin","os_login":"","proxy_user":"","host":"localhost","ip":"","db":""}} +{"audit_record":{"name":"Connect","record":"7_2024-09-03T01:53:14","timestamp":"2024-09-03T01:53:33Z","connection_id":"993","status":1156,"user":"","priv_user":"","os_login":"","proxy_user":"","host":"juju-da2225-8","ip":"10.207.85.214","db":""}} +{"audit_record":{"name":"Connect","record":"8_2024-09-03T01:53:14","timestamp":"2024-09-03T01:53:33Z","connection_id":"994","status":0,"user":"serverconfig","priv_user":"serverconfig","os_login":"","proxy_user":"","host":"juju-da2225-8","ip":"10.207.85.214","db":""}} +``` + +````{tab-set} +```{tab-item} VM +:sync: vm + +The logs are stored in the `/var/snap/charmed-mysql/common/var/log/mysql` directory, and are rotated every minute to the `/var/snap/charmed-mysql/common/var/log/mysql/archive_audit` directory. +``` + +```{tab-item} K8s +:sync: k8s + +The logs are stored in the `/var/log/mysql` directory of the mysql container, and it's rotated every minute to the `/var/log/mysql/archive_audit` directory. +``` +```` + +It's recommended to integrate the charm with {ref}`COS `, from where the logs can be easily persisted and queried using Loki and Grafana. + +## Configurations + +### `plugin-audit-enabled` + +The audit plugin is enabled by default in the charm, but it's possible to disable it by setting: + +````{tab-set} +```{tab-item} VM +:sync: vm + + juju config mysql plugin-audit-enabled=false +``` + +```{tab-item} K8s +:sync: k8s + + juju config mysql-k8s plugin-audit-enabled=false +``` +```` + +Valid values are `false` and `true` (default). By setting it to false, existing logs are still kept in the `archive_audit` directory. + +### `logs_audit_policy` + +Audit log policy: + +````{tab-set} +```{tab-item} VM +:sync: vm + + juju config mysql logs_audit_policy=queries +``` + +```{tab-item} K8s +:sync: k8s + + juju config mysql-k8s logs_audit_policy=queries +``` +```` + +Valid values are `logins` (default), `queries` and `all`. + +### `plugin-audit-strategy` + +By default the audit plugin writes logs in asynchronous mode for better performance. + +To ensure logs are written to disk on more timely fashion, this configuration can be set to semi-synchronous mode: + +````{tab-set} +```{tab-item} VM +:sync: vm + + juju config mysql plugin-audit-strategy=semi-async +``` + +```{tab-item} K8s +:sync: k8s + + juju config mysql-k8s plugin-audit-strategy=semi-async +``` +```` + +Valid values are `async` (default) and `semi-async`. + diff --git a/kubernetes/docs/explanation/logs/index.md b/docs/explanation/logs/index.md similarity index 80% rename from kubernetes/docs/explanation/logs/index.md rename to docs/explanation/logs/index.md index 9d38a4027..0004710ed 100644 --- a/kubernetes/docs/explanation/logs/index.md +++ b/docs/explanation/logs/index.md @@ -1,42 +1,82 @@ +--- +myst: + html_meta: + description: "Learn about Charmed MySQL logging: audit logs, error logs, log rotation configuration, and storage locations for VM and K8s deployments." +--- + +(logs)= # Logs -This explanation goes over the types of logging in MySQL and the configuration parameters for log rotation. +This explanation goes over the types of logging in MySQL and the configuration parameters for log +rotation. -The charm currently has audit and error logs enabled by default. All of these files are rotated if present into a separate dedicated archive folder under the logs directory. We do not yet support the rotation of binary logs (binlog, relay log, undo log, redo log, etc). +The charm currently has audit and error logs enabled by default. All of these files are rotated if +present into a separate dedicated archive folder under the logs directory. We do not yet support the rotation of binary logs (binlog, relay log, undo log, redo log, etc). ## Log types +````{tab-set} +```{tab-item} VM +:sync: vm + +The charm stores its logs in `/var/snap/charmed-mysql/common/var/log/mysql`. + + $ ls -lahR /var/snap/charmed-mysql/common/var/log/mysql + + # /var/log/mysql: + drwxrwx--- 2 mysql mysql 4.0K Oct 23 20:46 archive_audit + drwxrwx--- 2 mysql mysql 4.0K Oct 23 20:46 archive_error + -rw-r----- 1 mysql mysql 1.1K Oct 23 20:46 audit.log + -rw-r----- 1 mysql mysql 1.1K Oct 23 20:46 error.log + + # /var/snap/charmed-mysql/common/var/log/mysql/archive_audit: + -rw-r----- 1 snap_daemon root 43K Sep 3 01:24 audit.log-20240903_0124.gz + -rw-r----- 1 snap_daemon root 109K Sep 3 01:25 audit.log-20240903_0125.gz + + # /var/snap/charmed-mysql/common/var/log/mysql/archive_error: + + -rw-r----- 1 mysql mysql 8.7K Oct 23 20:44 error.log-43_2045.gz + -rw-r----- 1 mysql mysql 2.3K Oct 23 20:45 error.log-43_2046.gz +``` + +```{tab-item} K8s +:sync: k8s + The charm stores its logs in `/var/log/mysql`. -```shell -$ ls -lahR /var/log/mysql + $ ls -lahR /var/log/mysql -/var/log/mysql: -drwxrwx--- 2 mysql mysql 4.0K Oct 23 20:46 archive_audit -drwxrwx--- 2 mysql mysql 4.0K Oct 23 20:46 archive_error --rw-r----- 1 mysql mysql 1.1K Oct 23 20:46 audit.log --rw-r----- 1 mysql mysql 1.1K Oct 23 20:46 error.log + # /var/log/mysql: + drwxrwx--- 2 mysql mysql 4.0K Oct 23 20:46 archive_audit + drwxrwx--- 2 mysql mysql 4.0K Oct 23 20:46 archive_error + -rw-r----- 1 mysql mysql 1.1K Oct 23 20:46 audit.log + -rw-r----- 1 mysql mysql 1.1K Oct 23 20:46 error.log -/var/log/mysql/archive_audit: --rw-r----- 1 snap_daemon root 43K Sep 3 01:24 audit.log-20240903_0124.gz --rw-r----- 1 snap_daemon root 109K Sep 3 01:25 audit.log-20240903_0125.gz + # /var/log/mysql/archive_audit: + -rw-r----- 1 snap_daemon root 43K Sep 3 01:24 audit.log-20240903_0124.gz + -rw-r----- 1 snap_daemon root 109K Sep 3 01:25 audit.log-20240903_0125.gz -/var/log/mysql/archive_error: --rw-r----- 1 mysql mysql 8.7K Oct 23 20:44 error.log-43_2045.gz --rw-r----- 1 mysql mysql 2.3K Oct 23 20:45 error.log-43_2046.gz + # /var/log/mysql/archive_error: + -rw-r----- 1 mysql mysql 8.7K Oct 23 20:44 error.log-43_2045.gz + -rw-r----- 1 mysql mysql 2.3K Oct 23 20:45 error.log-43_2046.gz ``` +```` -It is recommended to set up a [COS integration] so that these log files can be streamed to Loki. This leads to better persistence and security of the logs. + +It is recommended to set up a {ref}`COS integration ` so that these log files can be streamed to Loki. This leads to better persistence and security of the logs. ### Audit logs + The Audit Log plugin allows all login/logout records to be stored in a log file.
+ Example of audit logs in JSON format with login/logout records ```json + {"audit_record":{"name":"Connect","record":"17_2024-09-03T01:52:14","timestamp":"2024-09-03T01:53:14Z","connection_id":"988","status":1156,"user":"","priv_user":"","os_login":"","proxy_user":"","host":"juju-da2225-8","ip":"10.207.85.214","db":""}} {"audit_record":{"name":"Connect","record":"18_2024-09-03T01:52:14","timestamp":"2024-09-03T01:53:14Z","connection_id":"989","status":0,"user":"serverconfig","priv_user":"serverconfig","os_login":"","proxy_user":"","host":"juju-da2225-8","ip":"10.207.85.214","db":""}} {"audit_record":{"name":"Quit","record":"1_2024-09-03T01:53:14","timestamp":"2024-09-03T01:53:14Z","connection_id":"989","status":0,"user":"serverconfig","priv_user":"serverconfig","os_login":"","proxy_user":"","host":"juju-da2225-8","ip":"10.207.85.214","db":""}} @@ -48,14 +88,17 @@ The Audit Log plugin allows all login/logout records to be stored in a log file. {"audit_record":{"name":"Connect","record":"7_2024-09-03T01:53:14","timestamp":"2024-09-03T01:53:33Z","connection_id":"993","status":1156,"user":"","priv_user":"","os_login":"","proxy_user":"","host":"juju-da2225-8","ip":"10.207.85.214","db":""}} {"audit_record":{"name":"Connect","record":"8_2024-09-03T01:53:14","timestamp":"2024-09-03T01:53:33Z","connection_id":"994","status":0,"user":"serverconfig","priv_user":"serverconfig","os_login":"","proxy_user":"","host":"juju-da2225-8","ip":"10.207.85.214","db":""}} ``` +
-For more details, see the [Audit Logs explanation]. +The audit log allows for more configuration. For details, see {ref}`audit-logs`. ### Error logs
+ Example of error logs with format time thread [label] [err_code] [subsystem] msg + ```shell 2023-10-24T23:28:07.048728Z mysqld_safe Number of processes running now: 0 2023-10-24T23:28:07.063027Z mysqld_safe mysqld restarted @@ -66,7 +109,7 @@ For more details, see the [Audit Logs explanation]. 2023-10-24T23:28:11.486308Z 0 [Warning] [MY-010068] [Server] CA certificate ca.pem is self signed. 2023-10-24T23:28:11.487473Z 0 [System] [MY-013602] [Server] Channel mysql_main configured to support TLS. Encrypted connections are now supported for this channel. 2023-10-24T23:28:11.538807Z 0 [System] [MY-011323] [Server] X Plugin ready for connections. Bind-address: '0.0.0.0' port: 33060, socket: /var/snap/charmed-mysql/common/var/run/mysqld/mysqlx.sock -2023-10-24T23:28:11.538957Z 0 [System] [MY-010931] [Server] /snap/charmed-mysql/69/usr/sbin/mysqld: ready for connections. Version: '8.0.34-0ubuntu0.22.04.1' socket: '/var/snap/charmed-mysql/common/var/run/mysqld/mysqld.sock' port: 3306 (Ubuntu). +2023-10-24T23:28:11.538957Z 0 [System] [MY-010931] [Server] /snap/charmed-mysql/69/usr/sbin/mysqld: ready for connections. Version: '8.0.34-0ubuntu0.22.04.1' socket: '/var/snap/charmed-mysql/common/var/run/mysqld/mysqld.sock' port: 3306 (Ubuntu). 2023-10-24T23:28:17.983851Z 12 [Warning] [MY-010604] [Repl] Neither --relay-log nor --relay-log-index were used; so replication may break when this MySQL server acts as a replica and has his hostname changed!! Please use '--relay-log=juju-9860bb-0-relay-bin' to avoid this problem. 2023-10-24T23:28:17.999093Z 12 [System] [MY-010597] [Repl] 'CHANGE REPLICATION SOURCE TO FOR CHANNEL 'mysqlsh.test' executed'. Previous state source_host='', source_port= 3306, source_log_file='', source_log_pos= 4, source_bind=''. New state source_host='juju-9860bb-0.lxd', source_port= 3306, source_log_file='', source_log_pos= 4, source_bind=''. 2023-10-24T23:28:18.025941Z 15 [Warning] [MY-010897] [Repl] Storing MySQL user name or password information in the connection metadata repository is not secure and is therefore not recommended. Please consider using the USER and PASSWORD connection options for START REPLICA; see the 'START REPLICA Syntax' in the MySQL Manual for more information. @@ -88,10 +131,15 @@ For more details, see the [Audit Logs explanation]. 2023-10-24T23:28:19.179289Z 28 [System] [MY-011566] [Repl] Plugin group_replication reported: 'Setting super_read_only=OFF.' 2023-10-24T23:28:19.179408Z 28 [System] [MY-013731] [Repl] Plugin group_replication reported: 'The member action "mysql_start_failover_channels_if_primary" for event "AFTER_PRIMARY_ELECTION" with priority "10" will be run.' 2023-10-24T23:28:19.179600Z 31 [System] [MY-011510] [Repl] Plugin group_replication reported: 'This server is working as primary member.' -2023-10-24T23:28:19.875216Z 12 [System] [MY-014010] [Repl] Plugin group_replication reported: 'Plugin 'group_replication' has been started.' +2023-10-24T23:28:19.875216Z 12 [System] [MY-014010] [Repl] Plugin group_replication reported: 'Plugin 'group_replication' has been started.' ``` +
+### Other logs + +Other logs (slow queries, general query) are currently disabled. + ## Log rotation configuration Following the configuration options exposed by the charm: @@ -135,11 +183,10 @@ The following are logrotate config values used for log rotation: ## High Level Design -There is a cron job on the machine where the charm exists that is triggered every minute and runs `logrotate`. The logrotate utility does *not* use `copytruncate`. Instead, the existing log file is moved into the archive directory by logrotate, and then the logrotate's postrotate script invokes `juju-run` (or `juju-exec` depending on the juju version) to dispatch a custom event. This custom event's handler flushes the MySQL log with the [FLUSH](https://dev.mysql.com/doc/refman/8.0/en/flush.html) statement that will result in a new and empty log file being created under `/var/log/mysql` and the rotated file's descriptor being closed. +There is a cron job on the machine where the charm exists that is triggered every minute and runs `logrotate`. The logrotate utility does *not* use `copytruncate`. Instead, the existing log file is moved into the archive directory by logrotate, and then the logrotate post-rotate script invokes `juju-run` (or `juju-exec` depending on the juju version) to dispatch a custom event. This custom event's handler flushes the MySQL log with the [FLUSH](https://dev.mysql.com/doc/refman/8.0/en/flush.html) statement that will result in a new and empty log file being created under `/var/log/mysql` and the rotated file's descriptor being closed. We use a custom event in juju to execute the FLUSH statement in order to avoid storing any credentials on the disk. The charm code has a mechanism that will retrieve credentials from the peer relation databag or juju secrets backend, if available, and keep these credentials in memory for the duration of the event handler. - [COS integration]: /how-to/monitoring-cos/enable-monitoring @@ -149,7 +196,6 @@ We use a custom event in juju to execute the FLUSH statement in order to avoid s ```{toctree} :titlesonly: :maxdepth: 2 -:hidden: Audit logs ``` diff --git a/machines/docs/explanation/roles.md b/docs/explanation/roles.md similarity index 80% rename from machines/docs/explanation/roles.md rename to docs/explanation/roles.md index a113a3a13..211d9cc52 100644 --- a/machines/docs/explanation/roles.md +++ b/docs/explanation/roles.md @@ -1,7 +1,16 @@ +--- +myst: + html_meta: + description: "Reference for predefined roles in Charmed MySQL, including charmed_backup, charmed_dba, charmed_router, and other custom MySQL roles." +--- + +(roles)= # Roles ```{note} -The following roles are available starting on revision 412 +The following roles are available in `8.0` starting from: +* Revision 412 for the {ref}`VM charm ` +* Revision 284 for the {ref}`K8s charm ` ``` There are several definitions of roles in Charmed MySQL: @@ -9,10 +18,11 @@ There are several definitions of roles in Charmed MySQL: * Predefined database-level roles ```{seealso} -[](/explanation/users) +{ref}`users` ``` ## MySQL roles + MySQL does not provide any built-in roles for users to get permissions from. ## Charmed MySQL instance-level roles @@ -44,12 +54,14 @@ mysql> SELECT host, user FROM mysql.user; +-----------+------------------+ ``` -Additionally, the role `charmed_router` is available to ease the integration with [Charmed MySQL Router](https://charmhub.io/mysql-router). +Additionally, the role `charmed_router` is available to ease the integration with Charmed MySQL Router ([VM](https://charmhub.io/mysql-router) | [K8s](https://charmhub.io/mysql-router-k8s)). + This role contains all the necessary permissions for a MySQL Router relation user to operate. ## Charmed MySQL database-level roles -Charmed MySQL also introduces database level roles, with permissions tied to each database that's created. +Charmed MySQL also introduces database level roles, with permissions tied to each database that is created. + Example for a database named `test`: ```text diff --git a/machines/docs/explanation/security/cryptography.md b/docs/explanation/security/cryptography.md similarity index 77% rename from machines/docs/explanation/security/cryptography.md rename to docs/explanation/security/cryptography.md index 3fefdbb45..e8e50ba9c 100644 --- a/machines/docs/explanation/security/cryptography.md +++ b/docs/explanation/security/cryptography.md @@ -1,13 +1,33 @@ +--- +myst: + html_meta: + description: "Learn about cryptographic mechanisms in Charmed MySQL including resource verification, signed commits, and TLS encryption for cluster connections." +--- +(cryptography)= # Cryptography This document describes the cryptography used by Charmed MySQL. ## Resource checksums +````{tab-set} +```{tab-item} VM +:sync: vm + Charmed MySQL and Charmed MySQL Router operators use pinned revisions of the [Charmed MySQL snap](https://github.com/canonical/charmed-mysql-snap) to provide reproducible and secure environments. The Charmed MySQL snap packages the MySQL workload along with the necessary dependencies and utilities required for the operators’ lifecycle. For more details, see the snap contents in the [snapcraft.yaml file](https://github.com/canonical/charmed-mysql-snap/blob/8.0/edge/snap/snapcraft.yaml). +``` + +```{tab-item} K8s +:sync: k8s + +Charmed MySQL K8s and Charmed MySQL Router K8s operators use a pinned version of the [Charmed MySQL rock](https://github.com/orgs/canonical/packages/container/package/charmed-mysql) to provide reproducible and secure environments. + +The rock is an OCI image derived from the respective snap. The Charmed MySQL K8s snap packages the MySQL workload along with the necessary dependencies and utilities required for the operators’ lifecycle. For more details, see the snap contents in the [snapcraft.yaml file](https://github.com/canonical/charmed-mysql-snap/blob/8.0/edge/snap/snapcraft.yaml). +``` +```` Every artifact bundled into the Charmed MySQL snap is verified against its MD5, SHA256, or SHA512 checksum after download. The installation of certified snap into the rock is ensured by snap primitives that verify their squashfs filesystems images GPG signature. For more information on the snap verification process, refer to the [snapcraft.io documentation](https://snapcraft.io/docs/assertions). @@ -15,7 +35,7 @@ Every artifact bundled into the Charmed MySQL snap is verified against its MD5, MySQL and its extra components (mysql-shell, xtrabackup, mysqld-exporter, mysqlrouter-exporter, percona-server-plugins, mysql-pitr-helper, etc.) are built by Canonical from upstream source codes into PPAs and stored on [Launchpad](https://launchpad.net/mysql). -Charmed MySQL snap is published using a GitHub repository workflow. +Charmed MySQL charms, snap and ROCK are built and released programmatically using release pipelines implemented via GitHub Actions in their respective repositories. All repositories in GitHub are set up with branch protection rules, requiring: diff --git a/docs/explanation/security/index.md b/docs/explanation/security/index.md new file mode 100644 index 000000000..72f702215 --- /dev/null +++ b/docs/explanation/security/index.md @@ -0,0 +1,156 @@ +--- +myst: + html_meta: + description: "Security hardening guide for Charmed MySQL covering cloud environments, Juju security, OS hardening, encryption, authentication, and monitoring." +--- + +(security-hardening)= +# Security hardening + +This document provides an overview of security features and guidance for hardening the security of [Charmed MySQL](https://charmhub.io/mysql) deployments, including setting up and managing a secure environment. + +## Environment + +The environment where Charmed MySQL operates can be divided into two components: + +1. Cloud +2. Juju + +### Cloud + +Charmed MySQL can be deployed on top of several clouds and virtualisation layers: + +| Cloud | Security guides | +|-----------|---------------------------------------------------------------------------------------| +| OpenStack | [OpenStack Security Guide] | +| AWS | [Best practices for security, identity and compliance], [AWS security credentials] | +| Azure | [Azure security best practices and patterns], [Managed identities for Azure resource] | +| GCP | [Google security overview], [Harden your cluster's security] | + +### Juju + +Juju is the component responsible for orchestrating the entire lifecycle, from deployment to Day 2 operations. For more information on Juju security hardening, see the +[Juju security page](https://documentation.ubuntu.com/juju/latest/explanation/juju-security/index.html) and the [How to harden your deployment](https://documentation.ubuntu.com/juju/3.6/howto/manage-your-juju-deployment/harden-your-juju-deployment/) guide. + +#### Cloud credentials + +When configuring cloud credentials to be used with Juju, ensure that users have the correct permissions to operate at the required level. Juju superusers responsible for bootstrapping and managing controllers require elevated permissions to manage several kinds of resources, such as virtual machines, networks, storages, etc. Please refer to the links below for more information on the policies required to be used depending on the cloud. + +| Cloud | Cloud user policies | +|-----------|-----------------------------------------------------------------| +| OpenStack | [OpenStack cloud and Juju] | +| AWS | [Juju AWS Permission], [AWS Instance Profiles], [Juju on AWS] | +| Azure | [Juju Azure Permission], [How to use Juju with Microsoft Azure] | +| GCP | [Google GCE cloud and Juju] | + +#### Juju users + +It is very important that Juju users are set up with minimal permissions depending on the scope of their operations. Please refer to the [User access levels](https://juju.is/docs/juju/user-permissions) documentation for more information on the access levels and corresponding abilities. + +Juju user credentials must be stored securely and rotated regularly to limit the chances of unauthorized access due to credentials leakage. + +## Applications + +In the following, we provide guidance on how to harden your deployment using: + +1. Operating system +2. Security upgrades +3. Encryption +4. Authentication +5. Monitoring and auditing + +### Operating system + +Charmed MySQL and Charmed MySQL Router run on top of Ubuntu 22.04 LTS (Jammy). Deploy a [Landscape Client Charm](https://charmhub.io/landscape-client?) to connect the underlying VM to a Landscape User Account to manage security upgrades and integrate [Ubuntu Pro](https://ubuntu.com/pro) subscriptions. + +### Security upgrades + +Charmed MySQL operator and Charmed MySQL Router operator install a pinned revision of the Charmed MySQL snap to provide reproducible and secure environments. + +New versions (revisions) of charmed operators can be released to upgrade workloads, the operator's code, or both. It is important to refresh the charm regularly to make sure the workload is as secure as possible. + +For more information on upgrading the charm, see {ref}`refresh` and [How to upgrade MySQL Router](https://charmhub.io/mysql-router/docs/h-upgrade?channel=dpe/edge), as well as the {ref}`release-notes`. + +### Encryption + +By default, encryption is optional for both external connections and internal communication between cluster members. To enforce encryption in transit, integrate Charmed MySQL with a TLS certificate provider. Please refer to the [Charming Security page](https://charmhub.io/topics/security-with-x-509-certificates) for more information on how to select the right certificate provider for your use case. + +Encryption in transit for backups is provided by the storage (Charmed MySQL is a client for the S3 storage). + +For more information on encryption, see {ref}`cryptography` and {ref}`enable-tls`. + +### Authentication + +Charmed MySQL uses the [caching_sha2_password](https://dev.mysql.com/doc/refman/8.0/en/caching-sha2-pluggable-authentication.html) plugin for authentication. + +### Monitoring + +Charmed MySQL provides native integration with the [Canonical Observability Stack (COS)](https://charmhub.io/topics/canonical-observability-stack). To reduce the blast radius of infrastructure disruptions, the general recommendation is to deploy COS and the observed application into separate environments, isolated from one another. Refer to the [COS production deployments best practices](https://charmhub.io/topics/canonical-observability-stack/reference/best-practices) for more information. + +For more information, see {ref}`enable-monitoring`, {ref}`enable-alert-rules`, and {ref}`enable-tracing`. + +### Security event logging + +Charmed MySQL comes with the audit log plugin enabled by default. + +````{tab-set} +```{tab-item} VM +:sync: vm + +The logs are stored in the `/var/snap/charmed-mysql/common/var/log/mysql` directory, and are rotated every minute to the `/var/snap/charmed-mysql/common/var/log/mysql/archive_audit` directory. + +We recommend setting the retention period to a value greater than the default (three days): + + juju config mysql logs_retention_period=14 # days + +By default, the audit log records logins and logouts. To include the SQL queries executed by each user: + + juju config mysql logs_audit_policy=all +``` + +```{tab-item} K8s +:sync: k8s + +The logs are stored in the `/var/log/mysql` directory of the mysql container, and it's rotated every minute to the `/var/log/mysql/archive_audit` directory. + +We recommend setting the retention period to a value greater than the default (three days): + + juju config mysql-k8s logs_retention_period=14 # days + +By default, the audit log records logins and logouts. To include the SQL queries executed by each user: + + juju config mysql-k8s logs_audit_policy=all +``` +```` + +See {ref}`audit-logs`. + +## Additional resources + +For details on the cryptography used by Charmed MySQL, see {ref}`cryptography`. + +```{toctree} +:titlesonly: +:maxdepth: 2 +:hidden: + +cryptography +``` + + +[OpenStack Security Guide]: https://docs.openstack.org/security-guide/ +[Best practices for security, identity and compliance]: https://aws.amazon.com/architecture/security-identity-compliance +[AWS security credentials]: https://docs.aws.amazon.com/IAM/latest/UserGuide/security-creds.html +[Azure security best practices and patterns]: https://learn.microsoft.com/en-us/azure/security/fundamentals/best-practices-and-patterns +[Managed identities for Azure resource]: https://learn.microsoft.com/en-us/entra/identity/managed-identities-azure-resources/ +[Google security overview]: https://cloud.google.com/kubernetes-engine/docs/concepts/security-overview +[Harden your cluster's security]: https://cloud.google.com/kubernetes-engine/docs/concepts/security-overview + + +[OpenStack cloud and Juju]: https://canonical-juju.readthedocs-hosted.com/en/latest/user/reference/cloud/list-of-supported-clouds/the-openstack-cloud-and-juju/ +[Juju AWS Permission]: https://discourse.charmhub.io/t/juju-aws-permissions/5307 +[AWS Instance Profiles]: https://discourse.charmhub.io/t/using-aws-instance-profiles-with-juju-2-9/5185 +[Juju on AWS]: https://juju.is/docs/juju/amazon-ec2 +[Juju Azure Permission]: https://juju.is/docs/juju/microsoft-azure +[How to use Juju with Microsoft Azure]: https://discourse.charmhub.io/t/how-to-use-juju-with-microsoft-azure/15219 +[Google GCE cloud and Juju]: https://canonical-juju.readthedocs-hosted.com/en/latest/user/reference/cloud/list-of-supported-clouds/the-google-gce-cloud-and-juju/ \ No newline at end of file diff --git a/machines/docs/explanation/users.md b/docs/explanation/users.md similarity index 77% rename from machines/docs/explanation/users.md rename to docs/explanation/users.md index 3dc949a90..da0147bd2 100644 --- a/machines/docs/explanation/users.md +++ b/docs/explanation/users.md @@ -1,3 +1,10 @@ +--- +myst: + html_meta: + description: "Understand the types of MySQL users in Charmed MySQL: internal operator users and relation users created for integrated applications." +--- + +(users)= # Users There are two main types of users in MySQL: @@ -13,12 +20,12 @@ The operator uses the following internal database users: * `root` - the initial/default MySQL user. Used for very initial bootstrap and restricted to local access. * `clusteradmin` - the user to manage replication in the MySQL InnoDB ClusterSet. * `serverconfig` - the user that operates MySQL instances. -* `monitoring` - the user for [COS integration](/how-to/monitoring-cos/enable-monitoring). -* `backups` - the user to [perform/list/restore backups](/how-to/back-up-and-restore/create-a-backup). +* `monitoring` - the user for {ref}`COS integration `. +* `backups` - the user to for {ref}`backup operations `. * `mysql_innodb_cluster_#######` - the [internal recovery users](https://dev.mysql.com/doc/mysql-shell/8.0/en/innodb-cluster-user-accounts.html#mysql-innodb-cluster-users-created) which enable connections between the servers in the cluster. Dedicated user created for each Juju unit/InnoDB Cluster member. * `mysql_innodb_cs_#######` - the internal recovery user which enable connections between MySQl InnoDB Clusters in ClusterSet. One user is created for entire MySQL ClusterSet. -The full list of internal users is available in charm [source code](https://github.com/canonical/mysql-operator/blob/main/src/constants.py). +The full list of internal users is available in charm's source code ([VM](https://github.com/canonical/mysql-operators/blob/8.0/edge/machines/src/constants.py) | [K8s](https://github.com/canonical/mysql-operators/blob/8.0/edge/kubernetes/src/constants.py)). ```{caution} It is forbidden to manage internal users, as they are dedicated to the operator’s logic. @@ -52,12 +59,12 @@ mysql> select Host,User,account_locked from mysql.user; Passwords for *internal* users can be rotated using the action `set-password` on the juju leader unit. ```{seealso} -[How to manage passwords](/how-to/manage-passwords) +{ref}`manage-passwords` ``` ## Relation users -The operator created a dedicated user for every application related/integrated with database. The username is composed by the relation ID and truncated uuid for the model, to ensure there is no username clash in cross model relations. Usernames are limited to 32 chars as per [MySQL limit](https://dev.mysql.com/doc/refman/8.0/en/user-names.html). +The operator created a dedicated user for every application related/integrated with database. The username is composed by the relation ID and truncated UUID for the model, to ensure there is no username clash in cross model relations. Usernames are limited to 32 chars as per [MySQL limit](https://dev.mysql.com/doc/refman/8.0/en/user-names.html). Relation users are removed on the juju relation/integration removal request. However, database data stays in place and can be reused on re-created relations (using new user credentials): @@ -72,7 +79,7 @@ mysql> select Host,User,account_locked from mysql.user where User like 'relation 2 row in set (0.00 sec) ``` -The extra user(s) will be created for relation with [mysql-router](https://charmhub.io/mysql-router) charm to provide necessary users for applications related via the `mysql-router` app: +The extra user(s) will be created for relation with the MySQL Router charm ([VM](https://charmhub.io/mysql-router) | [K8s](https://charmhub.io/mysql-router-k8s)) to provide necessary users for applications related via the `mysql-router` (or `mysql-router-k8s`) app: ```shell mysql> select Host,User,account_locked from mysql.user where User like 'mysql_router%'; @@ -86,15 +93,26 @@ mysql> select Host,User,account_locked from mysql.user where User like 'mysql_ro To rotate passwords for relation users, remove the relation and re-relate: -```shell -juju remove-relation mysql myclientapp -juju wait-for application mysql -juju relate mysql myclientapp +````{tab-set} +```{tab-item} VM +:sync: vm + + juju remove-relation mysql + juju wait-for application mysql + juju relate mysql ``` +```{tab-item} K8s +:sync: k8s + + juju remove-relation mysql-k8s + juju wait-for application mysql-k8s + juju relate mysql-k8s +``` +```` + ### Admin port user access The charm mainly uses the `serverconfig` user for internal operations. For connections with this user, a special admin port is used (port `33062`), which enables the charm to operate MySQL even when users connections are saturated. For further information on the administrative connection, refer to [MySQL docs](https://dev.mysql.com/doc/refman/8.0/en/administrative-connection-interface.html) on the topic. - diff --git a/machines/docs/how-to/back-up-and-restore/configure-s3-aws.md b/docs/how-to/back-up-and-restore/configure-s3-aws.md similarity index 51% rename from machines/docs/how-to/back-up-and-restore/configure-s3-aws.md rename to docs/how-to/back-up-and-restore/configure-s3-aws.md index f3b0d5e52..1c0ec9609 100644 --- a/machines/docs/how-to/back-up-and-restore/configure-s3-aws.md +++ b/docs/how-to/back-up-and-restore/configure-s3-aws.md @@ -1,3 +1,10 @@ +--- +myst: + html_meta: + description: "Configure the s3-integrator charm for AWS S3 to enable Charmed MySQL backups, including credentials, bucket setup, and application integration." +--- + +(configure-s3-aws)= # Configure S3 for AWS Charmed MySQL backups can be stored on any S3 compatible storage. S3 access and configurations are managed with the [`s3-integrator` charm](https://charmhub.io/s3-integrator). @@ -5,7 +12,7 @@ Charmed MySQL backups can be stored on any S3 compatible storage. S3 access and This guide will teach you how to deploy and configure the s3-integrator charm for [AWS S3](https://aws.amazon.com/s3/), send the configuration to a Charmed MySQL application, and update it. ```{seealso} -[](/how-to/back-up-and-restore/configure-s3-radosgw) +{ref}`configure-s3-radosgw` ``` ## Set up `s3-integrator` @@ -23,25 +30,57 @@ juju config s3-integrator \ ``` ```{note} -The amazon S3 endpoint must be specified as `s3..amazonaws.com ` within the first 24 hours of creating the bucket. For older buckets, the endpoint `s3.amazonaws.com` can be used. +The Amazon S3 endpoint must be specified as `s3..amazonaws.com ` within the first 24 hours of creating the bucket. For older buckets, the endpoint `s3.amazonaws.com` can be used. See [this post](https://repost.aws/knowledge-center/s3-http-307-response) for more information. ``` +```{admonition} Juju 2.9 users +:class: tip + +Remember that `juju run ` becomes `juju run-action --wait`. + +See also: {ref}`breaking-changes-juju` +``` + To pass these configurations to Charmed MySQL, relate the two applications: -```shell -juju integrate s3-integrator mysql + +````{tab-set} +```{tab-item} VM +:sync: vm + + juju relate s3-integrator mysql ``` +```{tab-item} K8s +:sync: k8s + + juju relate s3-integrator mysql-k8s +``` +```` + You can create, list, and restore backups now: -```shell -juju run mysql/leader list-backups -juju run mysql/leader create-backup -juju run mysql/leader list-backups -juju run mysql/leader restore backup-id= +````{tab-set} +```{tab-item} VM +:sync: vm + + juju run mysql/leader list-backups + juju run mysql/leader create-backup + juju run mysql/leader list-backups + juju run mysql/leader restore backup-id= +``` + +```{tab-item} K8s +:sync: k8s + + juju run mysql-k8s/leader list-backups + juju run mysql-k8s/leader create-backup + juju run mysql-k8s/leader list-backups + juju run mysql-k8s/leader restore backup-id= ``` +```` You can also update your S3 configuration options after relating: @@ -49,5 +88,5 @@ You can also update your S3 configuration options after relating: juju config s3-integrator