This might be a juju problem.
We should find out why the secret is not being found even though it should be there. Our caching is hiding problem a little because we do not get the secret from juju in the serialize_model function but instead return it from the cache. We only find the problem when calling ops's set_content on the actual juju secret.
unit-charmed-etcd-0: 01:50:03 DEBUG unit.charmed-etcd/0.juju-log etcd-peers:3: charms.data_platform_libs.v1.data_interfaces:Secret with label etcd-client.7.fa2f173949a38a15.mtls.secret not found
unit-charmed-etcd-0: 01:50:03 ERROR unit.charmed-etcd/0.juju-log etcd-peers:3: root:Uncaught exception while in charm code:
Traceback (most recent call last):
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/model.py", line 3637, in _run
result = subprocess.run(args, **kwargs) # type: ignore
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/usr/lib/python3.12/subprocess.py", line 571, in run
raise CalledProcessError(retcode, process.args,
subprocess.CalledProcessError: Command '('/var/lib/juju/tools/unit-charmed-etcd-0/secret-info-get', '--label', 'etcd-client.7.user.secret', '--format=json')' returned non-zero exit status 1.
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/model.py", line 4034, in _run_for_secret
return self._run(*args, return_output=return_output, use_json=use_json)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/model.py", line 3639, in _run
raise ModelError(e.stderr) from e
ops.model.ModelError: ERROR secret "etcd-client.7.user.secret" not found
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/lib/charms/data_platform_libs/v1/data_interfaces.py", line 891, in serialize_model
secret.set_content(full_content)
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/lib/charms/data_platform_libs/v1/data_interfaces.py", line 606, in set_content
self.meta.set_content(content)
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/model.py", line 1536, in set_content
self._id = self.get_info().id
^^^^^^^^^^^^^^^
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/model.py", line 1513, in get_info
return self._backend.secret_info_get(id=self.id, label=self.label)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/model.py", line 4046, in secret_info_get
result = self._run_for_secret('secret-info-get', *args, return_output=True, use_json=True)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/model.py", line 4037, in _run_for_secret
raise SecretNotFoundError() from e
ops.model.SecretNotFoundError
The above exception was the direct cause of the following exception:
Traceback (most recent call last):
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/src/charm.py", line 415, in <module>
ops.main(EtcdOperatorCharm) # type: ignore
^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/__init__.py", line 356, in __call__
return _main.main(charm_class=charm_class, use_juju_for_storage=use_juju_for_storage)
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/_main.py", line 504, in main
manager.run()
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/_main.py", line 488, in run
self._emit()
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/_main.py", line 423, in _emit
self._emit_charm_event(self.dispatcher.event_name)
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/_main.py", line 467, in _emit_charm_event
event_to_emit.emit(*args, **kwargs)
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/framework.py", line 351, in emit
framework._emit(event)
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/framework.py", line 924, in _emit
self._reemit(event_path)
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/ops/framework.py", line 1030, in _reemit
custom_handler(event)
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/src/events/etcd.py", line 359, in _on_peer_relation_changed
self._update_client_relations()
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/src/events/etcd.py", line 769, in _update_client_relations
self.charm.external_clients_manager.update_client_relations_data(
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/src/managers/external_clients.py", line 157, in update_client_relations_data
self.state.etcd_provides_event_handler.set_responses(relation.id, responses)
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/lib/charms/data_platform_libs/v1/data_interfaces.py", line 2540, in set_responses
self.interface.write_model(
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/lib/charms/data_platform_libs/v1/data_interfaces.py", line 1710, in write_model
write_model(
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/lib/charms/data_platform_libs/v1/data_interfaces.py", line 1790, in write_model
dumped = model.model_dump(
^^^^^^^^^^^^^^^^^
File "/var/lib/juju/agents/unit-charmed-etcd-0/charm/venv/lib/python3.12/site-packages/pydantic/main.py", line 463, in model_dump
return self.__pydantic_serializer__.to_python(
^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^^
pydantic_core._pydantic_core.PydanticSerializationError: Error calling function `serialize_model`: SecretNotFoundError:
This might be a juju problem.
Steps to reproduce
The issue is flaky and can be seen here, and here
Expected behavior
We should find out why the secret is not being found even though it should be there. Our caching is hiding problem a little because we do not get the secret from juju in the
serialize_modelfunction but instead return it from the cache. We only find the problem when calling ops'sset_contenton the actual juju secret.Actual behavior