diff --git a/.github/BOTMETA.yml b/.github/BOTMETA.yml index 2cd7bbeabe..43122899fb 100644 --- a/.github/BOTMETA.yml +++ b/.github/BOTMETA.yml @@ -216,6 +216,10 @@ files: maintainers: resmo $filters/to_time_unit.yml: maintainers: resmo + $filters/to_toml.py: + maintainers: milliams + $filters/to_toml.yml: + maintainers: milliams $filters/to_weeks.yml: maintainers: resmo $filters/to_yaml.py: @@ -1498,6 +1502,8 @@ files: maintainers: vbotka $plugin_utils/unsafe.py: maintainers: felixfontein + $plugin_utils/_tags.py: + maintainers: felixfontein $tests/a_module.py: maintainers: felixfontein $tests/ansible_type.py: diff --git a/.mypy.ini b/.mypy.ini index e3128f32ff..381cde50e1 100644 --- a/.mypy.ini +++ b/.mypy.ini @@ -216,6 +216,8 @@ ignore_missing_imports = True ignore_missing_imports = True [mypy-thycotic.*] ignore_missing_imports = True +[mypy-tomlkit.*] +ignore_missing_imports = True [mypy-univention.*] ignore_missing_imports = True [mypy-vexatapi.*] diff --git a/plugins/filter/to_toml.py b/plugins/filter/to_toml.py new file mode 100644 index 0000000000..58d9872e56 --- /dev/null +++ b/plugins/filter/to_toml.py @@ -0,0 +1,50 @@ +# Copyright (c) Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +from __future__ import annotations + +import typing as t +from collections.abc import Mapping + +from ansible.module_utils.common.collections import is_sequence +from ansible.module_utils.common.text.converters import to_text + +TOMLKIT_IMPORT_ERROR: ImportError | None +try: + from tomlkit import dumps +except ImportError as imp_exc: + TOMLKIT_IMPORT_ERROR = imp_exc +else: + TOMLKIT_IMPORT_ERROR = None + +from ansible.errors import AnsibleError, AnsibleFilterError + +from ansible_collections.community.general.plugins.plugin_utils._tags import remove_all_tags + + +def _stringify_keys(value: t.Any) -> t.Any: + """Recursively convert all keys to strings.""" + if isinstance(value, Mapping): + return {to_text(k): _stringify_keys(v) for k, v in value.items()} + if is_sequence(value): + return [_stringify_keys(e) for e in value] + return value + + +def to_toml(value: t.Mapping, *, redact_sensitive_values: bool = False) -> str: + """Serialize input as TOML.""" + if TOMLKIT_IMPORT_ERROR: + raise AnsibleError("tomlkit must be installed to use this plugin") from TOMLKIT_IMPORT_ERROR + if not isinstance(value, Mapping): + raise AnsibleFilterError("to_toml only accepts dictionaries.") + return dumps( + remove_all_tags(_stringify_keys(value), redact_sensitive_values=redact_sensitive_values), + ) + + +class FilterModule: + def filters(self): + return { + "to_toml": to_toml, + } diff --git a/plugins/filter/to_toml.yml b/plugins/filter/to_toml.yml new file mode 100644 index 0000000000..299f912f04 --- /dev/null +++ b/plugins/filter/to_toml.yml @@ -0,0 +1,42 @@ +# Copyright (c) Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +DOCUMENTATION: + name: to_toml + author: + - Matt Williaks (@milliams) + version_added: 12.3.0 + short_description: Convert variable to TOML string + description: + - Converts an Ansible variable into a TOML string representation. + - This filter functions as a wrapper to the L(Python TOML Kit library, https://pypi.org/project/tomlkit/)'s C(tomlkit.dumps) function. + requirements: + - tomlkit + positional: _input + options: + _input: + description: + - A variable or expression that returns a data structure. + type: dict + required: true + redact_sensitive_values: + description: + - If set to V(true), vaulted strings are replaced by V() instead of being decrypted. + - With future ansible-core versions, this can extend to other strings tagged as sensitive. + - B(Note) that with ansible-core 2.18 and before this might not yield the expected result + since these versions of ansible-core strip the vault information away from strings that are + part of more complex data structures specified in C(vars). + type: bool + default: false + +EXAMPLES: | + --- + # Dump variable in a template to create a TOML document + value: "{{ my_config | community.general.to_toml }}" + +RETURN: + _value: + description: + - The TOML serialized string representing the variable structure inputted. + type: string diff --git a/plugins/filter/to_yaml.py b/plugins/filter/to_yaml.py index b042cbd80f..7a52973adf 100644 --- a/plugins/filter/to_yaml.py +++ b/plugins/filter/to_yaml.py @@ -5,7 +5,6 @@ from __future__ import annotations import typing as t -from collections.abc import Mapping, Set from yaml import dump @@ -14,81 +13,7 @@ try: except ImportError: from yaml import SafeDumper # type: ignore -from ansible.module_utils.common.collections import is_sequence - -try: - # This is ansible-core 2.19+ - from ansible.parsing.vault import VaultHelper, VaultLib - from ansible.utils.vars import transform_to_native_types - - HAS_TRANSFORM_TO_NATIVE_TYPES = True -except ImportError: - HAS_TRANSFORM_TO_NATIVE_TYPES = False - -from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode -from ansible.utils.unsafe_proxy import AnsibleUnsafe - - -def _to_native_types_compat(value: t.Any, *, redact_value: str | None) -> t.Any: - """Compatibility function for ansible-core 2.18 and before.""" - if value is None: - return value - if isinstance(value, AnsibleUnsafe): - # This only works up to ansible-core 2.18: - return _to_native_types_compat(value._strip_unsafe(), redact_value=redact_value) # type: ignore - # But that's fine, since this code path isn't taken on ansible-core 2.19+ anyway. - if isinstance(value, Mapping): - return { - _to_native_types_compat(key, redact_value=redact_value): _to_native_types_compat( - val, redact_value=redact_value - ) - for key, val in value.items() - } - if isinstance(value, Set): - return {_to_native_types_compat(elt, redact_value=redact_value) for elt in value} - if is_sequence(value): - return [_to_native_types_compat(elt, redact_value=redact_value) for elt in value] - if isinstance(value, AnsibleVaultEncryptedUnicode): - if redact_value is not None: - return redact_value - # This only works up to ansible-core 2.18: - return value.data - # But that's fine, since this code path isn't taken on ansible-core 2.19+ anyway. - if isinstance(value, bytes): - return bytes(value) - if isinstance(value, str): - return str(value) - - return value - - -def _to_native_types(value: t.Any, *, redact: bool) -> t.Any: - if isinstance(value, Mapping): - return {_to_native_types(k, redact=redact): _to_native_types(v, redact=redact) for k, v in value.items()} - if is_sequence(value): - return [_to_native_types(e, redact=redact) for e in value] - if redact: - ciphertext = VaultHelper.get_ciphertext(value, with_tags=False) - if ciphertext and VaultLib.is_encrypted(ciphertext): - return "" - return transform_to_native_types(value, redact=redact) - - -def remove_all_tags(value: t.Any, *, redact_sensitive_values: bool = False) -> t.Any: - """ - Remove all tags from all values in the input. - - If ``redact_sensitive_values`` is ``True``, all sensitive values will be redacted. - """ - if HAS_TRANSFORM_TO_NATIVE_TYPES: - return _to_native_types(value, redact=redact_sensitive_values) - - return _to_native_types_compat( # type: ignore[unreachable] - value, - redact_value="" - if redact_sensitive_values - else None, # same string as in ansible-core 2.19 by transform_to_native_types() - ) +from ansible_collections.community.general.plugins.plugin_utils._tags import remove_all_tags def to_yaml( diff --git a/plugins/plugin_utils/_tags.py b/plugins/plugin_utils/_tags.py new file mode 100644 index 0000000000..48fc603387 --- /dev/null +++ b/plugins/plugin_utils/_tags.py @@ -0,0 +1,85 @@ +# Copyright (c) Contributors to the Ansible project +# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +# Note that this module util is **PRIVATE** to the collection. It can have breaking changes at any time. +# Do not use this from other collections or standalone plugins/modules! + +import typing as t +from collections.abc import Mapping, Set + +from ansible.module_utils.common.collections import is_sequence + +try: + # This is ansible-core 2.19+ + from ansible.parsing.vault import VaultHelper, VaultLib + from ansible.utils.vars import transform_to_native_types + + HAS_TRANSFORM_TO_NATIVE_TYPES = True +except ImportError: + HAS_TRANSFORM_TO_NATIVE_TYPES = False + +from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode +from ansible.utils.unsafe_proxy import AnsibleUnsafe + + +def _to_native_types_compat(value: t.Any, *, redact_value: str | None) -> t.Any: + """Compatibility function for ansible-core 2.18 and before.""" + if value is None: + return value + if isinstance(value, AnsibleUnsafe): + # This only works up to ansible-core 2.18: + return _to_native_types_compat(value._strip_unsafe(), redact_value=redact_value) # type: ignore + # But that's fine, since this code path isn't taken on ansible-core 2.19+ anyway. + if isinstance(value, Mapping): + return { + _to_native_types_compat(key, redact_value=redact_value): _to_native_types_compat( + val, redact_value=redact_value + ) + for key, val in value.items() + } + if isinstance(value, Set): + return {_to_native_types_compat(elt, redact_value=redact_value) for elt in value} + if is_sequence(value): + return [_to_native_types_compat(elt, redact_value=redact_value) for elt in value] + if isinstance(value, AnsibleVaultEncryptedUnicode): + if redact_value is not None: + return redact_value + # This only works up to ansible-core 2.18: + return value.data + # But that's fine, since this code path isn't taken on ansible-core 2.19+ anyway. + if isinstance(value, bytes): + return bytes(value) + if isinstance(value, str): + return str(value) + + return value + + +def _to_native_types(value: t.Any, *, redact: bool) -> t.Any: + if isinstance(value, Mapping): + return {_to_native_types(k, redact=redact): _to_native_types(v, redact=redact) for k, v in value.items()} + if is_sequence(value): + return [_to_native_types(e, redact=redact) for e in value] + if redact: + ciphertext = VaultHelper.get_ciphertext(value, with_tags=False) + if ciphertext and VaultLib.is_encrypted(ciphertext): + return "" + return transform_to_native_types(value, redact=redact) + + +def remove_all_tags(value: t.Any, *, redact_sensitive_values: bool = False) -> t.Any: + """ + Remove all tags from all values in the input. + + If ``redact_sensitive_values`` is ``True``, all sensitive values will be redacted. + """ + if HAS_TRANSFORM_TO_NATIVE_TYPES: + return _to_native_types(value, redact=redact_sensitive_values) + + return _to_native_types_compat( # type: ignore[unreachable] + value, + redact_value="" + if redact_sensitive_values + else None, # same string as in ansible-core 2.19 by transform_to_native_types() + ) diff --git a/tests/integration/targets/filter_to_toml/aliases b/tests/integration/targets/filter_to_toml/aliases new file mode 100644 index 0000000000..343f119da8 --- /dev/null +++ b/tests/integration/targets/filter_to_toml/aliases @@ -0,0 +1,5 @@ +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +azp/posix/3 diff --git a/tests/integration/targets/filter_to_toml/main.yml b/tests/integration/targets/filter_to_toml/main.yml new file mode 100644 index 0000000000..eb5a962dff --- /dev/null +++ b/tests/integration/targets/filter_to_toml/main.yml @@ -0,0 +1,84 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +- hosts: localhost + gather_facts: false + vars_files: + - vaulted_vars.yml + vars: + timestamp: 2025-01-02T03:04:05Z + bar: "foobarbaz" + tasks: + - name: Convert more complex data structure (from vars file) + set_fact: + complex: "{{ foobar | community.general.to_toml }}" + complex_redact: "{{ foobar | community.general.to_toml(redact_sensitive_values=true) }}" + + - assert: + that: + - complex == exp_complex + - complex_redact == exp_complex_redact + vars: + exp_complex: | + a_value = 123 + a_list = ["bar", 2025-02-03T04:05:06, "Hello!", true, false] + exp_complex_redact: | + a_value = 123 + a_list = ["", 2025-02-03T04:05:06, "Hello!", true, false] + + - name: Convert more complex data structure (from vars) + set_fact: + complex: "{{ data | community.general.to_toml }}" + complex_redact: "{{ data | community.general.to_toml(redact_sensitive_values=true) }}" + vars: + data: + foo: 123 + bar: 1.23 + baz: true + bam: foobar + bang: + - "{{ timestamp }}" + - "{{ bar }}" + - "{{ foo }}" + + - when: ansible_version.full is version("2.19", "<") + assert: + that: + - complex == exp_complex + # With ansible-core 2.18 and before, the vaulted string is decrypted before it reaches the filter, + # so the redaction does not work there. + - complex_redact == exp_complex + vars: + exp_complex: | + foo = 123 + bar = 1.23 + baz = true + bam = "foobar" + bang = ["2025-01-02 03:04:05+00:00", "foobarbaz", "bar"] + exp_complex_redact: | + foo = 123 + bar = 1.23 + baz = true + bam = "foobar" + bang = ["2025-01-02 03:04:05+00:00", "foobarbaz", "bar"] + + - when: ansible_version.full is version("2.19", ">=") + assert: + that: + - complex == exp_complex + - complex_redact == exp_complex_redact + vars: + exp_complex: | + foo = 123 + bar = 1.23 + baz = true + bam = "foobar" + bang = [2025-01-02T03:04:05Z, "foobarbaz", "bar"] + exp_complex_redact: | + foo = 123 + bar = 1.23 + baz = true + bam = "foobar" + bang = [2025-01-02T03:04:05Z, "foobarbaz", ""] diff --git a/tests/integration/targets/filter_to_toml/password b/tests/integration/targets/filter_to_toml/password new file mode 100644 index 0000000000..d97c5eada5 --- /dev/null +++ b/tests/integration/targets/filter_to_toml/password @@ -0,0 +1 @@ +secret diff --git a/tests/integration/targets/filter_to_toml/password.license b/tests/integration/targets/filter_to_toml/password.license new file mode 100644 index 0000000000..a1390a69ed --- /dev/null +++ b/tests/integration/targets/filter_to_toml/password.license @@ -0,0 +1,3 @@ +Copyright (c) Ansible Project +GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +SPDX-License-Identifier: GPL-3.0-or-later diff --git a/tests/integration/targets/filter_to_toml/runme.sh b/tests/integration/targets/filter_to_toml/runme.sh new file mode 100755 index 0000000000..0570805e3c --- /dev/null +++ b/tests/integration/targets/filter_to_toml/runme.sh @@ -0,0 +1,15 @@ +#!/usr/bin/env bash +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +set -eux + +source virtualenv.sh + +# Requirements have to be installed prior to running ansible-playbook +# because plugins and requirements are loaded before the task runs + +pip install tomlkit + +ansible-playbook --vault-password-file password main.yml "$@" diff --git a/tests/integration/targets/filter_to_toml/vaulted_vars.yml b/tests/integration/targets/filter_to_toml/vaulted_vars.yml new file mode 100644 index 0000000000..c8f7c6141d --- /dev/null +++ b/tests/integration/targets/filter_to_toml/vaulted_vars.yml @@ -0,0 +1,27 @@ +--- +# Copyright (c) Ansible Project +# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt) +# SPDX-License-Identifier: GPL-3.0-or-later + +foo: !vault | + $ANSIBLE_VAULT;1.1;AES256 + 32336431346561346535396563363438333131636539653331376466383331663838303835353862 + 3536306130663166393533626530646435383938323066320a303366613035323835373030303262 + 35633636653362393531653961396665663965356562346538643863336562393734376234313134 + 3562663234326435390a376464633234373636643538353562326133316439343863373333363265 + 6239 + +foobar: + a_value: 123 + a_list: + - !vault | + $ANSIBLE_VAULT;1.1;AES256 + 32336431346561346535396563363438333131636539653331376466383331663838303835353862 + 3536306130663166393533626530646435383938323066320a303366613035323835373030303262 + 35633636653362393531653961396665663965356562346538643863336562393734376234313134 + 3562663234326435390a376464633234373636643538353562326133316439343863373333363265 + 6239 + - 2025-02-03 04:05:06 + - Hello! + - true + - false