Add to_toml filter (#11423)

* Add to_toml filter

This is based heavily on the to_yaml filter, but
with a pared-down feature set.

* Protect import

* Don't quote datetime as a string

* Use Ansible error types

* Import correct error types

* Don't use AnsibleTypeError

It doesn't seem to be available on older Ansible
core versions.

* Fix antsibull-nox errors

* Install dependencies for to_toml integration test

Co-authored-by: Felix Fontein <felix@fontein.de>

* Reduce author list to main contributor

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update version added for to_toml

Co-authored-by: Felix Fontein <felix@fontein.de>

* Use AnsibleError for missing import

Co-authored-by: Felix Fontein <felix@fontein.de>

* Use AnsibleFilterError for runtime type check

Co-authored-by: Felix Fontein <felix@fontein.de>

* Move common code to plugin_utils/_tags.py

* Mark module util as private

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update BOTMETA for to_toml

Co-authored-by: Felix Fontein <felix@fontein.de>

* Fix typo

* Correct version number

Co-authored-by: Felix Fontein <felix@fontein.de>

* Use to_text for to_toml dict key conversions

Co-authored-by: Felix Fontein <felix@fontein.de>

* Add tomlkit requirement to docs

Co-authored-by: Felix Fontein <felix@fontein.de>

* Add missing import

* Add aliases for for to_toml integration test

---------

Co-authored-by: Felix Fontein <felix@fontein.de>

.github/BOTMETA.yml

@@ -216,6 +216,10 @@ files:
     maintainers: resmo
   $filters/to_time_unit.yml:
     maintainers: resmo
+  $filters/to_toml.py:
+    maintainers: milliams
+  $filters/to_toml.yml:
+    maintainers: milliams
   $filters/to_weeks.yml:
     maintainers: resmo
   $filters/to_yaml.py:
@@ -1498,6 +1502,8 @@ files:
     maintainers: vbotka
   $plugin_utils/unsafe.py:
     maintainers: felixfontein
+  $plugin_utils/_tags.py:
+    maintainers: felixfontein
   $tests/a_module.py:
     maintainers: felixfontein
   $tests/ansible_type.py:

.mypy.ini

@@ -216,6 +216,8 @@ ignore_missing_imports = True
 ignore_missing_imports = True
 [mypy-thycotic.*]
 ignore_missing_imports = True
+[mypy-tomlkit.*]
+ignore_missing_imports = True
 [mypy-univention.*]
 ignore_missing_imports = True
 [mypy-vexatapi.*]

plugins/filter/to_toml.py

@@ -0,0 +1,50 @@
+# Copyright (c) Contributors to the Ansible project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+from __future__ import annotations
+
+import typing as t
+from collections.abc import Mapping
+
+from ansible.module_utils.common.collections import is_sequence
+from ansible.module_utils.common.text.converters import to_text
+
+TOMLKIT_IMPORT_ERROR: ImportError | None
+try:
+    from tomlkit import dumps
+except ImportError as imp_exc:
+    TOMLKIT_IMPORT_ERROR = imp_exc
+else:
+    TOMLKIT_IMPORT_ERROR = None
+
+from ansible.errors import AnsibleError, AnsibleFilterError
+
+from ansible_collections.community.general.plugins.plugin_utils._tags import remove_all_tags
+
+
+def _stringify_keys(value: t.Any) -> t.Any:
+    """Recursively convert all keys to strings."""
+    if isinstance(value, Mapping):
+        return {to_text(k): _stringify_keys(v) for k, v in value.items()}
+    if is_sequence(value):
+        return [_stringify_keys(e) for e in value]
+    return value
+
+
+def to_toml(value: t.Mapping, *, redact_sensitive_values: bool = False) -> str:
+    """Serialize input as TOML."""
+    if TOMLKIT_IMPORT_ERROR:
+        raise AnsibleError("tomlkit must be installed to use this plugin") from TOMLKIT_IMPORT_ERROR
+    if not isinstance(value, Mapping):
+        raise AnsibleFilterError("to_toml only accepts dictionaries.")
+    return dumps(
+        remove_all_tags(_stringify_keys(value), redact_sensitive_values=redact_sensitive_values),
+    )
+
+
+class FilterModule:
+    def filters(self):
+        return {
+            "to_toml": to_toml,
+        }

plugins/filter/to_toml.yml

@@ -0,0 +1,42 @@
+# Copyright (c) Contributors to the Ansible project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+DOCUMENTATION:
+  name: to_toml
+  author:
+    - Matt Williaks (@milliams)
+  version_added: 12.3.0
+  short_description: Convert variable to TOML string
+  description:
+    - Converts an Ansible variable into a TOML string representation.
+    - This filter functions as a wrapper to the L(Python TOML Kit library, https://pypi.org/project/tomlkit/)'s C(tomlkit.dumps) function.
+  requirements:
+    - tomlkit
+  positional: _input
+  options:
+    _input:
+      description:
+        - A variable or expression that returns a data structure.
+      type: dict
+      required: true
+    redact_sensitive_values:
+      description:
+        - If set to V(true), vaulted strings are replaced by V(<redacted>) instead of being decrypted.
+        - With future ansible-core versions, this can extend to other strings tagged as sensitive.
+        - B(Note) that with ansible-core 2.18 and before this might not yield the expected result
+          since these versions of ansible-core strip the vault information away from strings that are
+          part of more complex data structures specified in C(vars).
+      type: bool
+      default: false
+
+EXAMPLES: |
+  ---
+  # Dump variable in a template to create a TOML document
+  value: "{{ my_config | community.general.to_toml }}"
+
+RETURN:
+  _value:
+    description:
+      - The TOML serialized string representing the variable structure inputted.
+    type: string

plugins/filter/to_yaml.py

@@ -5,7 +5,6 @@
 from __future__ import annotations
 
 import typing as t
-from collections.abc import Mapping, Set
 
 from yaml import dump
 
@@ -14,81 +13,7 @@ try:
 except ImportError:
     from yaml import SafeDumper  # type: ignore
 
-from ansible.module_utils.common.collections import is_sequence
+from ansible_collections.community.general.plugins.plugin_utils._tags import remove_all_tags
-
-try:
-    # This is ansible-core 2.19+
-    from ansible.parsing.vault import VaultHelper, VaultLib
-    from ansible.utils.vars import transform_to_native_types
-
-    HAS_TRANSFORM_TO_NATIVE_TYPES = True
-except ImportError:
-    HAS_TRANSFORM_TO_NATIVE_TYPES = False
-
-from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
-from ansible.utils.unsafe_proxy import AnsibleUnsafe
-
-
-def _to_native_types_compat(value: t.Any, *, redact_value: str | None) -> t.Any:
-    """Compatibility function for ansible-core 2.18 and before."""
-    if value is None:
-        return value
-    if isinstance(value, AnsibleUnsafe):
-        # This only works up to ansible-core 2.18:
-        return _to_native_types_compat(value._strip_unsafe(), redact_value=redact_value)  # type: ignore
-        # But that's fine, since this code path isn't taken on ansible-core 2.19+ anyway.
-    if isinstance(value, Mapping):
-        return {
-            _to_native_types_compat(key, redact_value=redact_value): _to_native_types_compat(
-                val, redact_value=redact_value
-            )
-            for key, val in value.items()
-        }
-    if isinstance(value, Set):
-        return {_to_native_types_compat(elt, redact_value=redact_value) for elt in value}
-    if is_sequence(value):
-        return [_to_native_types_compat(elt, redact_value=redact_value) for elt in value]
-    if isinstance(value, AnsibleVaultEncryptedUnicode):
-        if redact_value is not None:
-            return redact_value
-        # This only works up to ansible-core 2.18:
-        return value.data
-        # But that's fine, since this code path isn't taken on ansible-core 2.19+ anyway.
-    if isinstance(value, bytes):
-        return bytes(value)
-    if isinstance(value, str):
-        return str(value)
-
-    return value
-
-
-def _to_native_types(value: t.Any, *, redact: bool) -> t.Any:
-    if isinstance(value, Mapping):
-        return {_to_native_types(k, redact=redact): _to_native_types(v, redact=redact) for k, v in value.items()}
-    if is_sequence(value):
-        return [_to_native_types(e, redact=redact) for e in value]
-    if redact:
-        ciphertext = VaultHelper.get_ciphertext(value, with_tags=False)
-        if ciphertext and VaultLib.is_encrypted(ciphertext):
-            return "<redacted>"
-    return transform_to_native_types(value, redact=redact)
-
-
-def remove_all_tags(value: t.Any, *, redact_sensitive_values: bool = False) -> t.Any:
-    """
-    Remove all tags from all values in the input.
-
-    If ``redact_sensitive_values`` is ``True``, all sensitive values will be redacted.
-    """
-    if HAS_TRANSFORM_TO_NATIVE_TYPES:
-        return _to_native_types(value, redact=redact_sensitive_values)
-
-    return _to_native_types_compat(  # type: ignore[unreachable]
-        value,
-        redact_value="<redacted>"
-        if redact_sensitive_values
-        else None,  # same string as in ansible-core 2.19 by transform_to_native_types()
-    )
 
 
 def to_yaml(

plugins/plugin_utils/_tags.py

@@ -0,0 +1,85 @@
+# Copyright (c) Contributors to the Ansible project
+# GNU General Public License v3.0+ (see COPYING or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+# Note that this module util is **PRIVATE** to the collection. It can have breaking changes at any time.
+# Do not use this from other collections or standalone plugins/modules!
+
+import typing as t
+from collections.abc import Mapping, Set
+
+from ansible.module_utils.common.collections import is_sequence
+
+try:
+    # This is ansible-core 2.19+
+    from ansible.parsing.vault import VaultHelper, VaultLib
+    from ansible.utils.vars import transform_to_native_types
+
+    HAS_TRANSFORM_TO_NATIVE_TYPES = True
+except ImportError:
+    HAS_TRANSFORM_TO_NATIVE_TYPES = False
+
+from ansible.parsing.yaml.objects import AnsibleVaultEncryptedUnicode
+from ansible.utils.unsafe_proxy import AnsibleUnsafe
+
+
+def _to_native_types_compat(value: t.Any, *, redact_value: str | None) -> t.Any:
+    """Compatibility function for ansible-core 2.18 and before."""
+    if value is None:
+        return value
+    if isinstance(value, AnsibleUnsafe):
+        # This only works up to ansible-core 2.18:
+        return _to_native_types_compat(value._strip_unsafe(), redact_value=redact_value)  # type: ignore
+        # But that's fine, since this code path isn't taken on ansible-core 2.19+ anyway.
+    if isinstance(value, Mapping):
+        return {
+            _to_native_types_compat(key, redact_value=redact_value): _to_native_types_compat(
+                val, redact_value=redact_value
+            )
+            for key, val in value.items()
+        }
+    if isinstance(value, Set):
+        return {_to_native_types_compat(elt, redact_value=redact_value) for elt in value}
+    if is_sequence(value):
+        return [_to_native_types_compat(elt, redact_value=redact_value) for elt in value]
+    if isinstance(value, AnsibleVaultEncryptedUnicode):
+        if redact_value is not None:
+            return redact_value
+        # This only works up to ansible-core 2.18:
+        return value.data
+        # But that's fine, since this code path isn't taken on ansible-core 2.19+ anyway.
+    if isinstance(value, bytes):
+        return bytes(value)
+    if isinstance(value, str):
+        return str(value)
+
+    return value
+
+
+def _to_native_types(value: t.Any, *, redact: bool) -> t.Any:
+    if isinstance(value, Mapping):
+        return {_to_native_types(k, redact=redact): _to_native_types(v, redact=redact) for k, v in value.items()}
+    if is_sequence(value):
+        return [_to_native_types(e, redact=redact) for e in value]
+    if redact:
+        ciphertext = VaultHelper.get_ciphertext(value, with_tags=False)
+        if ciphertext and VaultLib.is_encrypted(ciphertext):
+            return "<redacted>"
+    return transform_to_native_types(value, redact=redact)
+
+
+def remove_all_tags(value: t.Any, *, redact_sensitive_values: bool = False) -> t.Any:
+    """
+    Remove all tags from all values in the input.
+
+    If ``redact_sensitive_values`` is ``True``, all sensitive values will be redacted.
+    """
+    if HAS_TRANSFORM_TO_NATIVE_TYPES:
+        return _to_native_types(value, redact=redact_sensitive_values)
+
+    return _to_native_types_compat(  # type: ignore[unreachable]
+        value,
+        redact_value="<redacted>"
+        if redact_sensitive_values
+        else None,  # same string as in ansible-core 2.19 by transform_to_native_types()
+    )

tests/integration/targets/filter_to_toml/aliases

@@ -0,0 +1,5 @@
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+azp/posix/3

tests/integration/targets/filter_to_toml/main.yml

@@ -0,0 +1,84 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- hosts: localhost
+  gather_facts: false
+  vars_files:
+    - vaulted_vars.yml
+  vars:
+    timestamp: 2025-01-02T03:04:05Z
+    bar: "foobarbaz"
+  tasks:
+    - name: Convert more complex data structure (from vars file)
+      set_fact:
+        complex: "{{ foobar | community.general.to_toml }}"
+        complex_redact: "{{ foobar | community.general.to_toml(redact_sensitive_values=true) }}"
+
+    - assert:
+        that:
+          - complex == exp_complex
+          - complex_redact == exp_complex_redact
+      vars:
+        exp_complex: |
+          a_value = 123
+          a_list = ["bar", 2025-02-03T04:05:06, "Hello!", true, false]
+        exp_complex_redact: |
+          a_value = 123
+          a_list = ["<redacted>", 2025-02-03T04:05:06, "Hello!", true, false]
+
+    - name: Convert more complex data structure (from vars)
+      set_fact:
+        complex: "{{ data | community.general.to_toml }}"
+        complex_redact: "{{ data | community.general.to_toml(redact_sensitive_values=true) }}"
+      vars:
+        data:
+          foo: 123
+          bar: 1.23
+          baz: true
+          bam: foobar
+          bang:
+            - "{{ timestamp }}"
+            - "{{ bar }}"
+            - "{{ foo }}"
+
+    - when: ansible_version.full is version("2.19", "<")
+      assert:
+        that:
+          - complex == exp_complex
+          # With ansible-core 2.18 and before, the vaulted string is decrypted before it reaches the filter,
+          # so the redaction does not work there.
+          - complex_redact == exp_complex
+      vars:
+        exp_complex: |
+          foo = 123
+          bar = 1.23
+          baz = true
+          bam = "foobar"
+          bang = ["2025-01-02 03:04:05+00:00", "foobarbaz", "bar"]
+        exp_complex_redact: |
+          foo = 123
+          bar = 1.23
+          baz = true
+          bam = "foobar"
+          bang = ["2025-01-02 03:04:05+00:00", "foobarbaz", "bar"]
+
+    - when: ansible_version.full is version("2.19", ">=")
+      assert:
+        that:
+          - complex == exp_complex
+          - complex_redact == exp_complex_redact
+      vars:
+        exp_complex: |
+          foo = 123
+          bar = 1.23
+          baz = true
+          bam = "foobar"
+          bang = [2025-01-02T03:04:05Z, "foobarbaz", "bar"]
+        exp_complex_redact: |
+          foo = 123
+          bar = 1.23
+          baz = true
+          bam = "foobar"
+          bang = [2025-01-02T03:04:05Z, "foobarbaz", "<redacted>"]

tests/integration/targets/filter_to_toml/password

@@ -0,0 +1 @@
+secret

tests/integration/targets/filter_to_toml/password.license

@@ -0,0 +1,3 @@
+Copyright (c) Ansible Project
+GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+SPDX-License-Identifier: GPL-3.0-or-later

tests/integration/targets/filter_to_toml/runme.sh

@@ -0,0 +1,15 @@
+#!/usr/bin/env bash
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+set -eux
+
+source virtualenv.sh
+
+# Requirements have to be installed prior to running ansible-playbook
+# because plugins and requirements are loaded before the task runs
+
+pip install tomlkit
+
+ansible-playbook --vault-password-file password main.yml "$@"

tests/integration/targets/filter_to_toml/vaulted_vars.yml

@@ -0,0 +1,27 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+foo: !vault |
+  $ANSIBLE_VAULT;1.1;AES256
+  32336431346561346535396563363438333131636539653331376466383331663838303835353862
+  3536306130663166393533626530646435383938323066320a303366613035323835373030303262
+  35633636653362393531653961396665663965356562346538643863336562393734376234313134
+  3562663234326435390a376464633234373636643538353562326133316439343863373333363265
+  6239
+
+foobar:
+  a_value: 123
+  a_list:
+    - !vault |
+      $ANSIBLE_VAULT;1.1;AES256
+      32336431346561346535396563363438333131636539653331376466383331663838303835353862
+      3536306130663166393533626530646435383938323066320a303366613035323835373030303262
+      35633636653362393531653961396665663965356562346538643863336562393734376234313134
+      3562663234326435390a376464633234373636643538353562326133316439343863373333363265
+      6239
+    - 2025-02-03 04:05:06
+    - Hello!
+    - true
+    - false