1
0
Fork 0
mirror of https://github.com/ansible-collections/community.general.git synced 2026-03-21 20:59:10 +00:00

merge_variables: extended merge capabilities added (#11536)

* merge_variables: extended merge capabilities added

This extension gives you more control over the variable merging process of the lookup plugin `merge_variables`. It closes the gap between Puppet's Hiera merging capabilities and the limitations of Ansible's default variable plugin `host_group_vars` regarding fragment-based value definition. You can now decide which merge strategy should be applied to dicts, lists, and other types. Furthermore, you can specify a merge strategy that should be applied in case of type conflicts.

The default behavior of the plugin has been preserved so that it is fully backward-compatible with the already implemented state.

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* Update changelogs/fragments/11536-merge-variables-extended-merging-capabilities.yml

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update plugins/lookup/merge_variables.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Periods added at the end of each choice description

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* Update plugins/lookup/merge_variables.py

Co-authored-by: Mark <40321020+m-a-r-k-e@users.noreply.github.com>

* ref: follow project standard for choice descriptions

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* ref: more examples added and refactoring

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* Update plugins/lookup/merge_variables.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* ref: some more comments to examples added

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* fix: unused import removed

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* ref: re-add "merge" to strategy map

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* Update comments

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* Specification of transformations solely as string

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* Comments updated

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* ref: `append_rp` and `prepend_rp` removed
feat: options dict for list transformations re-added
feat: allow setting `keep` for dedup transformation with possible values: `first` (default) and `last`

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* ref: improve options documentation

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* ref: documentation improved, avoiding words like newer or older in merge description

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* Update plugins/lookup/merge_variables.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* ref: "prio" replaced by "dict"

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

* feat: two integration tests added

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>

---------

Signed-off-by: Fiehe Christoph  <c.fiehe@eurodata.de>
Co-authored-by: Fiehe Christoph <c.fiehe@eurodata.de>
Co-authored-by: Felix Fontein <felix@fontein.de>
Co-authored-by: Mark <40321020+m-a-r-k-e@users.noreply.github.com>
This commit is contained in:
Christoph Fiehe 2026-03-19 22:45:44 +01:00 committed by GitHub
parent 25b5655be7
commit dae2157bb7
No known key found for this signature in database
GPG key ID: B5690EEEBB952194
6 changed files with 989 additions and 42 deletions

View file

@ -5,6 +5,7 @@
from __future__ import annotations
import unittest
from copy import deepcopy
from unittest.mock import patch
from ansible.errors import AnsibleError
@ -15,6 +16,14 @@ from ansible_collections.community.internal_test_tools.tests.unit.mock.loader im
from ansible_collections.community.general.plugins.lookup import merge_variables
merge_hash_data = {
"low_dict": {"a": {"a": {"x": "low_value", "y": "low_value", "list": ["low_value"]}}, "b": [1, 1, 2, 3]},
"high_dict": {
"a": {"a": {"y": "high_value", "z": "high_value", "list": ["high_value"]}},
"b": [3, 4, 4, {"5": "value"}],
},
}
class TestMergeVariablesLookup(unittest.TestCase):
class HostVarsMock(dict):
@ -33,7 +42,11 @@ class TestMergeVariablesLookup(unittest.TestCase):
self.merge_vars_lookup = merge_variables.LookupModule(loader=self.loader, templar=self.templar)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(AnsiblePlugin, "get_option", side_effect=[None, "ignore", "suffix", None])
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "deep", "append", "replace", "replace", []],
)
@patch.object(Templar, "template", side_effect=[["item1"], ["item3"]])
def test_merge_list(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
@ -44,7 +57,11 @@ class TestMergeVariablesLookup(unittest.TestCase):
self.assertEqual(results, [["item1", "item3"]])
@patch.object(AnsiblePlugin, "set_options")
@patch.object(AnsiblePlugin, "get_option", side_effect=[["initial_item"], "ignore", "suffix", None])
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[["initial_item"], "ignore", "suffix", None, "deep", "append", "replace", "replace", []],
)
@patch.object(Templar, "template", side_effect=[["item1"], ["item3"]])
def test_merge_list_with_initial_value(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
@ -55,7 +72,11 @@ class TestMergeVariablesLookup(unittest.TestCase):
self.assertEqual(results, [["initial_item", "item1", "item3"]])
@patch.object(AnsiblePlugin, "set_options")
@patch.object(AnsiblePlugin, "get_option", side_effect=[None, "ignore", "suffix", None])
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "deep", "append", "replace", "replace", []],
)
@patch.object(
Templar,
"template",
@ -76,7 +97,17 @@ class TestMergeVariablesLookup(unittest.TestCase):
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[{"initial_item": "random value", "list_item": ["test0"]}, "ignore", "suffix", None],
side_effect=[
{"initial_item": "random value", "list_item": ["test0"]},
"ignore",
"suffix",
None,
"deep",
"append",
"replace",
"replace",
[],
],
)
@patch.object(
Templar,
@ -105,7 +136,11 @@ class TestMergeVariablesLookup(unittest.TestCase):
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(AnsiblePlugin, "get_option", side_effect=[None, "warn", "suffix", None])
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "warn", "suffix", None, "deep", "append", "replace", "replace", []],
)
@patch.object(Templar, "template", side_effect=[{"item": "value1"}, {"item": "value2"}])
@patch.object(Display, "warning")
def test_merge_dict_non_unique_warning(self, mock_set_options, mock_get_option, mock_template, mock_display):
@ -118,7 +153,11 @@ class TestMergeVariablesLookup(unittest.TestCase):
self.assertEqual(results, [{"item": "value2"}])
@patch.object(AnsiblePlugin, "set_options")
@patch.object(AnsiblePlugin, "get_option", side_effect=[None, "error", "suffix", None])
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "error", "suffix", None, "deep", "append", "replace", "replace", []],
)
@patch.object(Templar, "template", side_effect=[{"item": "value1"}, {"item": "value2"}])
def test_merge_dict_non_unique_error(self, mock_set_options, mock_get_option, mock_template):
with self.assertRaises(AnsibleError):
@ -128,7 +167,11 @@ class TestMergeVariablesLookup(unittest.TestCase):
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(AnsiblePlugin, "get_option", side_effect=[None, "ignore", "suffix", None])
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "deep", "append", "replace", "replace", []],
)
@patch.object(Templar, "template", side_effect=[{"item1": "test", "list_item": ["test1"]}, ["item2", "item3"]])
def test_merge_list_and_dict(self, mock_set_options, mock_get_option, mock_template):
with self.assertRaises(AnsibleError):
@ -141,7 +184,11 @@ class TestMergeVariablesLookup(unittest.TestCase):
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(AnsiblePlugin, "get_option", side_effect=[None, "ignore", "suffix", ["all"]])
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", ["all"], "deep", "append", "replace", "replace", []],
)
@patch.object(
Templar,
"template",
@ -174,7 +221,11 @@ class TestMergeVariablesLookup(unittest.TestCase):
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(AnsiblePlugin, "get_option", side_effect=[None, "ignore", "suffix", ["dummy1"]])
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", ["dummy1"], "deep", "append", "replace", "replace", []],
)
@patch.object(
Templar,
"template",
@ -212,7 +263,11 @@ class TestMergeVariablesLookup(unittest.TestCase):
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(AnsiblePlugin, "get_option", side_effect=[None, "ignore", "suffix", ["dummy1", "dummy2"]])
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", ["dummy1", "dummy2"], "deep", "append", "replace", "replace", []],
)
@patch.object(
Templar,
"template",
@ -249,7 +304,11 @@ class TestMergeVariablesLookup(unittest.TestCase):
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(AnsiblePlugin, "get_option", side_effect=[None, "ignore", "suffix", ["dummy1", "dummy2"]])
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", ["dummy1", "dummy2"], "deep", "append", "replace", "replace", []],
)
@patch.object(
Templar,
"template",
@ -270,3 +329,371 @@ class TestMergeVariablesLookup(unittest.TestCase):
results = self.merge_vars_lookup.run(["__merge_var"], variables)
self.assertEqual(results, [["item1", "item5"]])
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "shallow", "replace", "replace", "replace", []],
)
@patch.object(
Templar, "template", side_effect=[deepcopy(merge_hash_data["low_dict"]), deepcopy(merge_hash_data["high_dict"])]
)
def test_merge_dict_shallow_and_list_replace(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
["__merge_dict_shallow_and_list_replace"],
{
"testdict1__merge_dict_shallow_and_list_replace": merge_hash_data["low_dict"],
"testdict2__merge_dict_shallow_and_list_replace": merge_hash_data["high_dict"],
},
)
self.assertEqual(results, [merge_hash_data["high_dict"]])
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "shallow", "keep", "replace", "replace", []],
)
@patch.object(
Templar, "template", side_effect=[deepcopy(merge_hash_data["low_dict"]), deepcopy(merge_hash_data["high_dict"])]
)
def test_merge_dict_shallow_and_list_keep(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
["__merge_dict_shallow_and_list_keep"],
{
"testdict1__merge_dict_shallow_and_list_keep": merge_hash_data["low_dict"],
"testdict2__merge_dict_shallow_and_list_keep": merge_hash_data["high_dict"],
},
)
self.assertEqual(results, [{"a": merge_hash_data["high_dict"]["a"], "b": merge_hash_data["low_dict"]["b"]}])
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "shallow", "append", "replace", "replace", []],
)
@patch.object(
Templar, "template", side_effect=[deepcopy(merge_hash_data["low_dict"]), deepcopy(merge_hash_data["high_dict"])]
)
def test_merge_dict_shallow_and_list_append(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
["__merge_dict_shallow_and_list_append"],
{
"testdict1__merge_dict_shallow_and_list_append": merge_hash_data["low_dict"],
"testdict2__merge_dict_shallow_and_list_append": merge_hash_data["high_dict"],
},
)
self.assertEqual(
results,
[
{
"a": merge_hash_data["high_dict"]["a"],
"b": merge_hash_data["low_dict"]["b"] + merge_hash_data["high_dict"]["b"],
}
],
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "shallow", "prepend", "replace", "replace", []],
)
@patch.object(
Templar, "template", side_effect=[deepcopy(merge_hash_data["low_dict"]), deepcopy(merge_hash_data["high_dict"])]
)
def test_merge_dict_shallow_and_list_prepend(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
["__merge_dict_shallow_and_list_prepend"],
{
"testdict1__merge_dict_shallow_and_list_prepend": merge_hash_data["low_dict"],
"testdict2__merge_dict_shallow_and_list_prepend": merge_hash_data["high_dict"],
},
)
self.assertEqual(
results,
[
{
"a": merge_hash_data["high_dict"]["a"],
"b": merge_hash_data["high_dict"]["b"] + merge_hash_data["low_dict"]["b"],
}
],
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "deep", "replace", "replace", "replace", []],
)
@patch.object(
Templar, "template", side_effect=[deepcopy(merge_hash_data["low_dict"]), deepcopy(merge_hash_data["high_dict"])]
)
def test_merge_dict_deep_and_list_replace(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
["__merge_dict_deep_and_list_replace"],
{
"testdict1__merge_dict_deep_and_list_replace": merge_hash_data["low_dict"],
"testdict2__merge_dict_deep_and_list_replace": merge_hash_data["high_dict"],
},
)
self.assertEqual(
results,
[
{
"a": {"a": {"x": "low_value", "y": "high_value", "z": "high_value", "list": ["high_value"]}},
"b": merge_hash_data["high_dict"]["b"],
}
],
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "deep", "keep", "replace", "replace", []],
)
@patch.object(
Templar, "template", side_effect=[deepcopy(merge_hash_data["low_dict"]), deepcopy(merge_hash_data["high_dict"])]
)
def test_merge_dict_deep_and_list_keep(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
["__merge_dict_deep_and_list_keep"],
{
"testdict1__merge_dict_deep_and_list_keep": merge_hash_data["low_dict"],
"testdict2__merge_dict_deep_and_list_keep": merge_hash_data["high_dict"],
},
)
self.assertEqual(
results,
[
{
"a": {"a": {"x": "low_value", "y": "high_value", "z": "high_value", "list": ["low_value"]}},
"b": merge_hash_data["low_dict"]["b"],
}
],
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "deep", "append", "replace", "replace", []],
)
@patch.object(
Templar, "template", side_effect=[deepcopy(merge_hash_data["low_dict"]), deepcopy(merge_hash_data["high_dict"])]
)
def test_merge_dict_deep_and_list_append(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
["__merge_dict_deep_and_list_append"],
{
"testdict1__merge_dict_deep_and_list_append": merge_hash_data["low_dict"],
"testdict2__merge_dict_deep_and_list_append": merge_hash_data["high_dict"],
},
)
self.assertEqual(
results,
[
{
"a": {
"a": {
"x": "low_value",
"y": "high_value",
"z": "high_value",
"list": ["low_value", "high_value"],
}
},
"b": merge_hash_data["low_dict"]["b"] + merge_hash_data["high_dict"]["b"],
}
],
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "deep", "prepend", "replace", "replace", []],
)
@patch.object(
Templar, "template", side_effect=[deepcopy(merge_hash_data["low_dict"]), deepcopy(merge_hash_data["high_dict"])]
)
def test_merge_dict_deep_and_list_prepend(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
["__merge_dict_deep_and_list_prepend"],
{
"testdict1__merge_dict_deep_and_list_prepend": merge_hash_data["low_dict"],
"testdict2__merge_dict_deep_and_list_prepend": merge_hash_data["high_dict"],
},
)
self.assertEqual(
results,
[
{
"a": {
"a": {
"x": "low_value",
"y": "high_value",
"z": "high_value",
"list": ["high_value", "low_value"],
}
},
"b": merge_hash_data["high_dict"]["b"] + merge_hash_data["low_dict"]["b"],
}
],
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "replace", "append", "keep", "keep", []],
)
@patch.object(
Templar, "template", side_effect=[deepcopy(merge_hash_data["low_dict"]), deepcopy(merge_hash_data["high_dict"])]
)
def test_merge_dict_replace(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
["__merge_dict_replace"],
{
"testdict1__merge_dict_replace": merge_hash_data["low_dict"],
"testdict2__merge_dict_replace": merge_hash_data["high_dict"],
},
)
self.assertEqual(results, [merge_hash_data["high_dict"]])
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "keep", "append", "replace", "replace", []],
)
@patch.object(
Templar, "template", side_effect=[deepcopy(merge_hash_data["low_dict"]), deepcopy(merge_hash_data["high_dict"])]
)
def test_merge_dict_keep(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
["__merge_dict_keep"],
{
"testdict1__merge_dict_keep": merge_hash_data["low_dict"],
"testdict2__merge_dict_keep": merge_hash_data["high_dict"],
},
)
self.assertEqual(results, [merge_hash_data["low_dict"]])
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "deep", "merge", "replace", "replace", []],
)
@patch.object(
Templar,
"template",
side_effect=[[{"a": "b", "c": "d"}, {"c": "d"}], [{"a": "x", "y": "z"}, {"c": "z"}, {"a": "y"}]],
)
def test_merge_list_deep(self, mock_set_options, mock_get_option, mock_template):
results = self.merge_vars_lookup.run(
["__merge_list_deep"],
{
"testdict1__merge_list_deep": [{"a": "b", "c": "d"}, {"c": "d"}],
"testdict2__merge_list_deep": [{"a": "x", "y": "z"}, {"c": "z"}, {"a": "y"}],
},
)
self.assertEqual(results, [[{"a": "x", "c": "d", "y": "z"}, {"c": "z"}, {"a": "y"}]])
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[None, "ignore", "suffix", None, "deep", "append", "replace", "replace", ["dedup"]],
)
@patch.object(
Templar, "template", side_effect=[deepcopy(merge_hash_data["low_dict"]), deepcopy(merge_hash_data["high_dict"])]
)
def test_merge_dict_deep_and_list_append_with_dedup_keep_first(
self, mock_set_options, mock_get_option, mock_template
):
results = self.merge_vars_lookup.run(
["__merge_dict_deep_and_list_append_with_dedup_keep_first"],
{
"testdict1__merge_dict_deep_and_list_append_with_dedup_keep_first": merge_hash_data["low_dict"],
"testdict2__merge_dict_deep_and_list_append_with_dedup_keep_first": merge_hash_data["high_dict"],
},
)
self.assertEqual(
results,
[
{
"a": {
"a": {
"x": "low_value",
"y": "high_value",
"z": "high_value",
"list": ["low_value", "high_value"],
}
},
"b": [1, 2, 3, 4, {"5": "value"}],
}
],
)
@patch.object(AnsiblePlugin, "set_options")
@patch.object(
AnsiblePlugin,
"get_option",
side_effect=[
None,
"ignore",
"suffix",
None,
"deep",
"prepend",
"replace",
"replace",
[{"name": "dedup", "options": {"keep": "last"}}],
],
)
@patch.object(
Templar, "template", side_effect=[deepcopy(merge_hash_data["low_dict"]), deepcopy(merge_hash_data["high_dict"])]
)
def test_merge_dict_deep_and_list_prepend_with_dedup_keep_last(
self, mock_set_options, mock_get_option, mock_template
):
results = self.merge_vars_lookup.run(
["__merge_dict_deep_and_list_prepend_with_dedup_keep_last"],
{
"testdict1__merge_dict_deep_and_list_prepend_with_dedup_keep_last": merge_hash_data["low_dict"],
"testdict2__merge_dict_deep_and_list_prepend_with_dedup_keep_last": merge_hash_data["high_dict"],
},
)
self.assertEqual(
results,
[
{
"a": {
"a": {
"x": "low_value",
"y": "high_value",
"list": ["high_value", "low_value"],
"z": "high_value",
}
},
"b": [4, {"5": "value"}, 1, 2, 3],
}
],
)