locale_gen: search for available locales in /usr/local as well (#11046)

* locale_gen: search for available locales in /usr/local as well

* better var name

* add test for /usr/local

* Apply suggestions from code review

Co-authored-by: Felix Fontein <felix@fontein.de>

* skip /usr/local/ for Archlinux

* improve/update documentation

* add license file for the custom locale

* add changelog frag

* Update plugins/modules/locale_gen.py

Co-authored-by: Felix Fontein <felix@fontein.de>

* Update changelogs/fragments/11046-locale-gen-usrlocal.yml

Co-authored-by: Felix Fontein <felix@fontein.de>

---------

Co-authored-by: Felix Fontein <felix@fontein.de>

changelogs/fragments/11046-locale-gen-usrlocal.yml

@@ -0,0 +1,2 @@
+minor_changes:
+  - locale_gen - extend the search for available locales to include ``/usr/local/share/i18n/SUPPORTED`` in Debian and Ubuntu systems (https://github.com/ansible-collections/community.general/issues/10964, https://github.com/ansible-collections/community.general/pull/11046).

plugins/modules/locale_gen.py

@@ -34,17 +34,20 @@ options:
       - Whether the locales shall be present.
     choices: [absent, present]
     default: present
+
 notes:
+  - Currently the module is B(only supported for Debian, Ubuntu, and Arch Linux) systems.
+  - This module requires the package C(locales) installed in Debian and Ubuntu systems.
   - If C(/etc/locale.gen) exists, the module assumes to be using the B(glibc) mechanism, else if C(/var/lib/locales/supported.d/)
     exists it assumes to be using the B(ubuntu_legacy) mechanism, else it raises an error.
-  - When using glibc mechanism, it manages locales by editing C(/etc/locale.gen) and running C(locale-gen).
+  - When using V(glibc) mechanism, it manages locales by editing C(/etc/locale.gen) and running C(locale-gen).
-  - When using ubuntu_legacy mechanism, it manages locales by editing C(/var/lib/locales/supported.d/local) and then running
+  - When using V(ubuntu_legacy) mechanism, it manages locales by editing C(/var/lib/locales/supported.d/local) and then running
     C(locale-gen).
-  - Please note that the code path that uses ubuntu_legacy mechanism has not been tested for a while, because Ubuntu is already
+  - Please note that the module asserts the availability of the locale by checking the files C(/usr/share/i18n/SUPPORTED) and
-    using the glibc mechanism. There is no support for that, given our inability to test it. Therefore, that mechanism is
+    C(/usr/local/share/i18n/SUPPORTED), but the C(/usr/local) one is not supported by Archlinux.
-    B(deprecated) and will be removed in community.general 13.0.0.
+  - Please note that the code path that uses V(ubuntu_legacy) mechanism has not been tested for a while, because recent versions of
-  - Currently the module is B(only supported for Debian and Ubuntu) systems.
+    Ubuntu is already using the V(glibc) mechanism. There is no support for V(ubuntu_legacy), given our inability to test it.
-  - This module requires the package C(locales) installed in Debian and Ubuntu systems.
+    Therefore, that mechanism is B(deprecated) and will be removed in community.general 13.0.0.
 """
 
 EXAMPLES = r"""
@@ -85,7 +88,7 @@ from ansible_collections.community.general.plugins.module_utils.locale_gen impor
 ETC_LOCALE_GEN = "/etc/locale.gen"
 VAR_LIB_LOCALES = "/var/lib/locales/supported.d"
 VAR_LIB_LOCALES_LOCAL = os.path.join(VAR_LIB_LOCALES, "local")
-SUPPORTED_LOCALES = "/usr/share/i18n/SUPPORTED"
+SUPPORTED_LOCALES = ["/usr/share/i18n/SUPPORTED", "/usr/local/share/i18n/SUPPORTED"]
 LOCALE_NORMALIZATION = {
     ".utf8": ".UTF-8",
     ".eucjp": ".EUC-JP",
@@ -111,7 +114,7 @@ class LocaleGen(StateModuleHelper):
     )
 
     def __init_module__(self):
-        self.MECHANISMS = dict(
+        self.mechanisms = dict(
             ubuntu_legacy=dict(
                 available=SUPPORTED_LOCALES,
                 apply_change=self.apply_change_ubuntu_legacy,
@@ -156,19 +159,21 @@ class LocaleGen(StateModuleHelper):
         checking either :
         * if the locale is present in /etc/locales.gen
         * or if the locale is present in /usr/share/i18n/SUPPORTED"""
-        regexp = r"^\s*#?\s*(?P<locale>\S+[\._\S]+) (?P<charset>\S+)\s*$"
-        locales_available = self.MECHANISMS[self.vars.mechanism]["available"]
 
-        re_compiled = re.compile(regexp)
+        self.vars.set("available_lines", [], verbosity=4)
-        with open(locales_available, "r") as fd:
+        available_locale_entry_re_matches = []
-            lines = fd.readlines()
+        for locale_path in self.mechanisms[self.vars.mechanism]["available"]:
-        res = [re_compiled.match(line) for line in lines]
+            if os.path.exists(locale_path):
-        self.vars.set("available_lines", lines, verbosity=4)
+                with open(locale_path, "r") as fd:
+                    self.vars.available_lines.extend(fd.readlines())
+
+        re_locale_entry = re.compile(r"^\s*#?\s*(?P<locale>\S+[\._\S]+) (?P<charset>\S+)\s*$")
+        available_locale_entry_re_matches.extend([re_locale_entry.match(line) for line in self.vars.available_lines])
 
         locales_not_found = []
         for locale in self.vars.name:
             # Check if the locale is not found in any of the matches
-            if not any(match and match.group("locale") == locale for match in res):
+            if not any(match and match.group("locale") == locale for match in available_locale_entry_re_matches):
                 locales_not_found.append(locale)
 
         # locale may be installed but not listed in the file, for example C.UTF-8 in some systems
@@ -219,38 +224,41 @@ class LocaleGen(StateModuleHelper):
             re_search = re.compile(search_string)
             locale_regexes.append([re_search, new_string])
 
-        for i in range(len(lines)):
+        def search_replace(line):
             for [search, replace] in locale_regexes:
-                lines[i] = search.sub(replace, lines[i])
+                line = search.sub(replace, line)
+            return line
+
+        lines = [search_replace(line) for line in lines]
 
         # Write the modified content back to the file
         with open(ETC_LOCALE_GEN, "w") as fw:
             fw.writelines(lines)
 
-    def apply_change_glibc(self, targetState, names):
+    def apply_change_glibc(self, target_state, names):
         """Create or remove locale.
 
         Keyword arguments:
-        targetState -- Desired state, either present or absent.
+        target_state -- Desired state, either present or absent.
         names -- Names list including encoding such as de_CH.UTF-8.
         """
 
-        self.set_locale_glibc(names, enabled=(targetState == "present"))
+        self.set_locale_glibc(names, enabled=(target_state == "present"))
 
         runner = locale_gen_runner(self.module)
         with runner() as ctx:
             ctx.run()
 
-    def apply_change_ubuntu_legacy(self, targetState, names):
+    def apply_change_ubuntu_legacy(self, target_state, names):
         """Create or remove locale.
 
         Keyword arguments:
-        targetState -- Desired state, either present or absent.
+        target_state -- Desired state, either present or absent.
         names -- Name list including encoding such as de_CH.UTF-8.
         """
         runner = locale_gen_runner(self.module)
 
-        if targetState == "present":
+        if target_state == "present":
             # Create locale.
             # Ubuntu's patched locale-gen automatically adds the new locale to /var/lib/locales/supported.d/local
             with runner() as ctx:
@@ -273,7 +281,7 @@ class LocaleGen(StateModuleHelper):
     def __state_fallback__(self):
         if self.vars.state_tracking == self.vars.state:
             return
-        self.MECHANISMS[self.vars.mechanism]["apply_change"](self.vars.state, self.vars.name)
+        self.mechanisms[self.vars.mechanism]["apply_change"](self.vars.state, self.vars.name)
 
 
 def main():

tests/integration/targets/locale_gen/files/en_US@iso

@@ -0,0 +1,120 @@
+comment_char %
+escape_char /
+
+% This file is part of the GNU C Library and contains locale data.
+% The Free Software Foundation does not claim any copyright interest
+% in the locale data contained in this file.  The foregoing does not
+% affect the license of the GNU C Library as a whole.  It does not
+% exempt you from the conditions of the license if your use would
+% otherwise be governed by that license.
+
+LC_IDENTIFICATION
+title      "English locale for the USA with ISO formats"
+language   "American English"
+
+category "i18n:2012";LC_IDENTIFICATION
+category "i18n:2012";LC_CTYPE
+category "i18n:2012";LC_COLLATE
+category "i18n:2012";LC_TIME
+category "i18n:2012";LC_NUMERIC
+category "i18n:2012";LC_MONETARY
+category "i18n:2012";LC_MESSAGES
+category "i18n:2012";LC_PAPER
+category "i18n:2012";LC_NAME
+category "i18n:2012";LC_ADDRESS
+category "i18n:2012";LC_TELEPHONE
+category "i18n:2012";LC_MEASUREMENT
+END LC_IDENTIFICATION
+
+LC_TIME
+day   "Sunday";/
+      "Monday";/
+      "Tuesday";/
+      "Wednesday";/
+      "Thursday";/
+      "Friday";/
+      "Saturday"
+abday "Sun";/
+      "Mon";/
+      "Tue";/
+      "Wed";/
+      "Thu";/
+      "Fri";/
+      "Sat"
+
+mon   "January";/
+      "February";/
+      "March";/
+      "April";/
+      "May";/
+      "June";/
+      "July";/
+      "August";/
+      "September";/
+      "October";/
+      "November";/
+      "December"
+abmon "Jan";/
+      "Feb";/
+      "Mar";/
+      "Apr";/
+      "May";/
+      "Jun";/
+      "Jul";/
+      "Aug";/
+      "Sep";/
+      "Oct";/
+      "Nov";/
+      "Dec"
+
+date_fmt      "%a %d %b %Y %T %Z"
+d_t_fmt       "%a %d %b %Y %T"
+d_fmt         "%Y-%m-%d"
+t_fmt         "%T"
+t_fmt_ampm    ""
+am_pm         "";""
+week          7;19971130;4
+first_weekday 2
+END LC_TIME
+
+LC_CTYPE
+copy "en_US"
+END LC_CTYPE
+
+LC_COLLATE
+copy "en_US"
+END LC_COLLATE
+
+LC_MONETARY
+copy "en_US"
+END LC_MONETARY
+
+LC_NUMERIC
+decimal_point "."
+thousands_sep ","
+grouping      3;3
+END LC_NUMERIC
+
+LC_MESSAGES
+copy "en_US"
+END LC_MESSAGES
+
+LC_PAPER
+copy "i18n"
+END LC_PAPER
+
+LC_TELEPHONE
+copy "en_US"
+END LC_TELEPHONE
+
+LC_MEASUREMENT
+copy "i18n"
+END LC_MEASUREMENT
+
+LC_NAME
+copy "en_US"
+END LC_NAME
+
+LC_ADDRESS
+copy "en_US"
+END LC_ADDRESS

tests/integration/targets/locale_gen/files/en_US@iso.license

@@ -0,0 +1,3 @@
+Copyright (c) Ansible Project
+GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+SPDX-License-Identifier: GPL-3.0-or-later

tests/integration/targets/locale_gen/tasks/11046-usrlocal.yml

@@ -0,0 +1,85 @@
+---
+# Copyright (c) Ansible Project
+# GNU General Public License v3.0+ (see LICENSES/GPL-3.0-or-later.txt or https://www.gnu.org/licenses/gpl-3.0.txt)
+# SPDX-License-Identifier: GPL-3.0-or-later
+
+- name: Arch-less block
+  when: ansible_distribution not in ['Archlinux']
+  block:
+    - name: Is the locale we're going to test against installed?
+      command: locale -a
+      register: initial_state
+
+    - name: Make sure the locale is not available
+      community.general.locale_gen:
+        name: en_US@iso
+        state: absent
+      ignore_errors: true
+      register: not_available
+
+    - name: Ensure /usr/local/share/i18n/
+      ansible.builtin.file:
+        path: /usr/local/share/i18n/locales
+        state: directory
+        mode: '0755'
+
+    - name: Back up /etc/locale.gen
+      ansible.builtin.copy:
+        src: /etc/locale.gen
+        dest: /etc/locale.gen.bkp
+        remote_src: true
+
+    - name: Add line to /etc/locale.gen
+      ansible.builtin.shell: >
+        echo "# en_US@iso UTF-8" >> /etc/locale.gen
+
+    - name: Copy custom locale
+      ansible.builtin.copy:
+        dest: /usr/local/share/i18n/locales/en_US@iso
+        src: en_US@iso
+        mode: '0644'
+
+    - name: Add custom locale to SUPPORTED
+      ansible.builtin.copy:
+        dest: /usr/local/share/i18n/SUPPORTED
+        content: |
+          en_US@iso UTF-8
+        mode: '0644'
+
+    - name: Make sure the locale is available
+      community.general.locale_gen:
+        name: en_US@iso
+        state: absent
+      register: available
+
+    - name: Make sure the locale is installed
+      community.general.locale_gen:
+        name: en_US@iso
+        state: present
+      register: installed
+
+    - name: Check assertions
+      ansible.builtin.assert:
+        that:
+          - not_available is failed
+          - >
+            "locales you have entered are not available on your system: en_US@iso" in not_available.msg
+          - available is not changed
+          - installed is changed
+
+  always:
+    - name: Make sure the locale is not installed
+      community.general.locale_gen:
+        name: en_US@iso
+        state: absent
+
+    - name: Remove /usr/local/share/i18n/
+      ansible.builtin.file:
+        path: /usr/local/share/i18n/
+        state: absent
+
+    - name: Restore /etc/locale.gen
+      ansible.builtin.copy:
+        src: /etc/locale.gen.bkp
+        dest: /etc/locale.gen
+        remote_src: true

tests/integration/targets/locale_gen/tasks/main.yml

@@ -17,3 +17,6 @@
   loop: "{{ locale_list_basic }}"
   loop_control:
     loop_var: locale_basic
+
+- name: Run tests for 11046
+  ansible.builtin.include_tasks: 11046-usrlocal.yml