test(ssh): cover key rotation with Molecule

Replace a private key while retaining its public key file to detect stale derived keys.
Verify the replacement public key, ownership and file permissions.

Co-authored-by: Copilot <copilot@github.com>

.gitea/workflows/ansible-linters.yml

@@ -12,11 +12,11 @@ jobs:
     runs-on:
     - ubuntu-latest
     steps:
-    - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
     - name: Run ansible-lint
-      uses: ansible/ansible-lint@665d9e07a1943254d2910faffc106adaf7ea7294 # v26.8.0
+      uses: ansible/ansible-lint@9bce2bfb8f4a984e6b13767bc0d6bf0a49492164 # v26.10.1
       with:
-        args: "--config-file .ansible-lint"
+        args: "--config-file .ansible-lint.yml"
         # The molecule scenario is linted as well, so its collections are required beside the ones of the role.
         requirements_file: "molecule/default/collections.yml"
         setup_python: "true"

.gitea/workflows/markdown-linters.yml

@@ -12,7 +12,7 @@ jobs:
     runs-on:
     - ubuntu-latest
     steps:
-    - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
     - uses: DavidAnson/markdownlint-cli2-action@ded1f9488f68a970bc66ea5619e13e9b52e601cd # v23.2.0
       with:
         globs: '**/*.md'

.gitea/workflows/molecule.yml

@@ -16,7 +16,7 @@ jobs:
     runs-on: ubuntu-latest-amd64
     steps:
     # The scenario includes the role by its directory name, whose parent is used as roles path.
-    - uses: actions/checkout@d23441a48e516b6c34aea4fa41551a30e30af803 # v6.1.0
+    - uses: actions/checkout@3d3c42e5aac5ba805825da76410c181273ba90b1 # v7.0.1
       with:
         path: unix-users
     - name: Install molecule

README.md

@@ -138,10 +138,11 @@ environment variables can be espcilitly defined, to consume it during execution
 > [!IMPORTANT]
 > To allow consuming environment variables must be set `PermitUserEnvironment yes` in `/etc/ssh/sshd_config`.
 
-The private key `toor@toor-pc.ed25519.key` must be stored in `ssh/private_keys`. The public key will be automatically
-extracted from the private key.
+Private keys are configured using `filename` and `content`; the role writes them to `~/.ssh/<filename>` and extracts
+their public keys to `~/.ssh/<filename>.pub`. Store private key content encrypted with Ansible Vault.
 
-The public keys `claire@claire-pc.pub` as well as `bob@bob-pc.pub` must be stored in `ssh/authorized_keys`.
+Entries in `authorized_keys` are also added to `~/.ssh/authorized_keys`, including their optional command and environment
+settings.
 
 ```yaml
 unix_users:
@@ -156,14 +157,18 @@ unix_users:
         StrictHostKeyChecking: "no"
         UserKnownHostFile: /dev/null
       authorized_keys:
-      - filename: claire@claire-pc.pub
+      - content: ssh-ed25519 AAAAC3NzaC... claire
       - command: /usr/local/bin/upload-file.sh
         envs:
         - key: SSH_KEY_NAME
           value: bob@bob-pc
-        filename: bob@bob-pc.pub
+        content: ssh-ed25519 AAAAC3NzaC... bob
       private_keys:
-      - toor@toor-pc.ed25519.key
+      - filename: toor@toor-pc.ed25519.key
+        content: |
+          -----BEGIN OPENSSH PRIVATE KEY-----
+          ...
+          -----END OPENSSH PRIVATE KEY-----
     shell: /bin/bash
     password: toor
     group: toor

defaults/main.yml

@@ -9,20 +9,24 @@ unix_groups: {}
 #     state: absent
 
 unix_users: {}
-#   alice:
-#     state: "present"
-#     name: "Alice"
-#     uid: "1000"
-#     ssh:
-#       config:
-#       - Host: "*"
-#         StrictHostKeyChecking: "no"
-#         UserKnownHostFile: /dev/null
-#       authorized_keys:
-#       - filename: alice@alice-pc.pub
-#         # command: "" # optional
-#       private_keys:
-#       - alice@alice-pc.ed25519.key
+# alice:
+#   state: "present"
+#   name: "Alice"
+#   uid: "1000"
+#   ssh:
+#     config:
+#     - Host: "*"
+#       StrictHostKeyChecking: "no"
+#       UserKnownHostFile: /dev/null
+#     authorized_keys:
+#     - content: ssh-ed25519 AAAAC3NzaC... alice
+#       # command: "" # optional
+#     private_keys:
+#     - filename: alice@alice-pc.ed25519.key
+#       content: |
+#         -----BEGIN OPENSSH PRIVATE KEY-----
+#         ...
+#         -----END OPENSSH PRIVATE KEY-----
 #     home: /home/alice
 #     btrfs: false
 #     netrc:

molecule/default/converge.yml

@@ -28,13 +28,15 @@
           - Host: "*"
             StrictHostKeyChecking: "no"
           authorized_keys:
-          - filename: molecule.pub
+          - content: "{{ lookup('ansible.builtin.file', molecule_scenario_directory + '/files/ssh/authorized_keys/molecule.pub', rstrip=false) }}"
             command: "/usr/bin/true"
             envs:
             - key: EDITOR
               value: vi
+          - content: "{{ lookup('ansible.builtin.file', molecule_scenario_directory + '/files/ssh/authorized_keys/molecule.pub', rstrip=false) }}"
           private_keys:
-          - molecule.ed25519.key
+          - filename: alice.ed25519.key
+            content: "{{ lookup('ansible.builtin.file', molecule_scenario_directory + '/files/ssh/private_keys/molecule.ed25519.key', rstrip=false) }}"
         netrc:
         - machine: hostname.local
           login: alice

molecule/default/create.yml

@@ -37,10 +37,22 @@
     - "{{ _authorized_key }}"
 
   # The key pair is generated instead of committed, private key material does not belong into a repository.
-  - name: Generate the ssh key pair the role is fed with
+  - name: Generate the SSH key pairs for initial configuration and rotation
     ansible.builtin.command:
-      cmd: "ssh-keygen -t ed25519 -N '' -C molecule -f {{ _private_key }}"
-      creates: "{{ _private_key }}"
+      argv:
+      - ssh-keygen
+      - -t
+      - ed25519
+      - -N
+      - ''
+      - -C
+      - molecule
+      - -f
+      - "{{ item }}"
+      creates: "{{ item }}"
+    loop:
+    - "{{ _private_key }}"
+    - "{{ _private_key }}.replacement"
 
   - name: Offer the public key as authorized key fixture
     ansible.builtin.copy:

molecule/default/verify.yml

@@ -9,8 +9,8 @@
       /home/molecule-alice/.ssh: "0700"
       /home/molecule-alice/.ssh/authorized_keys: "0600"
       /home/molecule-alice/.ssh/config: "0644"
-      /home/molecule-alice/.ssh/molecule.ed25519.key: "0600"
-      /home/molecule-alice/.ssh/molecule.ed25519.key.pub: "0644"
+      /home/molecule-alice/.ssh/alice.ed25519.key: "0600"
+      /home/molecule-alice/.ssh/alice.ed25519.key.pub: "0644"
       /home/molecule-alice/.forward: "0644"
       /home/molecule-alice/.netrc: "0600"
       /home/molecule-alice/.bashrc.d: "0755"
@@ -72,7 +72,7 @@
     register: _alice_keys
     loop:
     - "{{ _alice_home }}/.ssh/authorized_keys"
-    - "{{ _alice_home }}/.ssh/molecule.ed25519.key.pub"
+    - "{{ _alice_home }}/.ssh/alice.ed25519.key.pub"
 
   - name: Assert that the authorized key carries its options and matches the extracted public key
     vars:
@@ -83,6 +83,7 @@
       - _authorized_key.startswith('command="/usr/bin/true",environment="EDITOR=vi" ')
       - _public_key.startswith('ssh-ed25519 ')
       - _public_key.split()[1] == _authorized_key.split()[2]
+      - (_alice_keys.results[0].content | b64decode).splitlines() | select('match', '^ssh-ed25519 ') | list | length == 1
       fail_msg: "the authorized key of molecule-alice does not match the key extracted from its private key"
 
   - name: Read the user and group database
@@ -141,3 +142,57 @@
       - _bob_files.results[0].stat.exists
       - not _bob_files.results[1:] | map(attribute='stat.exists') | select | list
       fail_msg: "molecule-bob has files that were never declared for him"
+
+- name: Verify SSH key rotation
+  hosts: all
+  vars:
+    _replacement_key: "{{ molecule_scenario_directory }}/files/ssh/private_keys/molecule.ed25519.key.replacement"
+    unix_groups: {}
+    unix_users:
+      molecule-alice:
+        group: molecule-alice
+        password: alice
+        ssh:
+          private_keys:
+          - filename: alice.ed25519.key
+            content: "{{ lookup('ansible.builtin.file', _replacement_key, rstrip=false) }}"
+        state: present
+  tasks:
+  - name: Replace the private key while keeping the existing public key file
+    ansible.builtin.include_role:
+      name: unix-users
+
+  - name: Read the public key after rotation
+    ansible.builtin.slurp:
+      src: /home/molecule-alice/.ssh/alice.ed25519.key.pub
+    register: _rotated_public_key
+
+  - name: Assert the existing public key matches the replacement key and differs from the original
+    vars:
+      _expected_public_key: "{{ lookup('ansible.builtin.file', _replacement_key + '.pub') }}"
+    ansible.builtin.assert:
+      that:
+      - (_rotated_public_key.content | b64decode).split()[1] == _expected_public_key.split()[1]
+      - _rotated_public_key.content != _alice_keys.results[1].content
+
+  - name: Stat the rotated private and public keys
+    ansible.builtin.stat:
+      path: "{{ item.path }}"
+    loop:
+    - path: /home/molecule-alice/.ssh/alice.ed25519.key
+      mode: '0600'
+    - path: /home/molecule-alice/.ssh/alice.ed25519.key.pub
+      mode: '0644'
+    loop_control:
+      label: "{{ item.path }}"
+    register: _rotated_key_files
+
+  - name: Assert rotated keys retain their owner and permissions
+    ansible.builtin.assert:
+      that:
+      - item.stat.pw_name == 'molecule-alice'
+      - item.stat.gr_name == 'molecule-alice'
+      - item.stat.mode == item.item.mode
+    loop: "{{ _rotated_key_files.results }}"
+    loop_control:
+      label: "{{ item.item.path }}"

tasks/create_unix_user.yml

@@ -23,10 +23,17 @@
       msg: "Determined device {{ _unix_users_btrfs_device.stdout }} does not have a btrfs filesystem"
     when: _unix_users_device_filesystem != 'btrfs'
 
+  # Select by UUID, device matching fails for symlinked devices like /dev/mapper/* (community.general#11331).
+  - name: "Find btrfs filesystem UUID"
+    ansible.builtin.command:
+      cmd: "findmnt -no UUID -T {{ _unix_users_home }}"
+    register: _unix_users_btrfs_uuid
+    changed_when: false
+
   # The subvolume stays root owned until the user exists. It is chowned further below, after the user was created.
   - name: "Create btrfs volume for unix user: {{ unix_user.key }}"
     community.general.btrfs_subvolume:
-      filesystem_device: "{{ _unix_users_btrfs_device.stdout }}"
+      filesystem_uuid: "{{ _unix_users_btrfs_uuid.stdout }}"
       name: "{{ _unix_users_home }}"
       state: present
 
@@ -49,7 +56,7 @@
         unix_user.value.btrfs
   block:
   - name: "Copy skel files"
-    ansible.builtin.include_tasks: copy_skel_file.yaml
+    ansible.builtin.include_tasks: copy_skel_file.yml
     loop_control:
       loop_var: skel_file
     with_items:
@@ -88,11 +95,10 @@
     state: absent
   when: unix_user.value.ssh.authorized_keys is not defined or unix_user.value.ssh.authorized_keys | length <= 0
 
-# The relative source is resolved against the files directory of the playbook, like the lookup in authorized_keys.j2.
 - name: "Create private SSH keys for unix user: {{ unix_user.key }}"
   ansible.builtin.copy:
-    src: "ssh/private_keys/{{ item }}"
-    dest: "{{ _unix_users_home }}/.ssh/{{ item }}"
+    content: "{{ item.content }}"
+    dest: "{{ _unix_users_home }}/.ssh/{{ item.filename }}"
     owner: "{{ unix_user.key }}"
     group: "{{ unix_user.value.group | default('users') }}"
     mode: "0600"
@@ -102,21 +108,26 @@
   when: unix_user.value.ssh.private_keys is defined and unix_user.value.ssh.private_keys | length > 0
 
 - name: "Extract public SSH keys from private keys for unix user: {{ unix_user.key }}"
-  ansible.builtin.shell:
-    cmd: "ssh-keygen -y -f {{ _unix_users_home }}/.ssh/{{ item }} > {{ _unix_users_home }}/.ssh/{{ item }}.pub"
-    creates: "{{ _unix_users_home }}/.ssh/{{ item }}.pub"
+  ansible.builtin.command:
+    argv: [ "ssh-keygen", "-y", "-f", "{{ _unix_users_home }}/.ssh/{{ item.filename }}" ]
+  changed_when: false
+  register: _unix_users_ssh_pub_key_extraction
+  loop_control:
+    label: "{{ item.filename }}"
   with_items:
   - "{{ unix_user.value.ssh.private_keys }}"
   when: unix_user.value.ssh.private_keys is defined and unix_user.value.ssh.private_keys | length > 0
 
-- name: "Correct permissions of public SSH keys for unix user: {{ unix_user.key }}"
-  ansible.builtin.file:
-    path: "{{ _unix_users_home }}/.ssh/{{ item }}.pub"
+- name: "Write derived public SSH keys for unix user: {{ unix_user.key }}"
+  ansible.builtin.copy:
+    content: "{{ item.stdout }}\n"
+    dest: "{{ _unix_users_home }}/.ssh/{{ item.item.filename }}.pub"
     owner: "{{ unix_user.key }}"
     group: "{{ unix_user.value.group | default('users') }}"
     mode: "0644"
-  with_items:
-  - "{{ unix_user.value.ssh.private_keys }}"
+  loop_control:
+    label: "{{ item.item.filename }}"
+  loop: "{{ _unix_users_ssh_pub_key_extraction.results }}"
   when: unix_user.value.ssh.private_keys is defined and unix_user.value.ssh.private_keys | length > 0
 
 - name: "Create custom SSH client config for unix user: {{ unix_user.key }}"
@@ -164,7 +175,7 @@
 
 - name: "Create shell rc files"
   when: unix_user.value.shell_rc_files is defined
-  ansible.builtin.include_tasks: create_shell_rc_file.yaml
+  ansible.builtin.include_tasks: create_shell_rc_file.yml
   with_items:
   - "{{ unix_user.value.shell_rc_files }}"
   loop_control:

tasks/main.yml

@@ -1,7 +1,7 @@
 ---
 
 - name: Remove unix user
-  ansible.builtin.include_tasks: remove_unix_user.yaml
+  ansible.builtin.include_tasks: remove_unix_user.yml
   with_dict: "{{ unix_users }}"
   loop_control:
     loop_var: unix_user
@@ -11,7 +11,7 @@
         unix_user.value.state == 'absent'
 
 - name: Remove unix groups
-  ansible.builtin.include_tasks: remove_unix_group.yaml
+  ansible.builtin.include_tasks: remove_unix_group.yml
   with_dict: "{{ unix_groups }}"
   loop_control:
     loop_var: unix_group
@@ -21,7 +21,7 @@
         unix_group.value.state == 'absent'
 
 - name: Create unix groups
-  ansible.builtin.include_tasks: create_unix_group.yaml
+  ansible.builtin.include_tasks: create_unix_group.yml
   with_dict: "{{ unix_groups }}"
   loop_control:
     loop_var: unix_group
@@ -33,7 +33,7 @@
         )
 
 - name: Create unix users
-  ansible.builtin.include_tasks: create_unix_user.yaml
+  ansible.builtin.include_tasks: create_unix_user.yml
   no_log: true
   with_dict: "{{ unix_users }}"
   loop_control:

templates/authorized_keys.j2

@@ -21,8 +21,8 @@
     {% endif %}
   {% endif %}
   {% if _args | length > 0 %}
-{{ _args | join(',') }} {{ lookup('file', 'ssh/authorized_keys/' + authorized_key.filename ) }}
+{{ _args | join(',') }} {{ authorized_key.content | trim }}
   {% else %}
-{{ lookup('file', 'ssh/authorized_keys/' + authorized_key.filename ) }}
+{{ authorized_key.content | trim }}
   {% endif %}
 {% endfor %}