NS8 Hermes Agent module

[Stll0 (Stefano Fancello)]

@capote

Jun 04 11:43:30 daho-ns8 agent@hermes-agent2[3427]: chown: der Eigentümer von './systemd/user/hermes@.service.bak' wird geändert: Die Operation ist nicht erlaubt
Jun 04 11:43:30 daho-ns8 agent@hermes-agent2[3427]: chown: der Eigentümer von './systemd/user/hermes@.service.d' wird geändert: Die Operation ist nicht erlaubt

always wrong owner on files you added, chown them

[capote (Marko)]

Perhaps The error:

chown: the owner of “./systemd/user/hermes@.service.bak” is being changed: The operation is not ermittelnd

The extract-image script includes set -e and, at the end, chown -cR --reference=. ... If a file does not belong to the user (e.g. a .bak file created by systemd), chown fails and the script terminates.

[capote (Marko)]

chown -R hermes-agent2:hermes-agent2 /home/hermes-agent2/.config/systemd/user/hermes@.service.bak
chown -R hermes-agent2:hermes-agent2 /home/hermes-agent2/.config/systemd/user/hermes@.service.d

api-cli run update-module --data '{"module_url":"ghcr.io/nethserver/hermes-agent:0.5.0","instances":["hermes-agent2"],"force":true}'

The update is currently in progress. Hopefully it will run smoothly until the end.

[capote (Marko)]

root@daho-ns8:/# runagent -m hermes-agent2 podman ps -a
ERRO[0000] User-selected graph driver "overlay" overwritten by graph driver "vfs" from database - delete libpod local files ("/home/hermes-agent2/.local/share/containers/storage") to resolve.  May prevent use of images created by other tools
CONTAINER ID  IMAGE                                         COMMAND               CREATED         STATUS         PORTS                      NAMES
dac201234dc0  localhost/podman-pause:5.4.2-1766335321                             12 hours ago    Up 12 hours                               hermes-pod-1-infra
22faf8d24727  ghcr.io/nethserver/hermes-agent-hermes:0.5.0  hermes gateway ru...  11 minutes ago  Up 11 minutes                             hermes-1
9d06a951ea27  ghcr.io/nethserver/hermes-agent-socket:0.5.0  -d -d UNIX-LISTEN...  11 minutes ago  Up 11 minutes                             hermes-socket-1
66a72193fc57  ghcr.io/nethserver/hermes-agent-auth:0.5.0    python /app/authp...  11 minutes ago  Up 11 minutes  127.0.0.1:20005->9119/tcp  hermes-auth
root@daho-ns8:/# runagent -m hermes-agent2 env | grep -i hermes_image
HERMES_AGENT_HERMES_IMAGE=ghcr.io/nethserver/hermes-agent-hermes:0.5.0
PREV_HERMES_AGENT_HERMES_IMAGE=ghcr.io/stell0/hermes-agent-hermes:0.4.1
root@daho-ns8:/# loginctl user-status hermes-agent2 | head -20
hermes-agent2 (1020)
   Since: Wed 2026-06-03 23:49:23 CEST; 2h 40min ago
   State: lingering
Sessions: 17
  Linger: yes
    Unit: user-1020.slice
          └─user@1020.service
            ├─app.slice
            │ ├─agent.service
            │ │ └─3427 /usr/local/bin/agent --agentid=module/hermes-agent2 --actionsdir=/usr/local/agent/actions --actionsdir=/home/hermes-agent2/.config/actions --eventsdir=/home/hermes-agent2/.config/events
            │ ├─app-hermes.slice
            │ │ └─hermes@1.service
            │ │   ├─ 5283 /usr/bin/slirp4netns --mtu=65520 --enable-sandbox --enable-seccomp --enable-ipv6 -c -r 3 -e 4 --netns-type=path /run/user/1020/netns/netns-0dadcd9a-64b7-a0f9-0ff6-2aa1305f01b9 tap0
            │ │   ├─ 5287 /usr/bin/conmon --api-version 1 -c dac201234dc08c8e95dae650a774c2adf5aa4df254cd6a1e01217853995cbe57 -u dac201234dc08c8e95dae650a774c2adf5aa4df254cd6a1e01217853995cbe57 -r /usr/bin/crun -b /home/hermes-agent2/.local/share/containers/storage/vfs-containers/dac201234dc08c8e95dae650a774c2adf5aa4df254cd6a1e01217853995cbe57/userdata -p /run/user/1020/containers/vfs-containers/dac201234dc08c8e95dae650a774c2adf5aa4df254cd6a1e01217853995cbe57/userdata/pidfile -n hermes-pod-1-infra --exit-dir /run/user/1020/libpod/tmp/exits --persist-dir /run/user/1020/libpod/tmp/persist/dac201234dc08c8e95dae650a774c2adf5aa4df254cd6a1e01217853995cbe57 --full-attach -s -l journald --log-level warning --syslog --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/user/1020/containers/vfs-containers/dac201234dc08c8e95dae650a774c2adf5aa4df254cd6a1e01217853995cbe57/userdata/oci-log --conmon-pidfile /run/user/1020/containers/vfs-containers/dac201234dc08c8e95dae650a774c2adf5aa4df254cd6a1e01217853995cbe57/userdata/conmon.pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /home/hermes-agent2/.local/share/containers/storage --exit-command-arg --runroot --exit-command-arg /run/user/1020/containers --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/user/1020/libpod/tmp --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /home/hermes-agent2/.local/share/containers/storage/volumes --exit-command-arg --db-backend --exit-command-arg boltdb --exit-command-arg --transient-store=false --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg vfs --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --stopped-only --exit-command-arg dac201234dc08c8e95dae650a774c2adf5aa4df254cd6a1e01217853995cbe57
            │ │   ├─84627 /usr/bin/podman run --name hermes-1 --replace --rm --sdnotify=conmon --cgroups=no-conmon --pod hermes-pod-1 --env-file /home/hermes-agent2/.config/state/agents/1/agent.env --env-file /home/hermes-agent2/.config/state/secrets/1.env --env API_SERVER_ENABLED=true --env API_SERVER_HOST=127.0.0.1 -v /home/hermes-agent2/.local/share/containers/storage/volumes/hermes-agents-home/_data/1:/opt/data --tz=UTC ghcr.io/nethserver/hermes-agent-hermes:0.5.0 hermes gateway run
            │ │   └─84646 /usr/bin/conmon --api-version 1 -c 22faf8d247279e2523165f6116a3f5030c946adb1e919d5a252b94841fa6be2b -u 22faf8d247279e2523165f6116a3f5030c946adb1e919d5a252b94841fa6be2b -r /usr/bin/crun -b /home/hermes-agent2/.local/share/containers/storage/vfs-containers/22faf8d247279e2523165f6116a3f5030c946adb1e919d5a252b94841fa6be2b/userdata -p /run/user/1020/containers/vfs-containers/22faf8d247279e2523165f6116a3f5030c946adb1e919d5a252b94841fa6be2b/userdata/pidfile -n hermes-1 --exit-dir /run/user/1020/libpod/tmp/exits --persist-dir /run/user/1020/libpod/tmp/persist/22faf8d247279e2523165f6116a3f5030c946adb1e919d5a252b94841fa6be2b --full-attach -s -l journald --log-level warning --syslog --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/user/1020/containers/vfs-containers/22faf8d247279e2523165f6116a3f5030c946adb1e919d5a252b94841fa6be2b/userdata/oci-log --conmon-pidfile /run/user/1020/containers/vfs-containers/22faf8d247279e2523165f6116a3f5030c946adb1e919d5a252b94841fa6be2b/userdata/conmon.pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /home/hermes-agent2/.local/share/containers/storage --exit-command-arg --runroot --exit-command-arg /run/user/1020/containers --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/user/1020/libpod/tmp --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /home/hermes-agent2/.local/share/containers/storage/volumes --exit-command-arg --db-backend --exit-command-arg boltdb --exit-command-arg --transient-store=false --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg vfs --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --stopped-only --exit-command-arg --rm --exit-command-arg 22faf8d247279e2523165f6116a3f5030c946adb1e919d5a252b94841fa6be2b
            │ ├─app-hermes\x2dsocket.slice
            │ │ └─hermes-socket@1.service
            │ │   ├─84669 /usr/bin/podman run --name hermes-socket-1 --replace --rm --sdnotify=conmon --cgroups=no-conmon --pod hermes-pod-1 --volume /home/hermes-agent2/.config/state/dashboard-sockets:/sockets:z --tz=UTC ghcr.io/nethserver/hermes-agent-socket:0.5.0 -d -d UNIX-LISTEN:/sockets/agent-1.sock,fork,unlink-early,mode=0660 TCP-CONNECT:127.0.0.1:9120
            │ │   └─84682 /usr/bin/conmon --api-version 1 -c 9d06a951ea2733662d0397fdc014e7858adbf5a698176ff4df0b0652bb6fe970 -u 9d06a951ea2733662d0397fdc014e7858adbf5a698176ff4df0b0652bb6fe970 -r /usr/bin/crun -b /home/hermes-agent2/.local/share/containers/storage/vfs-containers/9d06a951ea2733662d0397fdc014e7858adbf5a698176ff4df0b0652bb6fe970/userdata -p /run/user/1020/containers/vfs-containers/9d06a951ea2733662d0397fdc014e7858adbf5a698176ff4df0b0652bb6fe970/userdata/pidfile -n hermes-socket-1 --exit-dir /run/user/1020/libpod/tmp/exits --persist-dir /run/user/1020/libpod/tmp/persist/9d06a951ea2733662d0397fdc014e7858adbf5a698176ff4df0b0652bb6fe970 --full-attach -s -l journald --log-level warning --syslog --runtime-arg --log-format=json --runtime-arg --log --runtime-arg=/run/user/1020/containers/vfs-containers/9d06a951ea2733662d0397fdc014e7858adbf5a698176ff4df0b0652bb6fe970/userdata/oci-log --conmon-pidfile /run/user/1020/containers/vfs-containers/9d06a951ea2733662d0397fdc014e7858adbf5a698176ff4df0b0652bb6fe970/userdata/conmon.pid --exit-command /usr/bin/podman --exit-command-arg --root --exit-command-arg /home/hermes-agent2/.local/share/containers/storage --exit-command-arg --runroot --exit-command-arg /run/user/1020/containers --exit-command-arg --log-level --exit-command-arg warning --exit-command-arg --cgroup-manager --exit-command-arg systemd --exit-command-arg --tmpdir --exit-command-arg /run/user/1020/libpod/tmp --exit-command-arg --network-config-dir --exit-command-arg "" --exit-command-arg --network-backend --exit-command-arg netavark --exit-command-arg --volumepath --exit-command-arg /home/hermes-agent2/.local/share/containers/storage/volumes --exit-command-arg --db-backend --exit-command-arg boltdb --exit-command-arg --transient-store=false --exit-command-arg --runtime --exit-command-arg crun --exit-command-arg --storage-driver --exit-command-arg vfs --exit-command-arg --events-backend --exit-command-arg journald --exit-command-arg container --exit-command-arg cleanup --exit-command-arg --stopped-only --exit-command-arg --rm --exit-command-arg 9d06a951ea2733662d0397fdc014e7858adbf5a698176ff4df0b0652bb6fe970
root@daho-ns8:/#

and from my MacBook:

❯ curl -sk -o /dev/null -w "%{http_code}" https://hermes.sub.domain.de/hermes-1/
200%

The update seems to have worked well.

What I can’t quite explain myself

ERRO[0000] User-selected graph driver "overlay" overwritten by graph driver "vfs" from database - delete libpod local files ("/home/hermes-agent2/.local/share/containers/storage") to resolve.  May prevent use of images created by other tools

[Stll0 (Stefano Fancello)]

@capote

ERRO[0000] User-selected graph driver "overlay" overwritten by graph driver "vfs" from database - delete libpod local files ("/home/hermes-agent2/.local/share/containers/storage") to resolve.  May prevent use of images created by other tools

[capote (Marko)]

within the logs:

2026-06-04 10:04:37,122 INFO gateway.run: Reconnecting api_server (attempt 6)...
2026-06-04 10:04:37,125 ERROR gateway.platforms.api_server: [Api_Server] Refusing to start: API_SERVER_KEY is required for the API server, including loopback-only binds on 127.0.0.1.
2026-06-04 10:04:37,136 INFO gateway.run: Reconnect api_server failed, next retry in 300s
2026-06-04 10:06:52,208 INFO gateway.memory_monitor: [MEMORY] rss=237MB gc=(1148, 2, 0) threads=8 uptime=900s
2026-06-04 10:07:04,756 INFO cron.scheduler: Job '03cc5120d1c7' (no_agent): empty stdout — silent run
2026-06-04 10:07:04,767 INFO cron.scheduler: Job '03cc5120d1c7': agent returned [SILENT] — skipping delivery

[capote (Marko)]

Hermes Agent NS8 Module: Update 0.4.1 → 0.5.0 — Bug Report & Resolution

Date: 2026-06-04
System: NethServer 8 (daho-ns8, 192.168.3.21)
Module: hermes-agent2
Container: hermes-1 (pod: hermes-pod-1)
Image transition: ghcr.io/stell0/hermes-agent-*:0.4.1 → Package hermes-agent · GitHub

---

What did we do?

We performed a module image update of the Hermes Agent NS8 module from version 0.4.1 (maintained by Stell0) to version 0.5.0 (maintained by NethServer upstream). This is a standard NS8 module lifecycle operation using the api-cli run update-module command.

Update Command

api-cli run update-module --data '{
  "module_url": "ghcr.io/nethserver/hermes-agent:0.5.0",
  "instances": ["hermes-agent2"],
  "force": true
}'

Architecture Overview

The module runs as a rootless Podman pod with 4 containers:

Container	Image	Role
hermes-pod-1-infra	podman-pause:5.4.2	infra
hermes-1	nethserver/hermes-agent-hermes:0.5.0	gateway + dashboard
hermes-socket-1	nethserver/hermes-agent-socket:0.5.0	dashboard socket
hermes-auth	nethserver/hermes-agent-auth:0.5.0	auth proxy (:20005)

The module was already in production with Telegram bot, API server, and DEVONthink MCP integration active.

---

How did we do it?

Step 1: Initial Update Attempt — extract-image chown Failure

The first api-cli run update-module failed at the module agent level with exit code 1. The image pull succeeded, but the extract-image helper script (which extracts the module’s actions, systemd units, and templates from the new image into the module’s config directory) aborted.

Diagnosis — checking the module agent logs:

journalctl -t agent@hermes-agent2 --no-pager -n 100

Key error found:

chown: changing ownership of './systemd/user/hermes@.service.bak': Operation not permitted
...
subprocess.CalledProcessError: Command '('extract-image', 'ghcr.io/nethserver/hermes-agent:0.5.0')' returned non-zero exit status 1.
task/module/hermes-agent2/...: action "update-module" status is "aborted" (1) at step 05pullimages

Root Cause: The extract-image script (/usr/local/agent/bin/extract-image) uses set -e and runs chown -cR --reference=. . at the end. Any files owned by root inside the module user’s config tree block this. Files like hermes@.service.bak and hermes@.service.d/ (created by systemctl edit running as root) had root ownership.

Fix:

# Identify and fix ownership issues
find /home/hermes-agent2/.config -not -user hermes-agent2 2>/dev/null

# Fix affected files
chown -R hermes-agent2:hermes-agent2 /home/hermes-agent2/.config/systemd/user/hermes@.service.bak
chown -R hermes-agent2:hermes-agent2 /home/hermes-agent2/.config/systemd/user/hermes@.service.d

After this, re-running update-module succeeded (returned {}).

Step 2: Post-Update — API_SERVER_KEY Missing

The container restarted with the new image, but the API server refused to start:

ERROR gateway.platforms.api_server: [Api_Server] Refusing to start: API_SERVER_KEY is
required for the API server, including loopback-only binds on 127.0.0.1.

Root Cause: The old 0.4.1 image did not require API_SERVER_KEY. The new 0.5.0 image enforces it when API_SERVER_ENABLED=true. The original service unit references --env-file %S/state/secrets/%i.env but systemd’s %S (state directory) resolution was inconsistent — it resolved to /home/hermes-agent2/.config/state/ while the container startup process sometimes looked at different paths.

Fix (two parts):

Part 1: Add API_SERVER_KEY to the secrets file:

runagent -m hermes-agent2 bash -c 'echo "API_SERVER_KEY=$(grep HERMES_AGENT_SECRET /home/hermes-agent2/.config/state/secrets/1.env | cut -d= -f2)" >> /home/hermes-agent2/.config/state/secrets/1.env'

Part 2: Set the key directly as --env in the systemd override.conf (more reliable than --env-file with %S expansion):

[Service]
ExecStart=
ExecStart=runagent /usr/bin/podman run \
--name hermes-%i \
--replace \
--rm \
--sdnotify=conmon \
--cgroups=no-conmon \
--pod hermes-pod-%i \
--env-file /home/hermes-agent2/.config/state/agents/%i/agent.env \
--env-file /home/hermes-agent2/.config/state/secrets/%i.env \
--env API_SERVER_ENABLED=true \
--env API_SERVER_HOST=127.0.0.1 \
--env API_SERVER_KEY=M-Y-K-E-Y \
-v /home/hermes-agent2/.local/share/containers/storage/volumes/hermes-agents-home/_data/%i:/opt/data \
--tz=${TIMEZONE} \
${HERMES_AGENT_HERMES_IMAGE} \
hermes gateway run

Note: API_SERVER_KEY uses the same value as HERMES_AGENT_SECRET — no new secret is created.

Step 3: PID 1 Gateway Restart Limitation

After adding the key to the service override, we discovered that hermes gateway restart does not work inside the container. The gateway runs as PID 1, and PID 1 cannot restart itself.

Fix: Always restart the container from the host using systemd:

runagent -m hermes-agent2 systemctl --user restart hermes@1.service

Step 4: Restore DEVONthink MCP SSH Tunnel

After the container restart, the SSH tunnel to the Mac (192.168.3.155) for DEVONthink MCP was no longer running. The cron watchdog (every 5 min) would eventually restart it, but MCP servers are only initialized at container startup — if the tunnel isn’t running during container boot, MCP never connects.

Manual tunnel start inside the container:

runagent -m hermes-agent2 podman exec -d hermes-1 ssh -o StrictHostKeyChecking=no -o UserKnownHostsFile=/dev/null -o ServerAliveInterval=30 -o ServerAliveCountMax=3 -i /opt/data/.ssh/id_ed25519 -L 8421:localhost:8420 -N USER@192.168.3.155

Verification:

runagent -m hermes-agent2 podman exec hermes-1 ps aux | grep ssh

The resolution for the timing issue is documented but not fully automated — the cron watchdog (job 03cc5120d1c7, every 5 min, no_agent=true) maintains the tunnel, and a container restart with the tunnel already running is required for MCP to connect.

---

What results did we verify?

Container State

$ runagent -m hermes-agent2 podman ps -a
CONTAINER ID  IMAGE                                         COMMAND               NAMES
dac201234dc0  localhost/podman-pause:5.4.2                   ...                   hermes-pod-1-infra
22faf8d24727  ghcr.io/nethserver/hermes-agent-hermes:0.5.0  hermes gateway run    hermes-1
9d06a951ea27  ghcr.io/nethserver/hermes-agent-socket:0.5.0  socat dashboard       hermes-socket-1
66a72193fc57  ghcr.io/nethserver/hermes-agent-auth:0.5.0    python authproxy      hermes-auth

All 4 containers running with the new 0.5.0 images.

Environment Variables

$ runagent -m hermes-agent2 env | grep HERMES_IMAGE
HERMES_AGENT_HERMES_IMAGE=ghcr.io/nethserver/hermes-agent-hermes:0.5.0

API Server

$ runagent -m hermes-agent2 podman exec hermes-1 grep api_server /opt/data/logs/gateway.log
2026-06-04 12:14:31,362 INFO gateway.run: Connecting to api_server...
2026-06-04 12:14:31,379 INFO gateway.platforms.api_server: [Api_Server] API server listening on http://127.0.0.1:8642 (model: hermes-agent)
2026-06-04 12:14:31,387 INFO gateway.run: ✓ api_server connected

Telegram Gateway

This conversation is running through the Telegram gateway — confirmed active.

Dashboard / Web UI

$ curl -sk -o /dev/null -w "%{http_code}" https://hermes.sub.domain.de/hermes-1/
200

Dashboard responds with HTTP 200 (login page).

DEVONthink MCP

7 databases found: Eingang, Wissenspeicher, Dokumente, Archiv, Genealogie, Handbücher

All 59 DEVONthink MCP tools available.

SSH Tunnel

$ runagent -m hermes-agent2 podman exec hermes-1 ps aux | grep ssh
hermes   14  0.0  0.0  14340  9804 ?  Ss  12:14  0:00 ssh ... -L 8421:localhost:8420 -N USER@192.168.3.155

Two SSH tunnel processes running (watchdog + manual start). Port 8421 forwarding to localhost:8420 on Mac.

Graph Driver Warning (benign)

Every Podman command shows:

ERRO[0000] User-selected graph driver "overlay" overwritten by graph driver "vfs"

This is a cosmetic warning only — the storage database was initialized with vfs while the system now prefers overlay. All containers run without issues. A reset would fix it but is not necessary.

---

Issues Encountered Summary

#	Issue	Severity	Resolution
1	extract-image chown fails on root-owned files	Blocking	chown the problematic files to module user
2	API_SERVER_KEY missing in 0.5.0	Blocking	Add via --env in override.conf (not %S env-file)
3	PID 1 cannot hermes gateway restart	Limitation	Always use systemctl --user restart from host
4	MCP tunnel timing (tunnel after boot)	Annoyance	Start tunnel before container; cron watchdog maintains
5	Graph driver overlay→vfs warning	Cosmetic	Ignore — no functional impact

---

Lessons Learned (Do Not Repeat)

1. Never run hermes gateway restart inside the container — PID 1 blocks it. Use systemctl --user restart hermes@1.service from the host.
2. Never rely on %S expansion in systemd --env-file for the module user — use absolute paths in override.conf.
3. The --mount type=volume,subpath= option is not supported by Podman 4.3.1 (Debian 12). Always use bind mounts -v /path:/path in override.conf.
4. SSH tunnel belongs inside the container (not on the host) so 127.0.0.1:8421 resolves correctly.
5. Always do a chown check before running update-module: find /home/<module-user>/.config -not -user <module-user> 2>/dev/null.

[Stll0 (Stefano Fancello)]

I’m not sure about that, have you run hermes update? what is your hermes version?

hermes@hermes-pod-2:/opt/hermes$ hermes --version
Hermes Agent v0.15.1 (2026.5.29)

check difference between your runagent -m hermes-agent2 systemctl --user cat hermes@

and this:

/home/hermes-agent9/.config/systemd/user/hermes@.service

[Unit]
Description=Hermes gateway %i
Requires=hermes-pod@%i.service
After=hermes-pod@%i.service

[Service]
Environment=PODMAN_SYSTEMD_UNIT=%n
EnvironmentFile=-%S/state/environment
EnvironmentFile=-%S/state/agents/%i/agent.env
WorkingDirectory=%S/state
Restart=always
KillMode=none
Type=notify
NotifyAccess=all
ExecStartPre=/usr/local/bin/runagent discover-smarthost
ExecStartPre=/usr/local/bin/runagent sync-agent-runtime --agent-id %i
ExecStartPre=-/usr/bin/podman rm --force hermes-%i
ExecStart=runagent /usr/bin/podman run 
–name hermes-%i 
–replace 
–rm 
–sdnotify=conmon 
–cgroups=no-conmon 
–pod hermes-pod-%i 
–env-file %S/state/agents/%i/agent.env 
–env-file %S/state/secrets/%i.env 
–env API_SERVER_ENABLED=true 
–env API_SERVER_HOST=127.0.0.1 
–mount type=volume,src=hermes-agents-home,dst=/opt/data,subpath=%i 
–tz=${TIMEZONE} 
${HERMES_AGENT_HERMES_IMAGE} 
hermes gateway run
ExecStop=/usr/bin/podman stop --ignore --time 10 hermes-%i
ExecStopPost=-/usr/bin/podman rm --force hermes-%i
TimeoutStopSec=70

[Install]
WantedBy=default.target

I think that your custom fragment introduced some differences: I added API_SERVER_HOST=127.0.0.1 because it doesn’t requires the API_SERVER_KEY when binded only to localhost

BTW is not a big deal to add the key, I’ll do it next release. I’m just afraid you are diverging too much from the “standard”

[capote (Marko)]

Thank you for your response.

hermes@hermes-pod-1:/opt/hermes$ hermes --version
Hermes Agent v0.15.1 (2026.5.29)
Project: /opt/hermes
Python: 3.13.5
OpenAI SDK: 2.24.0
Update available: 1 commit behind — run 'docker pull nousresearch/hermes-agent:latest'

BTW is not a big deal to add the key, I’ll do it next release. I’m just afraid you are diverging too much from the “standard”

I think so too, as I don’t know any better and rely on support from Hermes.

After all, the analysis and solution aren’t mine; they were found by Hermes themselves.

I hope that with each new release I’ll have fewer problems and will be able to use the standard version.

Just to be on the safe side, I’ve set up a backup and restore system so that, if necessary, I can reinstall the module and restore my customisations (e.g. knowledge).

It would be helpful if, in due course, you could specify which files need to be restored without damaging the standard configuration.

In any case, the system is currently running without any issues.

I hope I’m not creating unnecessary extra work for you, but rather providing you with some insights from the alpha or beta testing, so to speak.

Here is the detailed diff analysis of the two configurations.

"/home/hermes-agent2/.config/systemd/user/hermes@.service

# /home/hermes-agent2/.config/systemd/user/hermes@.service
[Unit]
Description=Hermes gateway %i
Requires=hermes-pod@%i.service
After=hermes-pod@%i.service

[Service]
Environment=PODMAN_SYSTEMD_UNIT=%n
EnvironmentFile=-%S/state/environment
EnvironmentFile=-%S/state/agents/%i/agent.env
WorkingDirectory=%S/state
Restart=always
KillMode=none
Type=notify
NotifyAccess=all
ExecStartPre=/usr/local/bin/runagent discover-smarthost
ExecStartPre=/usr/local/bin/runagent sync-agent-runtime --agent-id %i
ExecStartPre=-/usr/bin/podman rm --force hermes-%i
ExecStart=runagent /usr/bin/podman run \
                --name hermes-%i \
                --replace \
                --rm \
                --sdnotify=conmon \
                --cgroups=no-conmon \
                --pod hermes-pod-%i \
                --env-file %S/state/agents/%i/agent.env \
                --env-file %S/state/secrets/%i.env \
                --env API_SERVER_ENABLED=true \
                --env API_SERVER_HOST=127.0.0.1 \
                --mount type=volume,src=hermes-agents-home,dst=/opt/data,subpath=%i \
                --tz=${TIMEZONE} \
                ${HERMES_AGENT_HERMES_IMAGE} \
                hermes gateway run
ExecStop=/usr/bin/podman stop --ignore --time 10 hermes-%i
ExecStopPost=-/usr/bin/podman rm --force hermes-%i
TimeoutStopSec=70

[Install]
WantedBy=default.target

# /home/hermes-agent2/.config/systemd/user/hermes@.service.d/override.conf
[Service]
ExecStart=
ExecStart=runagent /usr/bin/podman run \
--name hermes-%i \
--replace \
--rm \
--sdnotify=conmon \
--cgroups=no-conmon \
--pod hermes-pod-%i \
--env-file /home/hermes-agent2/.config/state/agents/%i/agent.env \
--env-file /home/hermes-agent2/.config/state/secrets/%i.env \
--env API_SERVER_ENABLED=true \
--env API_SERVER_HOST=127.0.0.1 \
--env API_SERVER_KEY=my-key \
-v /home/hermes-agent2/.local/share/containers/storage/volumes/hermes-agents-home/_data/%i:/opt/data \
--tz=${TIMEZONE} \
${HERMES_AGENT_HERMES_IMAGE} \
hermes gateway run

the formale diff

--- /home/hermes-agent9/.config/systemd/user/hermes@.service
+++ /home/hermes-agent2/.config/systemd/user/hermes@.service
@@ -15,21 +15,21 @@
 ExecStartPre=/usr/local/bin/runagent discover-smarthost
 ExecStartPre=/usr/local/bin/runagent sync-agent-runtime --agent-id %i
 ExecStartPre=-/usr/bin/podman rm --force hermes-%i
-ExecStart=runagent /usr/bin/podman run 
-–name hermes-%i 
-–replace 
-–rm 
-–sdnotify=conmon 
-–cgroups=no-conmon 
-–pod hermes-pod-%i 
-–env-file %S/state/agents/%i/agent.env 
-–env-file %S/state/secrets/%i.env 
-–env API_SERVER_ENABLED=true 
-–env API_SERVER_HOST=127.0.0.1 
-–mount type=volume,src=hermes-agents-home,dst=/opt/data,subpath=%i 
-–tz=${TIMEZONE} 
-${HERMES_AGENT_HERMES_IMAGE} 
-hermes gateway run
+ExecStart=runagent /usr/bin/podman run \
+                --name hermes-%i \
+                --replace \
+                --rm \
+                --sdnotify=conmon \
+                --cgroups=no-conmon \
+                --pod hermes-pod-%i \
+                --env-file %S/state/agents/%i/agent.env \
+                --env-file %S/state/secrets/%i.env \
+                --env API_SERVER_ENABLED=true \
+                --env API_SERVER_HOST=127.0.0.1 \
+                --mount type=volume,src=hermes-agents-home,dst=/opt/data,subpath=%i \
+                --tz=${TIMEZONE} \
+                ${HERMES_AGENT_HERMES_IMAGE} \
+                hermes gateway run
 ExecStop=/usr/bin/podman stop --ignore --time 10 hermes-%i
 ExecStopPost=-/usr/bin/podman rm --force hermes-%i
 TimeoutStopSec=70

but I can’t evaluate that

[Stll0 (Stefano Fancello)]

those are the same, mine had a formatting problem. But use systemctl --user cat hermes@ because that is where your customization are shown

[capote (Marko)]

@Stll0

systemctl --user cat hermes@

root@daho-ns8:~# runagent -m hermes-agent2 systemctl --user cat hermes@
# /home/hermes-agent2/.config/systemd/user/hermes@.service
[Unit]
Description=Hermes gateway %i
Requires=hermes-pod@%i.service
After=hermes-pod@%i.service

[Service]
Environment=PODMAN_SYSTEMD_UNIT=%n
EnvironmentFile=-%S/state/environment
EnvironmentFile=-%S/state/agents/%i/agent.env
WorkingDirectory=%S/state
Restart=always
KillMode=none
Type=notify
NotifyAccess=all
ExecStartPre=/usr/local/bin/runagent discover-smarthost
ExecStartPre=/usr/local/bin/runagent sync-agent-runtime --agent-id %i
ExecStartPre=-/usr/bin/podman rm --force hermes-%i
ExecStart=runagent /usr/bin/podman run \
                --name hermes-%i \
                --replace \
                --rm \
                --sdnotify=conmon \
                --cgroups=no-conmon \
                --pod hermes-pod-%i \
                --env-file %S/state/agents/%i/agent.env \
                --env-file %S/state/secrets/%i.env \
                --env API_SERVER_ENABLED=true \
                --env API_SERVER_HOST=127.0.0.1 \
                --mount type=volume,src=hermes-agents-home,dst=/opt/data,subpath=%i \
                --tz=${TIMEZONE} \
                ${HERMES_AGENT_HERMES_IMAGE} \
                hermes gateway run
ExecStop=/usr/bin/podman stop --ignore --time 10 hermes-%i
ExecStopPost=-/usr/bin/podman rm --force hermes-%i
TimeoutStopSec=70

[Install]
WantedBy=default.target

# /home/hermes-agent2/.config/systemd/user/hermes@.service.d/override.conf
[Service]
ExecStart=
ExecStart=runagent /usr/bin/podman run \
--name hermes-%i \
--replace \
--rm \
--sdnotify=conmon \
--cgroups=no-conmon \
--pod hermes-pod-%i \
--env-file /home/hermes-agent2/.config/state/agents/%i/agent.env \
--env-file /home/hermes-agent2/.config/state/secrets/%i.env \
--env API_SERVER_ENABLED=true \
--env API_SERVER_HOST=127.0.0.1 \
--env API_SERVER_KEY=my-key \
-v /home/hermes-agent2/.local/share/containers/storage/volumes/hermes-agents-home/_data/%i:/opt/data \
--tz=${TIMEZONE} \
${HERMES_AGENT_HERMES_IMAGE} \
hermes gateway run