Initial commit

This commit is contained in:
mdaleo404 committed 2026-11-11 10:35:11 +00:00
1 parent 43ecc9db8c
commit ad14160b1f
42 files changed
+17460

No files matched your search

+1
View File
@@ -0,0 +1 @@
* @mdaleo404
+91
View File
@@ -0,0 +1,91 @@
name: CI
on:
pull_request:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
name: lint-and-types
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.13"
- name: Install Poetry
run: |
# Pinned to the release that generated poetry.lock: the content hash in the
# lock file depends on Poetry's hashing rules, so a different Poetry can
# report the lock as stale. Invoked as a module so the interpreter's Poetry
# is used even if an older one is earlier on PATH.
pip install "poetry==2.5.0"
python -m poetry --version
- name: Install dev dependencies
run: python -m poetry install --with dev
- name: Lint (ruff)
run: python -m poetry run ruff check .
- name: Type check (mypy)
run: python -m poetry run mypy
tests:
name: tests (python ${{ matrix.python-version }})
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
# The package requires Python 3.12 or newer.
python-version: ["3.12", "3.13", "3.14"]
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ matrix.python-version }}
- name: Install Poetry
run: |
# Pinned to the release that generated poetry.lock: the content hash in the
# lock file depends on Poetry's hashing rules, so a different Poetry can
# report the lock as stale. Invoked as a module so the interpreter's Poetry
# is used even if an older one is earlier on PATH.
pip install "poetry==2.5.0"
python -m poetry --version
- name: Install project and dev dependencies
run: python -m poetry install --with dev
# The generated-role validation tests run real ansible-lint and
# ansible-playbook --syntax-check, so the pinned collections must be present.
- name: Install pinned Ansible collections
run: >
python -m poetry run ansible-galaxy collection install
--requirements-file tests/requirements.yml
--collections-path collections
- name: Run tests
run: python -m poetry run pytest -q
+1
View File
@@ -0,0 +1 @@
* @mdaleo404
+2
View File
@@ -0,0 +1,2 @@
github: mdaleo404
ko_fi: mdaleo404
+91
View File
@@ -0,0 +1,91 @@
name: CI
on:
pull_request:
workflow_dispatch:
permissions:
contents: read
concurrency:
group: ${{ github.workflow }}-${{ github.ref }}
cancel-in-progress: true
jobs:
lint:
name: lint-and-types
runs-on: ubuntu-latest
timeout-minutes: 15
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: "3.13"
- name: Install Poetry
run: |
# Pinned to the release that generated poetry.lock: the content hash in the
# lock file depends on Poetry's hashing rules, so a different Poetry can
# report the lock as stale. Invoked as a module so the interpreter's Poetry
# is used even if an older one is earlier on PATH.
pip install "poetry==2.5.0"
python -m poetry --version
- name: Install dev dependencies
run: python -m poetry install --with dev
- name: Lint (ruff)
run: python -m poetry run ruff check .
- name: Type check (mypy)
run: python -m poetry run mypy
tests:
name: tests (python ${{ matrix.python-version }})
runs-on: ubuntu-latest
timeout-minutes: 20
strategy:
fail-fast: false
matrix:
# The package requires Python 3.12 or newer.
python-version: ["3.12", "3.13", "3.14"]
steps:
- name: Checkout code
uses: actions/checkout@11bd71901bbe5b1630ceea73d27597364c9af683 # v4.2.2
with:
persist-credentials: false
- name: Set up Python
uses: actions/setup-python@a26af69be951a213d495a4c3e4e4022e16d87065 # v5.6.0
with:
python-version: ${{ matrix.python-version }}
- name: Install Poetry
run: |
# Pinned to the release that generated poetry.lock: the content hash in the
# lock file depends on Poetry's hashing rules, so a different Poetry can
# report the lock as stale. Invoked as a module so the interpreter's Poetry
# is used even if an older one is earlier on PATH.
pip install "poetry==2.5.0"
python -m poetry --version
- name: Install project and dev dependencies
run: python -m poetry install --with dev
# The generated-role validation tests run real ansible-lint and
# ansible-playbook --syntax-check, so the pinned collections must be present.
- name: Install pinned Ansible collections
run: >
python -m poetry run ansible-galaxy collection install
--requirements-file tests/requirements.yml
--collections-path collections
- name: Run tests
run: python -m poetry run pytest -q
+2
View File
@@ -174,3 +174,5 @@ cython_debug/
# PyPI configuration file # PyPI configuration file
.pypirc .pypirc
# Various
collections/
+27
View File
@@ -0,0 +1,27 @@
repos:
- repo: https://github.com/pre-commit/pre-commit-hooks
rev: v6.0.0
hooks:
- id: trailing-whitespace
name: Enforce removal of trailing whitespace
- id: end-of-file-fixer
name: Enforce end of the file format
- id: check-yaml
name: Check YAML syntax
- id: check-toml
name: Check TOML syntax
- id: check-merge-conflict
name: Check for merge conflict markers
- repo: local
hooks:
- id: ruff
name: Lint with ruff
entry: poetry run ruff check --fix
language: system
types: [python]
- id: mypy
name: Type-check with mypy
entry: poetry run mypy
language: system
pass_filenames: false
types_or: [python, pyi]
+256
View File
@@ -1,2 +1,258 @@
[![Licence](https://img.shields.io/badge/GPL--3.0--or--later-orange?label=Licence)](https://git.sysmd.uk/mdaleo404/compose2role/src/branch/main/LICENSE)
[![Gitea Release](https://img.shields.io/gitea/v/release/mdaleo404/compose2role?gitea_url=https%3A%2F%2Fgit.sysmd.uk%2F&style=flat&color=orange&logo=gitea)](https://git.sysmd.uk/mdaleo404/compose2role/releases)
[![pre-commit](https://img.shields.io/badge/pre--commit-enabled-blue?logo=pre-commit&style=flat)](https://git.sysmd.uk/mdaleo404/compose2role/src/branch/main/.pre-commit-config.yaml)
# compose2role # compose2role
Generate Ansible roles from Docker Compose projects discovered on remote hosts.
`compose2role` inspects a Docker Compose project over SSH and writes an Ansible role that
reproduces it. The role restores the captured Compose files, dotenv files, and file-backed secrets
byte for byte (comments, quoting, ordering, and line endings included), then applies them with
`docker compose` under the original project name. Applying a role to its source host is an in-place
adoption, not a migration.
It is deliberately **not** a Compose rewriter, normalizer, or migration tool. It is remote-only:
`generate` and `audit` require `--host` plus exactly one of `--project` or `--all`; a local-stack
invocation is a usage error. The daemon is named, not guessed (`--docker-context`), and privilege is
explicit (`--sudo` is never inferred).
Angle-bracket tokens such as `<host>`, `<project>`, and `<role>` are placeholders.
## Documentation
The in-depth reference lives on the [SysMD](https://www.sysmd.uk) site. This README covers the model, requirements, quick
start, and generated role at a glance.
| Page | Covers |
| --- | --- |
| [Installation and configuration](https://www.sysmd.uk/docs/compose2role/install/) | Requirements, install, configuration discovery, environment variables |
| [Remote workflow](https://www.sysmd.uk/docs/compose2role/workflow/) | Discovery, capture, Docker context selection, audit, safe regeneration |
| [Command reference](https://www.sysmd.uk/docs/compose2role/reference/) | Options, JSON output, exit codes |
| [Generated role reference](https://www.sysmd.uk/docs/compose2role/generated-role/) | Layout, variables, tasks, playbook boundary |
| [Secrets and SOPS](https://www.sysmd.uk/docs/compose2role/secrets/) | Capture, classification, secret policy, encryption |
| [Safety and limitations](https://www.sysmd.uk/docs/compose2role/safety/) | Security boundaries, refusals, unsupported features |
| [Troubleshooting](https://www.sysmd.uk/docs/compose2role/troubleshooting/) | SSH, Docker, SOPS, drift, and policy failures |
| [Development and validation](https://www.sysmd.uk/docs/compose2role/development/) | Tests, linting, type checks, Ansible validation |
## What it does
- Discovers Compose projects that a remote Docker daemon already knows about.
- Captures the ordered Compose files, the root `.env`, contained `env_file` inputs, and contained
file-backed Compose secrets.
- Canonicalizes the project with the remote Compose implementation and classifies effective
environment values as secret or plain.
- Reports stable, value-free `C2Rxxx` findings and audits drift against an existing role.
- Renders the complete plan before touching the output directory, writes atomically, and refuses to
overwrite a differing or unmanaged role unless `--force` is given.
- Records the selected Docker context so the role deploys to the same daemon.
It does not read or convert local Compose stacks, rewrite captured source, copy bind-mounted data,
follow paths or symlinks outside the project root, support Compose `include`, build contexts,
service `extends`, `profiles`, file-backed configs, or secret-shaped non-environment mapping keys,
create or guess a SOPS configuration, persist plaintext secret values, tear down an existing stack,
or generate playbooks.
## Requirements
- **Controller** — Python >= 3.12, an OpenSSH client, `sops`, and `gpg` when your SOPS backend uses
it.
- **Target host** — a running Docker daemon with Compose v2 or newer, an SSH account that can run
`docker compose ls`, `docker compose config`, and `docker ps`, and passwordless `sudo -n` when
`--sudo` is selected.
- **SOPS** — an operator-owned configuration with a creation rule matching `vars/secrets.sops.yml`.
- **To apply generated roles** — Ansible and the collections the role declares
(`community.docker` always, `community.sops` when the role contains secrets). Neither Ansible nor
its collections are needed to generate roles.
Rootless daemons are common; name the daemon with a Docker context. See
[Docker context selection](https://www.sysmd.uk/docs/compose2role/workflow/#docker-context-selection).
## Install
### From PyPI
```console
pipx install compose2role
# or
pip install compose2role
```
`pip` installs only the Python package. `ssh` and `sops` are external binaries and must already be
on `PATH`; `gpg` is needed when your SOPS backend uses it. Ansible and its collections are not
needed to generate roles.
### From source
```console
poetry install
poetry run compose2role --version
```
The package installs a `compose2role` console script. Examples in this document use
`poetry run compose2role` because they assume a checkout; if you installed the published package,
run `compose2role` directly.
For generated-role validation, install the pinned test collections into the checkout:
```console
poetry run ansible-galaxy collection install \
--requirements-file tests/requirements.yml --collections-path collections
```
`collections/` is local, gitignored test state.
## Quick start
Replace the placeholders with your own host, raw project name, role name, and paths. Each step is
safe to repeat.
### 1. Check the toolchain
```console
poetry run compose2role doctor --host <host> \
--sops-config <path/to/.sops.yaml>
```
A green host result means `discover` will reach the same daemon. `doctor` prints one line per check
with a remediation hint, then `N ok, N warnings, N errors`.
### 2. Discover projects
```console
poetry run compose2role discover --host <host> --docker-context <context>
```
Copy the raw project name exactly; it is the value for `--project`. Use `--format json` for
machine-readable output, or omit `--docker-context` to use the remote CLI's default context.
### 3. Audit before writing
```console
poetry run compose2role audit --host <host> --project <project> -v
```
Audit resolves the project through the same pipeline as generation, reports findings, and compares
the captured source with any existing role. It writes nothing and never encrypts anything.
### 4. Preview the role
```console
poetry run compose2role generate --host <host> --project <project> \
--output <path/to/roles> --sops-config <path/to/.sops.yaml> --check
```
`--check` renders the full plan and lists planned files without writing. Use `--diff` instead to
review value-free `added`, `changed`, `unchanged`, and `withheld` paths.
### 5. Generate the role
```console
poetry run compose2role generate --host <host> --project <project> \
--output <path/to/roles> --sops-config <path/to/.sops.yaml>
```
An unchanged project prints `unchanged` and writes nothing. A differing or unmanaged role is
refused unless `--force` is given, which backs up the previous role and swaps atomically. Use
`--all` instead of `--project` to generate every discovered project; each succeeds or fails
independently.
### 6. Write and run a playbook
Generation writes roles only. Write the play that applies one:
```yaml
---
- name: Deploy <role>
hosts: <host>
become: true # rootful daemons; remove or replace it for a rootless daemon
roles:
- <role>
```
The role's tasks are tagged `source`, `secrets`, and `compose`:
```console
ansible-playbook -i <inventory> <your-playbook>.yml --roles-path <roles> --tags source,secrets
ansible-playbook -i <inventory> <your-playbook>.yml --roles-path <roles> --tags compose
```
To deploy to a different host, override `<role>_project_dir` and any external prerequisites.
## Commands at a glance
| Command | Purpose | Key options |
| --- | --- | --- |
| `doctor` | Diagnose the controller, configuration, roles, or a remote host. | `--host`, `--sudo`, `--docker-context`, `--format` |
| `discover` | List the projects the remote daemon knows about. | `--host`, `--docker-context`, `--format text\|table\|json` |
| `audit` | Resolve a project, report findings, and compare drift; writes nothing. | `--host`, `--project`/`--all`, `--fail-on`, `-v` |
| `generate` | Render and write roles. | `--host`, `--project`/`--all`, `--output`, `--sops-config`, `--check`, `--diff`, `--force` |
All commands accept `--config` where meaningful. See the
[command reference](https://www.sysmd.uk/docs/compose2role/reference/) for the full option list,
JSON shapes, and exit codes.
## Generated role
```text
<roles>/<role>/
├── defaults/main.yml
├── tasks/main.yml
├── meta/main.yml
├── meta/requirements.yml
├── .compose2role.json
├── templates/ # captured source, mirroring original relative paths
└── vars/
└── secrets.sops.yml # only when the role has secrets
```
`meta/requirements.yml` declares the collections the role needs. Install them on the Ansible
controller that applies the role:
```console
ansible-galaxy collection install -r <roles>/<role>/meta/requirements.yml
```
`defaults/main.yml` defines `<role>_project_dir`, the original raw name as
`<role>_compose_project_name`, and `<role>_docker_context` when one was selected. Captured non-secret
values become lower-case role variables; secret values live only in `vars/secrets.sops.yml`.
`.compose2role.json` records provenance: generator version, mode, names, source paths, external
prerequisites, and per-file hashes, never a secret value.
See the [generated role reference](https://www.sysmd.uk/docs/compose2role/generated-role/) for
variable naming, task details, and linting.
## Safety
- Secret values may exist in controller memory while the plan is rendered, but never reach plaintext
role files, defaults, templates, provenance, logs, diffs, findings, exception text, or command
output. Only encrypted `vars/secrets.sops.yml` is persisted.
- The SOPS configuration is operator-owned; the tool never copies it or invents recipients.
- Captured inputs must be regular, non-symlinked files inside the project root. Bind-mounted data is
never read or copied.
- Generated roles apply with `state: present` and `remove_orphans: false`; they never remove
services, volumes, networks, or images.
See [Safety and limitations](https://www.sysmd.uk/docs/compose2role/safety/) for the full refusal
rules and [Secrets and SOPS](https://www.sysmd.uk/docs/compose2role/secrets/) for classification
and policy details.
## Development
```console
poetry run pre-commit run --all-files
poetry run pytest -q
poetry run ruff check .
poetry run mypy
```
The pre-commit hooks run ruff, mypy, and file hygiene; pytest is not a hook. Enable them on
commit once with `poetry run pre-commit install`.
The suite runs real `ansible-lint` and `ansible-playbook --syntax-check` against generated roles.
Without the pinned collections in `collections/`, those tests fail. Remote tests stub `ssh` and
`sops` through `PATH`, so they need no real host, network, sudo, or secrets. `mypy` checks only
`src` in strict mode.
See [Development and validation](https://www.sysmd.uk/docs/compose2role/development/) for focused
tests and the dedicated Ansible environment setup.
Generated
+1547
View File
File diff suppressed because it is too large. Load diff
+72
View File
@@ -0,0 +1,72 @@
[project]
name = "compose2role"
version = "0.1.0"
description = "Generate Ansible roles from Docker Compose projects"
authors = [
{ name = "Marco D'Aleo", email = "marco@marcodaleo.com" }
]
license = "GPL-3.0-or-later"
readme = "README.md"
requires-python = ">=3.12,<4.0"
dependencies = [
"click>=8.1.8,<9.0",
"jinja2>=3.1.6,<4.0",
"ruamel.yaml>=0.18.16,<0.19",
]
keywords = [
"ansible",
"docker",
"docker-compose",
"sops",
"ssh",
"cli",
"sysadmin",
]
classifiers = [
"Environment :: Console",
"Intended Audience :: System Administrators",
"Operating System :: POSIX :: Linux",
"Programming Language :: Python :: 3",
"Programming Language :: Python :: 3.12",
"Programming Language :: Python :: 3.13",
"Programming Language :: Python :: 3.14",
"Topic :: System :: Systems Administration",
]
[project.urls]
Homepage = "https://git.sysmd.uk/mdaleo404/compose2role"
Repository = "https://git.sysmd.uk/mdaleo404/compose2role"
[project.scripts]
compose2role = "compose2role.cli:main"
[build-system]
requires = ["poetry-core>=2.0.0,<3.0.0"]
build-backend = "poetry.core.masonry.api"
[tool.poetry]
packages = [{ include = "compose2role", from = "src" }]
[dependency-groups]
dev = [
"ansible-lint>=26.10.1,<27.0",
"mypy>=1.15,<2.0",
"pytest>=8.3,<9.0",
"ruff>=0.11,<1.0",
"pre-commit (>=4.6,<5.0)",
]
[tool.pytest.ini_options]
testpaths = ["tests"]
[tool.ruff]
target-version = "py312"
line-length = 100
[tool.ruff.lint]
select = ["E", "F", "I", "UP"]
[tool.mypy]
files = ["src"]
python_version = "3.12"
strict = true
+1
View File
@@ -0,0 +1 @@
"""Compose projects to Ansible roles."""
+775
View File
@@ -0,0 +1,775 @@
"""The compose2role command-line interface."""
from __future__ import annotations
import json
import os
import sys
from dataclasses import replace
from difflib import unified_diff
from pathlib import Path
import click
from compose2role.config import HostProfile, load_repository_config
from compose2role.discover import discover_projects, validate_docker_context
from compose2role.doctor import DoctorCheck, DoctorReport, diagnose, target_for
from compose2role.drift import inspect_drift
from compose2role.emit import external_prerequisites, normalize_remote_role_name
from compose2role.errors import (
Compose2RoleError,
ConfigurationError,
DockerContextError,
GenerationError,
)
from compose2role.findings import Finding, Severity
from compose2role.layout import AnsibleLayout, discover_layout
from compose2role.remote import resolve_remote_projects
from compose2role.remote_generate import generate_remote_projects
from compose2role.resolve import (
AuditOutcome,
AuditResult,
GeneratedProject,
GenerationOutcome,
GenerationResult,
PlanDiff,
ProjectDrift,
ProjectFailure,
ResolvedProject,
)
from compose2role.secret_policy import SecretPolicy, load_secret_policy
class _Command(click.Command):
"""Click command that lists its full short help instead of an ellipsis."""
def get_short_help_str(self, limit: int = 45) -> str:
help_text = self.help or ""
paragraph_end = help_text.find("\n\n")
if paragraph_end != -1:
help_text = help_text[:paragraph_end]
return " ".join(help_text.split())
class _Group(click.Group):
"""Click group that renders every command description without truncation."""
command_class = _Command
@click.group(cls=_Group, context_settings={"max_content_width": 100})
@click.version_option(package_name="compose2role", message="%(prog)s %(version)s")
def main() -> None:
"""Generate Ansible roles that reproduce remote Docker Compose projects.
Captures a project over SSH and writes a role that restores its exact Compose files,
dotenv inputs, and file-backed secrets, then applies them with docker compose. Remote-only:
generate and audit require --host plus exactly one of --project or --all.
"""
def _color_enabled() -> bool:
return sys.stdout.isatty() and "NO_COLOR" not in os.environ
class _ConfigurationClickError(click.ClickException):
"""Report configuration and layout failures as usage errors."""
exit_code = 2
def _layout(
config_path: Path | None,
*,
explicit_output: Path | None = None,
explicit_sops_config: Path | None = None,
explicit_secret_policy: Path | None = None,
host: str | None = None,
) -> AnsibleLayout:
config = load_repository_config(config_path, start=Path.cwd())
return discover_layout(
config,
start=Path.cwd(),
explicit_output=explicit_output,
explicit_sops_config=explicit_sops_config,
explicit_secret_policy=explicit_secret_policy,
host=host,
)
def _configured_sudo(explicit: bool | None, profile: HostProfile) -> bool:
if explicit is not None:
return explicit
return bool(profile.sudo)
def _docker_context(explicit: str | None, profile: HostProfile) -> str | None:
selected = explicit if explicit is not None else profile.docker_context
if selected is None:
return None
try:
return validate_docker_context(selected)
except DockerContextError as error:
raise _ConfigurationClickError(str(error)) from error
def _status(ok: bool, message: str) -> None:
glyph = "✓" if ok else "✗"
click.secho(f"{glyph} {message}", fg="green" if ok else "red", color=_color_enabled())
def _visible_findings(findings: tuple[Finding, ...], verbose: bool) -> tuple[Finding, ...]:
if verbose:
return findings
return tuple(finding for finding in findings if finding.severity == Severity.ERROR)
def _echo_findings(findings: tuple[Finding, ...], verbose: bool, *, indent: str = "") -> None:
for finding in _visible_findings(findings, verbose):
click.echo(
f"{indent}{finding.location}: {finding.severity.upper()} "
f"{finding.id}: {finding.message}"
)
def _echo_summary(ok_count: int, noun: str, failed_count: int) -> None:
summary = f"{ok_count} {noun}"
if failed_count:
summary = f"{summary}, {failed_count} failed"
click.echo()
click.echo(summary)
@main.command()
@click.option(
"--output",
"output_directory",
"-o",
type=click.Path(path_type=Path),
help="Roles directory to write into (default: resolved from --config or ansible.cfg).",
)
@click.option(
"--config",
"config_path",
type=click.Path(path_type=Path),
help="Repository configuration file to load.",
)
@click.option(
"--check",
"check_only",
is_flag=True,
help="List the planned files without writing or encrypting anything.",
)
@click.option(
"--diff",
"diff_only",
is_flag=True,
help="Show value-free changes; write only together with --force.",
)
@click.option(
"--force",
"force",
is_flag=True,
help="Replace a differing role after backing it up under .compose2role/backups/.",
)
@click.option(
"--host",
help="Remote OpenSSH alias or host:port destination (required).",
)
@click.option(
"--sudo",
"use_sudo",
is_flag=True,
default=None,
help="Run remote commands with sudo -n.",
)
@click.option(
"--docker-context",
"docker_context",
help="Remote Docker context to use (default: the remote CLI default).",
)
@click.option(
"--project",
"project_name",
help="Raw Compose project name to generate (exactly one project).",
)
@click.option(
"--all",
"all_projects",
is_flag=True,
help="Generate every discovered project.",
)
@click.option(
"--secret-policy",
"secret_policy_path",
type=click.Path(path_type=Path),
help="TOML secret-classification overrides.",
)
@click.option(
"--sops-config",
"sops_config_path",
type=click.Path(path_type=Path),
help="SOPS creation-rule file for encrypting secrets (or COMPOSE2ROLE_SOPS_CONFIG).",
)
@click.option(
"--verbose",
"-v",
is_flag=True,
help="Also show warning and info findings.",
)
@click.option(
"--format",
"output_format",
type=click.Choice(("text", "json")),
default="text",
help="Output format (default: text).",
)
def generate(
output_directory: Path | None,
config_path: Path | None,
check_only: bool,
diff_only: bool,
force: bool,
host: str | None,
use_sudo: bool | None,
docker_context: str | None,
project_name: str | None,
all_projects: bool,
secret_policy_path: Path | None,
sops_config_path: Path | None,
verbose: bool,
output_format: str,
) -> None:
"""Generate Ansible roles from remote Compose projects.
Requires --host and exactly one of --project or --all. Every run, including --check, needs an
existing SOPS configuration and the sops executable. A byte-identical role is left alone; a
differing or unmanaged role is refused unless --force is given.
"""
if host is None:
raise click.UsageError(
"local generation was removed; use --host with exactly one of --project or --all"
)
if check_only and diff_only:
raise click.UsageError("--check and --diff are mutually exclusive")
if project_name is None and not all_projects:
raise click.UsageError("remote generate requires exactly one of --project or --all")
if project_name is not None and all_projects:
raise click.UsageError("remote generate accepts only one of --project or --all")
text_output = output_format == "text"
if text_output:
click.echo(f"Discovering projects on {host}...")
try:
layout = _layout(
config_path,
explicit_output=output_directory,
explicit_sops_config=sops_config_path,
explicit_secret_policy=secret_policy_path,
host=host,
)
secret_policy = _load_secret_policy(layout.secret_policy)
result = generate_remote_projects(
host,
layout.roles_directory,
project_name=project_name,
all_projects=all_projects,
use_sudo=_configured_sudo(use_sudo, layout.host),
docker_context=_docker_context(docker_context, layout.host),
secret_policy=secret_policy,
sops_config_path=layout.sops_config,
check_only=check_only,
diff_only=diff_only,
force=force,
on_outcome=(
(lambda outcome: _echo_generation_outcome(outcome, verbose))
if text_output
else None
),
)
except ConfigurationError as error:
raise _ConfigurationClickError(str(error)) from error
except Compose2RoleError as error:
raise click.ClickException(str(error)) from error
if output_format == "json":
click.echo(json.dumps(result.as_json()))
else:
_echo_generation_summary(result)
if result.failures:
raise click.exceptions.Exit(1)
def _echo_plan_diff(diff: PlanDiff) -> None:
for file_diff in diff.files:
if file_diff.state == "unchanged":
continue
if file_diff.withheld:
reason = (
f": {file_diff.withheld_reason}"
if file_diff.withheld_reason is not None
else ""
)
click.echo(
f" {file_diff.path}: {file_diff.change} (content withheld{reason})"
)
continue
old_lines = (file_diff.old_text or "").splitlines()
new_lines = (file_diff.new_text or "").splitlines()
from_file = "/dev/null" if file_diff.change == "added" else f"a/{file_diff.path}"
rendered = list(
unified_diff(
old_lines,
new_lines,
fromfile=from_file,
tofile=f"b/{file_diff.path}",
lineterm="",
)
)
for line in rendered:
click.echo(f" {line}")
if file_diff.change == "changed" and not any(
line.startswith(("+", "-")) for line in rendered
):
click.echo(f" {file_diff.path}: changed")
def _echo_generation_outcome(outcome: GenerationOutcome, verbose: bool) -> None:
if isinstance(outcome, GeneratedProject):
if outcome.checked:
action = "checked"
elif outcome.diff is not None and outcome.diff_only:
action = "unchanged" if outcome.diff.identical else "diff"
elif outcome.unchanged:
action = "unchanged"
else:
action = "generated"
_status(True, f"{outcome.name} -> {outcome.role_name} ({action})")
_echo_prerequisites(outcome.project)
_echo_findings(outcome.project.findings, verbose, indent=" ")
if outcome.diff is not None:
_echo_plan_diff(outcome.diff)
if outcome.checked:
for path in outcome.files:
click.echo(f" {path}")
return
role = f" -> {outcome.role_name}" if outcome.role_name is not None else ""
_status(False, f"{outcome.name}{role}: FAILED ({outcome.category}): {outcome.message}")
_echo_findings(outcome.findings, verbose, indent=" ")
def _echo_prerequisites(project: ResolvedProject) -> None:
prerequisites = external_prerequisites(project.canonical_document)
if prerequisites:
rendered = ", ".join(f"external {kind} {name}" for kind, name in prerequisites)
click.echo(f" prerequisites: {rendered}")
def _echo_generation_summary(result: GenerationResult) -> None:
generated = tuple(
outcome for outcome in result.outcomes if isinstance(outcome, GeneratedProject)
)
if any(outcome.checked for outcome in generated):
noun = "checked"
elif generated and all(outcome.diff is not None and outcome.diff_only for outcome in generated):
noun = "unchanged" if all(outcome.unchanged for outcome in generated) else "diff"
elif generated and all(outcome.unchanged for outcome in generated):
noun = "unchanged"
else:
noun = "generated"
_echo_summary(len(generated), noun, len(result.failures))
@main.command()
@click.option(
"--format",
"output_format",
type=click.Choice(("text", "json")),
default="text",
help="Output format (default: text).",
)
@click.option(
"--fail-on",
type=click.Choice(("error", "warning")),
help="Exit 3 when a finding at this severity or above exists.",
)
@click.option(
"--verbose",
"-v",
is_flag=True,
help="Also show warning and info findings.",
)
@click.option(
"--host",
help="Remote OpenSSH alias or host:port destination (required).",
)
@click.option(
"--sudo",
"use_sudo",
is_flag=True,
default=None,
help="Run remote commands with sudo -n.",
)
@click.option(
"--docker-context",
"docker_context",
help="Remote Docker context to use (default: the remote CLI default).",
)
@click.option(
"--project",
"project_name",
help="Raw Compose project name to audit (exactly one project).",
)
@click.option(
"--all",
"all_projects",
is_flag=True,
help="Audit every discovered project.",
)
@click.option(
"--secret-policy",
"secret_policy_path",
type=click.Path(path_type=Path),
help="TOML secret-classification overrides.",
)
@click.option(
"--config",
"config_path",
type=click.Path(path_type=Path),
help="Repository configuration file to load.",
)
def audit(
output_format: str,
fail_on: str | None,
verbose: bool,
host: str | None,
use_sudo: bool | None,
docker_context: str | None,
project_name: str | None,
all_projects: bool,
secret_policy_path: Path | None,
config_path: Path | None,
) -> None:
"""Audit remote Compose projects without writing anything.
Uses the same pipeline as generate and compares the captured source with an existing role's
provenance. Requires --host and exactly one of --project or --all. Text output shows errors
only; add -v for more, or use --format json.
"""
if host is None:
raise click.UsageError(
"local audit was removed; use --host with exactly one of --project or --all"
)
if project_name is None and not all_projects:
raise click.UsageError("remote audit requires exactly one of --project or --all")
if project_name is not None and all_projects:
raise click.UsageError("remote audit accepts only one of --project or --all")
text_output = output_format == "text"
if text_output:
click.echo(f"Discovering projects on {host}...")
try:
layout = _layout(config_path, explicit_secret_policy=secret_policy_path, host=host)
secret_policy = _load_secret_policy(layout.secret_policy)
result = resolve_remote_projects(
host,
project_name=project_name,
all_projects=all_projects,
use_sudo=_configured_sudo(use_sudo, layout.host),
secret_policy=secret_policy,
docker_context=_docker_context(docker_context, layout.host),
on_outcome=(
(
lambda outcome: _echo_audit_outcome(
_outcome_with_drift(outcome, layout.roles_directory), verbose
)
)
if text_output
else None
),
)
result = _audit_result_with_drift(result, layout.roles_directory)
except ConfigurationError as error:
raise _ConfigurationClickError(str(error)) from error
except Compose2RoleError as error:
raise click.ClickException(str(error)) from error
if output_format == "json":
click.echo(json.dumps(result.as_json()))
else:
drifted = sum(
1
for project in result.projects
if project.drift is not None and project.drift.state == "source changed"
)
noun = f"ok, {drifted} drift" if all_projects and drifted else "ok"
_echo_summary(len(result.projects), noun, len(result.failures))
if result.failures:
raise click.exceptions.Exit(1)
findings = tuple(finding for project in result.projects for finding in project.findings)
if fail_on is not None and _has_failure(findings, fail_on):
raise click.exceptions.Exit(3)
def _load_secret_policy(secret_policy_path: Path | None) -> SecretPolicy | None:
selected_path = secret_policy_path
if selected_path is None:
environment_path = os.environ.get("COMPOSE2ROLE_SECRET_POLICY")
selected_path = Path(environment_path) if environment_path else None
return load_secret_policy(selected_path) if selected_path is not None else None
def _echo_audit_status(name: str, drift: ProjectDrift) -> None:
_status(True, f"{name} ({_drift_label(drift)})")
def _drift_label(drift: ProjectDrift) -> str:
if drift.changed:
return f"{drift.state}: {', '.join(drift.changed)}"
return drift.state
def _outcome_with_drift(outcome: AuditOutcome, roles_directory: Path) -> AuditOutcome:
if not isinstance(outcome, ResolvedProject):
return outcome
try:
role_name = normalize_remote_role_name(outcome.project.name)
except GenerationError:
drift = ProjectDrift(
"missing",
role=outcome.project.name,
reason="project name cannot be normalized into an Ansible role name",
)
else:
drift = inspect_drift(outcome, roles_directory / role_name)
return replace(outcome, drift=drift)
def _audit_result_with_drift(result: AuditResult, roles_directory: Path) -> AuditResult:
outcomes = tuple(
_outcome_with_drift(outcome, roles_directory) for outcome in result.outcomes
)
return AuditResult(
projects=tuple(
outcome for outcome in outcomes if isinstance(outcome, ResolvedProject)
),
failures=tuple(
outcome for outcome in outcomes if isinstance(outcome, ProjectFailure)
),
outcomes=outcomes,
)
def _echo_audit_outcome(outcome: AuditOutcome, verbose: bool) -> None:
if isinstance(outcome, ResolvedProject):
if outcome.drift is not None:
_echo_audit_status(outcome.project.name, outcome.drift)
else:
_status(True, outcome.project.name)
_echo_findings(outcome.findings, verbose, indent=" ")
return
_status(False, f"{outcome.name}: FAILED ({outcome.category}): {outcome.message}")
def _has_failure(findings: tuple[Finding, ...], fail_on: str) -> bool:
failure_severities = {
"error": frozenset({Severity.ERROR}),
"warning": frozenset({Severity.ERROR, Severity.WARNING}),
}
return any(finding.severity in failure_severities[fail_on] for finding in findings)
@main.command()
@click.option(
"--host",
required=True,
help="Remote OpenSSH alias or host:port destination.",
)
@click.option(
"--sudo",
"use_sudo",
is_flag=True,
default=None,
help="Run the remote Docker command with sudo -n.",
)
@click.option(
"--docker-context",
"docker_context",
help="Remote Docker context to use (default: the remote CLI default).",
)
@click.option(
"--format",
"output_format",
type=click.Choice(("text", "table", "json")),
default="text",
help="Output format (table is an alias for text).",
)
@click.option(
"--config",
"config_path",
type=click.Path(path_type=Path),
help="Repository configuration file to load.",
)
def discover(
host: str,
use_sudo: bool | None,
docker_context: str | None,
output_format: str,
config_path: Path | None,
) -> None:
"""List Compose projects running on a remote host.
Prints raw project names, statuses, and config files; copy a raw name for use with --project.
Discovery only reads Docker metadata and never changes the host.
"""
if output_format in {"text", "table"}:
click.echo(f"Discovering projects on {host}...")
try:
layout = _layout(config_path, host=host)
projects = discover_projects(
host,
use_sudo=_configured_sudo(use_sudo, layout.host),
docker_context=_docker_context(docker_context, layout.host),
)
except ConfigurationError as error:
raise _ConfigurationClickError(str(error)) from error
except Compose2RoleError as error:
raise click.ClickException(str(error)) from error
if output_format == "json":
click.echo(json.dumps({"projects": [project.as_json() for project in projects]}))
return
for project in projects:
_status(True, f"{project.name} {project.status} {', '.join(project.config_files)}")
_echo_summary(len(projects), "found", 0)
@main.command()
@click.option(
"--host",
help="Also check this remote host (the path discover takes: SSH, daemon, Compose).",
)
@click.option(
"--sudo",
"use_sudo",
is_flag=True,
default=None,
help="Run remote check commands with sudo -n (requires --host).",
)
@click.option(
"--docker-context",
"docker_context",
help="Remote Docker context to use (default: the remote CLI default).",
)
@click.option(
"--config",
"config_path",
type=click.Path(path_type=Path),
help="Repository configuration file to load.",
)
@click.option(
"--output",
"output_directory",
type=click.Path(path_type=Path),
help="Roles directory to inspect (default: resolved layout).",
)
@click.option(
"--secret-policy",
"secret_policy_path",
type=click.Path(path_type=Path),
help="TOML secret-classification overrides.",
)
@click.option(
"--sops-config",
"sops_config_path",
type=click.Path(path_type=Path),
help="SOPS creation-rule file to validate (or COMPOSE2ROLE_SOPS_CONFIG).",
)
@click.option(
"--format",
"output_format",
type=click.Choice(("text", "json")),
default="text",
help="Output format (default: text).",
)
def doctor(
host: str | None,
use_sudo: bool | None,
docker_context: str | None,
config_path: Path | None,
output_directory: Path | None,
secret_policy_path: Path | None,
sops_config_path: Path | None,
output_format: str,
) -> None:
"""Check the toolchain, configuration, and generated roles.
Without --host this checks the controller and the generated roles; with --host it checks only
the host path discover takes: SSH resolution, the selected daemon, and Docker Compose. Warnings
never fail the run, a failed required check exits 1, and a configuration error exits 2.
"""
if host is None and use_sudo:
raise click.UsageError("--sudo requires --host")
if host is None and docker_context is not None:
raise click.UsageError("--docker-context requires --host")
target = target_for(host)
configuration_failure = False
try:
layout = _layout(
config_path,
explicit_output=output_directory,
explicit_sops_config=sops_config_path,
explicit_secret_policy=secret_policy_path,
host=host,
)
selected_context = _docker_context(docker_context, layout.host)
except ConfigurationError as error:
configuration_failure = True
report = DoctorReport(
target=target,
checks=(
DoctorCheck(
"config",
"error",
str(error),
hint="fix the configuration file or pass --config",
required=True,
),
),
)
else:
report = diagnose(
layout,
host=host,
use_sudo=_configured_sudo(use_sudo, layout.host),
docker_context=selected_context,
environ=os.environ,
)
if output_format == "json":
click.echo(json.dumps(report.as_json()))
else:
_echo_doctor_report(report)
if configuration_failure:
raise click.exceptions.Exit(2)
if report.failed:
raise click.exceptions.Exit(1)
def _echo_doctor_report(report: DoctorReport) -> None:
statuses = {
"ok": ("✓", "green"),
"warning": ("!", "yellow"),
"error": ("✗", "red"),
}
for check in report.checks:
glyph, color = statuses[check.status]
line = f"{glyph} {check.name}: {check.detail}"
if check.hint is not None:
line = f"{line} — {check.hint}"
click.secho(line, fg=color, color=_color_enabled())
summary = report.summary()
click.echo()
click.echo(
f"{summary['ok']} ok, {summary['warnings']} warnings, {summary['errors']} errors"
)
+206
View File
@@ -0,0 +1,206 @@
"""Discover and validate the repository configuration file."""
from __future__ import annotations
import os
import tomllib
from collections.abc import Mapping
from dataclasses import dataclass, field
from pathlib import Path
from compose2role.errors import ConfigurationError
CONFIG_FILENAME = ".compose2role.toml"
ANSIBLE_CONFIG_FILENAME = "ansible.cfg"
_DEFAULT_KEYS = frozenset(
{
"output",
"sops_config",
"secret_policy",
}
)
_HOST_KEYS = frozenset({"docker_context", "sudo"})
@dataclass(frozen=True)
class HostProfile:
"""Per-host settings that override repository defaults."""
sudo: bool | None = None
docker_context: str | None = None
@dataclass(frozen=True)
class RepositoryConfig:
"""A validated repository configuration rooted at a directory."""
root: Path
path: Path | None = None
output: Path | None = None
sops_config: Path | None = None
secret_policy: Path | None = None
hosts: Mapping[str, HostProfile] = field(default_factory=dict)
def host(self, name: str | None) -> HostProfile:
"""Return the selected host profile, or an empty profile."""
if name is None:
return HostProfile()
return self.hosts.get(name, HostProfile())
def find_repository_config(start: Path | None = None) -> Path | None:
"""Find the nearest `.compose2role.toml`, then the nearest `ansible.cfg`."""
for filename in (CONFIG_FILENAME, ANSIBLE_CONFIG_FILENAME):
found = find_file_upwards(start, filename)
if found is not None:
return found
return None
def find_file_upwards(start: Path | None, filename: str) -> Path | None:
"""Find FILENAME in START or its nearest ancestor directory."""
for directory in ancestor_directories(start):
candidate = directory / filename
if candidate.is_file():
return candidate
return None
def ancestor_directories(start: Path | None = None) -> tuple[Path, ...]:
"""Return START and each ancestor, treating a non-existent path as a directory."""
anchor = (start if start is not None else Path.cwd()).expanduser().resolve()
if anchor.is_file():
anchor = anchor.parent
directories: list[Path] = []
current = anchor
while True:
directories.append(current)
if current == current.parent:
return tuple(directories)
current = current.parent
def load_repository_config(
explicit_path: Path | None = None,
*,
start: Path | None = None,
environ: Mapping[str, str] | None = None,
) -> RepositoryConfig:
"""Load the selected configuration, falling back to a discovered config root."""
environment = os.environ if environ is None else environ
selected_path = explicit_path
if selected_path is None:
environment_path = environment.get("COMPOSE2ROLE_CONFIG")
selected_path = Path(environment_path) if environment_path else None
if selected_path is None:
selected_path = find_repository_config(start)
if selected_path is None:
root = (start if start is not None else Path.cwd()).expanduser().resolve()
return RepositoryConfig(root=root)
selected_path = selected_path.expanduser()
if not selected_path.is_file():
raise ConfigurationError(f"configuration file is unreadable: {selected_path}")
if selected_path.name == ANSIBLE_CONFIG_FILENAME:
return RepositoryConfig(root=selected_path.parent.resolve(), path=selected_path)
return _load_toml(selected_path)
def _load_toml(path: Path) -> RepositoryConfig:
try:
content = path.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError) as error:
raise ConfigurationError(f"configuration file is unreadable: {path}") from error
try:
document: object = tomllib.loads(content)
except tomllib.TOMLDecodeError as error:
raise ConfigurationError(f"configuration file is not valid TOML: {path}") from error
root = path.parent.resolve()
settings = _mapping(document, f"configuration file must be a table: {path}")
unknown = sorted(set(settings) - {"defaults", "hosts"})
if unknown:
raise ConfigurationError(f"unknown configuration key in {path}: {unknown[0]}")
defaults = _mapping(
settings.get("defaults", {}), f"configuration defaults must be a table: {path}"
)
unknown_defaults = sorted(set(defaults) - _DEFAULT_KEYS)
if unknown_defaults:
raise ConfigurationError(
f"unknown configuration key in {path}: {unknown_defaults[0]}"
)
return RepositoryConfig(
root=root,
path=path,
output=_path_setting(defaults, "output", root, path),
sops_config=_path_setting(defaults, "sops_config", root, path),
secret_policy=_path_setting(defaults, "secret_policy", root, path),
hosts=_host_profiles(settings.get("hosts"), path),
)
def _host_profiles(value: object, path: Path) -> Mapping[str, HostProfile]:
if value is None:
return {}
hosts = _mapping(value, f"configuration hosts must be a table: {path}")
profiles: dict[str, HostProfile] = {}
for name, host_value in hosts.items():
settings = _mapping(host_value, f"host profile must be a table: {path}")
unknown = sorted(set(settings) - _HOST_KEYS)
if unknown:
raise ConfigurationError(f"unknown host key in {path}: {unknown[0]}")
profiles[name] = HostProfile(
sudo=_bool_setting(settings, "sudo", path),
docker_context=_string_setting(settings, "docker_context", path),
)
return profiles
def _mapping(value: object, message: str) -> Mapping[str, object]:
if not isinstance(value, dict):
raise ConfigurationError(message)
result: dict[str, object] = {}
for key, item in value.items():
if not isinstance(key, str):
raise ConfigurationError(message)
result[key] = item
return result
def _string_setting(
settings: Mapping[str, object], key: str, path: Path
) -> str | None:
value = settings.get(key)
if value is None:
return None
if not isinstance(value, str):
raise ConfigurationError(f"configuration key {key!r} must be a string: {path}")
return value
def _bool_setting(settings: Mapping[str, object], key: str, path: Path) -> bool | None:
value = settings.get(key)
if value is None:
return None
if not isinstance(value, bool):
raise ConfigurationError(f"configuration key {key!r} must be a boolean: {path}")
return value
def _path_setting(
settings: Mapping[str, object], key: str, root: Path, path: Path
) -> Path | None:
value = _string_setting(settings, key, path)
if value is None:
return None
return _resolve_path(value, root)
def _resolve_path(value: str, root: Path) -> Path:
path = Path(value).expanduser()
if path.is_absolute():
return path
return root / path
+277
View File
@@ -0,0 +1,277 @@
"""Discover Compose projects on a remote Docker host through OpenSSH."""
from __future__ import annotations
import json
import re
from dataclasses import dataclass, replace
from json import JSONDecodeError
from subprocess import TimeoutExpired, run
from compose2role.errors import DockerContextError, RemoteCommandError
_SSH_TIMEOUT_SECONDS = 30
_DOCKER_CONTEXT_NAME = re.compile(r"[A-Za-z0-9][A-Za-z0-9_.-]*")
_WORKING_DIRECTORY_FORMAT = (
"'{{.Label \"com.docker.compose.project\"}}\t"
"{{.Label \"com.docker.compose.project.working_dir\"}}'"
)
@dataclass(frozen=True)
class DiscoveredProject:
"""The remote Compose project details Docker exposes without a filesystem scan."""
name: str
status: str
config_files: tuple[str, ...]
working_directories: tuple[str, ...] = ()
def as_json(self) -> dict[str, str | list[str]]:
"""Return the stable JSON form emitted by the CLI."""
return {
"name": self.name,
"status": self.status,
"config_files": list(self.config_files),
}
def discover_projects(
host: str,
*,
use_sudo: bool = False,
include_working_directories: bool = False,
docker_context: str | None = None,
) -> tuple[DiscoveredProject, ...]:
"""Ask Docker Compose on HOST for its known project config files."""
remote_command = docker_command(
("compose", "ls", "--all", "--format", "json"), docker_context=docker_context
)
if use_sudo:
remote_command = ("sudo", "-n", *remote_command)
command = ssh_command(host, remote_command)
try:
result = run(
command,
check=False,
capture_output=True,
text=True,
timeout=_SSH_TIMEOUT_SECONDS,
)
except FileNotFoundError as error:
raise RemoteCommandError(
"OpenSSH client executable 'ssh' was not found", category="ssh-client"
) from error
except TimeoutExpired as error:
raise RemoteCommandError(
f"SSH discovery timed out for host {host!r}", category="ssh-timeout"
) from error
if result.returncode != 0:
if result.returncode == 255:
raise RemoteCommandError(
_error_message(
f"SSH connection or remote command failed for host {host!r} (exit 255)",
result.stderr,
),
category="ssh",
)
if use_sudo:
raise RemoteCommandError(
_error_message(
f"remote Docker discovery failed for host {host!r}; "
"ensure the SSH user has passwordless sudo access to Docker",
result.stderr,
),
category="remote-discovery",
)
raise RemoteCommandError(
_error_message(
f"remote Docker discovery failed for host {host!r}; "
"the SSH user may not be able to reach the daemon: "
"join the docker group, configure passwordless sudo, "
"or name a Docker context with --docker-context",
result.stderr,
),
category="remote-discovery",
)
projects = _parse_projects(result.stdout)
if not include_working_directories:
return projects
working_directories = _discover_working_directories(
host, use_sudo=use_sudo, docker_context=docker_context
)
return tuple(
replace(project, working_directories=working_directories.get(project.name, ()))
for project in projects
)
def _discover_working_directories(
host: str, *, use_sudo: bool, docker_context: str | None
) -> dict[str, tuple[str, ...]]:
remote_command: tuple[str, ...] = docker_command(
("ps", "--all", "--format", _WORKING_DIRECTORY_FORMAT),
docker_context=docker_context,
)
if use_sudo:
remote_command = ("sudo", "-n", *remote_command)
try:
result = run(
ssh_command(host, remote_command),
check=False,
capture_output=True,
text=True,
timeout=_SSH_TIMEOUT_SECONDS,
)
except (FileNotFoundError, TimeoutExpired):
return {}
if result.returncode != 0:
return {}
return _parse_working_directories(result.stdout)
def _parse_working_directories(output: str) -> dict[str, tuple[str, ...]]:
try:
json_payload: object = json.loads(output)
except JSONDecodeError:
json_payload = None
if isinstance(json_payload, list):
# A JSON listing cannot carry the tab-separated label format, so labels are unavailable.
return {}
directories: dict[str, set[str]] = {}
for line in output.splitlines():
if not line:
continue
if line.count("\t") != 1:
raise RemoteCommandError(
"remote Docker returned invalid Compose project working-directory labels",
category="invalid-discovery",
)
project, directory = line.split("\t")
if not project and not directory:
continue
if (
not project
or not directory
or _has_control_characters(project)
or _has_control_characters(directory)
):
raise RemoteCommandError(
"remote Docker returned invalid Compose project working-directory labels",
category="invalid-discovery",
)
directories.setdefault(project, set()).add(directory)
return {project: tuple(sorted(values)) for project, values in directories.items()}
def validate_docker_context(context: str) -> str:
"""Return CONTEXT when it is one safe argv word, otherwise reject it."""
if not _DOCKER_CONTEXT_NAME.fullmatch(context):
raise DockerContextError(
f"invalid Docker context name: {context!r}; "
"use a letter or digit followed by letters, digits, '_', '.', or '-'"
)
return context
def docker_command(
command: tuple[str, ...], *, docker_context: str | None
) -> tuple[str, ...]:
"""Build a remote Docker argv, naming the context when one is selected."""
if docker_context is None:
return ("docker", *command)
return ("docker", "--context", docker_context, *command)
def ssh_command(host: str, remote_command: tuple[str, ...]) -> list[str]:
"""Build an OpenSSH argv without changing SSH's configured connection behavior."""
destination, port = _parse_destination(host)
command = ["ssh"]
if port is not None:
command.extend(("-p", port))
command.extend(("--", destination, *remote_command))
return command
def _error_message(message: str, stderr: str) -> str:
detail = stderr.strip()
return f"{message}: {detail}" if detail else message
def _parse_projects(output: str) -> tuple[DiscoveredProject, ...]:
try:
payload: object = json.loads(output)
except JSONDecodeError as error:
raise RemoteCommandError(
"remote Docker Compose returned invalid JSON", category="invalid-json"
) from error
if not isinstance(payload, list):
raise RemoteCommandError(
"remote Docker Compose returned an invalid project list", category="invalid-discovery"
)
projects: list[DiscoveredProject] = []
for item in payload:
if not isinstance(item, dict):
raise RemoteCommandError(
"remote Docker Compose returned an invalid project entry",
category="invalid-discovery",
)
name: object = item.get("Name")
status: object = item.get("Status", "")
config_files: object = item.get("ConfigFiles", "")
if not isinstance(name, str) or not name or _has_control_characters(name):
raise RemoteCommandError(
"remote Docker Compose returned a project without a name",
category="invalid-discovery",
)
if not isinstance(status, str) or _has_control_characters(status):
raise RemoteCommandError(
"remote Docker Compose returned an invalid project status",
category="invalid-discovery",
)
projects.append(
DiscoveredProject(
name=name,
status=status,
config_files=_config_files(config_files),
)
)
return tuple(projects)
def _config_files(value: object) -> tuple[str, ...]:
if isinstance(value, str):
paths = tuple(path for path in value.split(",") if path)
elif isinstance(value, list) and all(isinstance(path, str) for path in value):
paths = tuple(value)
else:
raise RemoteCommandError(
"remote Docker Compose returned invalid config file paths", category="invalid-discovery"
)
if any(_has_control_characters(path) for path in paths):
raise RemoteCommandError(
"remote Docker Compose returned invalid config file paths", category="invalid-discovery"
)
return paths
def _has_control_characters(value: str) -> bool:
return any(ord(character) < 32 or ord(character) == 127 for character in value)
def _parse_destination(host: str) -> tuple[str, str | None]:
if not host:
raise RemoteCommandError("SSH host must not be empty")
if host.count(":") != 1:
return host, None
destination, possible_port = host.rsplit(":", maxsplit=1)
if not possible_port.isascii() or not possible_port.isdecimal():
return host, None
port_number = int(possible_port)
if not destination or not 1 <= port_number <= 65535:
raise RemoteCommandError(f"SSH port is invalid for host {host!r}")
return destination, possible_port
+807
View File
@@ -0,0 +1,807 @@
"""Preflight checks for tools, repository configuration, and generated roles."""
from __future__ import annotations
import hashlib
import json
import os
import re
import shutil
from collections.abc import Mapping
from dataclasses import dataclass
from json import JSONDecodeError
from pathlib import Path, PurePosixPath
from stat import S_IMODE
from subprocess import CompletedProcess, TimeoutExpired, run
from typing import Literal
from compose2role.discover import docker_command, ssh_command
from compose2role.errors import GenerationError, SecretPolicyError
from compose2role.layout import AnsibleLayout
from compose2role.remote_emit import (
normalized_compose_hash,
normalized_dotenv_hash,
normalized_file_secret_hash,
)
from compose2role.secret_policy import load_secret_policy
from compose2role.sops import SOPS_TARGET, validate_sops_config
from compose2role.source import FileMetadata
DoctorStatus = Literal["ok", "warning", "error"]
DistroFamily = Literal["debian", "rhel"] | None
RoleDriftState = Literal["unchanged", "source changed", "source unavailable", "invalid"]
OS_RELEASE_PATH = Path("/etc/os-release")
_PROVENANCE_FILE = ".compose2role.json"
_BACKUP_DIRECTORY = Path(".compose2role") / "backups"
_REQUIRED_COLLECTIONS = ("community.docker", "community.sops")
_TIMEOUT_SECONDS = 30
# Single-quoted because OpenSSH joins the remote argv with spaces and the remote shell re-splits it.
_DOCKER_INFO_FORMAT = "'{{json .}}'"
@dataclass(frozen=True)
class DoctorCheck:
"""One value-free diagnostic result with an optional remediation hint."""
name: str
status: DoctorStatus
detail: str
hint: str | None = None
required: bool = False
def as_json(self) -> dict[str, object]:
"""Return the stable JSON form of one check."""
payload: dict[str, object] = {
"detail": self.detail,
"name": self.name,
"required": self.required,
"status": self.status,
}
if self.hint is not None:
payload["hint"] = self.hint
return payload
@dataclass(frozen=True)
class DoctorReport:
"""Every check for one selected mode and its exit-relevant summary."""
target: dict[str, str]
checks: tuple[DoctorCheck, ...]
@property
def failed(self) -> bool:
"""Return whether a check required for the selected mode failed."""
return any(check.status == "error" and check.required for check in self.checks)
def as_json(self) -> dict[str, object]:
"""Return the value-free doctor envelope."""
return {
"checks": [check.as_json() for check in self.checks],
"summary": self.summary(),
"target": dict(self.target),
}
def summary(self) -> dict[str, object]:
"""Return the status counts shared by text and JSON output."""
return {
"errors": sum(1 for check in self.checks if check.status == "error"),
"failed": self.failed,
"ok": sum(1 for check in self.checks if check.status == "ok"),
"warnings": sum(1 for check in self.checks if check.status == "warning"),
}
def target_for(host: str | None) -> dict[str, str]:
"""Return the selected mode's value-free JSON target."""
if host is None:
return {"kind": "local"}
return {"kind": "remote", "host": host}
@dataclass(frozen=True)
class _Distro:
family: DistroFamily
def diagnose(
layout: AnsibleLayout,
*,
host: str | None = None,
use_sudo: bool = False,
docker_context: str | None = None,
environ: Mapping[str, str] | None = None,
os_release_path: Path | None = None,
) -> DoctorReport:
"""Run every check required by the selected local or remote mode."""
environment = os.environ if environ is None else environ
distro = _read_distro(OS_RELEASE_PATH if os_release_path is None else os_release_path)
checks: list[DoctorCheck] = []
if host is None:
checks.extend((_config_check(layout), _layout_check(layout)))
checks.append(_secret_policy_check(layout))
checks.append(_output_check(layout.roles_directory))
checks.extend(
(
_sops_check(environment, distro),
_gpg_check(environment, distro),
_sops_rule_check(layout.sops_config),
)
)
galaxy = _resolve_ansible_tool("ansible-galaxy", environment)
checks.extend(
(
_ansible_tool_check("ansible-galaxy", galaxy, distro),
_collections_check(galaxy, layout, environment, distro),
_ansible_tool_check(
"ansible-lint",
_resolve_ansible_tool("ansible-lint", environment),
distro,
),
_ansible_tool_check(
"ansible-playbook",
_resolve_ansible_tool("ansible-playbook", environment),
distro,
),
_roles_check(layout.roles_directory),
_drift_check(layout.roles_directory),
_backups_check(layout.roles_directory),
)
)
else:
ssh = _resolve_executable("ssh", "COMPOSE2ROLE_SSH_BIN", environment)
checks.extend(
_remote_checks(ssh, host, use_sudo, docker_context, environment, distro)
)
target = target_for(host)
return DoctorReport(target=target, checks=tuple(checks))
def _config_check(layout: AnsibleLayout) -> DoctorCheck:
if layout.config.path is not None:
detail = f"repository configuration: {layout.config.path}"
else:
detail = f"no repository configuration discovered; using {layout.root}"
return DoctorCheck("config", "ok", detail)
def _layout_check(layout: AnsibleLayout) -> DoctorCheck:
detail = f"roles {layout.roles_directory}; inventory {len(layout.inventory)} path(s)"
return DoctorCheck("layout", "ok", detail)
def _secret_policy_check(layout: AnsibleLayout) -> DoctorCheck:
if layout.secret_policy is None:
return DoctorCheck("secret policy", "ok", "no secret policy configured")
try:
load_secret_policy(layout.secret_policy)
except SecretPolicyError as error:
return DoctorCheck(
"secret policy",
"warning",
str(error),
hint=f"fix the secret policy at {layout.secret_policy}",
)
return DoctorCheck("secret policy", "ok", f"secret policy loaded: {layout.secret_policy}")
def _output_check(roles_directory: Path) -> DoctorCheck:
writable, checked = _writable_directory(roles_directory)
if writable:
return DoctorCheck("output", "ok", f"roles directory is writable: {checked}")
return DoctorCheck(
"output",
"error",
f"roles directory is not writable: {checked}",
hint=f"make {checked} writable or pass --output",
required=True,
)
def _remote_checks(
ssh: str | None,
host: str,
use_sudo: bool,
docker_context: str | None,
environment: Mapping[str, str],
distro: _Distro,
) -> tuple[DoctorCheck, ...]:
if ssh is None:
return (
DoctorCheck(
"ssh",
"error",
"OpenSSH client executable 'ssh' was not found",
hint=_package_hint(_openssh_package(distro), distro),
required=True,
),
)
checks = [
DoctorCheck("ssh", "ok", f"OpenSSH client executable: {ssh}"),
_ssh_resolution_check(ssh, host, environment),
_remote_docker_check(ssh, host, use_sudo, docker_context, environment),
_remote_compose_check(ssh, host, use_sudo, docker_context, environment),
]
return tuple(checks)
def _ssh_resolution_check(
ssh: str, host: str, environment: Mapping[str, str]
) -> DoctorCheck:
command = ssh_command(host, ())
argv = [ssh, "-G", *command[1:]]
result = _run(argv, environment)
if result is None or result.returncode != 0:
return DoctorCheck(
"ssh resolution",
"error",
f"ssh -G {host} failed",
hint="check the SSH host alias and OpenSSH configuration",
required=True,
)
fields = _ssh_config_fields(result.stdout)
detail = (
f"ssh -G {host} resolved hostname {fields.get('hostname', 'unknown')}, "
f"user {fields.get('user', 'unknown')}, port {fields.get('port', 'unknown')}"
)
return DoctorCheck("ssh resolution", "ok", detail)
def _remote_docker_check(
ssh: str,
host: str,
use_sudo: bool,
docker_context: str | None,
environment: Mapping[str, str],
) -> DoctorCheck:
info = _run_remote(
ssh,
host,
docker_command(
("info", "--format", _DOCKER_INFO_FORMAT), docker_context=docker_context
),
use_sudo=use_sudo,
environment=environment,
)
if info is None or info.returncode != 0:
return DoctorCheck(
"remote docker",
"error",
f"remote Docker daemon is not reachable on {host}",
hint=(
"join the docker group, configure passwordless sudo, "
"or name a context with --docker-context"
),
required=True,
)
mode = _daemon_mode(info.stdout)
projects = _remote_project_count(ssh, host, use_sudo, docker_context, environment)
detail = (
f"remote Docker daemon is {mode}"
if mode is not None
else "remote Docker daemon is reachable"
)
if projects is None:
detail = f"{detail}; visible Compose projects unavailable"
else:
detail = f"{detail}; {projects} visible Compose project(s)"
return DoctorCheck("remote docker", "ok", detail)
def _remote_project_count(
ssh: str,
host: str,
use_sudo: bool,
docker_context: str | None,
environment: Mapping[str, str],
) -> int | None:
result = _run_remote(
ssh,
host,
docker_command(
("compose", "ls", "--all", "--format", "json"), docker_context=docker_context
),
use_sudo=use_sudo,
environment=environment,
)
if result is None or result.returncode != 0:
return None
try:
payload: object = json.loads(result.stdout)
except JSONDecodeError:
return None
return len(payload) if isinstance(payload, list) else None
def _run_remote(
ssh: str,
host: str,
remote_command: tuple[str, ...],
*,
use_sudo: bool,
environment: Mapping[str, str],
) -> CompletedProcess[str] | None:
if use_sudo:
remote_command = ("sudo", "-n", *remote_command)
command = ssh_command(host, remote_command)
return _run([ssh, *command[1:]], environment)
def _remote_compose_check(
ssh: str,
host: str,
use_sudo: bool,
docker_context: str | None,
environment: Mapping[str, str],
) -> DoctorCheck:
result = _run_remote(
ssh,
host,
docker_command(("compose", "version"), docker_context=docker_context),
use_sudo=use_sudo,
environment=environment,
)
if result is None or result.returncode != 0:
return DoctorCheck(
"remote compose",
"error",
f"remote docker compose version failed on {host}",
hint="verify SSH access and install the Docker Compose v2 or newer plugin",
required=True,
)
version = _compose_version(result.stdout)
major = _compose_major(version)
if major is None:
return DoctorCheck(
"remote compose",
"error",
f"remote Docker Compose version is unreadable: {version}",
hint="install the Docker Compose v2 or newer plugin on the host",
required=True,
)
if major < 2:
return DoctorCheck(
"remote compose",
"error",
f"remote Docker Compose is {version}; compose2role requires v2 or newer",
hint="install the Docker Compose v2 or newer plugin on the host",
required=True,
)
return DoctorCheck("remote compose", "ok", f"remote docker compose {version}")
def _sops_check(environment: Mapping[str, str], distro: _Distro) -> DoctorCheck:
executable = _resolve_executable("sops", "COMPOSE2ROLE_SOPS_BIN", environment)
if executable is None:
return DoctorCheck(
"sops",
"warning",
"sops executable was not found",
hint=_package_hint("sops", distro),
)
result = _run([executable, "--version"], environment)
if result is None or result.returncode != 0:
return DoctorCheck(
"sops",
"warning",
"sops --version failed",
hint="verify the sops installation",
)
return DoctorCheck("sops", "ok", _first_line(result.stdout) or f"sops: {executable}")
def _gpg_check(environment: Mapping[str, str], distro: _Distro) -> DoctorCheck:
executable = _resolve_executable("gpg", None, environment)
if executable is None:
package = "gnupg2" if distro.family == "rhel" else "gnupg"
return DoctorCheck(
"gpg",
"warning",
"gpg executable was not found",
hint=_package_hint(package, distro),
)
return DoctorCheck("gpg", "ok", f"gpg executable: {executable}")
def _sops_rule_check(path: Path | None) -> DoctorCheck:
if path is None:
return DoctorCheck("sops rule", "ok", "no SOPS configuration configured")
try:
validate_sops_config(path)
except GenerationError as error:
return DoctorCheck(
"sops rule",
"error",
str(error),
hint=f"define a creation rule for {SOPS_TARGET} in {path}",
required=True,
)
return DoctorCheck("sops rule", "ok", f"creation rule covers {SOPS_TARGET}: {path}")
def _ansible_tool_check(name: str, executable: str | None, distro: _Distro) -> DoctorCheck:
if executable is None:
package = "ansible-lint" if name == "ansible-lint" else _ansible_package(distro)
return DoctorCheck(
name,
"warning",
f"{name} executable was not found",
hint=_package_hint(package, distro),
)
return DoctorCheck(name, "ok", f"{name} executable: {executable}")
def _collections_check(
galaxy: str | None,
layout: AnsibleLayout,
environment: Mapping[str, str],
distro: _Distro,
) -> DoctorCheck:
if galaxy is None:
return DoctorCheck(
"ansible collections",
"warning",
"installed collections were not checked because ansible-galaxy was not found",
hint=_package_hint(_ansible_package(distro), distro),
)
result = _run([galaxy, "collection", "list", "--format", "json"], environment)
if result is None or result.returncode != 0:
return DoctorCheck(
"ansible collections",
"warning",
"ansible-galaxy collection list failed",
hint="verify the ansible-galaxy installation",
)
installed = _installed_collections(result.stdout)
if installed is None:
return DoctorCheck(
"ansible collections",
"warning",
"ansible-galaxy returned invalid collection data",
)
for name, version in _local_collections(layout, environment).items():
installed.setdefault(name, version)
missing = [name for name in _REQUIRED_COLLECTIONS if name not in installed]
if missing:
return DoctorCheck(
"ansible collections",
"warning",
f"missing collections: {', '.join(missing)}",
hint=f"install with: ansible-galaxy collection install {' '.join(missing)}",
)
found = ", ".join(
f"{name} {installed[name]}".rstrip() for name in _REQUIRED_COLLECTIONS
)
return DoctorCheck("ansible collections", "ok", f"found {found}")
def _roles_check(roles_directory: Path) -> DoctorCheck:
roles = _managed_roles(roles_directory)
if not roles:
return DoctorCheck("roles", "ok", "no generated roles yet")
return DoctorCheck("roles", "ok", f"{len(roles)} generated role(s)")
def _drift_check(roles_directory: Path) -> DoctorCheck:
managed = _managed_roles(roles_directory)
if not managed:
return DoctorCheck("drift", "ok", "no managed roles to compare")
counts: dict[RoleDriftState, int] = {
"unchanged": 0,
"source changed": 0,
"source unavailable": 0,
"invalid": 0,
}
for role_directory in managed:
counts[_role_drift(role_directory)] += 1
detail = (
f"{counts['unchanged']} unchanged, {counts['source changed']} source changed, "
f"{counts['source unavailable']} source unavailable"
)
if counts["invalid"]:
detail = f"{detail}, {counts['invalid']} unreadable provenance"
if counts["source changed"] or counts["invalid"]:
return DoctorCheck(
"drift",
"warning",
detail,
hint="run compose2role audit for an authoritative drift report",
)
return DoctorCheck("drift", "ok", detail)
def _backups_check(roles_directory: Path) -> DoctorCheck:
directory = roles_directory / _BACKUP_DIRECTORY
if not directory.is_dir():
return DoctorCheck("backups", "ok", "no backups")
try:
count = sum(1 for entry in directory.iterdir() if entry.is_dir())
except OSError:
count = 0
total = _directory_size(directory)
hint = f"prune with: rm -r {directory}" if count else None
return DoctorCheck(
"backups",
"ok",
f"{count} backup(s), {_format_size(total)}",
hint=hint,
)
def _managed_roles(roles_directory: Path) -> tuple[Path, ...]:
try:
entries = sorted(roles_directory.iterdir())
except OSError:
return ()
return tuple(
entry
for entry in entries
if entry.is_dir()
and not entry.is_symlink()
and not entry.name.startswith(".")
and (entry / _PROVENANCE_FILE).is_file()
)
def _role_drift(role_directory: Path) -> RoleDriftState:
"""Compare recorded provenance hashes without re-resolving the project.
``audit`` remains the authoritative check because it re-captures the source.
Doctor only needs a cheap, offline signal for each existing role.
"""
try:
document: object = json.loads(
(role_directory / _PROVENANCE_FILE).read_text(encoding="utf-8")
)
except (OSError, UnicodeDecodeError, JSONDecodeError):
return "invalid"
if not isinstance(document, dict):
return "invalid"
source = document.get("source")
files = document.get("files")
if not isinstance(source, dict) or not isinstance(files, dict):
return "invalid"
root_value = source.get("root")
if not isinstance(root_value, str) or not root_value:
return "invalid"
root = Path(root_value)
if not root.is_dir():
return "source unavailable"
for path, entry in files.items():
if not isinstance(path, str) or not isinstance(entry, dict):
return "invalid"
if not _recorded_file_matches(root, path, entry):
return "source changed"
return "unchanged"
def _recorded_file_matches(
root: Path, relative_path: str, entry: Mapping[str, object]
) -> bool:
expected = entry.get("sha256")
if not isinstance(expected, str):
return False
relative = PurePosixPath(relative_path)
if relative.is_absolute() or ".." in relative.parts:
return False
local_path = root.joinpath(*relative.parts)
try:
local_path.resolve().relative_to(root.resolve())
metadata = local_path.stat()
content = local_path.read_bytes()
except (OSError, ValueError):
return False
if hashlib.sha256(content).hexdigest() == expected:
return True
file_metadata = FileMetadata(
mode=S_IMODE(metadata.st_mode), uid=metadata.st_uid, gid=metadata.st_gid
)
return expected in {
normalized_compose_hash(content),
normalized_dotenv_hash(content),
normalized_file_secret_hash(file_metadata),
}
def _writable_directory(path: Path) -> tuple[bool, Path]:
candidate = path
while not candidate.exists() and candidate != candidate.parent:
candidate = candidate.parent
if not candidate.is_dir():
return False, candidate
return os.access(candidate, os.W_OK | os.X_OK), candidate
def _resolve_executable(
name: str, override_name: str | None, environment: Mapping[str, str]
) -> str | None:
search_path = environment.get("PATH", os.defpath)
if override_name is not None:
override = environment.get(override_name)
if override:
return shutil.which(override, path=search_path)
return shutil.which(name, path=search_path)
def _resolve_ansible_tool(name: str, environment: Mapping[str, str]) -> str | None:
override = environment.get("COMPOSE2ROLE_ANSIBLE_BIN")
if override:
candidate = Path(override).expanduser() / name
if candidate.is_file() and os.access(candidate, os.X_OK):
return str(candidate)
return None
return shutil.which(name, path=environment.get("PATH", os.defpath))
def _installed_collections(output: str) -> dict[str, str] | None:
try:
payload: object = json.loads(output)
except JSONDecodeError:
return None
if not isinstance(payload, dict):
return None
found: dict[str, str] = {}
for path_value in payload.values():
if not isinstance(path_value, dict):
continue
for name, version_value in path_value.items():
if not isinstance(name, str):
continue
version = ""
if isinstance(version_value, dict):
raw_version = version_value.get("version")
if isinstance(raw_version, str):
version = raw_version
found[name] = version
return found
def _run(argv: list[str], environment: Mapping[str, str]) -> CompletedProcess[str] | None:
try:
return run(
argv,
check=False,
capture_output=True,
env=dict(environment),
text=True,
timeout=_TIMEOUT_SECONDS,
)
except (FileNotFoundError, OSError, TimeoutExpired):
return None
def _read_distro(path: Path) -> _Distro:
try:
content = path.read_text(encoding="utf-8")
except (OSError, UnicodeDecodeError):
return _Distro(None)
values: dict[str, str] = {}
for line in content.splitlines():
stripped = line.strip()
if not stripped or stripped.startswith("#") or "=" not in stripped:
continue
key, _, value = stripped.partition("=")
value = value.strip()
if len(value) >= 2 and value[0] == value[-1] and value[0] in "\"'":
value = value[1:-1]
values[key.strip()] = value
tokens = {values.get("ID", "").lower(), *values.get("ID_LIKE", "").lower().split()}
if tokens & {"debian", "ubuntu"}:
return _Distro("debian")
if tokens & {"rhel", "fedora", "centos", "rocky", "almalinux", "ol"}:
return _Distro("rhel")
return _Distro(None)
def _package_hint(package: str, distro: _Distro) -> str:
if distro.family == "debian":
return f"install with: sudo apt-get install {package}"
if distro.family == "rhel":
suffix = " (EPEL)" if package == "sops" else ""
return f"install with: sudo dnf install {package}{suffix}"
return f"install {package} with your package manager"
def _ansible_package(distro: _Distro) -> str:
return "ansible-core" if distro.family == "rhel" else "ansible"
def _openssh_package(distro: _Distro) -> str:
return "openssh-clients" if distro.family == "rhel" else "openssh-client"
def _directory_size(path: Path) -> int:
total = 0
for root, _directories, files in os.walk(path, followlinks=False):
for name in files:
try:
total += (Path(root) / name).stat().st_size
except OSError:
continue
return total
def _format_size(size: int) -> str:
if size < 1024:
return f"{size} B"
value = size / 1024
for unit in ("KiB", "MiB"):
if value < 1024:
return f"{value:.1f} {unit}"
value /= 1024
return f"{value:.1f} GiB"
def _compose_version(text: str) -> str:
version = _first_line(text) or "version unknown"
prefix = "Docker Compose version "
if version.startswith(prefix):
version = version[len(prefix) :]
return version
def _compose_major(version: str) -> int | None:
match = re.match(r"v?(\d+)", version)
return int(match.group(1)) if match is not None else None
def _ssh_config_fields(output: str) -> dict[str, str]:
fields: dict[str, str] = {}
for line in output.splitlines():
key, separator, value = line.partition(" ")
if separator and key and value:
fields.setdefault(key, value.strip())
return fields
def _daemon_mode(output: str) -> str | None:
"""Return rootless or rootful from `docker info`; None when the payload is unusable."""
try:
payload: object = json.loads(output)
except JSONDecodeError:
return None
if not isinstance(payload, dict):
return None
options = payload.get("SecurityOptions")
if not isinstance(options, list):
return None
return "rootless" if "name=rootless" in options else "rootful"
def _local_collections(
layout: AnsibleLayout, environment: Mapping[str, str]
) -> dict[str, str]:
"""Find collections installed beside the repository or on ANSIBLE_COLLECTIONS_PATH."""
roots: list[Path] = [
Path(entry).expanduser()
for entry in environment.get("ANSIBLE_COLLECTIONS_PATH", "").split(os.pathsep)
if entry
]
roots.append(layout.root / "collections")
found: dict[str, str] = {}
for root in roots:
for base in (root / "ansible_collections", root):
try:
namespaces = sorted(base.iterdir())
except OSError:
continue
for namespace in namespaces:
if not namespace.is_dir():
continue
try:
collections = sorted(namespace.iterdir())
except OSError:
continue
for collection in collections:
if collection.is_dir():
found.setdefault(f"{namespace.name}.{collection.name}", "")
return found
def _first_line(text: str) -> str:
for line in text.splitlines():
stripped = line.strip()
if stripped:
return stripped[:120]
return ""
+98
View File
@@ -0,0 +1,98 @@
"""Compare current captured source hashes with generated-role provenance."""
from __future__ import annotations
import hashlib
import json
from pathlib import Path, PurePosixPath
from compose2role.remote_emit import (
normalized_compose_hash,
normalized_dotenv_hash,
normalized_file_secret_hash,
)
from compose2role.resolve import ProjectDrift, ProjectSnapshot, ResolvedProject
_PROVENANCE_FILE = ".compose2role.json"
def inspect_drift(project: ResolvedProject, role_directory: Path) -> ProjectDrift:
"""Return the value-free drift state for one project and its role directory."""
role = role_directory.name
if not role_directory.exists():
return ProjectDrift("missing", role=role, reason="role has not been generated")
if not role_directory.is_dir():
return ProjectDrift("unmanaged", role=role, reason="role path is not a directory")
provenance_path = role_directory / _PROVENANCE_FILE
if not provenance_path.is_file():
return ProjectDrift(
"unmanaged", role=role, reason="role has no .compose2role.json provenance"
)
try:
document: object = json.loads(provenance_path.read_text(encoding="utf-8"))
except (OSError, UnicodeDecodeError, json.JSONDecodeError):
return ProjectDrift(
"unmanaged", role=role, reason="role provenance is unreadable"
)
if not isinstance(document, dict):
return ProjectDrift(
"unmanaged", role=role, reason="role provenance is unreadable"
)
files = document.get("files")
if not isinstance(files, dict):
return ProjectDrift(
"unmanaged", role=role, reason="role provenance has no file hashes"
)
changed: set[str] = set()
recorded: set[str] = set()
for path, entry in files.items():
if not isinstance(path, str) or not isinstance(entry, dict):
return ProjectDrift(
"unmanaged", role=role, reason="role provenance has invalid file hashes"
)
expected = entry.get("sha256")
if not isinstance(expected, str):
return ProjectDrift(
"unmanaged", role=role, reason="role provenance has invalid file hashes"
)
recorded.add(path)
actual = _current_hash(project.source, PurePosixPath(path), entry.get("redacted") is True)
if actual != expected:
changed.add(path)
changed.update(_source_paths(project.source) - recorded)
if changed:
return ProjectDrift(
"source changed", role=role, changed=tuple(sorted(changed))
)
return ProjectDrift("unchanged", role=role)
def _current_hash(
snapshot: ProjectSnapshot, relative_path: PurePosixPath, redacted: bool
) -> str | None:
for source in snapshot.files:
if source.relative_path == relative_path:
if redacted:
return normalized_compose_hash(source.content)
return hashlib.sha256(source.content).hexdigest()
for source in snapshot.dotenv_files:
if source.relative_path == relative_path:
if redacted:
return normalized_dotenv_hash(source.content)
return hashlib.sha256(source.content).hexdigest()
for file_secret in snapshot.file_secrets:
if file_secret.source.relative_path == relative_path:
if redacted:
return normalized_file_secret_hash(file_secret.source.metadata)
return hashlib.sha256(file_secret.source.content).hexdigest()
return None
def _source_paths(snapshot: ProjectSnapshot) -> set[str]:
paths = {str(source.relative_path) for source in snapshot.files}
paths.update(str(source.relative_path) for source in snapshot.dotenv_files)
paths.update(str(file_secret.source.relative_path) for file_secret in snapshot.file_secrets)
return paths
+38
View File
@@ -0,0 +1,38 @@
"""The source-faithful role emitter for remote generation.
This module is the stable import surface; the rendering implementation lives in
:mod:`compose2role.remote_emit`.
"""
from __future__ import annotations
from compose2role.remote_emit import (
GeneratedFile,
PlanComparison,
RolePlan,
RoleWriteResult,
compare_plan,
diff_plan,
external_prerequisites,
normalize_remote_role_name,
preview_remote_role,
with_encrypted_secrets,
write_role,
)
from compose2role.resolve import FileDiff, PlanDiff
__all__ = [
"FileDiff",
"GeneratedFile",
"PlanComparison",
"PlanDiff",
"RolePlan",
"RoleWriteResult",
"compare_plan",
"diff_plan",
"external_prerequisites",
"normalize_remote_role_name",
"preview_remote_role",
"with_encrypted_secrets",
"write_role",
]
+33
View File
@@ -0,0 +1,33 @@
"""Domain errors reported by the command-line interface."""
class Compose2RoleError(Exception):
"""Base class for expected compose2role failures."""
class ConfigurationError(Compose2RoleError):
"""The repository configuration or Ansible layout is invalid or unreadable."""
class DockerContextError(Compose2RoleError):
"""The selected Docker context name is not a valid one."""
class ComposeParseError(Compose2RoleError):
"""The compose document is invalid or outside the supported subset."""
class GenerationError(Compose2RoleError):
"""The target Ansible role cannot be generated safely."""
class RemoteCommandError(Compose2RoleError):
"""An SSH-mediated Docker Compose command failed."""
def __init__(self, message: str, category: str = "remote-command") -> None:
super().__init__(message)
self.category = category
class SecretPolicyError(Compose2RoleError):
"""The controller-side secret-classification policy is invalid."""
+131
View File
@@ -0,0 +1,131 @@
"""Audit findings derived from the typed Compose model."""
from __future__ import annotations
from dataclasses import dataclass
from enum import StrEnum
from compose2role.model import SourcePosition, Stack
class Severity(StrEnum):
"""The impact level assigned to a finding."""
ERROR = "error"
WARNING = "warning"
INFO = "info"
@dataclass(frozen=True)
class Finding:
"""One stable, machine-readable diagnostic for a compose project."""
id: str
severity: Severity
message: str
location: SourcePosition
service: str | None = None
def as_json(self) -> dict[str, str | None]:
"""Return the stable JSON representation used by the CLI."""
return {
"id": self.id,
"severity": self.severity.value,
"message": self.message,
"location": str(self.location),
"service": self.service,
}
def audit_stack(stack: Stack) -> tuple[Finding, ...]:
"""Return all audit findings supported by the current ruleset."""
findings: list[Finding] = []
for service in stack.services:
if not service.image.is_pinned:
findings.append(
Finding(
id="C2R001",
severity=Severity.WARNING,
message="image is not pinned to a tag or digest",
location=service.location,
service=service.name,
)
)
for environment in service.environment:
if environment.is_secret:
findings.append(
Finding(
id="C2R002",
severity=Severity.ERROR,
message=(
f"{environment.name} is an inline secret-shaped environment variable; "
"move it to SOPS"
),
location=environment.location,
service=service.name,
)
)
if service.is_privileged:
findings.append(
Finding(
id="C2R009",
severity=Severity.WARNING,
message="service enables privileged mode",
location=service.location,
service=service.name,
)
)
if service.uses_host_network:
findings.append(
Finding(
id="C2R010",
severity=Severity.WARNING,
message="service uses host networking",
location=service.location,
service=service.name,
)
)
if service.runs_as_root:
findings.append(
Finding(
id="C2R011",
severity=Severity.WARNING,
message="service runs as root",
location=service.location,
service=service.name,
)
)
if not service.has_healthcheck:
findings.append(
Finding(
id="C2R012",
severity=Severity.INFO,
message="service has no healthcheck",
location=service.location,
service=service.name,
)
)
if not service.has_restart_policy:
findings.append(
Finding(
id="C2R016",
severity=Severity.WARNING,
message="service has no restart policy",
location=service.location,
service=service.name,
)
)
return tuple(findings)
def classified_secret_finding(
service: str, key: str, reason: str, location: SourcePosition
) -> Finding:
"""Report a remote secret classification without exposing its effective value."""
return Finding(
id="C2R025",
severity=Severity.INFO,
message=f"{key} is classified as a secret ({reason})",
location=location,
service=service,
)
+159
View File
@@ -0,0 +1,159 @@
"""Discover the Ansible repository layout."""
from __future__ import annotations
import configparser
import os
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path
from compose2role.config import HostProfile, RepositoryConfig, find_file_upwards
from compose2role.errors import ConfigurationError
@dataclass(frozen=True)
class AnsibleConfig:
"""The documented subset of `ansible.cfg` used by the tool."""
path: Path
roles_path: tuple[Path, ...] = ()
collections_path: tuple[Path, ...] = ()
inventory: tuple[Path, ...] = ()
@dataclass(frozen=True)
class AnsibleLayout:
"""Repository paths derived from configuration and Ansible conventions."""
root: Path
config: RepositoryConfig
ansible_config: AnsibleConfig | None
roles_directory: Path
inventory: tuple[Path, ...]
collections_path: tuple[Path, ...]
sops_config: Path | None
secret_policy: Path | None
host: HostProfile
def find_ansible_config(start: Path | None = None) -> Path | None:
"""Find the nearest `ansible.cfg` walking up from START."""
return find_file_upwards(start, "ansible.cfg")
def load_ansible_config(path: Path) -> AnsibleConfig:
"""Read only `[defaults]` `roles_path`, `collections_path`, and `inventory`."""
parser = configparser.ConfigParser(interpolation=None)
try:
with path.open(encoding="utf-8") as handle:
parser.read_file(handle)
except (OSError, UnicodeDecodeError, configparser.Error) as error:
raise ConfigurationError(f"Ansible configuration is unreadable: {path}") from error
if not parser.has_section("defaults"):
return AnsibleConfig(path=path)
return AnsibleConfig(
path=path,
roles_path=_split_config_paths(
parser.get("defaults", "roles_path", fallback=""), path.parent
),
collections_path=_split_config_paths(
parser.get("defaults", "collections_path", fallback=""), path.parent
),
inventory=_split_config_paths(
parser.get("defaults", "inventory", fallback=""), path.parent
),
)
def discover_layout(
config: RepositoryConfig,
*,
start: Path | None = None,
explicit_output: Path | None = None,
explicit_sops_config: Path | None = None,
explicit_secret_policy: Path | None = None,
host: str | None = None,
environ: Mapping[str, str] | None = None,
) -> AnsibleLayout:
"""Resolve repository paths with flags first, then environment, then config."""
environment = os.environ if environ is None else environ
root = config.root
anchor = (start if start is not None else root).expanduser().resolve()
ansible_path = find_ansible_config(anchor)
ansible_config = load_ansible_config(ansible_path) if ansible_path is not None else None
roles_directory = (
_selected_path(explicit_output, environment, None, anchor)
or config.output
or (ansible_config.roles_path[0] if ansible_config and ansible_config.roles_path else None)
or _first_existing((root / "ansible" / "roles", root / "roles"))
or root / "roles"
)
sops_config = (
_selected_path(explicit_sops_config, environment, None, anchor)
or _selected_path(None, environment, "COMPOSE2ROLE_SOPS_CONFIG", anchor)
or config.sops_config
)
secret_policy = (
_selected_path(explicit_secret_policy, environment, None, anchor)
or _selected_path(None, environment, "COMPOSE2ROLE_SECRET_POLICY", anchor)
or config.secret_policy
)
if ansible_config is not None and ansible_config.inventory:
inventory = ansible_config.inventory
else:
discovered_inventory = _first_existing(
(root / "inventory", root / "ansible" / "inventory")
)
inventory = () if discovered_inventory is None else (discovered_inventory,)
return AnsibleLayout(
root=root,
config=config,
ansible_config=ansible_config,
roles_directory=roles_directory,
inventory=inventory,
collections_path=ansible_config.collections_path if ansible_config else (),
sops_config=sops_config,
secret_policy=secret_policy,
host=config.host(host),
)
def _selected_path(
explicit: Path | None,
environment: Mapping[str, str],
environment_name: str | None,
base: Path,
) -> Path | None:
if explicit is not None:
path = explicit.expanduser()
return path if path.is_absolute() else base / path
if environment_name is None:
return None
value = environment.get(environment_name)
if not value:
return None
path = Path(value).expanduser()
return path if path.is_absolute() else base / path
def _first_existing(candidates: tuple[Path, ...]) -> Path | None:
for candidate in candidates:
if candidate.exists():
return candidate
return None
def _split_config_paths(value: str, base: Path) -> tuple[Path, ...]:
paths: list[Path] = []
for entry in value.replace(",", os.pathsep).split(os.pathsep):
entry = entry.strip()
if not entry:
continue
path = Path(entry).expanduser()
paths.append(path.resolve() if path.is_absolute() else (base / path).resolve())
return tuple(paths)
+247
View File
@@ -0,0 +1,247 @@
"""The typed Compose model used by remote audit."""
from __future__ import annotations
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import PurePosixPath
from compose2role.errors import ComposeParseError
@dataclass(frozen=True)
class SourcePosition:
"""A one-based location in a source compose file."""
path: PurePosixPath
line: int
column: int
def __str__(self) -> str:
return f"{self.path}:{self.line}:{self.column}"
@dataclass(frozen=True)
class Image:
"""An image repository and its tag or digest."""
source: str
repository: str | None
reference: str | None
separator: str | None
@property
def uses_interpolation(self) -> bool:
"""Whether Compose must resolve the image expression itself."""
return "${" in self.source
@property
def is_pinned(self) -> bool:
"""Whether the image has a tag other than ``latest`` or a digest."""
if self.uses_interpolation:
return True
return self.separator == "@" or self.reference != "latest"
@dataclass(frozen=True)
class Port:
"""A resolved port mapping."""
target: str
published: str | None
location: SourcePosition
@dataclass(frozen=True)
class EnvironmentVariable:
"""An inline, mapping-form environment variable from a Compose service."""
name: str
value: str
is_secret: bool
location: SourcePosition
@dataclass(frozen=True)
class Service:
"""A Compose service."""
name: str
image: Image
ports: tuple[Port, ...]
environment: tuple[EnvironmentVariable, ...]
is_privileged: bool
uses_host_network: bool
runs_as_root: bool
has_healthcheck: bool
has_restart_policy: bool
option_names: frozenset[str]
location: SourcePosition
@dataclass(frozen=True)
class Stack:
"""A complete Compose project."""
services: tuple[Service, ...]
option_names: frozenset[str]
name: str | None = None
def parse_canonical_stack(document: object, *, source_path: PurePosixPath) -> Stack:
"""Parse Docker Compose's canonical JSON model for remote audit.
Canonical JSON has no useful source positions, so every derived value uses the supplied
synthetic source position. Remote secret classification is intentionally performed separately
from this parser.
"""
document_mapping = _canonical_mapping(document, "remote Docker Compose returned invalid JSON")
services_value = document_mapping.get("services")
services_mapping = _canonical_mapping(
services_value, "remote Docker Compose returned no services"
)
if not services_mapping:
raise ComposeParseError("remote Docker Compose returned no services")
services = tuple(
_parse_canonical_service(source_path, service_name, service_value)
for service_name, service_value in services_mapping.items()
)
name_value = document_mapping.get("name")
if name_value is not None and not isinstance(name_value, str):
raise ComposeParseError("remote Docker Compose returned an invalid project name")
return Stack(
services=services,
option_names=frozenset(document_mapping),
name=name_value,
)
def _canonical_mapping(value: object, message: str) -> dict[str, object]:
if not isinstance(value, dict):
raise ComposeParseError(message)
mapping: dict[str, object] = {}
for key, item in value.items():
if not isinstance(key, str):
raise ComposeParseError(message)
mapping[key] = item
return mapping
def _parse_canonical_service(
source_path: PurePosixPath, service_name: str, service_value: object
) -> Service:
service = _canonical_mapping(service_value, "remote Docker Compose returned an invalid service")
position = SourcePosition(path=source_path, line=1, column=1)
image_value = service.get("image")
if not isinstance(image_value, str) or not image_value:
raise ComposeParseError("remote Docker Compose returned a service without an image")
ports_value = service.get("ports", [])
if not isinstance(ports_value, list):
raise ComposeParseError("remote Docker Compose returned invalid service ports")
environment_value = _canonical_mapping(
service.get("environment", {}), "remote Docker Compose returned invalid service environment"
)
environment: list[EnvironmentVariable] = []
for name, value in environment_value.items():
if not name or not isinstance(value, str) or _has_unresolved_interpolation(value):
raise ComposeParseError("remote Docker Compose returned unresolved environment values")
environment.append(
EnvironmentVariable(
name=name,
value=value,
is_secret=False,
location=position,
)
)
return Service(
name=service_name,
image=_parse_canonical_image(image_value),
ports=tuple(_parse_canonical_port(value, position) for value in ports_value),
environment=tuple(environment),
is_privileged=service.get("privileged") is True,
uses_host_network=service.get("network_mode") == "host",
runs_as_root=_runs_as_root(service.get("user")),
has_healthcheck=_has_healthcheck(service),
has_restart_policy=_has_restart_policy(service.get("restart")),
option_names=frozenset(service),
location=position,
)
def _parse_canonical_image(image: str) -> Image:
if _has_unresolved_interpolation(image):
raise ComposeParseError("remote Docker Compose returned an unresolved image reference")
if "@" in image:
repository, reference = image.split("@", maxsplit=1)
if repository and reference:
return Image(source=image, repository=repository, reference=reference, separator="@")
raise ComposeParseError("remote Docker Compose returned an invalid image reference")
last_slash = image.rfind("/")
last_colon = image.rfind(":")
if last_colon > last_slash:
repository, reference = image.rsplit(":", maxsplit=1)
if repository and reference:
return Image(source=image, repository=repository, reference=reference, separator=":")
raise ComposeParseError("remote Docker Compose returned an invalid image reference")
return Image(source=image, repository=image, reference="latest", separator=":")
def _has_unresolved_interpolation(value: str) -> bool:
"""Return whether a canonical value still contains an active Compose interpolation."""
index = 0
while index < len(value):
if value[index] != "$":
index += 1
continue
if index + 1 == len(value):
return False
following = value[index + 1]
if following == "$":
index += 2
continue
if following == "{" or following == "_" or (
following.isascii() and following.isalpha()
):
return True
index += 1
return False
def _parse_canonical_port(value: object, position: SourcePosition) -> Port:
port = _canonical_mapping(value, "remote Docker Compose returned an invalid port mapping")
target = port.get("target")
published = port.get("published")
if isinstance(target, bool) or not isinstance(target, (int, str)):
raise ComposeParseError("remote Docker Compose returned an invalid port mapping")
if isinstance(published, bool) or not isinstance(published, (int, str, type(None))):
raise ComposeParseError("remote Docker Compose returned an invalid port mapping")
return Port(
target=str(target),
published=None if published is None else str(published),
location=position,
)
def _runs_as_root(user: object) -> bool:
return user is None or user == "" or user == "0" or user == 0
def _has_healthcheck(service: Mapping[str, object]) -> bool:
if "healthcheck" not in service:
return False
healthcheck: object = service.get("healthcheck")
if not isinstance(healthcheck, Mapping):
return True
if healthcheck.get("disable") is True:
return False
test: object = healthcheck.get("test")
return test != "NONE"
def _has_restart_policy(restart: object) -> bool:
return isinstance(restart, str) and restart not in {"", "no"}
+25
View File
@@ -0,0 +1,25 @@
"""Transport-neutral role and Compose project name normalization."""
from __future__ import annotations
import re
def normalize_role_name(raw: str) -> str:
"""Derive a deterministic valid Ansible role name."""
normalized = re.sub(r"[^a-z0-9_]+", "_", raw.lower()).strip("_")
if not normalized:
return "compose_project"
if not normalized[0].isalpha():
return f"compose_{normalized}"
return normalized
def normalize_compose_project_name(raw: str) -> str:
"""Derive a deterministic Compose-compatible project name."""
normalized = re.sub(r"[^a-z0-9_-]+", "", raw.lower()).strip("_-")
if not normalized:
return "compose"
if not normalized[0].isalnum():
return f"compose_{normalized}"
return normalized
+601
View File
@@ -0,0 +1,601 @@
"""Resolve remote Docker Compose projects without persisting their source data."""
from __future__ import annotations
import base64
import binascii
import json
import os
import shlex
from collections.abc import Callable
from json import JSONDecodeError
from pathlib import PurePosixPath
from select import select
from subprocess import DEVNULL, PIPE, Popen, TimeoutExpired, run
from time import monotonic
from typing import IO, NoReturn
from compose2role.discover import DiscoveredProject, discover_projects, ssh_command
from compose2role.errors import RemoteCommandError
from compose2role.resolve import (
AuditOutcome,
AuditResult,
Canonicalizer,
CanonicalResult,
ProjectCapture,
ProjectFailure,
ProjectResolutionError,
ResolvedProject,
SourceFile,
relative_source_path,
resolve_project,
)
from compose2role.secret_policy import SecretPolicy
from compose2role.source import FileMetadata
_SSH_TIMEOUT_SECONDS = 30
_DISCOVERY_MESSAGES = {
"ssh": "SSH discovery failed",
"ssh-client": "OpenSSH client executable 'ssh' was not found",
"ssh-timeout": "SSH discovery timed out",
"remote-discovery": "remote Docker Compose discovery failed",
"invalid-json": "remote Docker Compose discovery returned invalid JSON",
"invalid-discovery": "remote Docker Compose discovery returned invalid project data",
}
_REMOTE_OPERATION_PREFIX = """\
set -eu
main() {
base64 -d | {
IFS= read -r operation || exit 64
case "$operation" in
config)
IFS= read -r project || exit 64
IFS= read -r root || exit 64
set --
while IFS= read -r file; do
[ -n "$file" ] || exit 64
set -- "$@" -f "$file"
done
exec env -i PATH="$PATH" %(docker)s compose --project-name "$project" \\
--project-directory "$root" "$@" config --format json
;;
*)
exit 64
;;
esac
}
}
main <<'__C2R_PAYLOAD__'
"""
_REMOTE_OPERATION_SUFFIX = """\
__C2R_PAYLOAD__
"""
_REMOTE_SOURCE_SESSION_SCRIPT = """\
# compose2role-source-session-v1
set -eu
source_state() {
source_path=$1
source_root=$2
root_physical=$(CDPATH= cd -P "$source_root" && pwd -P) || return 65
parent_physical=$(CDPATH= cd -P "$(dirname "$source_path")" && pwd -P) || parent_physical=
if [ "$root_physical" != / ] && [ -n "$parent_physical" ] \
&& [ "$parent_physical" != "$root_physical" ] \
&& [ "${parent_physical#"$root_physical"/}" = "$parent_physical" ]; then
printf '%s\\n' escape
elif [ -L "$source_path" ]; then
printf '%s\\n' symlink
elif [ -f "$source_path" ]; then
printf '%s\\n' file
elif [ -e "$source_path" ]; then
printf '%s\\n' other
else
printf '%s\\n' missing
fi
}
main() {
printf '%s\\n' ready
while IFS=' ' read -r marker request_id request_operation extra; do
[ "$marker" = request ] || exit 64
[ -n "$request_id" ] && [ -z "$extra" ] || exit 64
case "$request_id" in
''|*[!0-9]*) exit 64 ;;
esac
IFS= read -r path || exit 64
IFS= read -r root || exit 64
IFS= read -r trailer || exit 64
[ "$trailer" = "end $request_id" ] || exit 64
if ! state=$(source_state "$path" "$root"); then
printf 'result %s error\\nend %s\\n' "$request_id" "$request_id"
continue
fi
case "$request_operation" in
capture)
case "$state" in
file)
if ! metadata=$(stat -c '%a %u %g' -- "$path"); then
printf 'result %s error\\nend %s\\n' "$request_id" "$request_id"
continue
fi
if ! encoded=$(base64 < "$path"); then
printf 'result %s error\\nend %s\\n' "$request_id" "$request_id"
continue
fi
printf 'result %s file %s\\n' "$request_id" "$metadata"
printf '%s' "$encoded" | tr -d '\\n'
printf '\\nend %s\\n' "$request_id"
;;
escape|symlink|other|missing)
printf 'result %s %s\\nend %s\\n' "$request_id" "$state" "$request_id"
;;
*) exit 65 ;;
esac
;;
stat)
printf 'result %s %s\\nend %s\\n' "$request_id" "$state" "$request_id"
;;
*) exit 64 ;;
esac
done
}
# compose2role-source-session-ready
main
"""
class _RemoteSourceSession:
"""One framed SSH session for all source reads needed by a remote project."""
def __init__(self, host: str, *, use_sudo: bool) -> None:
remote_command: tuple[str, ...] = ("sh", "-s")
if use_sudo:
remote_command = ("sudo", "-n", *remote_command)
try:
process: Popen[bytes] = Popen(
ssh_command(host, remote_command),
stdin=PIPE,
stdout=PIPE,
stderr=DEVNULL,
)
except FileNotFoundError as error:
raise RemoteProjectError(
"ssh-client", "OpenSSH client executable 'ssh' was not found"
) from error
if process.stdin is None or process.stdout is None:
process.kill()
raise RemoteProjectError("source", "remote source session could not be started")
self._process = process
self._stdin: IO[bytes] = process.stdin
self._stdout: IO[bytes] = process.stdout
self._line_buffer = b""
self._next_request_id = 0
self._closed = False
try:
self._write(_REMOTE_SOURCE_SESSION_SCRIPT)
if self._read_line() != "ready":
raise RemoteProjectError("source", "remote source session response is invalid")
except RemoteProjectError:
self.abort()
raise
def capture(self, root: PurePosixPath, path: PurePosixPath) -> SourceFile | str:
"""Capture PATH and its metadata, or return its source-boundary state."""
return self._request("capture", root, path)
def stat(self, root: PurePosixPath, path: PurePosixPath) -> str:
"""Return the source-boundary state of PATH without reading its bytes."""
state = self._request("stat", root, path)
if not isinstance(state, str):
raise RemoteProjectError("source", "remote source session response is invalid")
return state
def close(self) -> None:
"""Finish the session and reject an abnormal remote exit or trailing output."""
if self._closed:
return
try:
self._stdin.close()
except OSError:
pass
trailing = self._line_buffer + self._read_to_eof()
try:
returncode = self._process.wait(timeout=_SSH_TIMEOUT_SECONDS)
except TimeoutExpired as error:
self.abort()
raise RemoteProjectError("timeout", "remote operation timed out") from error
self._closed = True
if returncode == 255:
raise RemoteProjectError("ssh", "SSH connection or remote command failed")
if returncode != 0:
raise RemoteProjectError("source", "remote source validation failed")
if trailing:
raise RemoteProjectError("source", "remote source session response is invalid")
def abort(self) -> None:
"""Close a failed session without replacing its original value-free error."""
if self._closed:
return
self._closed = True
try:
self._stdin.close()
except OSError:
pass
if self._process.poll() is None:
self._process.terminate()
try:
self._process.wait(timeout=_SSH_TIMEOUT_SECONDS)
except TimeoutExpired:
self._process.kill()
self._process.wait()
def _request(
self, operation: str, root: PurePosixPath, path: PurePosixPath
) -> SourceFile | str:
request_id = self._next_request_id
self._next_request_id += 1
self._write(f"request {request_id} {operation}\n{path}\n{root}\nend {request_id}\n")
header = self._read_line().split()
if len(header) < 3 or header[:2] != ["result", str(request_id)]:
raise RemoteProjectError("source", "remote source session response is invalid")
state = header[2]
if state not in {"error", "escape", "file", "missing", "other", "symlink"}:
raise RemoteProjectError("source", "remote source session response is invalid")
if state == "error":
self._expect_end(request_id)
raise RemoteProjectError("source", "remote source request failed")
if operation == "stat":
if len(header) != 3:
raise RemoteProjectError("source", "remote source session response is invalid")
self._expect_end(request_id)
return state
if operation != "capture":
raise RemoteProjectError("source", "remote source session response is invalid")
if state != "file":
if len(header) != 3:
raise RemoteProjectError("source", "remote source session response is invalid")
self._expect_end(request_id)
return state
if len(header) != 6:
raise RemoteProjectError("source", "remote source session response is invalid")
encoded_content = self._read_line()
self._expect_end(request_id)
metadata, content = _parse_captured_source_file(
f"file {' '.join(header[3:])}\n{encoded_content}"
)
return SourceFile(
relative_path=relative_source_path(root, path),
content=content,
metadata=metadata,
)
def _expect_end(self, request_id: int) -> None:
if self._read_line() != f"end {request_id}":
raise RemoteProjectError("source", "remote source session response is invalid")
def _write(self, value: str) -> None:
try:
self._stdin.write(value.encode("utf-8"))
self._stdin.flush()
except (BrokenPipeError, OSError) as error:
self._raise_closed(error)
def _read_line(self) -> str:
deadline = monotonic() + _SSH_TIMEOUT_SECONDS
while True:
separator = self._line_buffer.find(b"\n")
if separator != -1:
line = self._line_buffer[:separator]
self._line_buffer = self._line_buffer[separator + 1 :]
try:
return line.decode("utf-8")
except UnicodeDecodeError as error:
raise RemoteProjectError(
"source", "remote response was not valid UTF-8"
) from error
chunk = self._read_chunk(deadline)
if not chunk:
self._raise_closed()
self._line_buffer += chunk
def _read_to_eof(self) -> bytes:
deadline = monotonic() + _SSH_TIMEOUT_SECONDS
chunks = bytearray()
while True:
chunk = self._read_chunk(deadline)
if not chunk:
return bytes(chunks)
chunks += chunk
def _read_chunk(self, deadline: float) -> bytes:
timeout = deadline - monotonic()
if timeout <= 0:
self.abort()
raise RemoteProjectError("timeout", "remote operation timed out")
readable, _, _ = select((self._stdout,), (), (), timeout)
if not readable:
self.abort()
raise RemoteProjectError("timeout", "remote operation timed out")
try:
return os.read(self._stdout.fileno(), 8192)
except OSError as error:
self._raise_closed(error)
def _raise_closed(self, error: OSError | None = None) -> NoReturn:
try:
returncode = self._process.wait(timeout=_SSH_TIMEOUT_SECONDS)
except TimeoutExpired:
self.abort()
returncode = None
if returncode == 255:
message = ("ssh", "SSH connection or remote command failed")
else:
message = ("source", "remote source session closed unexpectedly")
if error is None:
raise RemoteProjectError(*message)
raise RemoteProjectError(*message) from error
RemoteProjectError = ProjectResolutionError
class RemoteCapture(ProjectCapture):
"""Adapt the framed SSH source session to the shared capture seam."""
def __init__(self, host: str, *, use_sudo: bool) -> None:
self._session = _RemoteSourceSession(host, use_sudo=use_sudo)
def capture(self, root: PurePosixPath, path: PurePosixPath) -> SourceFile | str:
"""Capture PATH and its metadata, or return its source-boundary state."""
return self._session.capture(root, path)
def stat(self, root: PurePosixPath, path: PurePosixPath) -> str:
"""Return the source-boundary state of PATH without reading its bytes."""
return self._session.stat(root, path)
def close(self) -> None:
"""Finish the session and reject an abnormal remote exit."""
self._session.close()
def abort(self) -> None:
"""Discard the session without replacing the original value-free error."""
self._session.abort()
class RemoteComposeCanonicalizer(Canonicalizer):
"""Resolve a remote project through Docker Compose's canonical JSON model."""
def __init__(
self, host: str, *, use_sudo: bool, docker_context: str | None = None
) -> None:
self._host = host
self._use_sudo = use_sudo
self._docker_context = docker_context
def canonicalize(
self,
identity: DiscoveredProject,
root: PurePosixPath,
files: tuple[PurePosixPath, ...],
) -> CanonicalResult:
"""Return the effective Compose document for the captured project."""
config_files = tuple(str(root.joinpath(*file.parts)) for file in files)
payload = "\n".join(("config", identity.name, str(root), *config_files)) + "\n"
output = _run_remote_config(
self._host,
identity.name,
payload,
use_sudo=self._use_sudo,
docker_context=self._docker_context,
)
try:
document: object = json.loads(output)
except JSONDecodeError as error:
raise RemoteProjectError(
"invalid-json", "remote Docker Compose returned invalid canonical JSON"
) from error
return CanonicalResult(document=_canonical_document(document))
def resolve_remote_projects(
host: str,
*,
project_name: str | None,
all_projects: bool,
use_sudo: bool,
secret_policy: SecretPolicy | None,
docker_context: str | None = None,
on_outcome: Callable[[AuditOutcome], None] | None = None,
) -> AuditResult:
"""Resolve a selected remote project or all discovered projects in isolation."""
try:
projects = discover_projects(
host,
use_sudo=use_sudo,
include_working_directories=True,
docker_context=docker_context,
)
except RemoteCommandError as error:
raise RemoteProjectError(
error.category,
_DISCOVERY_MESSAGES.get(error.category, "remote Docker Compose discovery failed"),
) from error
if project_name is not None:
selected_projects = tuple(project for project in projects if project.name == project_name)
if not selected_projects:
failure = ProjectFailure(
name=project_name,
category="unknown-project",
message="the requested remote Compose project was not discovered",
)
if on_outcome is not None:
on_outcome(failure)
return AuditResult(
projects=(),
failures=(failure,),
outcomes=(failure,),
)
elif all_projects:
selected_projects = projects
else:
raise ValueError("remote project selection is required")
resolved: list[ResolvedProject] = []
failures: list[ProjectFailure] = []
outcomes: list[AuditOutcome] = []
for project in selected_projects:
try:
resolution = _resolve_remote_project(
host,
project,
use_sudo=use_sudo,
secret_policy=secret_policy,
docker_context=docker_context,
)
resolved.append(resolution)
outcome: AuditOutcome = resolution
except RemoteProjectError as error:
failure = ProjectFailure(
name=project.name,
category=error.category,
message=str(error),
)
failures.append(failure)
outcome = failure
outcomes.append(outcome)
if on_outcome is not None:
on_outcome(outcome)
return AuditResult(
projects=tuple(resolved),
failures=tuple(failures),
outcomes=tuple(outcomes),
)
def _resolve_remote_project(
host: str,
project: DiscoveredProject,
*,
use_sudo: bool,
secret_policy: SecretPolicy | None,
docker_context: str | None,
) -> ResolvedProject:
return resolve_project(
RemoteCapture(host, use_sudo=use_sudo),
RemoteComposeCanonicalizer(
host, use_sudo=use_sudo, docker_context=docker_context
),
project,
secret_policy,
)
def _canonical_document(value: object) -> dict[str, object]:
if not isinstance(value, dict):
raise RemoteProjectError(
"invalid-model", "remote Docker Compose returned an unsupported effective model"
)
document: dict[str, object] = {}
for key, item in value.items():
if not isinstance(key, str):
raise RemoteProjectError(
"invalid-model", "remote Docker Compose returned an unsupported effective model"
)
document[key] = item
return document
def _parse_captured_source_file(output: str) -> tuple[FileMetadata, bytes]:
"""Decode one captured file header and its base64 content line."""
header, separator, encoded_content = output.partition("\n")
if not separator:
raise RemoteProjectError("source", "remote source metadata is invalid")
fields = header.split()
if len(fields) != 4 or fields[0] != "file":
raise RemoteProjectError("source", "remote source metadata is invalid")
mode, uid, gid = fields[1:]
if (
not mode
or len(mode) > 4
or any(character not in "01234567" for character in mode)
or len(uid) > 10
or not uid.isascii()
or not uid.isdecimal()
or len(gid) > 10
or not gid.isascii()
or not gid.isdecimal()
):
raise RemoteProjectError("source", "remote source metadata is invalid")
try:
content = base64.b64decode(encoded_content.strip(), validate=True)
except (binascii.Error, ValueError) as error:
raise RemoteProjectError("source", "remote source content is invalid") from error
return FileMetadata(mode=int(mode, 8), uid=int(uid), gid=int(gid)), content
def _run_remote_config(
host: str,
project_name: str,
payload: str,
*,
use_sudo: bool,
docker_context: str | None,
) -> str:
return _run_remote_operation(
host,
project_name,
payload,
use_sudo=use_sudo,
docker_context=docker_context,
category="remote-compose",
failure_message=f"remote Compose resolution failed for project {project_name!r}",
)
def _run_remote_operation(
host: str,
project_name: str,
payload: str,
*,
use_sudo: bool,
category: str,
failure_message: str,
docker_context: str | None = None,
) -> str:
remote_command: tuple[str, ...] = ("sh", "-s")
if use_sudo:
remote_command = ("sudo", "-n", *remote_command)
try:
result = run(
ssh_command(host, remote_command),
check=False,
capture_output=True,
input=_operation_script(payload, docker_context),
text=True,
encoding="utf-8",
timeout=_SSH_TIMEOUT_SECONDS,
)
except FileNotFoundError as error:
raise RemoteProjectError(
"ssh-client", "OpenSSH client executable 'ssh' was not found"
) from error
except TimeoutExpired as error:
raise RemoteProjectError("timeout", "remote operation timed out") from error
except UnicodeDecodeError as error:
raise RemoteProjectError(category, "remote response was not valid UTF-8") from error
if result.returncode == 255:
raise RemoteProjectError("ssh", "SSH connection or remote command failed")
if result.returncode != 0:
raise RemoteProjectError(category, failure_message)
return result.stdout
def _operation_script(payload: str, docker_context: str | None = None) -> str:
encoded_payload = base64.b64encode(payload.encode("utf-8")).decode("ascii")
docker = (
"docker"
if docker_context is None
else f"docker --context {shlex.quote(docker_context)}"
)
prefix = _REMOTE_OPERATION_PREFIX % {"docker": docker}
return f"{prefix}{encoded_payload}\n{_REMOTE_OPERATION_SUFFIX}"
+807
View File
@@ -0,0 +1,807 @@
"""Render and atomically write source-faithful Compose roles."""
from __future__ import annotations
import hashlib
import json
import os
import re
import shutil
import tempfile
from dataclasses import dataclass
from datetime import UTC, datetime
from importlib.metadata import version
from pathlib import Path, PurePosixPath
from typing import Literal
from ruamel.yaml import YAML
from ruamel.yaml.error import YAMLError
from compose2role.errors import GenerationError
from compose2role.naming import normalize_role_name
from compose2role.resolve import FileDiff, PlanDiff, ResolvedProject
from compose2role.source import FileMetadata
from compose2role.source_fidelity import (
SourceRoleMaterialization,
SourceTemplate,
materialize_source_role,
)
_ROLE_NAME = re.compile(r"[a-z][a-z0-9_]*")
_WriteState = Literal["generated", "unchanged"]
_PROVENANCE_FILE = PurePosixPath(".compose2role.json")
@dataclass(frozen=True)
class GeneratedFile:
"""One text or binary file in a generated role."""
relative_path: PurePosixPath
content: str | bytes
classified_secret: bool = False
@dataclass(frozen=True)
class RolePlan:
"""A fully rendered role that has not been written yet."""
role_name: str
files: tuple[GeneratedFile, ...]
secret_values: dict[str, str]
@dataclass(frozen=True)
class PlanComparison:
"""The value-free difference between a rendered plan and an existing role."""
role_exists: bool
unmanaged: bool
added: tuple[str, ...]
changed: tuple[str, ...]
@property
def identical(self) -> bool:
"""Return whether the existing role matches every planned file byte for byte."""
return self.role_exists and not self.added and not self.changed
@dataclass(frozen=True)
class RoleWriteResult:
"""The outcome of placing one rendered role."""
role_directory: Path
state: _WriteState
backup_directory: Path | None = None
def normalize_remote_role_name(project_name: str) -> str:
"""Derive a deterministic valid Ansible role name from a raw Compose project name."""
role_name = normalize_role_name(project_name)
if not _ROLE_NAME.fullmatch(role_name):
raise GenerationError("remote project name cannot be normalized into an Ansible role name")
return role_name
def preview_remote_role(
project: ResolvedProject, role_name: str, *, docker_context: str | None = None
) -> RolePlan:
"""Render a source-faithful remote role without writing it."""
_validate_role_name(role_name)
materialization = materialize_source_role(project, role_name)
return _preview_role(project, role_name, materialization, docker_context)
def _preview_role(
project: ResolvedProject,
role_name: str,
materialization: SourceRoleMaterialization,
docker_context: str | None,
) -> RolePlan:
has_secrets = bool(materialization.secret_values)
prerequisites = external_prerequisites(project.canonical_document)
files: list[GeneratedFile] = [
GeneratedFile(
PurePosixPath("defaults/main.yml"),
_render_source_defaults(
role_name,
materialization,
str(project.source.root),
project.project.name,
docker_context,
),
),
GeneratedFile(
PurePosixPath("tasks/main.yml"),
_render_source_tasks(
role_name,
materialization,
tuple(source.relative_path for source in project.source.files),
has_secrets,
docker_context,
),
),
GeneratedFile(PurePosixPath("meta/main.yml"), _render_meta(has_secrets)),
GeneratedFile(
PurePosixPath("meta/requirements.yml"), _render_requirements(has_secrets)
),
GeneratedFile(
PurePosixPath(".compose2role.json"),
_render_provenance(project, role_name, prerequisites, materialization),
),
]
files.extend(
GeneratedFile(
PurePosixPath("templates").joinpath(*template.relative_template_path.parts),
template.content,
classified_secret=template.has_secrets,
)
for template in materialization.templates
)
return RolePlan(
role_name=role_name,
files=tuple(files),
secret_values=materialization.secret_values,
)
def with_encrypted_secrets(plan: RolePlan, encrypted_secrets: bytes) -> RolePlan:
"""Attach real encrypted SOPS output to a rendered remote role plan."""
if not plan.secret_values:
raise GenerationError("remote role has no secrets to encrypt")
return RolePlan(
role_name=plan.role_name,
files=(
*plan.files,
GeneratedFile(
PurePosixPath("vars/secrets.sops.yml"),
encrypted_secrets,
classified_secret=True,
),
),
secret_values=plan.secret_values,
)
def compare_plan(plan: RolePlan, role_directory: Path) -> PlanComparison:
"""Compare a rendered plan with an existing role without writing anything."""
if role_directory.is_symlink() or (
role_directory.exists() and not role_directory.is_dir()
):
raise GenerationError(f"existing role path must be a directory: {role_directory}")
planned = _planned_files(plan)
if not role_directory.exists():
return PlanComparison(
role_exists=False,
unmanaged=False,
added=tuple(sorted(str(path) for path in planned)),
changed=(),
)
existing = _existing_role_files(role_directory)
return PlanComparison(
role_exists=True,
unmanaged=_PROVENANCE_FILE not in existing,
added=tuple(sorted(str(path) for path in planned if path not in existing)),
changed=tuple(
sorted(
str(path)
for path, content in planned.items()
if path in existing and existing[path] != content
)
),
)
def diff_plan(plan: RolePlan, role_directory: Path) -> PlanDiff:
"""Compare a rendered plan with an existing role for value-free review."""
if role_directory.is_symlink() or (
role_directory.exists() and not role_directory.is_dir()
):
raise GenerationError(f"existing role path must be a directory: {role_directory}")
planned = {
generated_file.relative_path: generated_file for generated_file in plan.files
}
if not role_directory.exists():
existing: dict[PurePosixPath, bytes | None] = {}
role_exists = False
unmanaged = False
else:
existing = _existing_role_files(role_directory)
role_exists = True
unmanaged = _PROVENANCE_FILE not in existing
files: list[FileDiff] = []
for path in sorted(planned, key=str):
generated_file = planned[path]
content = _generated_file_bytes(generated_file)
if path not in existing:
change: Literal["added", "changed", "unchanged"] = "added"
previous = b""
elif existing[path] == content:
change = "unchanged"
previous = b""
else:
change = "changed"
previous = existing[path] or b""
withheld, withheld_reason = _withheld(generated_file)
if unmanaged:
withheld, withheld_reason = True, None
if withheld:
files.append(
FileDiff(
path=str(path),
change=change,
withheld=True,
withheld_reason=withheld_reason,
)
)
elif change == "unchanged":
files.append(FileDiff(path=str(path), change=change))
else:
files.append(
FileDiff(
path=str(path),
change=change,
old_text=_decode_diff_text(previous),
new_text=_decode_diff_text(content),
)
)
return PlanDiff(role_exists=role_exists, unmanaged=unmanaged, files=tuple(files))
_DIFF_SAFE_PATHS = frozenset(
{
"defaults/main.yml",
"tasks/main.yml",
".compose2role.json",
"README.md",
"site.yml",
"site.yaml",
}
)
def _is_safe_diff_path(path: PurePosixPath) -> bool:
if str(path) in _DIFF_SAFE_PATHS:
return True
return bool(path.parts) and path.parts[0] == "meta"
def _withheld(generated_file: GeneratedFile) -> tuple[bool, str | None]:
if generated_file.classified_secret:
return True, "classified secret"
if not _is_safe_diff_path(generated_file.relative_path):
return True, None
return False, None
def _decode_diff_text(content: bytes) -> str:
return content.decode("utf-8", errors="replace")
def write_role(
plan: RolePlan, output_directory: Path, *, force: bool = False
) -> RoleWriteResult:
"""Place a rendered role, refusing an existing differing role unless forced."""
if output_directory.is_symlink() or (
output_directory.exists() and not output_directory.is_dir()
):
raise GenerationError("role output must be a directory")
role_directory = output_directory / plan.role_name
if role_directory.is_symlink() or (
role_directory.exists() and not role_directory.is_dir()
):
raise GenerationError(f"refusing to overwrite non-directory role path: {role_directory}")
if not role_directory.exists():
try:
output_directory.mkdir(parents=True, exist_ok=True)
except OSError as error:
raise GenerationError("role output could not be written") from error
_install_new_role(plan, output_directory, role_directory)
return RoleWriteResult(role_directory, "generated")
comparison = compare_plan(plan, role_directory)
if comparison.identical:
return RoleWriteResult(role_directory, "unchanged")
if not force:
raise GenerationError(_refusal_message(comparison, role_directory))
backup_directory = _replace_role(plan, output_directory, role_directory)
return RoleWriteResult(role_directory, "generated", backup_directory=backup_directory)
def _refusal_message(comparison: PlanComparison, role_directory: Path) -> str:
if comparison.unmanaged:
return (
f"refusing to overwrite unmanaged role {role_directory} "
"(no .compose2role.json); use --force to replace it"
)
return (
f"refusing to overwrite role {role_directory}: {_describe_changes(comparison)}; "
"use --diff to review or --force to replace it"
)
def _describe_changes(comparison: PlanComparison) -> str:
changes: list[str] = []
if comparison.added:
changes.append(f"added: {', '.join(comparison.added)}")
if comparison.changed:
changes.append(f"changed: {', '.join(comparison.changed)}")
return "; ".join(changes) if changes else "no file differences"
def _planned_files(plan: RolePlan) -> dict[PurePosixPath, bytes]:
return {
generated_file.relative_path: _generated_file_bytes(generated_file)
for generated_file in plan.files
}
def _generated_file_bytes(generated_file: GeneratedFile) -> bytes:
if isinstance(generated_file.content, bytes):
return generated_file.content
return generated_file.content.encode("utf-8")
def _existing_role_files(role_directory: Path) -> dict[PurePosixPath, bytes | None]:
files: dict[PurePosixPath, bytes | None] = {}
for directory, _, names in os.walk(role_directory, followlinks=False):
for name in names:
path = Path(directory, name)
relative_path = PurePosixPath(path.relative_to(role_directory).as_posix())
files[relative_path] = _read_existing_file(path)
return files
def _read_existing_file(path: Path) -> bytes | None:
try:
if path.is_symlink() or not path.is_file():
return None
return path.read_bytes()
except OSError:
return None
def _install_new_role(
plan: RolePlan, output_directory: Path, role_directory: Path
) -> None:
staging_directory = _stage_plan(plan, output_directory)
try:
if role_directory.exists() or role_directory.is_symlink():
raise GenerationError(f"refusing to overwrite existing role: {role_directory}")
os.replace(staging_directory, role_directory)
except OSError as error:
raise GenerationError("role output could not be written") from error
finally:
_discard_staging(staging_directory)
def _replace_role(
plan: RolePlan, output_directory: Path, role_directory: Path
) -> Path:
staging_directory = _stage_plan(plan, output_directory)
backups_directory = output_directory / ".compose2role" / "backups"
try:
backups_directory.mkdir(parents=True, exist_ok=True)
except OSError as error:
_discard_staging(staging_directory)
raise GenerationError("role output could not be written") from error
backup_directory = backups_directory / f"{plan.role_name}-{_utc_timestamp()}"
try:
os.replace(role_directory, backup_directory)
except OSError as error:
_discard_staging(staging_directory)
raise GenerationError("role output could not be written") from error
try:
os.replace(staging_directory, role_directory)
except OSError as error:
try:
os.replace(backup_directory, role_directory)
except OSError as restore_error:
raise GenerationError(
"role output could not be written and the previous role could not be "
f"restored; backup retained at {backup_directory}"
) from restore_error
raise GenerationError(
"role output could not be written; previous role restored"
) from error
finally:
_discard_staging(staging_directory)
return backup_directory
def _stage_plan(plan: RolePlan, output_directory: Path) -> Path:
try:
staging_directory = Path(
tempfile.mkdtemp(prefix=f".{plan.role_name}.", dir=output_directory)
)
except OSError as error:
raise GenerationError("role output could not be written") from error
try:
for generated_file in plan.files:
destination = staging_directory.joinpath(*generated_file.relative_path.parts)
destination.parent.mkdir(parents=True, exist_ok=True)
if isinstance(generated_file.content, bytes):
destination.write_bytes(generated_file.content)
else:
destination.write_text(generated_file.content, encoding="utf-8")
except OSError as error:
_discard_staging(staging_directory)
raise GenerationError("role output could not be written") from error
return staging_directory
def _discard_staging(staging_directory: Path) -> None:
if staging_directory.exists():
shutil.rmtree(staging_directory, ignore_errors=True)
def _utc_timestamp() -> str:
return datetime.now(UTC).strftime("%Y%m%dT%H%M%S%fZ")
def _validate_role_name(role_name: str) -> None:
if not _ROLE_NAME.fullmatch(role_name):
raise GenerationError("remote role name is invalid")
def _render_source_defaults(
role_name: str,
materialization: SourceRoleMaterialization,
project_dir: str,
compose_project_name: str,
docker_context: str | None,
) -> str:
lines = [
"---",
f"{role_name}_project_dir: {json.dumps(project_dir)}",
f"{role_name}_compose_project_name: {json.dumps(compose_project_name)}",
]
if docker_context is not None:
lines.append(f"{role_name}_docker_context: {json.dumps(docker_context)}")
for variable in materialization.variables:
if not variable.is_secret:
lines.append(f"{variable.name}: {json.dumps(variable.value)}")
secret_names = [variable.name for variable in materialization.variables if variable.is_secret]
if secret_names:
lines.append(f"{role_name}_required_secrets:")
lines.extend(f" - {name}" for name in secret_names)
lines.append(f'{role_name}_secrets_file: "{{{{ role_path }}}}/vars/secrets.sops.yml"')
return "\n".join(lines) + "\n"
def _render_source_tasks(
role_name: str,
materialization: SourceRoleMaterialization,
compose_files: tuple[PurePosixPath, ...],
has_secrets: bool,
docker_context: str | None,
) -> str:
source_files = (
*(template.source for template in materialization.templates),
*(file_secret.source for file_secret in materialization.file_secrets),
)
directories = sorted(
{
source.relative_path.parent
for source in source_files
if source.relative_path.parent != PurePosixPath(".")
},
key=str,
)
directory_tasks = ""
if directories:
rendered_directories = "\n".join(
f' - "{{{{ {role_name}_project_dir }}}}/{directory}"' for directory in directories
)
directory_tasks = f"""\
- name: Create {role_name} source directories
tags:
- source
ansible.builtin.file:
path: "{{{{ item }}}}"
state: directory
mode: "0755"
loop:
{rendered_directories}
"""
rendered_sources = "\n\n".join(
_render_source_file_task(role_name, template) for template in materialization.templates
)
rendered_compose_files = "\n".join(
f" - {json.dumps(str(compose_file))}" for compose_file in compose_files
)
secret_tasks = _render_source_secret_tasks(role_name) if has_secrets else ""
file_secret_tasks = _render_file_secret_tasks(role_name, materialization)
context_line = (
f' cli_context: "{{{{ {role_name}_docker_context }}}}"\n'
if docker_context is not None
else ""
)
return f"""\
---
- name: Ensure {role_name} Compose project directory exists
tags:
- source
ansible.builtin.file:
path: "{{{{ {role_name}_project_dir }}}}"
state: directory
mode: "0755"
{secret_tasks}{directory_tasks}{rendered_sources}
{file_secret_tasks}- name: Deploy {role_name} Compose project
tags:
- compose
community.docker.docker_compose_v2:
project_src: "{{{{ {role_name}_project_dir }}}}"
project_name: "{{{{ {role_name}_compose_project_name }}}}"
{context_line} files:
{rendered_compose_files}
state: present
pull: missing
recreate: auto
remove_orphans: false
"""
def _render_source_file_task(role_name: str, template: SourceTemplate) -> str:
"""Render one captured source file through a literal template task."""
relative_path = template.source.relative_path
metadata = template.source.metadata
destination = f"{{{{ {role_name}_project_dir }}}}/{relative_path}"
return f"""\
- name: {json.dumps(f"Render {relative_path}")}
tags:
- source
ansible.builtin.template:
src: {json.dumps(str(template.relative_template_path))}
dest: {json.dumps(destination)}
owner: "{metadata.uid}"
group: "{metadata.gid}"
mode: "{metadata.mode:04o}"
newline_sequence: {json.dumps(template.newline_sequence)}
"""
def _render_source_secret_tasks(role_name: str) -> str:
return f"""\
- name: Load {role_name} SOPS secrets
tags:
- secrets
community.sops.load_vars:
file: "{{{{ {role_name}_secrets_file }}}}"
name: {role_name}_secrets
no_log: true
- name: Assert {role_name} required secrets are present
tags:
- secrets
ansible.builtin.assert:
that: item in {role_name}_secrets
fail_msg: "Missing secret: {{{{ item }}}}"
loop: "{{{{ {role_name}_required_secrets }}}}"
loop_control:
label: "{{{{ item }}}}"
"""
def _render_file_secret_tasks(
role_name: str, materialization: SourceRoleMaterialization
) -> str:
if not materialization.file_secrets:
return ""
rendered_secrets = "\n".join(
(
f' - path: {json.dumps(str(file_secret.source.relative_path))}\n'
f' key: {json.dumps(file_secret.variable_name)}\n'
f' owner: {json.dumps(str(file_secret.source.metadata.uid))}\n'
f' group: {json.dumps(str(file_secret.source.metadata.gid))}\n'
f' mode: "{file_secret.source.metadata.mode:04o}"'
)
for file_secret in materialization.file_secrets
)
return f"""\
- name: Render {role_name} file-backed Compose secrets
tags:
- secrets
ansible.builtin.shell:
cmd: |
set -eu
target="{{{{ ({role_name}_project_dir ~ '/' ~ item.path) | quote }}}}"
owner="{{{{ item.owner }}}}"
group="{{{{ item.group }}}}"
mode="{{{{ item.mode }}}}"
if [ -L "$target" ] || [ -d "$target" ]; then
printf '%s\\n' 'unsafe file-secret target' >&2
exit 4
fi
temporary=$(mktemp "$target.compose2role.XXXXXX")
trap 'rm -f "$temporary"' EXIT
base64 --decode > "$temporary"
target_state="$(stat -c '%a:%u:%g' "$target" 2>/dev/null || true)"
expected_mode="$(printf '%o' "$((0$mode))")"
expected_state="$expected_mode:$owner:$group"
if [ -f "$target" ] && cmp -s "$temporary" "$target"; then
if [ "$target_state" = "$expected_state" ]; then
exit 0
fi
fi
install -o "$owner" -g "$group" -m "$mode" "$temporary" "$target"
exit 3
stdin: "{{{{ {role_name}_secrets[item.key] }}}}"
stdin_add_newline: false
register: {role_name}_file_secret_result
changed_when: {role_name}_file_secret_result.rc == 3
failed_when: {role_name}_file_secret_result.rc not in [0, 3]
no_log: true
loop:
{rendered_secrets}
loop_control:
label: "{{{{ item.path }}}}"
"""
def _render_meta(has_secrets: bool) -> str:
collections = [" - community.docker"]
if has_secrets:
collections.append(" - community.sops")
return "\n".join(("---", "collections:", *collections, "dependencies: []", ""))
def _render_requirements(has_secrets: bool) -> str:
collections = [" - name: community.docker", " version: 5.4.0"]
if has_secrets:
collections.extend((" - name: community.sops", " version: 2.5.0"))
return "\n".join(("---", "collections:", *collections, ""))
def _render_provenance(
project: ResolvedProject,
role_name: str,
prerequisites: tuple[tuple[str, str], ...],
materialization: SourceRoleMaterialization,
) -> str:
redacted_document = _redact_environment_values(project.canonical_document)
canonical_json = json.dumps(redacted_document, sort_keys=True, separators=(",", ":"))
config_files = list(project.project.config_files)
source_root = str(project.source.root)
provenance = {
"external_prerequisites": [
{"kind": kind, "name": name} for kind, name in prerequisites
],
"files": _source_file_hashes(project, materialization),
"generator": "compose2role",
"mode": "remote",
"project": project.project.name,
"role": role_name,
"source": {"config_files": config_files, "root": source_root},
"structural_hash": hashlib.sha256(canonical_json.encode("utf-8")).hexdigest(),
"version": version("compose2role"),
}
return json.dumps(provenance, indent=2, sort_keys=True) + "\n"
def _source_file_hashes(
project: ResolvedProject, materialization: SourceRoleMaterialization
) -> dict[str, dict[str, object]]:
secret_paths = {
template.source.relative_path
for template in materialization.templates
if template.has_secrets
}
entries: dict[str, dict[str, object]] = {}
for source in project.source.files:
if source.relative_path in secret_paths:
entries[str(source.relative_path)] = {
"redacted": True,
"sha256": normalized_compose_hash(source.content),
}
else:
entries[str(source.relative_path)] = {
"redacted": False,
"sha256": hashlib.sha256(source.content).hexdigest(),
}
for source in project.source.dotenv_files:
if source.relative_path in secret_paths:
entries[str(source.relative_path)] = {
"redacted": True,
"sha256": normalized_dotenv_hash(source.content),
}
else:
entries[str(source.relative_path)] = {
"redacted": False,
"sha256": hashlib.sha256(source.content).hexdigest(),
}
for file_secret in project.source.file_secrets:
entries[str(file_secret.source.relative_path)] = {
"redacted": True,
"sha256": normalized_file_secret_hash(file_secret.source.metadata),
}
return entries
def normalized_compose_hash(content: bytes) -> str:
try:
document = YAML(typ="rt").load(content.decode("utf-8"))
except (UnicodeDecodeError, YAMLError):
return _hash_json({"redacted": "compose"})
return _hash_json(_redact_environment_values(document))
def normalized_dotenv_hash(content: bytes) -> str:
try:
text = content.decode("utf-8")
except UnicodeDecodeError:
return _hash_json({"redacted": "dotenv"})
keys = [
match.group(1)
for line in text.splitlines()
if (match := _DOTENV_KEY.match(line)) is not None
]
return _hash_json({"dotenv_keys": keys})
def normalized_file_secret_hash(metadata: FileMetadata) -> str:
return _hash_json(
{
"content": "<redacted-file-secret>",
"gid": metadata.gid,
"mode": metadata.mode,
"uid": metadata.uid,
}
)
def _hash_json(value: object) -> str:
encoded = json.dumps(
value, sort_keys=True, separators=(",", ":"), default=str
).encode("utf-8")
return hashlib.sha256(encoded).hexdigest()
_DOTENV_KEY = re.compile(r"[ \t]*(?:export[ \t]+)?([A-Za-z_][A-Za-z0-9_]*)[ \t]*=")
def _redact_environment_values(value: object, *, redact_values: bool = False) -> object:
if isinstance(value, dict):
return {
key: "<redacted-environment-value>"
if redact_values
else _redact_environment_values(item, redact_values=key == "environment")
for key, item in value.items()
}
if isinstance(value, list):
if redact_values:
return ["<redacted-environment-value>" for _ in value]
return [_redact_environment_values(item) for item in value]
if redact_values:
return "<redacted-environment-value>"
return value
def external_prerequisites(document: dict[str, object]) -> tuple[tuple[str, str], ...]:
"""Return external Compose configs and secrets declared by the canonical model."""
prerequisites: list[tuple[str, str]] = []
for kind, key in (("config", "configs"), ("secret", "secrets")):
value = document.get(key, {})
definitions = _mapping(value, "remote Compose model has invalid external resources")
for name, definition in definitions.items():
definition_mapping = _mapping(
definition, "remote Compose model has invalid external resources"
)
if definition_mapping.get("external") is True:
prerequisites.append((kind, name))
return tuple(prerequisites)
def _mapping(value: object, message: str) -> dict[str, object]:
if not isinstance(value, dict):
raise GenerationError(message)
for key in value:
if not isinstance(key, str):
raise GenerationError(message)
return value
+215
View File
@@ -0,0 +1,215 @@
"""Orchestrate safe, isolated remote role generation."""
from __future__ import annotations
from collections.abc import Callable
from dataclasses import dataclass
from pathlib import Path
from compose2role.emit import (
RolePlan,
diff_plan,
normalize_remote_role_name,
preview_remote_role,
with_encrypted_secrets,
write_role,
)
from compose2role.errors import GenerationError
from compose2role.remote import resolve_remote_projects
from compose2role.resolve import (
GeneratedProject,
GenerationFailure,
GenerationOutcome,
GenerationResult,
PlanDiff,
ProjectFailure,
ResolvedProject,
)
from compose2role.secret_policy import SecretPolicy
from compose2role.sops import encrypt_secret_values, resolve_sops_configuration
def generate_remote_projects(
host: str,
output_directory: Path,
*,
project_name: str | None,
all_projects: bool,
use_sudo: bool,
secret_policy: SecretPolicy | None,
sops_config_path: Path | None,
check_only: bool,
diff_only: bool = False,
force: bool = False,
docker_context: str | None = None,
on_outcome: Callable[[GenerationOutcome], None] | None = None,
) -> GenerationResult:
"""Generate selected remote projects while isolating individual project failures."""
sops_configuration = resolve_sops_configuration(sops_config_path)
resolution = resolve_remote_projects(
host,
project_name=project_name,
all_projects=all_projects,
use_sudo=use_sudo,
secret_policy=secret_policy,
docker_context=docker_context,
)
prepared = _prepare_projects(resolution.projects, docker_context)
_mark_output_conflicts(prepared, output_directory)
outcomes: list[GenerationOutcome] = []
def record(outcome: GenerationOutcome) -> None:
outcomes.append(outcome)
if on_outcome is not None:
on_outcome(outcome)
for outcome in resolution.outcomes:
if isinstance(outcome, ProjectFailure):
record(
GenerationFailure(
name=outcome.name,
role_name=None,
category=outcome.category,
message=outcome.message,
)
)
continue
prepared_project = prepared[outcome.project.name]
if isinstance(prepared_project, GenerationFailure):
record(prepared_project)
continue
if check_only:
record(
GeneratedProject(
project=outcome,
role_name=prepared_project.role_name,
checked=True,
files=_plan_files(prepared_project.plan),
)
)
continue
plan = prepared_project.plan
if plan.secret_values:
try:
encrypted_secrets = encrypt_secret_values(sops_configuration, plan.secret_values)
plan = with_encrypted_secrets(plan, encrypted_secrets)
except GenerationError as error:
record(
_generation_failure(outcome, prepared_project.role_name, "sops", str(error))
)
continue
plan_diff: PlanDiff | None = None
if diff_only:
plan_diff = diff_plan(plan, output_directory / prepared_project.role_name)
if not force:
record(
GeneratedProject(
project=outcome,
role_name=prepared_project.role_name,
checked=False,
files=_plan_files(plan),
unchanged=plan_diff.identical,
diff=plan_diff,
diff_only=True,
)
)
continue
try:
write_result = write_role(plan, output_directory, force=force)
except GenerationError as error:
record(
_generation_failure(outcome, prepared_project.role_name, "output", str(error))
)
continue
record(
GeneratedProject(
project=outcome,
role_name=prepared_project.role_name,
checked=False,
files=_plan_files(plan),
unchanged=write_result.state == "unchanged",
diff=plan_diff,
)
)
return GenerationResult(outcomes=tuple(outcomes))
@dataclass(frozen=True)
class _PreparedProject:
"""A role plan ready for collision checking and optional output."""
project: ResolvedProject
role_name: str
plan: RolePlan
def _plan_files(plan: RolePlan) -> tuple[str, ...]:
return tuple(str(generated_file.relative_path) for generated_file in plan.files)
def _prepare_projects(
projects: tuple[ResolvedProject, ...], docker_context: str | None
) -> dict[str, _PreparedProject | GenerationFailure]:
prepared: dict[str, _PreparedProject | GenerationFailure] = {}
for project in projects:
try:
role_name = normalize_remote_role_name(project.project.name)
plan = preview_remote_role(
project, role_name, docker_context=docker_context
)
except GenerationError as error:
prepared[project.project.name] = _generation_failure(
project, None, "generation", str(error)
)
else:
prepared[project.project.name] = _PreparedProject(project, role_name, plan)
return prepared
def _mark_output_conflicts(
prepared: dict[str, _PreparedProject | GenerationFailure], output_directory: Path
) -> None:
if output_directory.is_symlink() or (
output_directory.exists() and not output_directory.is_dir()
):
for project_name, value in prepared.items():
if isinstance(value, _PreparedProject):
prepared[project_name] = _generation_failure(
value.project,
value.role_name,
"output",
"remote role output must be a directory",
)
return
by_role_name: dict[str, list[_PreparedProject]] = {}
for value in prepared.values():
if isinstance(value, _PreparedProject):
by_role_name.setdefault(value.role_name, []).append(value)
for candidates in by_role_name.values():
if len(candidates) > 1:
for candidate in candidates:
prepared[candidate.project.project.name] = _generation_failure(
candidate.project,
candidate.role_name,
"output",
"normalized remote role name collides with another selected project",
)
continue
def _generation_failure(
project: ResolvedProject,
role_name: str | None,
category: str,
message: str,
) -> GenerationFailure:
return GenerationFailure(
name=project.project.name,
role_name=role_name,
category=category,
message=message,
findings=project.findings,
classifications=project.classifications,
)
+862
View File
@@ -0,0 +1,862 @@
"""Shared project model, capture seam, and resolution pipeline."""
from __future__ import annotations
from dataclasses import dataclass, field
from pathlib import PurePosixPath
from typing import Literal, Protocol
from ruamel.yaml import YAML
from ruamel.yaml.comments import CommentedMap, CommentedSeq
from ruamel.yaml.error import YAMLError
from compose2role.discover import DiscoveredProject
from compose2role.errors import Compose2RoleError, ComposeParseError
from compose2role.findings import Finding, audit_stack, classified_secret_finding
from compose2role.model import Stack, parse_canonical_stack
from compose2role.secret_policy import SecretPolicy
from compose2role.secrets import EnvironmentClassification, classify_environment, is_secret_key
from compose2role.source import FileMetadata
_UNSUPPORTED_DOCUMENT_FEATURES = {
"include": "C2R008: remote Compose include files are not supported",
}
_UNSUPPORTED_SERVICE_FEATURES = {
"build": "C2R005: remote build contexts are not supported",
"extends": "C2R006: remote service extends declarations are not supported",
"profiles": "C2R007: remote service profiles are not supported",
}
@dataclass(frozen=True)
class SourceFile:
"""One raw, root-relative source file retained only in controller memory."""
relative_path: PurePosixPath
content: bytes = field(repr=False)
metadata: FileMetadata
@dataclass(frozen=True)
class FileSecret:
"""One file-backed Compose secret retained only in controller memory."""
name: str
source: SourceFile = field(repr=False)
@dataclass(frozen=True)
class ProjectSnapshot:
"""An ordered source snapshot for a root-contained Compose project."""
root: PurePosixPath
files: tuple[SourceFile, ...] = field(repr=False)
dotenv_files: tuple[SourceFile, ...] = field(default=(), repr=False)
file_secrets: tuple[FileSecret, ...] = field(default=(), repr=False)
@dataclass(frozen=True)
class ResolvedProject:
"""One project resolved to canonical and source-faithful representations."""
project: DiscoveredProject
stack: Stack
canonical_document: dict[str, object]
findings: tuple[Finding, ...]
classifications: tuple[EnvironmentClassification, ...]
source: ProjectSnapshot = field(repr=False)
secret_policy: SecretPolicy | None = field(default=None, repr=False)
drift: ProjectDrift | None = field(default=None, repr=False)
def as_json(self) -> dict[str, object]:
"""Return the value-free audit JSON envelope for a successful project."""
return {
"classifications": [
classification.as_json()
for classification in self.classifications
if classification.is_secret
],
"drift": None if self.drift is None else self.drift.as_json(),
"error": None,
"findings": [finding.as_json() for finding in self.findings],
"name": self.project.name,
"role": None,
"status": "ok",
}
@dataclass(frozen=True)
class ProjectFailure:
"""A value-free operational failure for one project."""
name: str
category: str
message: str
def as_json(self) -> dict[str, object]:
"""Return the value-free audit JSON envelope for a failed project."""
return {
"classifications": [],
"drift": None,
"error": self.message,
"findings": [],
"name": self.name,
"role": None,
"status": "failed",
}
type AuditOutcome = ResolvedProject | ProjectFailure
@dataclass(frozen=True)
class AuditResult:
"""Audit successes and isolated per-project failures."""
projects: tuple[ResolvedProject, ...]
failures: tuple[ProjectFailure, ...]
outcomes: tuple[AuditOutcome, ...]
def as_json(self) -> dict[str, list[dict[str, object]]]:
"""Return the batch JSON envelope used by audit."""
return {"projects": [outcome.as_json() for outcome in self.outcomes]}
type DiffState = Literal["added", "changed", "unchanged", "withheld"]
type DriftState = Literal["unchanged", "source changed", "unmanaged", "missing"]
@dataclass(frozen=True)
class ProjectDrift:
"""The value-free drift state of one project against its generated role."""
state: DriftState
role: str
changed: tuple[str, ...] = ()
reason: str | None = None
def as_json(self) -> dict[str, object]:
"""Return paths and reasons only, never source content."""
payload: dict[str, object] = {
"changed": list(self.changed),
"role": self.role,
"state": self.state,
}
if self.reason is not None:
payload["reason"] = self.reason
return payload
@dataclass(frozen=True)
class FileDiff:
"""One value-free comparison result for a planned role file."""
path: str
change: Literal["added", "changed", "unchanged"]
withheld: bool = False
withheld_reason: str | None = None
old_text: str | None = field(default=None, repr=False)
new_text: str | None = field(default=None, repr=False)
@property
def state(self) -> DiffState:
"""Return the reported state, preferring the withheld category."""
if self.withheld and self.change != "unchanged":
return "withheld"
return self.change
@dataclass(frozen=True)
class PlanDiff:
"""The value-free difference between a rendered plan and an existing role."""
role_exists: bool
unmanaged: bool
files: tuple[FileDiff, ...]
@property
def identical(self) -> bool:
"""Return whether every planned file is unchanged."""
return self.role_exists and all(file.state == "unchanged" for file in self.files)
def as_json(self) -> dict[str, list[str]]:
"""Return path-only diff categories; content is never included."""
grouped: dict[str, list[str]] = {
"added": [],
"changed": [],
"unchanged": [],
"withheld": [],
}
for file in self.files:
grouped[file.state].append(file.path)
return grouped
@dataclass(frozen=True)
class GeneratedProject:
"""One project successfully generated or checked."""
project: ResolvedProject
role_name: str
checked: bool
files: tuple[str, ...]
unchanged: bool = False
diff: PlanDiff | None = None
diff_only: bool = False
@property
def name(self) -> str:
"""Return the raw Compose project name."""
return self.project.project.name
def as_json(self) -> dict[str, object]:
"""Return a value-free generation summary for this project."""
if self.checked:
status = "checked"
elif self.diff is not None and self.diff_only:
status = "unchanged" if self.diff.identical else "diff"
elif self.unchanged:
status = "unchanged"
else:
status = "generated"
summary: dict[str, object] = {
"classifications": [
classification.as_json()
for classification in self.project.classifications
if classification.is_secret
],
"error": None,
"findings": [finding.as_json() for finding in self.project.findings],
"name": self.name,
"role": self.role_name,
"status": status,
}
if self.checked:
summary["files"] = list(self.files)
if self.diff is not None:
summary["diff"] = self.diff.as_json()
return summary
@dataclass(frozen=True)
class GenerationFailure:
"""A value-free generation failure for one project."""
name: str
role_name: str | None
category: str
message: str
findings: tuple[Finding, ...] = ()
classifications: tuple[EnvironmentClassification, ...] = ()
def as_json(self) -> dict[str, object]:
"""Return a value-free generation summary for this failed project."""
return {
"classifications": [
classification.as_json()
for classification in self.classifications
if classification.is_secret
],
"error": self.message,
"findings": [finding.as_json() for finding in self.findings],
"name": self.name,
"role": self.role_name,
"status": "failed",
}
type GenerationOutcome = GeneratedProject | GenerationFailure
@dataclass(frozen=True)
class GenerationResult:
"""Ordered generated and failed project outcomes."""
outcomes: tuple[GenerationOutcome, ...]
@property
def failures(self) -> tuple[GenerationFailure, ...]:
"""Return every project that did not generate successfully."""
return tuple(
outcome for outcome in self.outcomes if isinstance(outcome, GenerationFailure)
)
def as_json(self) -> dict[str, list[dict[str, object]]]:
"""Return the generation batch envelope."""
return {"projects": [outcome.as_json() for outcome in self.outcomes]}
class ProjectResolutionError(Compose2RoleError):
"""A value-free error while resolving a particular project."""
def __init__(self, category: str, message: str) -> None:
super().__init__(message)
self.category = category
class ProjectCapture(Protocol):
"""A transport that captures root-contained source files."""
def capture(self, root: PurePosixPath, path: PurePosixPath) -> SourceFile | str:
"""Capture PATH and its metadata, or return its source-boundary state."""
...
def stat(self, root: PurePosixPath, path: PurePosixPath) -> str:
"""Return the source-boundary state of PATH without reading its bytes."""
...
def close(self) -> None:
"""Finish the capture session."""
...
def abort(self) -> None:
"""Discard the capture session without replacing the original error."""
...
@dataclass(frozen=True)
class CanonicalResult:
"""The effective Compose document plus an optional source-located typed stack."""
document: dict[str, object]
stack: Stack | None = None
class Canonicalizer(Protocol):
"""A transport that produces the effective Compose document."""
def canonicalize(
self,
identity: DiscoveredProject,
root: PurePosixPath,
files: tuple[PurePosixPath, ...],
) -> CanonicalResult:
"""Return the canonical Compose document for the captured project."""
...
@dataclass(frozen=True)
class _EnvironmentFile:
"""A service env_file reference that must remain within the project root."""
path: PurePosixPath
required: bool
@dataclass(frozen=True)
class _FileSecretInput:
"""A contained source file referenced by one Compose secret declaration."""
name: str
path: PurePosixPath
def resolve_project(
capture: ProjectCapture,
canonicalizer: Canonicalizer,
identity: DiscoveredProject,
secret_policy: SecretPolicy | None,
*,
extra_dotenv_paths: tuple[PurePosixPath, ...] = (),
) -> ResolvedProject:
"""Capture, canonicalize, classify, and audit one root-contained project."""
root = project_root(identity)
try:
source = _capture_source_snapshot(capture, identity, root)
source_documents = _source_documents(source)
_validate_source_documents(source_documents)
environment_files = _environment_files(source_documents, root)
dotenv_files, file_secrets = _capture_dependent_inputs(
capture,
root,
environment_files,
_file_secret_inputs(source_documents, root),
extra_dotenv_paths,
)
except Exception:
capture.abort()
raise
else:
capture.close()
source = ProjectSnapshot(
root=source.root,
files=source.files,
dotenv_files=dotenv_files,
file_secrets=file_secrets,
)
canonical = canonicalizer.canonicalize(
identity, root, tuple(file.relative_path for file in source.files)
)
canonical_document = canonical.document
if canonical.stack is not None:
stack = canonical.stack
else:
try:
stack = parse_canonical_stack(
canonical_document,
source_path=PurePosixPath("remote-compose.json"),
)
except ComposeParseError as error:
raise ProjectResolutionError(
"invalid-model", "remote Docker Compose returned an unsupported effective model"
) from error
_validate_canonical_topology(canonical_document)
classifications = tuple(
classify_environment(
identity.name,
service.name,
environment.name,
environment.value,
environment.location,
secret_policy,
)
for service in stack.services
for environment in service.environment
)
inline_secret_entries = {
(service.name, environment.name)
for service in stack.services
for environment in service.environment
if environment.is_secret
}
findings = (
*audit_stack(stack),
*(
classified_secret_finding(
classification.service,
classification.key,
classification.reason,
classification.location,
)
for classification in classifications
if classification.is_secret
and (classification.service, classification.key) not in inline_secret_entries
),
)
return ResolvedProject(
project=identity,
stack=stack,
canonical_document=canonical_document,
findings=findings,
classifications=classifications,
source=source,
secret_policy=secret_policy,
)
def project_root(project: DiscoveredProject) -> PurePosixPath:
"""Return the root that contains every declared Compose file."""
if not project.config_files:
raise ProjectResolutionError("source", "the remote project has no Compose files")
_safe_line(project.name, "project name")
config_files = tuple(_safe_remote_path(path) for path in project.config_files)
if len(project.working_directories) > 1:
raise ProjectResolutionError(
"source", "remote Compose project has conflicting working-directory labels"
)
root = (
_safe_project_root(project.working_directories[0])
if project.working_directories
else config_files[0].parent
)
for path in config_files:
relative_source_path(root, path)
return root
def _validate_canonical_topology(document: dict[str, object]) -> None:
services = _canonical_mapping(
document.get("services"), "remote Docker Compose returned an unsupported effective model"
)
for service in services.values():
service_mapping = _canonical_mapping(
service, "remote Docker Compose returned an unsupported effective model"
)
_canonical_volumes(service_mapping)
def _canonical_volumes(service: dict[str, object]) -> tuple[dict[str, object], ...]:
volumes = service.get("volumes", [])
if not isinstance(volumes, list):
raise ProjectResolutionError(
"invalid-model", "remote Docker Compose returned an unsupported effective model"
)
return tuple(
_canonical_mapping(
volume, "remote Docker Compose returned an unsupported effective model"
)
for volume in volumes
)
def _canonical_mapping(value: object, message: str) -> dict[str, object]:
if not isinstance(value, dict):
raise ProjectResolutionError("invalid-model", message)
mapping: dict[str, object] = {}
for key, item in value.items():
if not isinstance(key, str):
raise ProjectResolutionError("invalid-model", message)
mapping[key] = item
return mapping
def _safe_remote_path(path: str) -> PurePosixPath:
_safe_line(path, "Compose file path")
parsed_path = PurePosixPath(path)
if not parsed_path.is_absolute():
raise ProjectResolutionError("source", "remote Compose file paths must be absolute")
if ".." in parsed_path.parts:
raise ProjectResolutionError("source", "remote Compose file path escapes the project root")
return parsed_path
def _safe_project_root(path: str) -> PurePosixPath:
_safe_line(path, "Compose project working directory")
parsed_path = PurePosixPath(path)
if not parsed_path.is_absolute():
raise ProjectResolutionError(
"source", "remote Compose project working directory must be absolute"
)
if ".." in parsed_path.parts:
raise ProjectResolutionError(
"source", "remote Compose project working directory escapes the project root"
)
return parsed_path
def _safe_line(value: str, description: str) -> None:
if not value or any(ord(character) < 32 or ord(character) == 127 for character in value):
raise ProjectResolutionError(
"source", f"remote {description} contains unsupported characters"
)
def relative_source_path(root: PurePosixPath, path: PurePosixPath) -> PurePosixPath:
"""Return PATH relative to ROOT, rejecting an escape."""
try:
return path.relative_to(root)
except ValueError as error:
raise ProjectResolutionError(
"source", "remote Compose file path escapes the project root"
) from error
def _capture_source_snapshot(
capture: ProjectCapture,
project: DiscoveredProject,
root: PurePosixPath,
) -> ProjectSnapshot:
paths = tuple(_safe_remote_path(path) for path in project.config_files)
files = tuple(_required_source_file(capture.capture(root, path)) for path in paths)
return ProjectSnapshot(root=root, files=files)
def _required_source_file(capture: SourceFile | str) -> SourceFile:
if isinstance(capture, SourceFile):
return capture
if capture == "escape":
raise ProjectResolutionError("source", "remote Compose file path escapes the project root")
if capture == "symlink":
raise ProjectResolutionError("source", "remote Compose source file must not be a symlink")
raise ProjectResolutionError("source", "remote Compose source file is not a regular file")
def _captured_input_file(capture: SourceFile | str, message: str) -> SourceFile:
if isinstance(capture, SourceFile):
return capture
raise ProjectResolutionError("source", message)
def _source_documents(source: ProjectSnapshot) -> tuple[dict[str, object], ...]:
documents: list[dict[str, object]] = []
for file in source.files:
try:
document: object = YAML(typ="rt").load(file.content.decode("utf-8"))
except (UnicodeDecodeError, YAMLError) as error:
raise ProjectResolutionError(
"source", "remote Compose source could not be parsed safely"
) from error
_validate_source_tags(document)
documents.append(_source_mapping(document))
return tuple(documents)
def _source_mapping(value: object) -> dict[str, object]:
if not isinstance(value, dict):
raise ProjectResolutionError("source", "remote Compose source must be a mapping")
mapping: dict[str, object] = {}
for key, item in value.items():
if not isinstance(key, str):
raise ProjectResolutionError(
"source", "remote Compose source has invalid mapping keys"
)
mapping[key] = item
return mapping
def _validate_source_tags(value: object) -> None:
tag = getattr(value, "tag", None)
tag_value = getattr(tag, "value", None)
if tag_value is not None and (
tag_value not in {"!override", "!reset"}
or not isinstance(value, (CommentedMap, CommentedSeq))
):
raise ProjectResolutionError(
"source", "remote Compose source has an unsupported YAML tag"
)
if isinstance(value, dict):
for key, item in value.items():
_validate_source_tags(key)
_validate_source_tags(item)
elif isinstance(value, list):
for item in value:
_validate_source_tags(item)
def _environment_files(
documents: tuple[dict[str, object], ...], root: PurePosixPath
) -> tuple[_EnvironmentFile, ...]:
environment_files: list[_EnvironmentFile] = []
for document in documents:
services = _source_mapping(document.get("services", {}))
for service in services.values():
service_mapping = _source_mapping(service)
if "env_file" not in service_mapping:
continue
environment_files.extend(
_parse_environment_files(service_mapping["env_file"], root)
)
return tuple(environment_files)
def _validate_source_documents(documents: tuple[dict[str, object], ...]) -> None:
for document in documents:
for feature, message in _UNSUPPORTED_DOCUMENT_FEATURES.items():
if feature in document:
raise ProjectResolutionError("source", message)
_validate_file_backed_configs(document)
_reject_non_environment_secret_mapping_keys(document)
services = _source_mapping(document.get("services", {}))
for service in services.values():
service_mapping = _source_mapping(service)
for feature, message in _UNSUPPORTED_SERVICE_FEATURES.items():
if feature in service_mapping:
raise ProjectResolutionError("source", message)
def _validate_file_backed_configs(document: dict[str, object]) -> None:
definitions = _source_mapping(document.get("configs", {}))
for definition in definitions.values():
definition_mapping = _source_mapping(definition)
if "file" in definition_mapping:
raise ProjectResolutionError(
"topology", "C2R027: remote file-backed Compose configs are not supported"
)
def _reject_non_environment_secret_mapping_keys(document: dict[str, object]) -> None:
_walk_source_mappings(document, scan_keys=False)
def _walk_source_mappings(value: object, *, scan_keys: bool) -> None:
if isinstance(value, dict):
for key, item in value.items():
if not isinstance(key, str):
raise ProjectResolutionError(
"source", "remote Compose source has invalid mapping keys"
)
if scan_keys and is_secret_key(key):
raise ProjectResolutionError(
"topology",
"C2R029: remote non-environment secret-shaped mapping keys are not supported",
)
if key == "environment":
continue
if key in {"services", "configs", "secrets", "networks", "volumes", "depends_on"}:
_walk_source_identity_mapping(item)
else:
_walk_source_mappings(item, scan_keys=True)
elif isinstance(value, list):
for item in value:
_walk_source_mappings(item, scan_keys=scan_keys)
def _walk_source_identity_mapping(value: object) -> None:
if not isinstance(value, dict):
_walk_source_mappings(value, scan_keys=True)
return
for key, item in value.items():
if not isinstance(key, str):
raise ProjectResolutionError(
"source", "remote Compose source has invalid mapping keys"
)
_walk_source_mappings(item, scan_keys=False)
def _parse_environment_files(value: object, root: PurePosixPath) -> tuple[_EnvironmentFile, ...]:
if isinstance(value, (str, dict)):
values: list[object] = [value]
elif isinstance(value, list):
values = value
else:
raise ProjectResolutionError("source", "remote service env_file entries are invalid")
environment_files: list[_EnvironmentFile] = []
for item in values:
required = True
if isinstance(item, str):
reference = item
elif isinstance(item, dict):
item_mapping = _source_mapping(item)
reference_value = item_mapping.get("path")
required_value = item_mapping.get("required", True)
if not isinstance(reference_value, str) or not isinstance(required_value, bool):
raise ProjectResolutionError(
"source", "remote service env_file entries are invalid"
)
reference = reference_value
required = required_value
else:
raise ProjectResolutionError("source", "remote service env_file entries are invalid")
environment_files.append(_EnvironmentFile(_contained_path(root, reference), required))
return tuple(environment_files)
def _contained_path(
root: PurePosixPath, reference: str, *, subject: str = "environment file"
) -> PurePosixPath:
_safe_line(reference, f"{subject} path")
relative_path = PurePosixPath(reference)
if relative_path.is_absolute() or ".." in relative_path.parts:
raise ProjectResolutionError("source", f"remote {subject} path escapes the project root")
path = root.joinpath(*relative_path.parts)
try:
path.relative_to(root)
except ValueError as error:
raise ProjectResolutionError(
"source", f"remote {subject} path escapes the project root"
) from error
return path
def _file_secret_inputs(
documents: tuple[dict[str, object], ...], root: PurePosixPath
) -> tuple[_FileSecretInput, ...]:
inputs: list[_FileSecretInput] = []
for document in documents:
definitions = _source_mapping(document.get("secrets", {}))
for name, definition in definitions.items():
if not isinstance(name, str) or not name:
raise ProjectResolutionError("source", "remote Compose secret names are invalid")
definition_mapping = _source_mapping(definition)
file_value = definition_mapping.get("file")
if file_value is None:
continue
if not isinstance(file_value, str) or not file_value:
raise ProjectResolutionError(
"source", "remote file-backed Compose secret input is invalid"
)
inputs.append(
_FileSecretInput(
name=name,
path=_contained_path(
root, file_value, subject="file-backed Compose secret"
),
)
)
return tuple(inputs)
def _capture_dependent_inputs(
capture: ProjectCapture,
root: PurePosixPath,
environment_files: tuple[_EnvironmentFile, ...],
secret_inputs: tuple[_FileSecretInput, ...],
extra_dotenv_paths: tuple[PurePosixPath, ...],
) -> tuple[tuple[SourceFile, ...], tuple[FileSecret, ...]]:
root_dotenv_path = root / ".env"
dotenv_files: list[SourceFile] = []
captured_dotenv_paths: set[PurePosixPath] = set()
root_dotenv_state = capture.stat(root, root_dotenv_path)
if root_dotenv_state == "file":
dotenv_files.append(
_captured_input_file(
capture.capture(root, root_dotenv_path),
"remote root dotenv input is not a regular file",
)
)
captured_dotenv_paths.add(root_dotenv_path)
elif root_dotenv_state == "escape":
raise ProjectResolutionError(
"source", "remote environment file path escapes the project root"
)
elif root_dotenv_state != "missing":
raise ProjectResolutionError("source", "remote root dotenv input is not a regular file")
for extra_path in extra_dotenv_paths:
absolute_path = root.joinpath(*extra_path.parts)
if absolute_path in captured_dotenv_paths:
continue
state = capture.stat(root, absolute_path)
if state == "file":
dotenv_files.append(
_captured_input_file(
capture.capture(root, absolute_path),
"remote environment file input is not a regular file",
)
)
captured_dotenv_paths.add(absolute_path)
continue
if state == "escape":
raise ProjectResolutionError(
"source", "remote environment file path escapes the project root"
)
raise ProjectResolutionError(
"source", "remote environment file input is not a regular file"
)
for environment_file in environment_files:
state = capture.stat(root, environment_file.path)
if state == "file":
if environment_file.path not in captured_dotenv_paths:
dotenv_files.append(
_captured_input_file(
capture.capture(root, environment_file.path),
"remote service env_file input is not a regular file",
)
)
captured_dotenv_paths.add(environment_file.path)
continue
if state == "escape":
raise ProjectResolutionError(
"source", "remote environment file path escapes the project root"
)
if state == "missing" and not environment_file.required:
continue
raise ProjectResolutionError(
"source", "remote service env_file input is not a regular file"
)
file_secrets: list[FileSecret] = []
for secret_input in secret_inputs:
state = capture.stat(root, secret_input.path)
if state == "file":
file_secrets.append(
FileSecret(
name=secret_input.name,
source=_captured_input_file(
capture.capture(root, secret_input.path),
"remote file-backed Compose secret input is not a regular file",
),
)
)
continue
if state == "escape":
raise ProjectResolutionError(
"source", "remote file-backed Compose secret path escapes the project root"
)
raise ProjectResolutionError(
"source", "remote file-backed Compose secret input is not a regular file"
)
return tuple(dotenv_files), tuple(file_secrets)
+175
View File
@@ -0,0 +1,175 @@
"""Controller-side secret policy overrides for remote environment secret classification."""
from __future__ import annotations
import tomllib
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from compose2role.errors import SecretPolicyError
_ENVIRONMENT_RULE_KEYS = frozenset({"project", "service", "key", "decision"})
_DOTENV_RULE_KEYS = frozenset({"project", "source", "key", "decision"})
@dataclass(frozen=True)
class SecretRule:
"""One exact environment classification override with no value data."""
project: str
service: str
key: str
decision: str
@dataclass(frozen=True)
class DotenvSecretRule:
"""One exact, value-free override for a copied dotenv entry."""
project: str
source: PurePosixPath
key: str
decision: str
@dataclass(frozen=True)
class SecretPolicy:
"""Exact controller-side secret policy rules."""
rules: tuple[SecretRule, ...]
dotenv_rules: tuple[DotenvSecretRule, ...] = ()
def decision_for(self, project: str, service: str, key: str) -> str | None:
"""Return the exact matching decision, if the secret policy has one."""
for rule in self.rules:
if (rule.project, rule.service, rule.key) == (project, service, key):
return rule.decision
return None
def dotenv_decision_for(
self, project: str, source: PurePosixPath, key: str
) -> str | None:
"""Return the exact matching copied-dotenv decision, if present."""
for rule in self.dotenv_rules:
if (rule.project, rule.source, rule.key) == (project, source, key):
return rule.decision
return None
def load_secret_policy(path: Path) -> SecretPolicy:
"""Load a secret policy without retaining arbitrary policy values in its interface."""
try:
document: object = tomllib.loads(path.read_text(encoding="utf-8"))
except (OSError, tomllib.TOMLDecodeError) as error:
raise SecretPolicyError("secret policy could not be read") from error
if not isinstance(document, dict):
raise SecretPolicyError("secret policy is invalid")
if frozenset(document) - {"rules", "dotenv_rules"}:
raise SecretPolicyError("secret policy must define only exact rule tables")
rules_value: object = document.get("rules", [])
dotenv_rules_value: object = document.get("dotenv_rules", [])
if not isinstance(rules_value, list) or not isinstance(dotenv_rules_value, list):
raise SecretPolicyError("secret policy must define rules")
if not rules_value and not dotenv_rules_value:
raise SecretPolicyError("secret policy must define rules")
rules: list[SecretRule] = []
decisions: dict[tuple[str, str, str], str] = {}
for rule_value in rules_value:
for rule in _parse_rule(rule_value):
identity = (rule.project, rule.service, rule.key)
previous_decision = decisions.get(identity)
if previous_decision is not None and previous_decision != rule.decision:
raise SecretPolicyError("secret policy has conflicting exact rules")
decisions[identity] = rule.decision
rules.append(rule)
dotenv_rules: list[DotenvSecretRule] = []
dotenv_decisions: dict[tuple[str, PurePosixPath, str], str] = {}
for rule_value in dotenv_rules_value:
for dotenv_rule in _parse_dotenv_rule(rule_value):
dotenv_identity = (dotenv_rule.project, dotenv_rule.source, dotenv_rule.key)
previous_decision = dotenv_decisions.get(dotenv_identity)
if previous_decision is not None and previous_decision != dotenv_rule.decision:
raise SecretPolicyError("secret policy has conflicting exact rules")
dotenv_decisions[dotenv_identity] = dotenv_rule.decision
dotenv_rules.append(dotenv_rule)
return SecretPolicy(rules=tuple(rules), dotenv_rules=tuple(dotenv_rules))
def _rule_keys(value: object) -> tuple[str, ...]:
if isinstance(value, str):
if not value:
raise SecretPolicyError("secret policy rule keys must be non-empty strings")
return (value,)
if isinstance(value, list) and value:
keys: list[str] = []
for item in value:
if not isinstance(item, str) or not item:
raise SecretPolicyError("secret policy rule keys must be non-empty strings")
keys.append(item)
return tuple(keys)
raise SecretPolicyError("secret policy rule keys must be non-empty strings")
def _parse_rule(value: object) -> list[SecretRule]:
if not isinstance(value, dict):
raise SecretPolicyError("secret policy is invalid")
rule: dict[str, object] = {}
for key, item in value.items():
if not isinstance(key, str):
raise SecretPolicyError("secret policy is invalid")
rule[key] = item
if frozenset(rule) != _ENVIRONMENT_RULE_KEYS:
raise SecretPolicyError("secret policy rules must contain only exact match fields")
project = rule["project"]
service = rule["service"]
key = rule["key"]
decision = rule["decision"]
if (
not isinstance(project, str)
or not project
or not isinstance(service, str)
or not service
or not isinstance(decision, str)
or decision not in {"secret", "plain"}
):
raise SecretPolicyError("secret policy rules are invalid")
return [
SecretRule(project=project, service=service, key=item, decision=decision)
for item in _rule_keys(key)
]
def _parse_dotenv_rule(value: object) -> list[DotenvSecretRule]:
if not isinstance(value, dict):
raise SecretPolicyError("secret policy is invalid")
rule: dict[str, object] = {}
for key, item in value.items():
if not isinstance(key, str):
raise SecretPolicyError("secret policy is invalid")
rule[key] = item
if frozenset(rule) != _DOTENV_RULE_KEYS:
raise SecretPolicyError("dotenv secret policy rules must contain only exact match fields")
project = rule["project"]
source_value = rule["source"]
key = rule["key"]
decision = rule["decision"]
if (
not isinstance(project, str)
or not project
or not isinstance(source_value, str)
or not source_value
or not isinstance(decision, str)
or decision not in {"secret", "plain"}
):
raise SecretPolicyError("dotenv secret policy rules are invalid")
source = PurePosixPath(source_value)
if source.is_absolute() or ".." in source.parts or str(source) != source_value:
raise SecretPolicyError("dotenv secret policy source paths must be root-relative")
return [
DotenvSecretRule(project=project, source=source, key=item, decision=decision)
for item in _rule_keys(key)
]
+143
View File
@@ -0,0 +1,143 @@
"""Value-aware secret classification that never exposes effective values."""
from __future__ import annotations
import re
from dataclasses import dataclass
from typing import TYPE_CHECKING
if TYPE_CHECKING:
from compose2role.model import SourcePosition
from compose2role.secret_policy import SecretPolicy
_SECRET_COMPONENTS = frozenset(
{
"CREDENTIAL",
"CREDENTIALS",
"DSN",
"PASS",
"PASSWORD",
"SECRET",
"TOKEN",
}
)
_CREDENTIAL_URI = re.compile(r"^[A-Za-z][A-Za-z0-9+.-]*://[^\s/:@]+:[^\s/@]+@")
_JWT = re.compile(r"^[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}\.[A-Za-z0-9_-]{8,}$")
_OPAQUE_TOKEN = re.compile(r"^[A-Za-z0-9_-]{32,}$")
_OPAQUE_TOKEN_PREFIXES = ("sk-", "ghp_", "xoxb-")
@dataclass(frozen=True)
class EnvironmentClassification:
"""A value-free secret decision for an effective service environment entry."""
service: str
key: str
is_secret: bool
reason: str
location: SourcePosition
def as_json(self) -> dict[str, str]:
"""Return the classification report without the effective value."""
return {
"key": self.key,
"location": str(self.location),
"reason": self.reason,
"service": self.service,
}
def is_secret_key(key: str) -> bool:
"""Return whether an environment-variable name likely carries a secret.
``KEY_PREFIX`` deliberately does not match: matching arbitrary ``KEY`` substrings
creates too many false positives. A standalone ``KEY`` or a ``*_KEY`` suffix does
match; later configuration can allow-list public keys explicitly.
"""
normalized = key.upper()
components = frozenset(normalized.split("_"))
return (
bool(components & _SECRET_COMPONENTS)
or normalized == "KEY"
or normalized.endswith("_KEY")
)
def classify_environment(
project: str,
service: str,
key: str,
value: str,
location: SourcePosition,
secret_policy: SecretPolicy | None = None,
) -> EnvironmentClassification:
"""Classify an effective environment value without including it in the result."""
secret_policy_decision = (
secret_policy.decision_for(project, service, key)
if secret_policy is not None
else None
)
if secret_policy_decision is not None:
return EnvironmentClassification(
service=service,
key=key,
is_secret=secret_policy_decision == "secret",
reason=f"secret-policy:forced-{secret_policy_decision}",
location=location,
)
if is_secret_key(key):
return EnvironmentClassification(
service=service,
key=key,
is_secret=True,
reason="key-name",
location=location,
)
if _CREDENTIAL_URI.match(value):
return EnvironmentClassification(
service=service,
key=key,
is_secret=True,
reason="value:credential-uri",
location=location,
)
if "-----BEGIN" in value and "PRIVATE KEY-----" in value:
return EnvironmentClassification(
service=service,
key=key,
is_secret=True,
reason="value:private-key",
location=location,
)
if _JWT.match(value):
return EnvironmentClassification(
service=service,
key=key,
is_secret=True,
reason="value:jwt",
location=location,
)
if value.startswith(_OPAQUE_TOKEN_PREFIXES) or _is_mixed_opaque_token(value):
return EnvironmentClassification(
service=service,
key=key,
is_secret=True,
reason="value:opaque-token",
location=location,
)
return EnvironmentClassification(
service=service,
key=key,
is_secret=False,
reason="not-secret",
location=location,
)
def _is_mixed_opaque_token(value: str) -> bool:
return bool(
_OPAQUE_TOKEN.match(value)
and any(character.islower() for character in value)
and any(character.isupper() for character in value)
and any(character.isdigit() for character in value)
)
+138
View File
@@ -0,0 +1,138 @@
"""Validate and apply SOPS configuration for remote generation."""
from __future__ import annotations
import json
import os
import re
import shutil
from collections.abc import Mapping
from dataclasses import dataclass
from pathlib import Path, PurePosixPath
from subprocess import TimeoutExpired, run
from ruamel.yaml import YAML
from ruamel.yaml.error import YAMLError
from compose2role.errors import GenerationError
SOPS_TARGET = PurePosixPath("vars/secrets.sops.yml")
@dataclass(frozen=True)
class SopsConfiguration:
"""A validated operator-owned SOPS config and executable."""
path: Path
executable: str
def validate_sops_config(path: Path) -> None:
"""Validate operator-owned SOPS coverage without copying or encrypting anything."""
if not path.is_file():
raise GenerationError("SOPS config could not be read")
_validate_creation_rule(path)
def resolve_sops_configuration(explicit_path: Path | None) -> SopsConfiguration:
"""Validate the selected SOPS configuration before remote acquisition."""
selected_path = explicit_path
if selected_path is None:
environment_path = os.environ.get("COMPOSE2ROLE_SOPS_CONFIG")
selected_path = Path(environment_path) if environment_path else None
if selected_path is None or not selected_path.is_file():
raise GenerationError(
"remote generation requires an existing SOPS config via --sops-config "
"or COMPOSE2ROLE_SOPS_CONFIG"
)
executable = shutil.which("sops")
if executable is None:
raise GenerationError("remote generation requires the SOPS executable")
_validate_creation_rule(selected_path)
return SopsConfiguration(path=selected_path, executable=executable)
def encrypt_secret_values(configuration: SopsConfiguration, values: Mapping[str, str]) -> bytes:
"""Encrypt an in-memory secret mapping without creating a plaintext file."""
plaintext = json.dumps(dict(values), sort_keys=True).encode("utf-8")
try:
result = run(
(
configuration.executable,
"--config",
str(configuration.path),
"--encrypt",
"--input-type",
"yaml",
"--output-type",
"yaml",
"--filename-override",
str(SOPS_TARGET),
"/dev/stdin",
),
check=False,
capture_output=True,
input=plaintext,
timeout=30,
)
except (FileNotFoundError, TimeoutExpired, OSError) as error:
raise GenerationError("SOPS encryption failed") from error
if result.returncode != 0 or not result.stdout:
raise GenerationError("SOPS encryption failed")
_validate_encrypted_output(values, result.stdout)
return result.stdout
def _validate_encrypted_output(values: Mapping[str, str], output: bytes) -> None:
try:
document: object = YAML(typ="safe").load(output.decode("utf-8"))
except (UnicodeDecodeError, YAMLError) as error:
raise GenerationError("SOPS encryption output is invalid") from error
if not isinstance(document, dict):
raise GenerationError("SOPS encryption output is invalid")
plaintext_values = {value for value in values.values() if value}
for key, plaintext in values.items():
encrypted_value = document.get(key)
if plaintext == "":
# SOPS deliberately leaves empty strings unencrypted; there is nothing to protect.
if encrypted_value != "":
raise GenerationError("SOPS encryption output is invalid")
continue
if not (
isinstance(encrypted_value, str)
and encrypted_value.startswith("ENC[")
and encrypted_value.endswith("]")
):
raise GenerationError("SOPS encryption output is invalid")
for key, value in document.items():
if key != "sops" and isinstance(value, str) and value in plaintext_values:
raise GenerationError("SOPS encryption output is invalid")
def _validate_creation_rule(path: Path) -> None:
try:
document: object = YAML(typ="safe").load(path.read_text(encoding="utf-8"))
except (OSError, YAMLError, UnicodeDecodeError) as error:
raise GenerationError("SOPS config could not be read") from error
if not isinstance(document, dict):
raise GenerationError("SOPS config must define creation rules")
rules = document.get("creation_rules")
if not isinstance(rules, list):
raise GenerationError("SOPS config must define creation rules")
target = str(SOPS_TARGET)
for rule in rules:
if not isinstance(rule, dict):
continue
path_regex = rule.get("path_regex")
if path_regex is None:
return
if not isinstance(path_regex, str):
continue
try:
if re.search(path_regex, target) is not None:
return
except re.error as error:
raise GenerationError("SOPS config contains an invalid path_regex") from error
raise GenerationError("SOPS config has no creation rule for vars/secrets.sops.yml")
+14
View File
@@ -0,0 +1,14 @@
"""Captured Compose source metadata."""
from __future__ import annotations
from dataclasses import dataclass
@dataclass(frozen=True)
class FileMetadata:
"""Ownership and mode captured with one source file."""
mode: int
uid: int
gid: int
File diff suppressed because it is too large. Load diff
+6
View File
@@ -0,0 +1,6 @@
---
collections:
- name: community.docker
version: 5.4.0
- name: community.sops
version: 2.5.0
+69
View File
@@ -0,0 +1,69 @@
"""Adapters that let one-shot SSH test stubs serve the framed source session."""
from __future__ import annotations
from pathlib import Path
from shlex import quote
def legacy_source_session_bridge(legacy_ssh: Path) -> str:
"""Wrap a one-shot SSH fake so the resolver can keep one real source session open."""
return f"""\
#!/bin/sh
legacy={quote(str(legacy_ssh))}
case "$*" in
*"sh -s")
IFS= read -r first_line || exit 97
if [ "$first_line" != '# compose2role-source-session-v1' ]; then
(printf '%s\\n' "$first_line"; cat) | "$legacy" "$@"
exit $?
fi
while IFS= read -r line; do
[ "$line" = '# compose2role-source-session-ready' ] && break
done
IFS= read -r invocation || exit 97
[ "$invocation" = main ] || exit 97
printf '%s\\n' ready
while IFS=' ' read -r marker request_id operation extra; do
[ "$marker" = request ] || exit 97
[ -n "$request_id" ] && [ -z "$extra" ] || exit 97
IFS= read -r path || exit 97
IFS= read -r root || exit 97
IFS= read -r trailer || exit 97
[ "$trailer" = "end $request_id" ] || exit 97
encoded=$(printf '%s\\n%s\\n%s\\n' "$operation" "$path" "$root" | base64 | tr -d '\\n')
response=$(printf '%s\\n%s\\n%s\\n' x "$encoded" y | "$legacy" "$@")
status=$?
[ "$status" -eq 0 ] || exit "$status"
case "$operation" in
stat)
case "$response" in
escape|file|missing|other|symlink)
printf 'result %s %s\\nend %s\\n' "$request_id" "$response" "$request_id"
;;
*) exit 97 ;;
esac
;;
capture)
header=$(printf '%s\\n' "$response" | sed -n '1p')
case "$header" in
file[[:space:]]*)
content=$(printf '%s\\n' "$response" | sed '1d')
printf 'result %s %s\\n%s\\nend %s\\n' \
"$request_id" "$header" "$content" "$request_id"
;;
escape|missing|other|symlink)
printf 'result %s %s\\nend %s\\n' "$request_id" "$header" "$request_id"
;;
*) exit 97 ;;
esac
;;
*) exit 97 ;;
esac
done
exit 0
;;
*) exec "$legacy" "$@" ;;
esac
"""
+60
View File
@@ -0,0 +1,60 @@
from __future__ import annotations
from pathlib import Path
import pytest
from click.testing import CliRunner
from compose2role.cli import main
@pytest.mark.parametrize("command", ("generate", "audit"))
def test_local_invocation_exits_with_a_usage_error(command: str) -> None:
result = CliRunner().invoke(main, [command])
assert result.exit_code == 2
assert "was removed" in result.output
assert "--host" in result.output
@pytest.mark.parametrize("command", ("generate", "audit"))
def test_local_source_is_rejected(command: str, tmp_path: Path) -> None:
source = tmp_path / "compose.yaml"
source.write_text("services: {}\n", encoding="utf-8")
result = CliRunner().invoke(main, [command, str(source)])
assert result.exit_code == 2
assert "Got unexpected extra argument" in result.output
@pytest.mark.parametrize("command", ("generate", "audit"))
def test_help_lists_only_remote_options(command: str) -> None:
result = CliRunner().invoke(main, [command, "--help"])
assert result.exit_code == 0
for option in (
"--file",
"--project-dir",
"--env-file",
"--resolve-with",
"--name",
"--sops-pgp",
):
assert option not in result.output
@pytest.mark.parametrize(
"arguments",
(
("--playbook",),
("--playbook-dir", "playbooks"),
),
)
def test_generate_no_longer_offers_playbook_options(arguments: tuple[str, ...]) -> None:
result = CliRunner().invoke(
main, ["generate", "--host", "web1", "--project", "my-app", *arguments]
)
assert result.exit_code == 2
assert "No such option" in result.output
+145
View File
@@ -0,0 +1,145 @@
from __future__ import annotations
from pathlib import Path
import pytest
from compose2role.config import (
HostProfile,
find_repository_config,
load_repository_config,
)
from compose2role.errors import ConfigurationError
def _write(path: Path, content: str) -> Path:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
return path
def test_explicit_config_wins_over_environment_and_discovery(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = tmp_path / "repository"
_write(repository / ".compose2role.toml", "[defaults]\noutput = 'discovered'\n")
environment_config = _write(
tmp_path / "environment.toml", "[defaults]\noutput = 'environment'\n"
)
explicit_config = _write(tmp_path / "explicit.toml", "[defaults]\noutput = 'explicit'\n")
monkeypatch.setenv("COMPOSE2ROLE_CONFIG", str(environment_config))
config = load_repository_config(explicit_config, start=repository)
assert config.path == explicit_config
assert config.output == explicit_config.parent / "explicit"
def test_environment_config_wins_over_discovered_config(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = tmp_path / "repository"
_write(repository / ".compose2role.toml", "[defaults]\noutput = 'discovered'\n")
environment_config = _write(
tmp_path / "environment.toml", "[defaults]\noutput = 'environment'\n"
)
monkeypatch.setenv("COMPOSE2ROLE_CONFIG", str(environment_config))
config = load_repository_config(start=repository)
assert config.path == environment_config
assert config.output == environment_config.parent / "environment"
def test_nearest_compose2role_config_is_discovered_walking_up(tmp_path: Path) -> None:
repository = tmp_path / "repository"
discovered = _write(
repository / ".compose2role.toml", "[defaults]\noutput = 'ansible/roles'\n"
)
nested = repository / "stacks" / "web"
config = load_repository_config(start=nested, environ={})
assert config.path == discovered
assert config.output == repository / "ansible" / "roles"
def test_ansible_cfg_is_the_fallback_configuration_root(tmp_path: Path) -> None:
repository = tmp_path / "repository"
ansible_config = _write(repository / "ansible.cfg", "[defaults]\n")
nested = repository / "stacks" / "web"
config = load_repository_config(start=nested, environ={})
assert config.path == ansible_config
assert config.root == repository
def test_compose2role_config_beats_a_closer_ansible_cfg(tmp_path: Path) -> None:
_write(tmp_path / "ansible.cfg", "[defaults]\n")
discovered = _write(tmp_path / "stacks" / ".compose2role.toml", "[defaults]\n")
assert find_repository_config(tmp_path / "stacks" / "web") == discovered
def test_defaults_and_host_profiles_are_parsed(tmp_path: Path) -> None:
config_path = _write(
tmp_path / ".compose2role.toml",
"""\
[defaults]
output = "ansible/roles"
sops_config = "ansible/.sops.yaml"
secret_policy = "ansible/policy.toml"
[hosts.web1]
sudo = true
docker_context = "rootless"
""",
)
config = load_repository_config(config_path, environ={})
assert config.output == tmp_path / "ansible" / "roles"
assert config.sops_config == tmp_path / "ansible" / ".sops.yaml"
assert config.secret_policy == tmp_path / "ansible" / "policy.toml"
assert config.host("web1") == HostProfile(sudo=True, docker_context="rootless")
assert config.host("unknown") == HostProfile()
def test_host_docker_context_is_parsed(tmp_path: Path) -> None:
config_path = _write(
tmp_path / ".compose2role.toml",
'[hosts."web1"]\ndocker_context = "rootless"\n',
)
config = load_repository_config(config_path, environ={})
assert config.host("web1").docker_context == "rootless"
assert config.host("web1").sudo is None
@pytest.mark.parametrize(
"content",
(
"[defaults\n",
"[unknown]\nvalue = true\n",
"[defaults]\nbogus = true\n",
"[defaults]\nplaybook = 'yes'\n",
"[defaults]\nplaybook = true\n",
"[defaults]\nplaybook_dir = 'playbooks'\n",
"[defaults]\noutput = 1\n",
"[hosts.web1]\nbogus = true\n",
"[hosts.web1]\nsudo = 'yes'\n",
"[hosts.web1]\ndocker_context = 5\n",
),
)
def test_invalid_configuration_is_a_configuration_error(tmp_path: Path, content: str) -> None:
config_path = _write(tmp_path / ".compose2role.toml", content)
with pytest.raises(ConfigurationError):
load_repository_config(config_path, environ={})
def test_missing_explicit_configuration_is_a_configuration_error(tmp_path: Path) -> None:
with pytest.raises(ConfigurationError):
load_repository_config(tmp_path / "missing.toml", environ={})
+384
View File
@@ -0,0 +1,384 @@
"""Docker context selection at the command-line and generated-role seams."""
from __future__ import annotations
import base64
import json
import os
from pathlib import Path
import pytest
from click.testing import CliRunner
from ruamel.yaml import YAML
from ssh_stub import legacy_source_session_bridge
from compose2role.cli import main
_SOURCE_DOCUMENT = """\
services:
web:
image: "web:1"
"""
_CANONICAL_DOCUMENT = {"services": {"web": {"image": "web:1"}}}
_SOPS_CONFIG = """\
creation_rules:
- path_regex: vars/secrets\\.sops\\.yml$
"""
_SSH_STUB = """\
#!/bin/sh
request=$(cat)
if [ -n "${C2R_SSH_LOG:-}" ]; then
printf '%s\\n' "$*" >> "$C2R_SSH_LOG"
case "$request" in
*"docker --context rootless compose --project-name"*)
printf 'canonicalized\\n' >> "$C2R_SSH_LOG"
;;
esac
fi
case "$*" in
*"compose ls --all --format json"*)
printf '[{"Name":"my-app","Status":"running(1)","ConfigFiles":"/srv/my-app/compose.yaml"}]\\n'
exit 0
;;
*" ps --all --format "*)
printf '%s' "${C2R_WORKING_DIRECTORY_LABEL:-}"
exit 0
;;
esac
encoded=$(printf '%s\\n' "$request" | tail -n 2 | head -n 1)
payload=$(printf '%s' "$encoded" | base64 -d)
operation=$(printf '%s\\n' "$payload" | head -n 1)
case "$operation" in
capture)
printf '%s\\n' 'file 640 0 0'
printf '%s\\n' "$C2R_SOURCE_DOCUMENT"
;;
stat)
printf '%s\\n' missing
;;
config)
printf '%s\\n' "$C2R_CANONICAL_DOCUMENT"
;;
*)
exit 97
;;
esac
"""
def _install(bin_directory: Path, name: str, script: str) -> Path:
executable = bin_directory / name
executable.write_text(script, encoding="utf-8")
executable.chmod(0o755)
return executable
def _environment(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, *, hosts: str = ""
) -> tuple[Path, Path]:
"""Return the repository and SSH log for one isolated run."""
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
legacy_ssh = _install(bin_directory, "ssh-legacy", _SSH_STUB)
_install(bin_directory, "ssh", legacy_source_session_bridge(legacy_ssh))
_install(bin_directory, "sops", "#!/bin/sh\nexit 99\n")
monkeypatch.setenv("PATH", f"{bin_directory}:{os.environ['PATH']}")
ssh_log = tmp_path / "ssh.log"
monkeypatch.setenv("C2R_SSH_LOG", str(ssh_log))
monkeypatch.setenv(
"C2R_SOURCE_DOCUMENT",
base64.b64encode(_SOURCE_DOCUMENT.encode("utf-8")).decode("ascii"),
)
monkeypatch.setenv("C2R_CANONICAL_DOCUMENT", json.dumps(_CANONICAL_DOCUMENT))
sops_config = tmp_path / "sops.yaml"
sops_config.write_text(_SOPS_CONFIG, encoding="utf-8")
monkeypatch.setenv("COMPOSE2ROLE_SOPS_CONFIG", str(sops_config))
repository = tmp_path / "repository"
repository.mkdir()
(repository / ".compose2role.toml").write_text(
f'[defaults]\noutput = "roles"\n\n{hosts}', encoding="utf-8"
)
return repository, ssh_log
def _lines(ssh_log: Path) -> list[str]:
if not ssh_log.exists():
return []
return ssh_log.read_text(encoding="utf-8").splitlines()
def test_discover_passes_the_context_to_the_discovery_command(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository, ssh_log = _environment(tmp_path, monkeypatch)
result = CliRunner().invoke(
main,
[
"discover",
"--host",
"web1",
"--docker-context",
"rootless",
"--config",
str(repository / ".compose2role.toml"),
],
)
assert result.exit_code == 0, result.output
assert "-- web1 docker --context rootless compose ls --all --format json" in _lines(
ssh_log
)
def test_discover_without_a_context_keeps_the_plain_argv(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository, ssh_log = _environment(tmp_path, monkeypatch)
result = CliRunner().invoke(
main,
["discover", "--host", "web1", "--config", str(repository / ".compose2role.toml")],
)
assert result.exit_code == 0, result.output
assert "-- web1 docker compose ls --all --format json" in _lines(ssh_log)
def test_the_configured_context_applies_when_the_flag_is_absent(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository, ssh_log = _environment(
tmp_path, monkeypatch, hosts='[hosts."web1"]\ndocker_context = "rootless"\n'
)
result = CliRunner().invoke(
main,
["discover", "--host", "web1", "--config", str(repository / ".compose2role.toml")],
)
assert result.exit_code == 0, result.output
assert "-- web1 docker --context rootless compose ls --all --format json" in _lines(
ssh_log
)
def test_the_flag_wins_over_the_configured_context(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository, ssh_log = _environment(
tmp_path, monkeypatch, hosts='[hosts."web1"]\ndocker_context = "production"\n'
)
result = CliRunner().invoke(
main,
[
"discover",
"--host",
"web1",
"--docker-context",
"rootless",
"--config",
str(repository / ".compose2role.toml"),
],
)
assert result.exit_code == 0, result.output
lines = _lines(ssh_log)
assert "-- web1 docker --context rootless compose ls --all --format json" in lines
assert not any("production" in line for line in lines)
def test_audit_uses_the_context_for_the_working_directory_listing(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository, ssh_log = _environment(tmp_path, monkeypatch)
result = CliRunner().invoke(
main,
[
"audit",
"--host",
"web1",
"--all",
"--docker-context",
"rootless",
"--config",
str(repository / ".compose2role.toml"),
],
)
assert result.exit_code == 0, result.output
lines = _lines(ssh_log)
assert "-- web1 docker --context rootless compose ls --all --format json" in lines
assert any("docker --context rootless ps --all --format " in line for line in lines)
def test_generate_uses_the_context_for_discovery_and_canonicalization(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository, ssh_log = _environment(tmp_path, monkeypatch)
result = CliRunner().invoke(
main,
[
"generate",
"--host",
"web1",
"--project",
"my-app",
"--docker-context",
"rootless",
"--config",
str(repository / ".compose2role.toml"),
],
)
assert result.exit_code == 0, result.output
lines = _lines(ssh_log)
assert "-- web1 docker --context rootless compose ls --all --format json" in lines
assert "canonicalized" in lines
def test_generate_without_a_context_uses_the_plain_argv(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository, ssh_log = _environment(tmp_path, monkeypatch)
result = CliRunner().invoke(
main,
[
"generate",
"--host",
"web1",
"--project",
"my-app",
"--config",
str(repository / ".compose2role.toml"),
],
)
assert result.exit_code == 0, result.output
lines = _lines(ssh_log)
assert "-- web1 docker compose ls --all --format json" in lines
assert "canonicalized" not in lines
@pytest.mark.parametrize(
"arguments",
(
("discover", "--host", "web1"),
("audit", "--host", "web1", "--all"),
("generate", "--host", "web1", "--project", "my-app"),
("doctor", "--host", "web1"),
),
)
def test_an_invalid_context_name_is_a_usage_error_before_ssh(
arguments: tuple[str, ...], tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository, ssh_log = _environment(tmp_path, monkeypatch)
result = CliRunner().invoke(
main,
[
*arguments,
"--docker-context",
"not a context",
"--config",
str(repository / ".compose2role.toml"),
],
)
assert result.exit_code == 2, result.output
assert "not a context" in result.output
assert _lines(ssh_log) == []
def _generate(tmp_path: Path, repository: Path, output: str, *extra: str) -> object:
return CliRunner().invoke(
main,
[
"generate",
"--host",
"web1",
"--project",
"my-app",
"--output",
str(tmp_path / output),
"--config",
str(repository / ".compose2role.toml"),
*extra,
],
)
def test_a_generation_records_the_context_in_the_role(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository, _ssh_log = _environment(tmp_path, monkeypatch)
result = _generate(tmp_path, repository, "context-roles", "--docker-context", "rootless")
assert result.exit_code == 0, result.output
role = tmp_path / "context-roles" / "my_app"
defaults = YAML(typ="safe").load(
(role / "defaults" / "main.yml").read_text(encoding="utf-8")
)
tasks = YAML(typ="safe").load(
(role / "tasks" / "main.yml").read_text(encoding="utf-8")
)
assert defaults["my_app_docker_context"] == "rootless"
compose = next(task for task in tasks if task["name"] == "Deploy my_app Compose project")
assert compose["community.docker.docker_compose_v2"]["cli_context"] == (
"{{ my_app_docker_context }}"
)
assert "docker_context" not in (role / ".compose2role.json").read_text(encoding="utf-8")
def test_a_context_free_generation_leaves_every_other_file_unchanged(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository, _ssh_log = _environment(tmp_path, monkeypatch)
plain = _generate(tmp_path, repository, "plain-roles")
contextual = _generate(
tmp_path, repository, "context-roles", "--docker-context", "rootless"
)
assert plain.exit_code == 0, plain.output
assert contextual.exit_code == 0, contextual.output
plain_role = tmp_path / "plain-roles" / "my_app"
context_role = tmp_path / "context-roles" / "my_app"
plain_defaults = (plain_role / "defaults" / "main.yml").read_text(encoding="utf-8")
plain_tasks = (plain_role / "tasks" / "main.yml").read_text(encoding="utf-8")
assert plain_defaults == (
"---\n"
'my_app_project_dir: "/srv/my-app"\n'
'my_app_compose_project_name: "my-app"\n'
'my_app_web_image: "web"\n'
'my_app_web_image_tag: "1"\n'
)
assert "docker_context" not in plain_defaults
assert "cli_context" not in plain_tasks
plain_files = {
path.relative_to(plain_role): path.read_bytes()
for path in plain_role.rglob("*")
if path.is_file()
}
context_files = {
path.relative_to(context_role): path.read_bytes()
for path in context_role.rglob("*")
if path.is_file()
}
assert set(plain_files) == set(context_files)
for relative, content in plain_files.items():
if str(relative) in {"defaults/main.yml", "tasks/main.yml"}:
assert context_files[relative] != content
continue
assert context_files[relative] == content, relative
+606
View File
@@ -0,0 +1,606 @@
"""Doctor diagnostics at the command-line seam."""
from __future__ import annotations
import hashlib
import json
from pathlib import Path
import pytest
from click.testing import CliRunner
from compose2role import doctor as doctor_module
from compose2role.cli import main
_SOPS_STUB = """\
#!/bin/sh
printf 'sops 3.9.0\\n'
exit 0
"""
_GPG_STUB = """\
#!/bin/sh
exit 0
"""
_GALAXY_STUB = """\
#!/bin/sh
if [ "$*" = "collection list --format json" ]; then
printf '%s' '{"collections": {'
printf '%s' '"community.docker": {"version": "5.4.0"}, '
printf '%s' '"community.sops": {"version": "2.5.0"}}}'
exit 0
fi
exit 1
"""
_EMPTY_STUB = """\
#!/bin/sh
exit 0
"""
_SSH_STUB = """\
#!/bin/sh
if [ -n "${DOCTOR_SSH_LOG:-}" ]; then
printf '%s\\n' "$*" >> "$DOCTOR_SSH_LOG"
fi
case "$*" in
-G*)
printf 'hostname web1\\nuser operator\\nport 2222\\n'
exit 0
;;
*" info --format "*)
printf '%s\\n' "${DOCTOR_DOCKER_INFO:-\"\"}"
exit "${DOCTOR_DOCKER_INFO_STATUS:-0}"
;;
*"compose ls --all --format json"*)
printf '%s\\n' "${DOCTOR_COMPOSE_LS:-[]}"
exit 0
;;
*"compose version"*)
printf 'Docker Compose version %s\\n' "${DOCTOR_COMPOSE_VERSION:-v2.30.0}"
exit 0
;;
esac
exit 1
"""
_SOPS_CONFIG = """\
creation_rules:
- path_regex: vars/secrets\\.sops\\.yml$
"""
def _install(bin_directory: Path, name: str, script: str) -> Path:
executable = bin_directory / name
executable.write_text(script, encoding="utf-8")
executable.chmod(0o755)
return executable
def _install_ansible_tools(bin_directory: Path) -> None:
_install(bin_directory, "ansible-galaxy", _GALAXY_STUB)
_install(bin_directory, "ansible-lint", _EMPTY_STUB)
_install(bin_directory, "ansible-playbook", _EMPTY_STUB)
def _isolate(monkeypatch: pytest.MonkeyPatch, bin_directory: Path) -> None:
monkeypatch.setenv("PATH", str(bin_directory))
for name in (
"COMPOSE2ROLE_SSH_BIN",
"COMPOSE2ROLE_SOPS_BIN",
"COMPOSE2ROLE_ANSIBLE_BIN",
"COMPOSE2ROLE_SSH_LOG",
"DOCTOR_SSH_LOG",
"DOCTOR_DOCKER_INFO",
"DOCTOR_DOCKER_INFO_STATUS",
"DOCTOR_COMPOSE_LS",
"DOCTOR_COMPOSE_VERSION",
"ANSIBLE_COLLECTIONS_PATH",
):
monkeypatch.delenv(name, raising=False)
def _set_os_release(monkeypatch: pytest.MonkeyPatch, tmp_path: Path, content: str) -> None:
path = tmp_path / "os-release"
path.write_text(content, encoding="utf-8")
monkeypatch.setattr(doctor_module, "OS_RELEASE_PATH", path)
def _repository(tmp_path: Path, output: str = "roles", extra: str = "") -> Path:
repository = tmp_path / "repository"
repository.mkdir()
(repository / ".compose2role.toml").write_text(
f'[defaults]\noutput = "{output}"\n{extra}', encoding="utf-8"
)
return repository
def _invoke(arguments: list[str]) -> object:
return CliRunner().invoke(main, ["doctor", *arguments])
def _checks(result: object) -> dict[str, dict[str, object]]:
assert getattr(result, "exit_code") in {0, 1}
payload = json.loads(getattr(result, "output"))
return {check["name"]: check for check in payload["checks"]}
def test_doctor_local_reports_every_check(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _repository(tmp_path)
(repository / "roles").mkdir()
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
_install(bin_directory, "sops", _SOPS_STUB)
_install(bin_directory, "gpg", _GPG_STUB)
_install_ansible_tools(bin_directory)
_isolate(monkeypatch, bin_directory)
_set_os_release(monkeypatch, tmp_path, "ID=debian\nID_LIKE=debian\n")
result = _invoke(["--config", str(repository / ".compose2role.toml")])
assert getattr(result, "exit_code") == 0
output = getattr(result, "output")
assert "✓ config:" in output
assert "✓ layout:" in output
assert "✓ sops: sops 3.9.0" in output
assert "✓ ansible collections: found" in output
assert "0 warnings, 0 errors" in output
def test_doctor_controller_mode_keeps_its_check_set(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _repository(tmp_path)
(repository / "roles").mkdir()
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
_install(bin_directory, "sops", _SOPS_STUB)
_install(bin_directory, "gpg", _GPG_STUB)
_install_ansible_tools(bin_directory)
_isolate(monkeypatch, bin_directory)
_set_os_release(monkeypatch, tmp_path, "ID=debian\n")
result = _invoke(
["--config", str(repository / ".compose2role.toml"), "--format", "json"]
)
assert getattr(result, "exit_code") == 0, getattr(result, "output")
payload = json.loads(getattr(result, "output"))
assert [check["name"] for check in payload["checks"]] == [
"config",
"layout",
"secret policy",
"output",
"sops",
"gpg",
"sops rule",
"ansible-galaxy",
"ansible collections",
"ansible-lint",
"ansible-playbook",
"roles",
"drift",
"backups",
]
def test_doctor_host_mode_requires_ssh_and_reports_no_controller_checks(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _repository(tmp_path)
(repository / "roles").mkdir()
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
_isolate(monkeypatch, bin_directory)
_set_os_release(monkeypatch, tmp_path, 'ID="rhel"\nID_LIKE="fedora"\n')
result = _invoke(
["--host", "web1", "--config", str(repository / ".compose2role.toml"), "--format", "json"]
)
checks = _checks(result)
assert getattr(result, "exit_code") == 1
assert set(checks) == {"ssh"}
assert checks["ssh"]["status"] == "error"
assert checks["ssh"]["required"] is True
assert "sudo dnf install openssh-clients" in checks["ssh"]["hint"]
assert json.loads(getattr(result, "output"))["summary"]["failed"] is True
def test_doctor_remote_resolves_ssh_and_checks_remote_compose(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _repository(tmp_path)
(repository / "roles").mkdir()
ssh_log = tmp_path / "ssh.log"
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
_install(bin_directory, "ssh", _SSH_STUB)
_isolate(monkeypatch, bin_directory)
monkeypatch.setenv("DOCTOR_SSH_LOG", str(ssh_log))
_set_os_release(monkeypatch, tmp_path, "ID=debian\n")
result = _invoke(
["--host", "web1", "--config", str(repository / ".compose2role.toml")]
)
assert getattr(result, "exit_code") == 0
output = getattr(result, "output")
assert "✓ ssh resolution: ssh -G web1 resolved hostname web1, user operator, port 2222" in (
output
)
assert "✓ remote docker:" in output
assert "✓ remote compose: remote docker compose v2.30.0" in output
assert "sops" not in output
invocations = ssh_log.read_text(encoding="utf-8").splitlines()
assert "-G -- web1" in invocations
assert "-- web1 docker compose version" in invocations
def _host_environment(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch, **environment: str
) -> Path:
"""Install the host-mode stubs and return the repository root."""
repository = _repository(tmp_path)
(repository / "roles").mkdir()
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
_install(bin_directory, "ssh", _SSH_STUB)
_install(bin_directory, "sops", _SOPS_STUB)
_install(bin_directory, "gpg", _GPG_STUB)
_install_ansible_tools(bin_directory)
_isolate(monkeypatch, bin_directory)
for name, value in environment.items():
monkeypatch.setenv(name, value)
_set_os_release(monkeypatch, tmp_path, "ID=debian\n")
return repository
def test_doctor_remote_sudo_prefixes_every_remote_command(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _host_environment(tmp_path, monkeypatch, DOCTOR_SSH_LOG=str(tmp_path / "ssh.log"))
result = _invoke(
["--host", "web1", "--sudo", "--config", str(repository / ".compose2role.toml")]
)
assert getattr(result, "exit_code") == 0, getattr(result, "output")
invocations = (tmp_path / "ssh.log").read_text(encoding="utf-8")
assert "-- web1 sudo -n docker info" in invocations
assert "-- web1 sudo -n docker compose version" in invocations
def test_doctor_host_mode_applies_the_docker_context(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _host_environment(
tmp_path, monkeypatch, DOCTOR_SSH_LOG=str(tmp_path / "ssh.log")
)
result = _invoke(
[
"--host",
"web1",
"--docker-context",
"rootless",
"--config",
str(repository / ".compose2role.toml"),
]
)
assert getattr(result, "exit_code") == 0, getattr(result, "output")
invocations = (tmp_path / "ssh.log").read_text(encoding="utf-8")
assert "-- web1 docker --context rootless info" in invocations
assert "-- web1 docker --context rootless compose version" in invocations
def test_doctor_quotes_the_docker_info_format_for_the_remote_shell(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _host_environment(
tmp_path, monkeypatch, DOCTOR_SSH_LOG=str(tmp_path / "ssh.log")
)
result = _invoke(
["--host", "web1", "--config", str(repository / ".compose2role.toml")]
)
assert getattr(result, "exit_code") == 0, getattr(result, "output")
invocations = (tmp_path / "ssh.log").read_text(encoding="utf-8")
assert "-- web1 docker info --format '{{json .}}'" in invocations
def test_doctor_host_mode_reports_only_the_host_check_set(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _host_environment(tmp_path, monkeypatch)
result = _invoke(
["--host", "web1", "--format", "json", "--config", str(repository / ".compose2role.toml")]
)
assert getattr(result, "exit_code") == 0, getattr(result, "output")
payload = json.loads(getattr(result, "output"))
assert [check["name"] for check in payload["checks"]] == [
"ssh",
"ssh resolution",
"remote docker",
"remote compose",
]
def test_doctor_host_mode_fails_when_the_daemon_is_unreachable(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _host_environment(tmp_path, monkeypatch, DOCTOR_DOCKER_INFO_STATUS="1")
result = _invoke(
["--host", "web1", "--format", "json", "--config", str(repository / ".compose2role.toml")]
)
checks = _checks(result)
assert getattr(result, "exit_code") == 1
assert checks["remote docker"]["status"] == "error"
assert checks["remote docker"]["required"] is True
assert "not reachable" in str(checks["remote docker"]["detail"])
def test_doctor_host_mode_reports_the_daemon_mode_and_project_count(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _host_environment(
tmp_path,
monkeypatch,
DOCTOR_DOCKER_INFO='{"SecurityOptions":["name=rootless"]}',
DOCTOR_COMPOSE_LS='[{"Name":"one"},{"Name":"two"}]',
)
result = _invoke(
["--host", "web1", "--format", "json", "--config", str(repository / ".compose2role.toml")]
)
check = _checks(result)["remote docker"]
assert check["status"] == "ok"
assert "rootless" in str(check["detail"])
assert "2 visible Compose project(s)" in str(check["detail"])
def test_doctor_host_mode_accepts_compose_v5(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _host_environment(tmp_path, monkeypatch, DOCTOR_COMPOSE_VERSION="v5.7.0")
result = _invoke(
["--host", "web1", "--format", "json", "--config", str(repository / ".compose2role.toml")]
)
checks = _checks(result)
assert getattr(result, "exit_code") == 0
assert checks["remote compose"]["status"] == "ok"
assert "v5.7.0" in str(checks["remote compose"]["detail"])
def test_doctor_host_mode_rejects_compose_v1(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _host_environment(tmp_path, monkeypatch, DOCTOR_COMPOSE_VERSION="v1.29.2")
result = _invoke(
["--host", "web1", "--format", "json", "--config", str(repository / ".compose2role.toml")]
)
checks = _checks(result)
assert getattr(result, "exit_code") == 1
assert checks["remote compose"]["status"] == "error"
assert "v1.29.2" in str(checks["remote compose"]["detail"])
def test_doctor_finds_repo_local_collections(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _repository(tmp_path)
(repository / "roles").mkdir()
snapshot = repository / "collections" / "ansible_collections"
for name in ("community/docker", "community/sops"):
collection = snapshot / name
collection.mkdir(parents=True)
(collection / "galaxy.yml").write_text("namespace: community\n", encoding="utf-8")
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
_install(
bin_directory,
"ansible-galaxy",
"#!/bin/sh\nif [ \"$*\" = 'collection list --format json' ]; then\n"
" printf '%s' '{}'\n exit 0\nfi\nexit 1\n",
)
_install(bin_directory, "ansible-lint", _EMPTY_STUB)
_install(bin_directory, "ansible-playbook", _EMPTY_STUB)
_isolate(monkeypatch, bin_directory)
_set_os_release(monkeypatch, tmp_path, "ID=debian\n")
result = _invoke(["--config", str(repository / ".compose2role.toml")])
assert getattr(result, "exit_code") == 0, getattr(result, "output")
assert "✓ ansible collections: found community.docker, community.sops" in (
getattr(result, "output")
)
def test_doctor_honours_the_ansible_collections_path(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _repository(tmp_path)
(repository / "roles").mkdir()
external = tmp_path / "external-collections"
for name in ("community/docker", "community/sops"):
collection = external / "ansible_collections" / name
collection.mkdir(parents=True)
(collection / "galaxy.yml").write_text("namespace: community\n", encoding="utf-8")
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
_install(
bin_directory,
"ansible-galaxy",
"#!/bin/sh\nif [ \"$*\" = 'collection list --format json' ]; then\n"
" printf '%s' '{}'\n exit 0\nfi\nexit 1\n",
)
_install(bin_directory, "ansible-lint", _EMPTY_STUB)
_install(bin_directory, "ansible-playbook", _EMPTY_STUB)
_isolate(monkeypatch, bin_directory)
monkeypatch.setenv("ANSIBLE_COLLECTIONS_PATH", str(external))
_set_os_release(monkeypatch, tmp_path, "ID=debian\n")
result = _invoke(["--config", str(repository / ".compose2role.toml")])
assert getattr(result, "exit_code") == 0, getattr(result, "output")
assert "✓ ansible collections: found" in getattr(result, "output")
def test_doctor_distro_hints_name_the_package_manager(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _repository(tmp_path)
(repository / "roles").mkdir()
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
_isolate(monkeypatch, bin_directory)
_set_os_release(monkeypatch, tmp_path, 'ID="rocky"\nID_LIKE="rhel centos fedora"\n')
result = _invoke(["--config", str(repository / ".compose2role.toml")])
output = getattr(result, "output")
assert "sudo dnf install sops (EPEL)" in output
assert "sudo dnf install gnupg2" in output
def test_doctor_reports_configuration_and_secret_policy_failures(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
_isolate(monkeypatch, bin_directory)
_set_os_release(monkeypatch, tmp_path, "ID=debian\n")
invalid_config = tmp_path / "invalid.toml"
invalid_config.write_text("this is not = toml", encoding="utf-8")
config_result = _invoke(["--config", str(invalid_config)])
assert getattr(config_result, "exit_code") == 2
assert "✗ config:" in getattr(config_result, "output")
repository = _repository(tmp_path, extra='secret_policy = "policy.toml"\n')
(repository / "roles").mkdir()
(repository / "policy.toml").write_text("rules = 5\n", encoding="utf-8")
secret_policy_result = _invoke(["--config", str(repository / ".compose2role.toml")])
assert getattr(secret_policy_result, "exit_code") == 0
assert "! secret policy: secret policy must define rules" in getattr(
secret_policy_result, "output"
)
def test_doctor_reports_an_unwritable_output_directory(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _repository(tmp_path, output="blocked/roles")
(repository / "blocked").write_text("not a directory", encoding="utf-8")
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
_isolate(monkeypatch, bin_directory)
_set_os_release(monkeypatch, tmp_path, "ID=debian\n")
result = _invoke(["--config", str(repository / ".compose2role.toml")])
assert getattr(result, "exit_code") == 1
assert "✗ output: roles directory is not writable" in getattr(result, "output")
def test_doctor_validates_a_configured_sops_rule(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _repository(tmp_path)
(repository / "roles").mkdir()
invalid_config = tmp_path / "invalid-sops.yaml"
invalid_config.write_text("creation_rules: []\n", encoding="utf-8")
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
_install(bin_directory, "sops", _SOPS_STUB)
_isolate(monkeypatch, bin_directory)
_set_os_release(monkeypatch, tmp_path, "ID=debian\n")
result = _invoke(
[
"--config",
str(repository / ".compose2role.toml"),
"--sops-config",
str(invalid_config),
]
)
assert getattr(result, "exit_code") == 1
assert "✗ sops rule:" in getattr(result, "output")
def _write_role(roles_directory: Path, project_root: Path, content: bytes) -> Path:
project_root.mkdir()
(project_root / "compose.yaml").write_bytes(content)
role_directory = roles_directory / "my_app"
role_directory.mkdir()
provenance = {
"files": {
"compose.yaml": {
"redacted": False,
"sha256": hashlib.sha256(content).hexdigest(),
}
},
"generator": "compose2role",
"mode": "local",
"project": "my-app",
"role": "my_app",
"source": {
"config_files": [str(project_root / "compose.yaml")],
"root": str(project_root),
},
"structural_hash": "fixture",
"version": "0.1.0",
}
(role_directory / ".compose2role.json").write_text(
json.dumps(provenance), encoding="utf-8"
)
return role_directory
def test_doctor_reports_roles_drift_and_backups(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
repository = _repository(tmp_path)
roles_directory = repository / "roles"
roles_directory.mkdir()
project_root = tmp_path / "project"
_write_role(roles_directory, project_root, b"services:\n web:\n image: web:1\n")
backup = roles_directory / ".compose2role" / "backups" / "my_app-20260101T000000Z"
backup.mkdir(parents=True)
(backup / "compose.yaml").write_bytes(b"x" * 2048)
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
_isolate(monkeypatch, bin_directory)
_set_os_release(monkeypatch, tmp_path, "ID=debian\n")
result = _invoke(["--config", str(repository / ".compose2role.toml")])
assert getattr(result, "exit_code") == 0
output = getattr(result, "output")
assert "✓ roles: 1 generated role(s)" in output
assert "✓ drift: 1 unchanged, 0 source changed, 0 source unavailable" in output
assert "✓ backups: 1 backup(s), 2.0 KiB" in output
assert "prune with: rm -r" in output
(project_root / "compose.yaml").write_bytes(b"services:\n web:\n image: web:2\n")
drifted = _invoke(["--config", str(repository / ".compose2role.toml")])
assert getattr(drifted, "exit_code") == 0
assert "! drift: 0 unchanged, 1 source changed, 0 source unavailable" in getattr(
drifted, "output"
)
+218
View File
@@ -0,0 +1,218 @@
from __future__ import annotations
import json
import os
from importlib.metadata import version as package_version
from pathlib import Path
import pytest
from click.testing import CliRunner
from compose2role.cli import main
def test_version_reports_the_installed_package_version() -> None:
result = CliRunner().invoke(main, ["--version"], prog_name="compose2role")
assert result.exit_code == 0, result.output
assert result.output == f"compose2role {package_version('compose2role')}\n"
def _install_stub_ssh(tmp_path: Path, script: str) -> Path:
bin_directory = tmp_path / "bin"
bin_directory.mkdir()
ssh = bin_directory / "ssh"
ssh.write_text(script, encoding="utf-8")
ssh.chmod(0o755)
return bin_directory
def _ssh_environment(bin_directory: Path, arguments_file: Path | None = None) -> dict[str, str]:
environment = {"PATH": f"{bin_directory}:{os.environ['PATH']}"}
if arguments_file is not None:
environment["C2R_SSH_ARGUMENTS"] = str(arguments_file)
return environment
def test_discover_lists_remote_compose_projects_as_json(tmp_path: Path) -> None:
bin_directory = _install_stub_ssh(
tmp_path,
"""\
#!/bin/sh
printf '%s\\n' '[{"Name":"myapp","Status":"running(1)","ConfigFiles":"/srv/myapp/compose.yaml"}]'
""",
)
result = CliRunner().invoke(
main,
["discover", "--host", "web1", "--format", "json"],
env=_ssh_environment(bin_directory),
)
assert result.exit_code == 0, result.output
assert json.loads(result.output) == {
"projects": [
{
"config_files": ["/srv/myapp/compose.yaml"],
"name": "myapp",
"status": "running(1)",
}
]
}
def test_discover_prints_one_status_line_per_project(tmp_path: Path) -> None:
"""Text discovery streams a status line per project and closes with a count."""
bin_directory = _install_stub_ssh(
tmp_path,
"""\
#!/bin/sh
printf '%s\\n' '[{"Name":"myapp","Status":"running(1)",'
printf '%s\\n' '"ConfigFiles":"/srv/myapp/compose.yaml"}]'
""",
)
result = CliRunner().invoke(
main,
["discover", "--host", "web1"],
env=_ssh_environment(bin_directory),
)
assert result.exit_code == 0, result.output
assert result.output.splitlines() == [
"Discovering projects on web1...",
"✓ myapp running(1) /srv/myapp/compose.yaml",
"",
"1 found",
]
@pytest.mark.parametrize(
("arguments", "expected_ssh_arguments"),
(
(
["discover", "--host", "admin@web1.example:2222", "--format", "json"],
[
"-p",
"2222",
"--",
"admin@web1.example",
"docker",
"compose",
"ls",
"--all",
"--format",
"json",
],
),
(
[
"discover",
"--host",
"admin@web1.example:2222",
"--sudo",
"--format",
"json",
],
[
"-p",
"2222",
"--",
"admin@web1.example",
"sudo",
"-n",
"docker",
"compose",
"ls",
"--all",
"--format",
"json",
],
),
),
)
def test_discover_passes_fixed_ssh_arguments(
tmp_path: Path, arguments: list[str], expected_ssh_arguments: list[str]
) -> None:
arguments_file = tmp_path / "ssh-arguments"
bin_directory = _install_stub_ssh(
tmp_path,
"""\
#!/bin/sh
printf '%s\\n' "$@" > "$C2R_SSH_ARGUMENTS"
printf '%s\\n' '[]'
""",
)
result = CliRunner().invoke(
main,
arguments,
env=_ssh_environment(bin_directory, arguments_file),
)
assert result.exit_code == 0, result.output
assert arguments_file.read_text(encoding="utf-8").splitlines() == expected_ssh_arguments
@pytest.mark.parametrize(
("arguments", "expected_message"),
(
(
["discover", "--host", "web1", "--sudo"],
"ensure the SSH user has passwordless sudo access to Docker",
),
(
["discover", "--host", "web1"],
"join the docker group",
),
),
)
def test_discover_explains_remote_docker_access_failures(
tmp_path: Path, arguments: list[str], expected_message: str
) -> None:
bin_directory = _install_stub_ssh(tmp_path, "#!/bin/sh\nexit 1\n")
result = CliRunner().invoke(
main,
arguments,
env=_ssh_environment(bin_directory),
)
assert result.exit_code == 1
assert expected_message in result.output
def test_discover_enumerates_the_access_remedies_without_recommending_one(
tmp_path: Path,
) -> None:
bin_directory = _install_stub_ssh(tmp_path, "#!/bin/sh\nexit 1\n")
result = CliRunner().invoke(
main,
["discover", "--host", "web1"],
env=_ssh_environment(bin_directory),
)
assert result.exit_code == 1
assert "join the docker group" in result.output
assert "configure passwordless sudo" in result.output
assert "--docker-context" in result.output
assert "--sudo" not in result.output
def test_discover_reports_ssh_connection_failures_without_a_sudo_hint(tmp_path: Path) -> None:
bin_directory = _install_stub_ssh(
tmp_path,
"#!/bin/sh\nprintf '%s\\n' 'Host key verification failed.' >&2\nexit 255\n",
)
result = CliRunner().invoke(
main,
["discover", "--host", "web1", "--sudo"],
env=_ssh_environment(bin_directory),
)
assert result.exit_code == 1
assert "SSH connection or remote command failed" in result.output
assert "Host key verification failed." in result.output
assert "passwordless sudo" not in result.output
+200
View File
@@ -0,0 +1,200 @@
from __future__ import annotations
from pathlib import Path
import pytest
from click.testing import CliRunner
from compose2role.cli import main
from compose2role.config import RepositoryConfig, load_repository_config
from compose2role.layout import discover_layout, find_ansible_config, load_ansible_config
from compose2role.naming import normalize_compose_project_name, normalize_role_name
def _write(path: Path, content: str) -> Path:
path.parent.mkdir(parents=True, exist_ok=True)
path.write_text(content, encoding="utf-8")
return path
def _config(tmp_path: Path, content: str) -> RepositoryConfig:
return load_repository_config(_write(tmp_path / ".compose2role.toml", content), environ={})
def test_roles_directory_prefers_explicit_output(tmp_path: Path) -> None:
layout = discover_layout(
RepositoryConfig(root=tmp_path),
start=tmp_path,
explicit_output=tmp_path / "explicit",
environ={},
)
assert layout.roles_directory == tmp_path / "explicit"
def test_roles_directory_uses_config_defaults_before_ansible_cfg(tmp_path: Path) -> None:
_write(tmp_path / "ansible.cfg", "[defaults]\nroles_path = ansible/roles\n")
config = _config(tmp_path, "[defaults]\noutput = 'configured/roles'\n")
layout = discover_layout(config, start=tmp_path, environ={})
assert layout.roles_directory == tmp_path / "configured" / "roles"
def test_roles_directory_uses_the_first_ansible_cfg_roles_path_entry(tmp_path: Path) -> None:
_write(
tmp_path / "ansible.cfg",
"[defaults]\nroles_path = ansible/roles:extra/roles\n",
)
layout = discover_layout(RepositoryConfig(root=tmp_path), start=tmp_path, environ={})
assert layout.roles_directory == tmp_path / "ansible" / "roles"
def test_roles_directory_falls_back_to_existing_ansible_roles(tmp_path: Path) -> None:
(tmp_path / "ansible" / "roles").mkdir(parents=True)
layout = discover_layout(RepositoryConfig(root=tmp_path), start=tmp_path, environ={})
assert layout.roles_directory == tmp_path / "ansible" / "roles"
def test_roles_directory_falls_back_to_existing_roles(tmp_path: Path) -> None:
(tmp_path / "roles").mkdir()
layout = discover_layout(RepositoryConfig(root=tmp_path), start=tmp_path, environ={})
assert layout.roles_directory == tmp_path / "roles"
def test_roles_directory_falls_back_to_config_root_roles(tmp_path: Path) -> None:
layout = discover_layout(RepositoryConfig(root=tmp_path), start=tmp_path, environ={})
assert layout.roles_directory == tmp_path / "roles"
def test_ansible_cfg_only_reads_the_documented_defaults_keys(tmp_path: Path) -> None:
_write(
tmp_path / "ansible.cfg",
"""\
[defaults]
roles_path = documented/roles
unknown = ignored
[other]
roles_path = ignored/roles
""",
)
layout = discover_layout(RepositoryConfig(root=tmp_path), start=tmp_path, environ={})
assert layout.roles_directory == tmp_path / "documented" / "roles"
def test_ansible_cfg_paths_resolve_relative_to_the_config_file(tmp_path: Path) -> None:
ansible_config = _write(
tmp_path / "ansible.cfg",
"""\
[defaults]
roles_path = roles
collections_path = collections:more-collections
inventory = inventory/hosts.ini:inventory/extra.ini
""",
)
parsed = load_ansible_config(ansible_config)
assert parsed.roles_path == (tmp_path / "roles",)
assert parsed.collections_path == (tmp_path / "collections", tmp_path / "more-collections")
assert parsed.inventory == (
tmp_path / "inventory" / "hosts.ini",
tmp_path / "inventory" / "extra.ini",
)
def test_inventory_directory_is_discovered_without_ansible_cfg(tmp_path: Path) -> None:
inventory = tmp_path / "inventory"
inventory.mkdir()
layout = discover_layout(RepositoryConfig(root=tmp_path), start=tmp_path, environ={})
assert layout.inventory == (inventory,)
def test_find_ansible_config_walks_up(tmp_path: Path) -> None:
ansible_config = _write(tmp_path / "ansible.cfg", "[defaults]\n")
nested = tmp_path / "stacks" / "web"
assert find_ansible_config(nested) == ansible_config
def test_sops_config_precedence(tmp_path: Path) -> None:
config = RepositoryConfig(root=tmp_path, sops_config=tmp_path / "configured.yaml")
explicit = discover_layout(
config,
start=tmp_path,
explicit_sops_config=tmp_path / "explicit.yaml",
environ={},
)
environment = discover_layout(
config,
start=tmp_path,
environ={"COMPOSE2ROLE_SOPS_CONFIG": str(tmp_path / "environment.yaml")},
)
configured = discover_layout(config, start=tmp_path, environ={})
assert explicit.sops_config == tmp_path / "explicit.yaml"
assert environment.sops_config == tmp_path / "environment.yaml"
assert configured.sops_config == tmp_path / "configured.yaml"
def test_secret_policy_precedence(tmp_path: Path) -> None:
config = RepositoryConfig(root=tmp_path, secret_policy=tmp_path / "configured.toml")
explicit = discover_layout(
config,
start=tmp_path,
explicit_secret_policy=tmp_path / "explicit.toml",
environ={},
)
environment = discover_layout(
config,
start=tmp_path,
environ={"COMPOSE2ROLE_SECRET_POLICY": str(tmp_path / "environment.toml")},
)
configured = discover_layout(config, start=tmp_path, environ={})
assert explicit.secret_policy == tmp_path / "explicit.toml"
assert environment.secret_policy == tmp_path / "environment.toml"
assert configured.secret_policy == tmp_path / "configured.toml"
def test_host_profile_is_selected_for_the_target_host(tmp_path: Path) -> None:
config = _config(tmp_path, "[hosts.web1]\nsudo = true\n")
layout = discover_layout(config, start=tmp_path, host="web1", environ={})
other = discover_layout(config, start=tmp_path, host="web2", environ={})
assert layout.host.sudo is True
assert other.host.sudo is None
def test_normalizers_produce_valid_names() -> None:
assert normalize_role_name("My App") == "my_app"
assert normalize_role_name("123") == "compose_123"
assert normalize_compose_project_name("My App") == "myapp"
assert normalize_compose_project_name("---") == "compose"
def test_invalid_configuration_exits_with_usage_status(
tmp_path: Path, monkeypatch: pytest.MonkeyPatch
) -> None:
_write(tmp_path / ".compose2role.toml", "[defaults\n")
monkeypatch.chdir(tmp_path)
result = CliRunner().invoke(main, ["discover", "--host", "web1"])
assert result.exit_code == 2, result.output
assert "configuration" in result.output.lower()
File diff suppressed because it is too large. Load diff
File diff suppressed because it is too large. Load diff
+136
View File
@@ -0,0 +1,136 @@
from __future__ import annotations
from pathlib import Path, PurePosixPath
import pytest
from compose2role.errors import SecretPolicyError
from compose2role.secret_policy import load_secret_policy
def _write_policy(tmp_path: Path, content: str) -> Path:
path = tmp_path / "secret-policy.toml"
path.write_text(content, encoding="utf-8")
return path
def test_secret_policy_key_list_expands_to_one_exact_rule_per_key(tmp_path: Path) -> None:
policy = load_secret_policy(
_write_policy(
tmp_path,
"""\
[[rules]]
project = "myapp"
service = "web"
key = ["API_TOKEN", "DB_PASSWORD"]
decision = "plain"
""",
)
)
assert policy.decision_for("myapp", "web", "API_TOKEN") == "plain"
assert policy.decision_for("myapp", "web", "DB_PASSWORD") == "plain"
assert policy.decision_for("myapp", "web", "OTHER_KEY") is None
assert policy.decision_for("myapp", "worker", "API_TOKEN") is None
def test_secret_policy_dotenv_key_list_expands_to_one_exact_rule_per_key(tmp_path: Path) -> None:
policy = load_secret_policy(
_write_policy(
tmp_path,
"""\
[[dotenv_rules]]
project = "myapp"
source = ".env"
key = ["MAIL_PASSWORD", "SMTP_TOKEN"]
decision = "secret"
""",
)
)
assert policy.dotenv_decision_for("myapp", PurePosixPath(".env"), "MAIL_PASSWORD") == "secret"
assert policy.dotenv_decision_for("myapp", PurePosixPath(".env"), "SMTP_TOKEN") == "secret"
assert policy.dotenv_decision_for("myapp", PurePosixPath(".env"), "OTHER_KEY") is None
assert policy.dotenv_decision_for("myapp", PurePosixPath(".env.local"), "SMTP_TOKEN") is None
def test_secret_policy_single_key_rule_still_matches_exactly(tmp_path: Path) -> None:
policy = load_secret_policy(
_write_policy(
tmp_path,
"""\
[[rules]]
project = "myapp"
service = "web"
key = "API_TOKEN"
decision = "secret"
""",
)
)
assert policy.decision_for("myapp", "web", "API_TOKEN") == "secret"
assert policy.decision_for("myapp", "web", "API_TOKEN_2") is None
@pytest.mark.parametrize(
"key_value",
['[]', '["API_TOKEN", 7]', '["API_TOKEN", ""]', '""', "7"],
)
def test_secret_policy_rejects_invalid_environment_key_values(
tmp_path: Path, key_value: str
) -> None:
policy_path = _write_policy(
tmp_path,
f"""\
[[rules]]
project = "myapp"
service = "web"
key = {key_value}
decision = "secret"
""",
)
with pytest.raises(SecretPolicyError, match="keys must be non-empty strings"):
load_secret_policy(policy_path)
@pytest.mark.parametrize(
"key_value",
['[]', '["MAIL_PASSWORD", 7]', '["MAIL_PASSWORD", ""]', '""', "7"],
)
def test_secret_policy_rejects_invalid_dotenv_key_values(tmp_path: Path, key_value: str) -> None:
policy_path = _write_policy(
tmp_path,
f"""\
[[dotenv_rules]]
project = "myapp"
source = ".env"
key = {key_value}
decision = "secret"
""",
)
with pytest.raises(SecretPolicyError, match="keys must be non-empty strings"):
load_secret_policy(policy_path)
def test_secret_policy_rejects_conflicts_between_list_and_single_rules(tmp_path: Path) -> None:
policy_path = _write_policy(
tmp_path,
"""\
[[rules]]
project = "myapp"
service = "web"
key = ["API_TOKEN", "LOG_LEVEL"]
decision = "secret"
[[rules]]
project = "myapp"
service = "web"
key = "LOG_LEVEL"
decision = "plain"
""",
)
with pytest.raises(SecretPolicyError, match="conflicting exact rules"):
load_secret_policy(policy_path)