From b056d0d80ee87f9978fbb9ac401a156fae56ee01 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 10 Aug 2026 09:55:13 +0530 Subject: [PATCH 01/14] feat(docker): add attachable Hubble Compose add-on for the 3-node cluster Add docker-compose-hubble.yml, a Hubble-only add-on that joins the cluster's pre-created external network (HUGEGRAPH_NETWORK, default hugegraph-net) with no depends_on, so attaching, upgrading, or removing Hubble never recreates PD, Store, or Server containers. Attach flow uses an explicit project (-p hugegraph-hubble); the fresh flow brings up cluster plus Hubble in one command with both -f flags. Give the 3-node cluster the Server settings Hubble's PD mode requires: PD registration (HG_SERVER_CLUSTER/USE_PD/REST_URL per replica via a shared env anchor), a required shared auth token secret so tokens validate on every replica, and a required admin password. The Server healthcheck now probes the bound REST URL. Hubble reads the 3x3 topology from hugegraph-hubble-3x3.properties. Document the attach, fresh, and dev-override flows plus migration notes in docker/README.md, update the cluster call sites across the docs, and extend the CI compose contract checks to the cluster file and add-on. Image tags stay on latest until the 1.8.0 release publishes; pin via HUGEGRAPH_VERSION in docker/.env. --- .github/workflows/server-ci.yml | 90 +++++++ .../implementation_patterns_and_guidelines.md | 3 +- .serena/memories/key_file_locations.md | 3 +- .serena/memories/suggested_commands.md | 6 +- README.md | 2 +- docker/README.md | 246 ++++++++++++++++-- docker/docker-compose-3pd-3store-3server.yml | 47 +++- docker/docker-compose-hubble.yml | 47 ++++ docker/hugegraph-hubble-3x3.properties | 33 +++ hugegraph-pd/README.md | 4 +- hugegraph-pd/docs/configuration.md | 4 +- hugegraph-server/README.md | 5 +- .../hugegraph-dist/docker/README.md | 5 +- hugegraph-store/AGENTS.md | 2 +- hugegraph-store/docs/deployment-guide.md | 21 +- 15 files changed, 475 insertions(+), 43 deletions(-) create mode 100644 docker/docker-compose-hubble.yml create mode 100644 docker/hugegraph-hubble-3x3.properties diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 9c4e577d85..96dd6eac05 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -138,6 +138,96 @@ jobs: check_compose docker/docker-compose.yml always always check_compose docker/docker-compose.dev.yml build missing + check_cluster_compose() { + local cluster="docker/docker-compose-3pd-3store-3server.yml" + local addon="docker/docker-compose-hubble.yml" + local rendered + rendered="$(mktemp)" + + # Both cluster credentials are required and may not be empty. + if HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-secret \ + env -u HUGEGRAPH_ADMIN_PASSWORD \ + docker compose -f "$cluster" config -q >/dev/null 2>&1; then + echo "$cluster accepted an unset admin password" >&2 + return 1 + fi + if HUGEGRAPH_ADMIN_PASSWORD= \ + HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-secret \ + docker compose -f "$cluster" config -q >/dev/null 2>&1; then + echo "$cluster accepted an empty admin password" >&2 + return 1 + fi + if HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ + env -u HUGEGRAPH_AUTH_TOKEN_SECRET \ + docker compose -f "$cluster" config -q >/dev/null 2>&1; then + echo "$cluster accepted an unset token secret" >&2 + return 1 + fi + if HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ + HUGEGRAPH_AUTH_TOKEN_SECRET= \ + docker compose -f "$cluster" config -q >/dev/null 2>&1; then + echo "$cluster accepted an empty token secret" >&2 + return 1 + fi + + # The add-on alone must define Hubble and nothing else, join the + # shared external network, and need no credentials. + env -u HUGEGRAPH_ADMIN_PASSWORD -u HUGEGRAPH_AUTH_TOKEN_SECRET \ + docker compose -f "$addon" config --format json > "$rendered" + jq -e ' + (.services | keys) == ["hubble"] and + .networks."hg-net".external == true and + .networks."hg-net".name == "hugegraph-net" and + (.services.hubble | has("depends_on") | not) and + any(.services.hubble.volumes[]; + .target == "/hubble/conf/hugegraph-hubble.properties" and + (.source | endswith("hugegraph-hubble-3x3.properties"))) + ' "$rendered" >/dev/null + + # The combined render carries the PD-registration and auth + # settings on every server replica and keeps Hubble on loopback. + HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ + HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-secret \ + docker compose -f "$cluster" -f "$addon" \ + config --format json > "$rendered" + jq -e ' + .name == "hugegraph-3x3" and + (.services | keys | length) == 10 and + .networks."hg-net".external == true and + .networks."hg-net".name == "hugegraph-net" and + .services.server0.environment.HG_SERVER_USE_PD == "true" and + .services.server0.environment.HG_SERVER_CLUSTER == "hg" and + .services.server0.environment.HG_SERVER_REST_URL == + "http://server0:8080" and + .services.server1.environment.HG_SERVER_REST_URL == + "http://server1:8080" and + .services.server1.environment.HG_SERVER_AUTH_TOKEN_SECRET == + "ci-test-secret" and + .services.server2.environment.HG_SERVER_REST_URL == + "http://server2:8080" and + .services.server2.environment.HG_SERVER_AUTH_TOKEN_SECRET == + "ci-test-secret" and + .services.server0.environment.HG_SERVER_INIT_STORE_ENABLED == + "false" and + .services.server0.environment.PASSWORD == + "ci-test-password" and + .services.server0.environment.HG_SERVER_AUTH_TOKEN_SECRET == + "ci-test-secret" and + (.services.hubble | has("depends_on") | not) and + .services.hubble.pull_policy == "missing" and + (.services.hubble.healthcheck.test[1] | + contains("http://127.0.0.1:8088/about") and + contains("\"status\":200") and + contains("\"name\":\"hugegraph-hubble\"")) and + any(.services.hubble.ports[]; + .target == 8088 and .published == "8088" and + .host_ip == "127.0.0.1") + ' "$rendered" >/dev/null + rm -f "$rendered" + } + + check_cluster_compose + - name: Run check_port unit tests if: ${{ env.BACKEND == 'rocksdb' }} run: | diff --git a/.serena/memories/implementation_patterns_and_guidelines.md b/.serena/memories/implementation_patterns_and_guidelines.md index d04e33ce56..e5c5052547 100644 --- a/.serena/memories/implementation_patterns_and_guidelines.md +++ b/.serena/memories/implementation_patterns_and_guidelines.md @@ -40,7 +40,8 @@ ## Docker - Single-node: `docker/docker-compose.yml` (bridge network, pd+store+server) -- Cluster: `docker/docker-compose-3pd-3store-3server.yml` +- Cluster: `docker/docker-compose-3pd-3store-3server.yml` (external `hugegraph-net` network + `docker/.env` credentials) +- Hubble add-on for the cluster: `docker/docker-compose-hubble.yml` - Container logs: stdout-based ## CI Pipelines diff --git a/.serena/memories/key_file_locations.md b/.serena/memories/key_file_locations.md index 3f2a60dee0..38fa9cb85b 100644 --- a/.serena/memories/key_file_locations.md +++ b/.serena/memories/key_file_locations.md @@ -17,7 +17,8 @@ ## Docker - `docker/docker-compose.yml` — Single-node (bridge network, pd+store+server) -- `docker/docker-compose-3pd-3store-3server.yml` — 3-node cluster +- `docker/docker-compose-3pd-3store-3server.yml` — 3-node cluster (external `hugegraph-net` network, credentials required) +- `docker/docker-compose-hubble.yml` — Hubble add-on for the 3-node cluster - `docker/docker-compose.dev.yml` — Dev mode ## PD Module diff --git a/.serena/memories/suggested_commands.md b/.serena/memories/suggested_commands.md index 346304432f..1585fc4b29 100644 --- a/.serena/memories/suggested_commands.md +++ b/.serena/memories/suggested_commands.md @@ -47,7 +47,11 @@ bin/enable-auth.sh # Enable auth ## Docker ```bash cd docker && docker compose up -d # Single-node (bridge network) -cd docker && docker compose -f docker-compose-3pd-3store-3server.yml up -d # Cluster +# Cluster: needs one-time setup first (hugegraph-net network + docker/.env +# credentials) — see docker/README.md "3-Node Cluster Quickstart" +cd docker && docker compose -f docker-compose-3pd-3store-3server.yml up -d +# Hubble add-on for a running cluster +cd docker && docker compose -p hugegraph-hubble -f docker-compose-hubble.yml up -d ``` ## Distributed Build (BETA) diff --git a/README.md b/README.md index adf9792776..36ee9c57b2 100644 --- a/README.md +++ b/README.md @@ -201,7 +201,7 @@ For advanced Docker configurations, see: * [Docker README](./docker/README.md) * [Server Docker README](hugegraph-server/hugegraph-dist/docker/README.md) -> **Docker Desktop (Mac/Windows)**: The 3-node distributed cluster (`docker/docker-compose-3pd-3store-3server.yml`) uses Docker bridge networking and works on all platforms including Docker Desktop. Allocate at least 12 GB memory to Docker Desktop. +> **Docker Desktop (Mac/Windows)**: The 3-node distributed cluster (`docker/docker-compose-3pd-3store-3server.yml`) joins a pre-created external Docker network shared with the Hubble add-on (see the [Docker README](./docker/README.md) quickstart) and works on all platforms including Docker Desktop. Allocate at least 12 GB memory to Docker Desktop. > **Note**: Docker images are convenience releases, not **official ASF distribution artifacts**. See [ASF Release Distribution Policy](https://infra.apache.org/release-distribution.html#dockerhub) for details. > diff --git a/docker/README.md b/docker/README.md index 0ee1f586b6..ec8286205c 100644 --- a/docker/README.md +++ b/docker/README.md @@ -1,22 +1,27 @@ # HugeGraph Docker Deployment -This directory contains Docker Compose files for running HugeGraph: +This directory contains Docker Compose files and their configuration for +running HugeGraph: | File | Description | |------|-------------| | `docker-compose.yml` | PD, Store, Server, and Hubble using pre-built images | | `docker-compose.dev.yml` | PD, Store, and Server built from source, plus Hubble | | `docker-compose-3pd-3store-3server.yml` | 3-node distributed cluster (PD + Store + Server) | +| `docker-compose-hubble.yml` | Hubble add-on for the 3-node cluster (attachable to a running cluster) | +| `hugegraph-hubble.properties` | Hubble configuration mounted by the single-node files | +| `hugegraph-hubble-3x3.properties` | Hubble configuration mounted by the add-on; edit when attaching to a cluster with different hostnames | ## Prerequisites - **Docker Engine** 20.10+ (or Docker Desktop 4.x+) - **Docker Compose** v2 (included in Docker Desktop) - **OpenSSL CLI** (used to generate the initial administrator password) -- **Memory**: Allocate at least **12 GB** to Docker Desktop (Settings → Resources → Memory). The 3-node cluster runs 9 JVM processes (3 PD + 3 Store + 3 Server) which are memory-intensive. Insufficient memory causes OOM kills that appear as silent Raft failures. +- **Memory**: Allocate at least **12 GB** to Docker Desktop (Settings → Resources → Memory). The 3-node cluster runs 9 JVM processes (3 PD + 3 Store + 3 Server) which are memory-intensive — plus a tenth container when the Hubble add-on is attached. Insufficient memory causes OOM kills that appear as silent Raft failures. > [!IMPORTANT] > The 12 GB minimum is for Docker Desktop. On Linux with native Docker, ensure the host has at least 12 GB of free memory. + --- ## Single-Node Setup @@ -168,14 +173,87 @@ To validate local images without Compose replacing them with remote `latest`: ## 3-Node Cluster Quickstart +The cluster and the Hubble add-on share one named Docker network so Hubble +can attach to a running cluster without touching it. Treat that network as a +trust boundary: PD and Store expose unauthenticated control APIs on it (only +the Server layer authenticates), and any container on the host can join it +by declaring the well-known name. One-time setup: write the required +credentials to a mode-600 `docker/.env` and create the network. +The cluster file requires both credentials — the admin password enables +authentication, and every Server replica must share one token secret so a +token issued by any server validates on all of them. The `:?` guards fire +on every Compose subcommand, including `down`. + +```bash +( + set -eu + cd docker + command -v openssl >/dev/null 2>&1 || { echo "openssl not found" >&2; exit 1; } + [ -e .env ] || install -m 600 /dev/null .env + chmod 600 .env + # Keep appends on their own lines even if the file was hand-edited. + [ ! -s .env ] || [ -z "$(tail -c1 .env)" ] || printf '\n' >> .env + pat='^[[:space:]]*(export[[:space:]]+)?' + if ! grep -Eq "${pat}HUGEGRAPH_ADMIN_PASSWORD=" .env; then + admin_password="$(openssl rand -base64 12)" + printf "HUGEGRAPH_ADMIN_PASSWORD='%s'\n" "${admin_password}" >> .env + unset admin_password + fi + if ! grep -Eq "${pat}HUGEGRAPH_AUTH_TOKEN_SECRET=" .env; then + token_secret="$(openssl rand -hex 32)" + printf "HUGEGRAPH_AUTH_TOKEN_SECRET='%s'\n" "${token_secret}" >> .env + unset token_secret + fi + # The shared cluster network. To override the name, export + # HUGEGRAPH_NETWORK in this shell before running the block — a value + # in docker/.env is read by Compose, not by this script. + net="${HUGEGRAPH_NETWORK:-hugegraph-net}" + docker network inspect "${net}" >/dev/null 2>&1 || + docker network create "${net}" + env -u HUGEGRAPH_ADMIN_PASSWORD -u HUGEGRAPH_AUTH_TOKEN_SECRET \ + docker compose -f docker-compose-3pd-3store-3server.yml config --quiet +) +``` + +Then start the cluster: + ```bash cd docker -HUGEGRAPH_VERSION=1.7.0 docker compose -f docker-compose-3pd-3store-3server.yml up -d +docker compose -f docker-compose-3pd-3store-3server.yml up -d -# To stop and remove all data volumes (clean restart) +# To stop and remove all data volumes (clean restart). +# The external hugegraph-net network is intentionally left in place. +# If the Hubble add-on is running, see "Hubble for the 3-Node Cluster" +# for the teardown that matches how it was started. docker compose -f docker-compose-3pd-3store-3server.yml down -v ``` +Pin a release by setting `HUGEGRAPH_VERSION` in `docker/.env` — the +cluster, the Hubble add-on, and the single-node quickstart file all read +it from there, so those versions cannot drift apart +(`docker-compose.dev.yml` builds PD/Store/Server from source and defaults +Hubble to `hugegraph/hubble:latest`; set `HUBBLE_IMAGE` to pin it). +Unpinned, the images default to `latest`; note the authenticated PD/Hubble +integration requires a release newer than `1.7.x`. Because the cluster +files use `pull_policy: missing`, an already-pulled `latest` is never +refreshed by `up -d` — pull explicitly or pin to pick up new releases. + +> [!NOTE] +> Upgrading an existing 3-node deployment: +> - Create `docker/.env` (block above) before running any Compose command +> against an older stack, `down` included. +> - The cluster now joins the pre-created `hugegraph-net` network instead of +> a per-project bridge, so the first `up -d` recreates all nine containers. +> Named data volumes are unchanged and survive the move; the orphaned +> `hugegraph-3x3_hg-net` bridge can be removed with +> `docker network rm hugegraph-3x3_hg-net`. +> - Authentication is now enabled: previously unauthenticated clients of the +> graph APIs on ports 8080–8082 will start receiving 401 responses and +> must supply the `admin` credential from `docker/.env` (`/versions` and +> `/openapi.json` stay open, so they cannot serve as an auth smoke test). +> On a cluster whose volumes predate authentication, verify you can sign +> in before decommissioning any existing access path. + **Startup ordering** is enforced via `depends_on` with `condition: service_healthy`: 1. **PD nodes** start first and must pass healthchecks (`/v1/health`) @@ -205,6 +283,110 @@ curl http://localhost:8620/v1/partitions --- +## Hubble for the 3-Node Cluster + +`docker-compose-hubble.yml` defines only the Hubble service. It joins the +cluster's external network (`hugegraph-net` by default, override with +`HUGEGRAPH_NETWORK`) and has no `depends_on` on cluster services, so +starting, stopping, or upgrading Hubble never recreates or restarts PD, +Store, or Server containers. Hubble reads the cluster topology from +`hugegraph-hubble-3x3.properties`; adjust that file when attaching to a +cluster with different hostnames. + +Sign in at `http://localhost:8088` as `admin` with the +`HUGEGRAPH_ADMIN_PASSWORD` from `docker/.env`. Hubble binds to host +loopback by default (`HUBBLE_PUBLISH_HOST`, same caveats as the +single-node setup). + +The two flows below create Hubble in different Compose projects, so manage +Hubble with the same flags you started it with: the attach flow always uses +`-p hugegraph-hubble -f docker-compose-hubble.yml`, the combined flow always +uses both `-f` flags. The explicit `-p` keeps the attach project independent +of the directory name and of other Compose projects. + +Run one Hubble per host: the single-node stack and both add-on flows all +publish `127.0.0.1:8088` and name their container `hg-hubble`. The two +add-on flows are therefore mutually exclusive — starting one while the +other's Hubble exists fails with a container-name conflict, so `down` the +flow you are leaving before switching. + +### Attach to a running cluster + +With the 3-node cluster already up: + +```bash +cd docker +docker compose -p hugegraph-hubble -f docker-compose-hubble.yml up -d +``` + +Lifecycle commands in this flow operate on Hubble alone and leave the +cluster and the external network in place: + +```bash +cd docker +docker compose -p hugegraph-hubble -f docker-compose-hubble.yml ps +docker compose -p hugegraph-hubble -f docker-compose-hubble.yml down +``` + +To remove everything in this flow, take down Hubble first, then the cluster: + +```bash +cd docker +docker compose -p hugegraph-hubble -f docker-compose-hubble.yml down +docker compose -f docker-compose-3pd-3store-3server.yml down -v +``` + +### Fresh cluster plus Hubble in one command + +After the one-time network and `docker/.env` setup from the quickstart: + +```bash +cd docker +docker compose -f docker-compose-3pd-3store-3server.yml \ + -f docker-compose-hubble.yml up -d +``` + +Hubble has no startup dependency on the cluster, so it reports healthy while +PD, Store, and Server are still forming the cluster; wait until every service +shows healthy before signing in: + +```bash +cd docker +docker compose -f docker-compose-3pd-3store-3server.yml \ + -f docker-compose-hubble.yml ps +``` + +In this flow Hubble belongs to the cluster project — use the same pair of +`-f` flags for `ps`, `stop`, and `down`. The attach-flow `ps`/`stop`/`down` +commands manage a different, empty project and do nothing here, and the +cluster-only quickstart commands treat this Hubble as an orphan container +(`--remove-orphans` would delete it) — always pass both `-f` flags. + +### Local Hubble image for development + +Build the Hubble image from `hugegraph-toolchain` source, then replace only +the Hubble container. In the attach flow: + +```bash +cd docker +HUBBLE_IMAGE=local/hugegraph-hubble:dev \ +HUBBLE_PULL_POLICY=never \ +docker compose -p hugegraph-hubble -f docker-compose-hubble.yml up -d +``` + +If the stack was started with the combined command, replace only the +`hubble` service under that project instead: + +```bash +cd docker +HUBBLE_IMAGE=local/hugegraph-hubble:dev \ +HUBBLE_PULL_POLICY=never \ +docker compose -f docker-compose-3pd-3store-3server.yml \ + -f docker-compose-hubble.yml up -d --no-deps hubble +``` + +--- + ## Environment Variable Reference Configuration is injected via environment variables. The old `docker/configs/application-pd*.yml` and `docker/configs/application-store*.yml` files are no longer used. @@ -256,7 +438,7 @@ Configuration is injected via environment variables. The old `docker/configs/app |----------|----------|---------|-----------------------------|-------------| | `HG_SERVER_BACKEND` | Yes | — | `backend` in `hugegraph.properties` | Storage backend (e.g. `hstore`) | | `HG_SERVER_PD_PEERS` | Yes | — | `pd.peers` | PD cluster addresses (e.g. `pd0:8686,pd1:8686,pd2:8686`) | -| `HG_SERVER_CLUSTER` | No | — | `cluster` in `rest-server.properties` | PD discovery application name; single-node Compose uses `hg` to match Hubble | +| `HG_SERVER_CLUSTER` | No | — | `cluster` in `rest-server.properties` | PD discovery application name; both the single-node and 3-node Compose files use `hg` to match the Hubble configuration | | `HG_SERVER_USE_PD` | No | — | `usePD` in `rest-server.properties` | Enables Server PD registration and discovery | | `HG_SERVER_REST_URL` | No | — | `restserver.url` | Address registered with PD and used by clients | | `HG_SERVER_MIN_FREE_MEMORY` | No | — | `restserver.min_free_memory` | Minimum free-memory guard in MB; local Compose uses `0` | @@ -287,17 +469,22 @@ Configuration is injected via environment variables. The old `docker/configs/app > PD startup path uses the explicit `auth.admin_pa` value when it first creates > the administrator. Changing it later does not rotate an existing password. -The single-node Compose files also accept these deployment-level overrides: - -| Variable | Default | Description | -|----------|---------|-------------| -| `HUGEGRAPH_SERVER_IMAGE` | `hugegraph/server:` | Complete Server image reference | -| `HUGEGRAPH_SERVER_PULL_POLICY` | `always` (`build` for dev) | Server pull policy | -| `HUBBLE_IMAGE` | `hugegraph/hubble:` | Complete Hubble image reference | -| `HUBBLE_PULL_POLICY` | `always` (`missing` for dev) | Hubble pull policy | -| `HUBBLE_PUBLISH_HOST` | `127.0.0.1` | Hubble host bind address; remote access requires an HTTPS reverse proxy | -| `HUGEGRAPH_ADMIN_PASSWORD` | required (`docker/.env`) | Initial admin password; no public default is provided | -| `HUGEGRAPH_AUTH_TOKEN_SECRET` | generated | JWT signing secret; explicit values must be at least 32 bytes | +The Compose files also accept these deployment-level overrides; the +"Used by" column names the files that read each variable (single = the +single-node files, cluster = `docker-compose-3pd-3store-3server.yml`, +add-on = `docker-compose-hubble.yml`): + +| Variable | Used by | Default | Description | +|----------|---------|---------|-------------| +| `HUGEGRAPH_VERSION` | single (quickstart), cluster, add-on | `latest` | Shared image tag for PD, Store, Server, and Hubble; pin it in `docker/.env` so these files resolve the same release. The dev file builds from source and defaults Hubble to `latest`; set `HUBBLE_IMAGE` to pin it | +| `HUGEGRAPH_SERVER_IMAGE` | single | `hugegraph/server:` | Complete Server image reference | +| `HUGEGRAPH_SERVER_PULL_POLICY` | single | `always` (`build` for dev) | Server pull policy | +| `HUBBLE_IMAGE` | single, add-on | `hugegraph/hubble:` | Complete Hubble image reference | +| `HUBBLE_PULL_POLICY` | single, add-on | `always` (`missing` for dev and the add-on) | Hubble pull policy | +| `HUBBLE_PUBLISH_HOST` | single, add-on | `127.0.0.1` | Hubble host bind address; remote access requires an HTTPS reverse proxy | +| `HUGEGRAPH_NETWORK` | cluster, add-on | `hugegraph-net` | Pre-created external Docker network shared by the 3-node cluster and the Hubble add-on; the single-node files use their own project bridge instead | +| `HUGEGRAPH_ADMIN_PASSWORD` | single, cluster | required (`docker/.env`) | Initial admin password; no public default is provided | +| `HUGEGRAPH_AUTH_TOKEN_SECRET` | single, cluster | generated (single); **required** (cluster) | JWT signing secret; explicit values must be at least 32 bytes. The cluster file requires it so all Server replicas validate each other's tokens | When authentication is enabled and no token secret is supplied, the Server entrypoint generates a random secret and writes it to both authentication @@ -362,7 +549,16 @@ configuration file. ## Port Reference -The table below reflects the published host ports in `docker-compose-3pd-3store-3server.yml`. +The table below reflects the published host ports of the 3-node cluster +(`docker-compose-3pd-3store-3server.yml`) and its Hubble add-on +(`docker-compose-hubble.yml`). + +> [!IMPORTANT] +> Cluster ports bind all host interfaces and bypass host firewalls under +> Docker's port publishing; the PD and Store APIs among them are +> unauthenticated. Do not run this file on an untrusted network. Hubble is +> the exception and binds loopback only by default. + The single-node Compose file publishes `8620`, `8520`, `8080`, and Hubble `8088`; Hubble defaults to host loopback. @@ -387,6 +583,7 @@ The single-node Compose file publishes `8620`, `8520`, `8080`, and Hubble | server0 | 8080 | 8080 | HTTP | Graph API | | server1 | 8080 | 8081 | HTTP | Graph API | | server2 | 8080 | 8082 | HTTP | Graph API | +| hubble | 8088 | 8088 | HTTP | Hubble UI; loopback-only by default (`HUBBLE_PUBLISH_HOST`) | --- @@ -403,6 +600,19 @@ The single-node Compose file publishes `8620`, `8520`, `8080`, and Hubble ## Troubleshooting +### `network hugegraph-net declared as external, but could not be found` + +**Symptom**: 3-node cluster or Hubble add-on commands that create +containers (`up`, `run`, `create`) fail immediately with this error, with +the resolved network name in the message. `config` and `ps` do not check +the network, so they can succeed while `up` fails. + +**Cause**: The shared cluster network does not exist yet. It is declared +`external`, so Compose never creates it on its own. + +**Fix**: `docker network create hugegraph-net` (or the name you set via +`HUGEGRAPH_NETWORK`), then re-run the command. + ### Containers Exiting or Restarting (OOM Kills) **Symptom**: Containers exit with code 137, or restart loops. Raft logs show election timeouts. @@ -445,6 +655,6 @@ docker stats --no-stream **Symptom**: Stores cannot connect to PD, or Server cannot connect to Store. -**Cause**: Services are using `127.0.0.1` instead of container hostnames, or the `hg-net` bridge network is misconfigured. +**Cause**: Services are using `127.0.0.1` instead of container hostnames, or containers are attached to different Docker networks (the cluster and the Hubble add-on must share the pre-created `hugegraph-net`). **Fix**: Ensure all `HG_*` env vars use container hostnames (`pd0`, `store0`, etc.), not `127.0.0.1` or `localhost`. diff --git a/docker/docker-compose-3pd-3store-3server.yml b/docker/docker-compose-3pd-3store-3server.yml index fc7930351b..371fbf507a 100644 --- a/docker/docker-compose-3pd-3store-3server.yml +++ b/docker/docker-compose-3pd-3store-3server.yml @@ -17,9 +17,14 @@ name: hugegraph-3x3 +# The cluster network is shared with the Hubble add-on +# (docker-compose-hubble.yml), so it is external and must exist first: +# docker network create hugegraph-net +# Set HUGEGRAPH_NETWORK to use a differently named network. networks: hg-net: - driver: bridge + external: true + name: ${HUGEGRAPH_NETWORK:-hugegraph-net} volumes: hg-pd0-data: @@ -31,6 +36,8 @@ volumes: # ── Shared service defaults ────────────────────────────────────────── x-pd-common: &pd-common + # Pin a release via HUGEGRAPH_VERSION in docker/.env; unset, the image + # tags default to latest. All Compose files here read the same variable. image: hugegraph/pd:${HUGEGRAPH_VERSION:-latest} pull_policy: missing restart: unless-stopped @@ -58,6 +65,24 @@ x-store-common: &store-common retries: 40 start_period: 120s +# Shared Server environment; each server node adds its own +# HG_SERVER_REST_URL on top of this map. +x-server-env: &server-env + STORE_REST: store0:8520 + HG_SERVER_BACKEND: hstore + HG_SERVER_PD_PEERS: pd0:8686,pd1:8686,pd2:8686 + # Register every Server replica with PD under one application name so + # PD-aware clients such as Hubble discover the cluster as one logical + # Server; `hg` matches the Hubble configuration. + HG_SERVER_CLUSTER: hg + HG_SERVER_USE_PD: "true" + HG_SERVER_MIN_FREE_MEMORY: "0" + HG_SERVER_INIT_STORE_ENABLED: "false" + # All replicas must share one token secret so a token issued by any + # server validates on every other server. + HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:?Set a shared auth token secret} + PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:?Set a non-default admin password} + x-server-common: &server-common image: hugegraph/server:${HUGEGRAPH_VERSION:-latest} pull_policy: missing @@ -67,12 +92,10 @@ x-server-common: &server-common store0: { condition: service_healthy } store1: { condition: service_healthy } store2: { condition: service_healthy } - environment: - STORE_REST: store0:8520 - HG_SERVER_BACKEND: hstore - HG_SERVER_PD_PEERS: pd0:8686,pd1:8686,pd2:8686 healthcheck: - test: ["CMD-SHELL", "curl -fsS http://localhost:8080/versions >/dev/null || exit 1"] + # With HG_SERVER_REST_URL set, the REST server binds that URL's + # hostname rather than localhost, so probe the bound address. + test: ["CMD-SHELL", "curl -fsS $${HG_SERVER_REST_URL}/versions >/dev/null || exit 1"] interval: 10s timeout: 5s retries: 30 @@ -86,7 +109,6 @@ services: <<: *pd-common container_name: hg-pd0 hostname: pd0 - networks: [ hg-net ] environment: HG_PD_GRPC_HOST: pd0 HG_PD_GRPC_PORT: "8686" @@ -104,7 +126,6 @@ services: <<: *pd-common container_name: hg-pd1 hostname: pd1 - networks: [ hg-net ] environment: HG_PD_GRPC_HOST: pd1 HG_PD_GRPC_PORT: "8686" @@ -122,7 +143,6 @@ services: <<: *pd-common container_name: hg-pd2 hostname: pd2 - networks: [ hg-net ] environment: HG_PD_GRPC_HOST: pd2 HG_PD_GRPC_PORT: "8686" @@ -187,16 +207,25 @@ services: <<: *server-common container_name: hg-server0 hostname: server0 + environment: + <<: *server-env + HG_SERVER_REST_URL: http://server0:8080 ports: ["8080:8080"] server1: <<: *server-common container_name: hg-server1 hostname: server1 + environment: + <<: *server-env + HG_SERVER_REST_URL: http://server1:8080 ports: ["8081:8080"] server2: <<: *server-common container_name: hg-server2 hostname: server2 + environment: + <<: *server-env + HG_SERVER_REST_URL: http://server2:8080 ports: ["8082:8080"] diff --git a/docker/docker-compose-hubble.yml b/docker/docker-compose-hubble.yml new file mode 100644 index 0000000000..519cf6e8bf --- /dev/null +++ b/docker/docker-compose-hubble.yml @@ -0,0 +1,47 @@ +# +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. +# + +# Hubble add-on for the distributed cluster defined in +# docker-compose-3pd-3store-3server.yml. Requires the pre-created cluster +# network (docker network create hugegraph-net). See "Hubble for the +# 3-Node Cluster" in docker/README.md for the attach and combined flows. + +networks: + hg-net: + external: true + name: ${HUGEGRAPH_NETWORK:-hugegraph-net} + +services: + hubble: + # Pin a release via HUGEGRAPH_VERSION in docker/.env; unset, the image + # tag defaults to latest. + image: ${HUBBLE_IMAGE:-hugegraph/hubble:${HUGEGRAPH_VERSION:-latest}} + pull_policy: ${HUBBLE_PULL_POLICY:-missing} + container_name: hg-hubble + hostname: hubble + restart: unless-stopped + networks: [hg-net] + ports: + - "${HUBBLE_PUBLISH_HOST:-127.0.0.1}:8088:8088" + volumes: + - ./hugegraph-hubble-3x3.properties:/hubble/conf/hugegraph-hubble.properties:ro + healthcheck: + test: ["CMD-SHELL", "body=$$(curl -fsS http://127.0.0.1:8088/about) && printf '%s' \"$$body\" | grep -q '\"status\":200' && printf '%s' \"$$body\" | grep -q '\"name\":\"hugegraph-hubble\"'"] + interval: 10s + timeout: 5s + retries: 30 + start_period: 60s diff --git a/docker/hugegraph-hubble-3x3.properties b/docker/hugegraph-hubble-3x3.properties new file mode 100644 index 0000000000..5184dfc070 --- /dev/null +++ b/docker/hugegraph-hubble-3x3.properties @@ -0,0 +1,33 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +server.host=0.0.0.0 +server.port=8088 + +cluster=hg +idc=docker + +pd.enabled=true +# Bootstrap target; the live topology comes from PD discovery. If this +# replica is down when Hubble starts, point it at another server. +server.direct_url=http://server0:8080 +pd.peers=pd0:8686,pd1:8686,pd2:8686 +# PD REST endpoint for the operations view (single address). +pd.server=pd0:8620 + +operations.store.allowed_targets=[http://store0:8520,http://store1:8520,http://store2:8520] + +# Dashboard is not part of this Compose stack. +dashboard.address= diff --git a/hugegraph-pd/README.md b/hugegraph-pd/README.md index 794dba9b98..81da8a009d 100644 --- a/hugegraph-pd/README.md +++ b/hugegraph-pd/README.md @@ -155,9 +155,9 @@ raft: For detailed configuration options and production tuning, see [Configuration Guide](docs/configuration.md). -#### Docker Bridge Network Example +#### Docker Network Example -When running PD in Docker with bridge networking (e.g., `docker/docker-compose-3pd-3store-3server.yml`), configuration is injected via environment variables instead of editing `application.yml` directly. Container hostnames are used instead of IP addresses: +When running PD in Docker on a shared network (e.g., `docker/docker-compose-3pd-3store-3server.yml`, which joins the pre-created external `hugegraph-net` network), configuration is injected via environment variables instead of editing `application.yml` directly. Container hostnames are used instead of IP addresses: **pd0** container: ```bash diff --git a/hugegraph-pd/docs/configuration.md b/hugegraph-pd/docs/configuration.md index e3ae4f6f25..97d6f7945a 100644 --- a/hugegraph-pd/docs/configuration.md +++ b/hugegraph-pd/docs/configuration.md @@ -119,9 +119,9 @@ raft: peers-list: 192.168.1.10:8610,192.168.1.11:8610,192.168.1.12:8610 ``` -### Docker Bridge Network Deployment +### Docker Network Deployment -When deploying PD in Docker with bridge networking (e.g., `docker/docker-compose-3pd-3store-3server.yml`), container hostnames are used instead of IP addresses. Configuration is injected via `HG_PD_*` environment variables: +When deploying PD in Docker on a shared network (e.g., `docker/docker-compose-3pd-3store-3server.yml`, which joins the pre-created external `hugegraph-net` network), container hostnames are used instead of IP addresses. Configuration is injected via `HG_PD_*` environment variables: ```yaml # pd0 — set via HG_PD_RAFT_ADDRESS and HG_PD_RAFT_PEERS_LIST env vars diff --git a/hugegraph-server/README.md b/hugegraph-server/README.md index 819bece457..db0cb3ca9e 100644 --- a/hugegraph-server/README.md +++ b/hugegraph-server/README.md @@ -45,7 +45,10 @@ For a full distributed deployment, use the compose file in the `docker/` directo ```bash cd docker -HUGEGRAPH_VERSION=1.7.0 docker compose -f docker-compose-3pd-3store-3server.yml up -d +# One-time setup first: the shared hugegraph-net network and the required +# credentials in docker/.env — see the 3-Node Cluster Quickstart in the +# guide linked below. +docker compose -f docker-compose-3pd-3store-3server.yml up -d ``` See [docker/README.md](../docker/README.md) for the full setup guide. diff --git a/hugegraph-server/hugegraph-dist/docker/README.md b/hugegraph-server/hugegraph-dist/docker/README.md index 9214aa830e..70ce39b0c8 100644 --- a/hugegraph-server/hugegraph-dist/docker/README.md +++ b/hugegraph-server/hugegraph-dist/docker/README.md @@ -125,7 +125,10 @@ For a full distributed HugeGraph cluster with PD, Store, and Server, use the ```bash cd docker -HUGEGRAPH_VERSION=1.7.0 docker compose -f docker-compose-3pd-3store-3server.yml up -d +# One-time setup first: the shared hugegraph-net network and the required +# credentials in docker/.env — see the 3-Node Cluster Quickstart in the +# guide linked below. +docker compose -f docker-compose-3pd-3store-3server.yml up -d ``` See [docker/README.md](../../../docker/README.md) for the full setup guide, diff --git a/hugegraph-store/AGENTS.md b/hugegraph-store/AGENTS.md index 8b5ef46bab..bd4027ad1c 100644 --- a/hugegraph-store/AGENTS.md +++ b/hugegraph-store/AGENTS.md @@ -129,7 +129,7 @@ bin/restart-hugegraph-store.sh 2. HugeGraph Store cluster (3+ nodes) 3. Proper configuration pointing Store nodes to PD cluster -See Docker Compose examples in the repository root `../docker/` directory. Single-node quickstart (pre-built images): `../docker/docker-compose.yml`. Single-node dev build (from source): `../docker/docker-compose.dev.yml`. 3-node cluster: `../docker/docker-compose-3pd-3store-3server.yml`. See `../docker/README.md` for the full setup guide. +See Docker Compose examples in the repository root `../docker/` directory. Single-node quickstart (pre-built images): `../docker/docker-compose.yml`. Single-node dev build (from source): `../docker/docker-compose.dev.yml`. 3-node cluster: `../docker/docker-compose-3pd-3store-3server.yml`. Hubble add-on for the cluster: `../docker/docker-compose-hubble.yml`. See `../docker/README.md` for the full setup guide. ## Configuration Files diff --git a/hugegraph-store/docs/deployment-guide.md b/hugegraph-store/docs/deployment-guide.md index de07904d64..0de7861a38 100644 --- a/hugegraph-store/docs/deployment-guide.md +++ b/hugegraph-store/docs/deployment-guide.md @@ -678,10 +678,13 @@ For a production-like 3-node distributed deployment, use the compose file at `do ```bash cd docker -HUGEGRAPH_VERSION=1.7.0 docker compose -f docker-compose-3pd-3store-3server.yml up -d +# One-time setup first: the shared hugegraph-net network and the required +# credentials in docker/.env — see the 3-Node Cluster Quickstart in +# docker/README.md. +docker compose -f docker-compose-3pd-3store-3server.yml up -d ``` -The compose file uses a Docker bridge network (`hg-net`) with container hostnames for service discovery. Configuration is injected via environment variables using the `HG_*` prefix: +The compose file joins a pre-created external Docker network (`hugegraph-net`, override via `HUGEGRAPH_NETWORK`) shared with the Hubble add-on, with container hostnames for service discovery. Configuration is injected via environment variables using the `HG_*` prefix: **PD environment variables** (per node): @@ -709,13 +712,20 @@ environment: HG_STORE_DATA_PATH: /hugegraph-store/storage # maps to app.data-path ``` -**Server environment variables**: +**Server environment variables** (per node; `HG_SERVER_REST_URL` names the node itself): ```yaml environment: HG_SERVER_BACKEND: hstore # maps to backend HG_SERVER_PD_PEERS: pd0:8686,pd1:8686,pd2:8686 # maps to pd.peers STORE_REST: store0:8520 # used by wait-partition.sh + HG_SERVER_CLUSTER: hg # PD discovery application name + HG_SERVER_USE_PD: "true" # register with PD + HG_SERVER_REST_URL: http://server0:8080 # address registered with PD + HG_SERVER_MIN_FREE_MEMORY: "0" # disable free-memory guard locally + HG_SERVER_INIT_STORE_ENABLED: "false" # PD/HStore deployments skip init-store + HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:?Set a shared auth token secret} # same value on all replicas, from docker/.env + PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:?} # required; initial admin password, from docker/.env ``` **Startup ordering** is enforced via `depends_on` with `condition: service_healthy`: @@ -728,8 +738,9 @@ environment: **Deploy**: ```bash -# Start cluster (run from the docker/ directory) -HUGEGRAPH_VERSION=1.7.0 docker compose -f docker-compose-3pd-3store-3server.yml up -d +# Start cluster (run from the docker/ directory, after the one-time setup +# in docker/README.md) +docker compose -f docker-compose-3pd-3store-3server.yml up -d # Check status docker ps From 620f7ffc5cf857d40d1f7ff504a758eed60e4b0b Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 10 Aug 2026 21:47:31 +0530 Subject: [PATCH 02/14] ci(docker): assert HUGEGRAPH_NETWORK and HUGEGRAPH_VERSION overrides in compose checks Render the combined cluster+Hubble topology with non-default network and version values and assert the overridden network name and all four image tags, so CI fails if any Compose file stops honoring either override. The standalone add-on render keeps asserting the defaults and now strips any runner-level overrides for hermeticity. --- .github/workflows/server-ci.yml | 15 +++++++++++++-- 1 file changed, 13 insertions(+), 2 deletions(-) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 96dd6eac05..86538ed1a1 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -171,8 +171,10 @@ jobs: fi # The add-on alone must define Hubble and nothing else, join the - # shared external network, and need no credentials. + # shared external network with its default name, and need no + # credentials or overrides. env -u HUGEGRAPH_ADMIN_PASSWORD -u HUGEGRAPH_AUTH_TOKEN_SECRET \ + -u HUGEGRAPH_NETWORK -u HUGEGRAPH_VERSION \ docker compose -f "$addon" config --format json > "$rendered" jq -e ' (.services | keys) == ["hubble"] and @@ -186,15 +188,24 @@ jobs: # The combined render carries the PD-registration and auth # settings on every server replica and keeps Hubble on loopback. + # Rendered with non-default HUGEGRAPH_NETWORK/HUGEGRAPH_VERSION so + # CI fails if any file stops honoring the overrides (the add-on + # render above covers the defaults). HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-secret \ + HUGEGRAPH_NETWORK=ci-test-net \ + HUGEGRAPH_VERSION=ci-test-tag \ docker compose -f "$cluster" -f "$addon" \ config --format json > "$rendered" jq -e ' .name == "hugegraph-3x3" and (.services | keys | length) == 10 and .networks."hg-net".external == true and - .networks."hg-net".name == "hugegraph-net" and + .networks."hg-net".name == "ci-test-net" and + .services.pd0.image == "hugegraph/pd:ci-test-tag" and + .services.store0.image == "hugegraph/store:ci-test-tag" and + .services.server0.image == "hugegraph/server:ci-test-tag" and + .services.hubble.image == "hugegraph/hubble:ci-test-tag" and .services.server0.environment.HG_SERVER_USE_PD == "true" and .services.server0.environment.HG_SERVER_CLUSTER == "hg" and .services.server0.environment.HG_SERVER_REST_URL == From 0694ff79842e70780e30477e12a44a81b757602e Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Wed, 12 Aug 2026 20:49:30 +0530 Subject: [PATCH 03/14] fix(docker): address PR 3149 review findings --- .github/workflows/server-ci.yml | 148 +++++++++++++++++++------ docker/README.md | 111 ++++++++++++++++--- docker/docker-compose-hubble.yml | 6 + docker/hugegraph-hubble-3x3.properties | 19 +++- 4 files changed, 231 insertions(+), 53 deletions(-) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 86538ed1a1..ac31ea9a88 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -95,18 +95,20 @@ jobs: rendered="$(mktemp)" if env -u HUGEGRAPH_ADMIN_PASSWORD \ - docker compose -f "$file" config -q >/dev/null 2>&1; then + docker compose --env-file /dev/null -f "$file" config -q >/dev/null 2>&1; then echo "$file accepted an unset admin password" >&2 return 1 fi if HUGEGRAPH_ADMIN_PASSWORD= \ - docker compose -f "$file" config -q >/dev/null 2>&1; then + docker compose --env-file /dev/null -f "$file" config -q >/dev/null 2>&1; then echo "$file accepted an empty admin password" >&2 return 1 fi HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ - docker compose -f "$file" config --format json > "$rendered" + env -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY -u HUBBLE_PUBLISH_HOST \ + -u HUGEGRAPH_SERVER_IMAGE -u HUGEGRAPH_SERVER_PULL_POLICY \ + docker compose --env-file /dev/null -f "$file" config --format json > "$rendered" jq -e \ --arg server_policy "$server_policy" \ --arg hubble_policy "$hubble_policy" ' @@ -144,64 +146,110 @@ jobs: local rendered rendered="$(mktemp)" + # RETURN only: an EXIT trap would fire after this function's + # `local rendered` has gone out of scope, which `set -u` turns + # into an "unbound variable" error. A hard errexit abort can + # therefore still leak one temp file, which is acceptable on an + # ephemeral runner. + trap 'rm -f "$rendered"' RETURN + + # --env-file /dev/null on every invocation: Compose otherwise reads + # docker/.env automatically, and the quickstart tells operators to + # create one holding exactly the credentials these assertions + # control. Without it the checks pass in CI (which has no .env) but + # report false failures for anyone running them locally after + # following the quickstart, and the add-on render below — which + # deliberately unsets the variables to assert their defaults — would + # read whatever HUGEGRAPH_NETWORK/HUGEGRAPH_VERSION that .env holds. + # Pinning an empty env file makes both renders depend only on what + # each invocation sets explicitly. + # Both cluster credentials are required and may not be empty. - if HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-secret \ - env -u HUGEGRAPH_ADMIN_PASSWORD \ - docker compose -f "$cluster" config -q >/dev/null 2>&1; then - echo "$cluster accepted an unset admin password" >&2 - return 1 - fi - if HUGEGRAPH_ADMIN_PASSWORD= \ - HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-secret \ - docker compose -f "$cluster" config -q >/dev/null 2>&1; then - echo "$cluster accepted an empty admin password" >&2 - return 1 - fi - if HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ - env -u HUGEGRAPH_AUTH_TOKEN_SECRET \ - docker compose -f "$cluster" config -q >/dev/null 2>&1; then - echo "$cluster accepted an unset token secret" >&2 - return 1 - fi - if HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ - HUGEGRAPH_AUTH_TOKEN_SECRET= \ - docker compose -f "$cluster" config -q >/dev/null 2>&1; then - echo "$cluster accepted an empty token secret" >&2 - return 1 - fi + # Each case asserts the guard fired for the *intended* variable: + # a bare non-zero exit would also be produced by a YAML error, a + # renamed file, or a missing docker binary. + assert_guard() { # assert_guard + local var="$1" desc="$2" err + if err="$(docker compose --env-file /dev/null -f "$cluster" \ + config -q 2>&1)"; then + echo "$cluster accepted $desc" >&2 + return 1 + fi + case "$err" in + *"$var"*) : ;; + *) echo "$cluster rejected $desc, but not because of $var: $err" >&2 + return 1 ;; + esac + } + + ( unset HUGEGRAPH_ADMIN_PASSWORD + export HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-token-secret-32-bytes-long + assert_guard HUGEGRAPH_ADMIN_PASSWORD "an unset admin password" ) + ( export HUGEGRAPH_ADMIN_PASSWORD= + export HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-token-secret-32-bytes-long + assert_guard HUGEGRAPH_ADMIN_PASSWORD "an empty admin password" ) + ( unset HUGEGRAPH_AUTH_TOKEN_SECRET + export HUGEGRAPH_ADMIN_PASSWORD=ci-test-password + assert_guard HUGEGRAPH_AUTH_TOKEN_SECRET "an unset token secret" ) + ( export HUGEGRAPH_ADMIN_PASSWORD=ci-test-password + export HUGEGRAPH_AUTH_TOKEN_SECRET= + assert_guard HUGEGRAPH_AUTH_TOKEN_SECRET "an empty token secret" ) # The add-on alone must define Hubble and nothing else, join the # shared external network with its default name, and need no # credentials or overrides. env -u HUGEGRAPH_ADMIN_PASSWORD -u HUGEGRAPH_AUTH_TOKEN_SECRET \ -u HUGEGRAPH_NETWORK -u HUGEGRAPH_VERSION \ - docker compose -f "$addon" config --format json > "$rendered" + -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY -u HUBBLE_PUBLISH_HOST \ + docker compose --env-file /dev/null -f "$addon" config --format json > "$rendered" jq -e ' (.services | keys) == ["hubble"] and .networks."hg-net".external == true and .networks."hg-net".name == "hugegraph-net" and + (.services.hubble.networks | has("hg-net")) and (.services.hubble | has("depends_on") | not) and any(.services.hubble.volumes[]; .target == "/hubble/conf/hugegraph-hubble.properties" and (.source | endswith("hugegraph-hubble-3x3.properties"))) + and any(.services.hubble.volumes[]; + .source == "hg-hubble-db" and + .target == "/hubble/db") + and any(.services.hubble.volumes[]; + .source == "hg-hubble-upload-files" and + .target == "/hubble/upload-files") ' "$rendered" >/dev/null + # The cluster's own default network name must match the add-on's, + # or the attach flow and the cluster land on different networks. + # The combined render below pins an override, so it cannot catch a + # drifting default. + HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ + HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-token-secret-32-bytes-long \ + env -u HUGEGRAPH_NETWORK -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY \ + -u HUBBLE_PUBLISH_HOST \ + docker compose --env-file /dev/null -f "$cluster" \ + config --format json > "$rendered" + jq -e '.networks."hg-net".name == "hugegraph-net"' \ + "$rendered" >/dev/null + # The combined render carries the PD-registration and auth # settings on every server replica and keeps Hubble on loopback. # Rendered with non-default HUGEGRAPH_NETWORK/HUGEGRAPH_VERSION so # CI fails if any file stops honoring the overrides (the add-on # render above covers the defaults). HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ - HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-secret \ + HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-token-secret-32-bytes-long \ HUGEGRAPH_NETWORK=ci-test-net \ HUGEGRAPH_VERSION=ci-test-tag \ - docker compose -f "$cluster" -f "$addon" \ + env -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY -u HUBBLE_PUBLISH_HOST \ + docker compose --env-file /dev/null -f "$cluster" -f "$addon" \ config --format json > "$rendered" jq -e ' .name == "hugegraph-3x3" and (.services | keys | length) == 10 and .networks."hg-net".external == true and .networks."hg-net".name == "ci-test-net" and + all(.services[]; .networks | has("hg-net")) and .services.pd0.image == "hugegraph/pd:ci-test-tag" and .services.store0.image == "hugegraph/store:ci-test-tag" and .services.server0.image == "hugegraph/server:ci-test-tag" and @@ -213,17 +261,17 @@ jobs: .services.server1.environment.HG_SERVER_REST_URL == "http://server1:8080" and .services.server1.environment.HG_SERVER_AUTH_TOKEN_SECRET == - "ci-test-secret" and + "ci-test-token-secret-32-bytes-long" and .services.server2.environment.HG_SERVER_REST_URL == "http://server2:8080" and .services.server2.environment.HG_SERVER_AUTH_TOKEN_SECRET == - "ci-test-secret" and + "ci-test-token-secret-32-bytes-long" and .services.server0.environment.HG_SERVER_INIT_STORE_ENABLED == "false" and .services.server0.environment.PASSWORD == "ci-test-password" and .services.server0.environment.HG_SERVER_AUTH_TOKEN_SECRET == - "ci-test-secret" and + "ci-test-token-secret-32-bytes-long" and (.services.hubble | has("depends_on") | not) and .services.hubble.pull_policy == "missing" and (.services.hubble.healthcheck.test[1] | @@ -234,7 +282,39 @@ jobs: .target == 8088 and .published == "8088" and .host_ip == "127.0.0.1") ' "$rendered" >/dev/null - rm -f "$rendered" + + # Hubble's properties file is mounted, not rendered, so Compose + # validation alone cannot catch it drifting from the services it + # describes. Tie the two together: renaming a service, changing a + # container hostname, or moving a REST port must be reflected in + # both places or CI fails here. Values are derived from the + # rendered model (hostnames and ports included) so the assertions + # cannot silently agree with a stale file. + local props="docker/hugegraph-hubble-3x3.properties" + local pd_peers store_targets cluster_name pd_rest + assert_props() { # assert_props + grep -Fqx "$1" "$props" || + { echo "$props: expected line '$1' ($2)" >&2; return 1; } + } + cluster_name="$(jq -r '.services.server0.environment.HG_SERVER_CLUSTER' \ + "$rendered")" + pd_peers="$(jq -r '.services.server0.environment.HG_SERVER_PD_PEERS' \ + "$rendered")" + store_targets="[$(jq -r '[.services | to_entries[] + | select(.key | startswith("store")) + | "http://" + .value.hostname + ":" + + (.value.environment.HG_STORE_REST_PORT)] + | sort | join(",")' "$rendered")]" + pd_rest="$(jq -r --arg h "$(printf '%s' "${pd_peers}" | cut -d, -f1 | cut -d: -f1)" \ + '.services | to_entries[] + | select(.value.hostname == $h) + | .value.hostname + ":" + .value.environment.HG_PD_REST_PORT' \ + "$rendered")" + assert_props "cluster=${cluster_name}" "matches HG_SERVER_CLUSTER" + assert_props "pd.peers=${pd_peers}" "matches HG_SERVER_PD_PEERS" + assert_props "pd.server=${pd_rest}" "names a real PD REST endpoint" + assert_props "operations.store.allowed_targets=${store_targets}" \ + "lists every Store REST endpoint" } check_cluster_compose diff --git a/docker/README.md b/docker/README.md index ec8286205c..5aa2e4900f 100644 --- a/docker/README.md +++ b/docker/README.md @@ -175,10 +175,17 @@ To validate local images without Compose replacing them with remote `latest`: The cluster and the Hubble add-on share one named Docker network so Hubble can attach to a running cluster without touching it. Treat that network as a -trust boundary: PD and Store expose unauthenticated control APIs on it (only -the Server layer authenticates), and any container on the host can join it -by declaring the well-known name. One-time setup: write the required -credentials to a mode-600 `docker/.env` and create the network. +trust boundary: only the Server layer performs real authentication. Store +serves its control APIs with no credentials at all, and while PD's REST +control APIs do require an `Authorization` header, PD only checks that the +Basic-auth *user* is one of its internal service names and never validates +the password — so any client on the network can read and drive them. Note +that this stack depends on that behaviour: Hubble reaches PD as the service +name `hubble` with an empty password, so tightening PD's credential check +would also break Hubble's operations view. Any container on the host can +also join the network by declaring the well-known name. One-time setup: +write the required credentials to a mode-600 `docker/.env` and create the +network. The cluster file requires both credentials — the admin password enables authentication, and every Server replica must share one token secret so a token issued by any server validates on all of them. The `:?` guards fire @@ -234,9 +241,13 @@ it from there, so those versions cannot drift apart (`docker-compose.dev.yml` builds PD/Store/Server from source and defaults Hubble to `hugegraph/hubble:latest`; set `HUBBLE_IMAGE` to pin it). Unpinned, the images default to `latest`; note the authenticated PD/Hubble -integration requires a release newer than `1.7.x`. Because the cluster -files use `pull_policy: missing`, an already-pulled `latest` is never -refreshed by `up -d` — pull explicitly or pin to pick up new releases. +integration requires a release newer than `1.7.x`. On an older image the +Server silently ignores `PASSWORD` and `HG_SERVER_AUTH_TOKEN_SECRET`, comes +up healthy, and leaves you an unauthenticated cluster — the 401 check under +"Verify the cluster is healthy" below is what detects this, so run it. +Because the cluster files use `pull_policy: missing`, an already-pulled +`latest` is never refreshed by `up -d` — pull explicitly or pin to pick up +new releases. > [!NOTE] > Upgrading an existing 3-node deployment: @@ -274,13 +285,48 @@ curl http://localhost:8520/v1/health # Check Server (Graph API) curl http://localhost:8080/versions -# List registered stores via PD -curl http://localhost:8620/v1/stores +# Every PD endpoint except /v1/health, /actuator/* and /v1/prom/targets/* +# needs an Authorization header. PD only checks that the Basic-auth user is +# one of its internal service names, so the empty password below is enough +# — and it grants the full PD control plane, writes included, not just these +# reads. That is exactly why the shared network must be treated as a trust +# boundary. Without the header PD answers with an exception body, not data. +pd_auth="Authorization: Basic $(printf 'hubble:' | base64)" + +# List registered stores via PD (expect three, each "state":"Up") +curl -H "${pd_auth}" http://localhost:8620/v1/stores # List partitions -curl http://localhost:8620/v1/partitions +curl -H "${pd_auth}" http://localhost:8620/v1/partitions ``` +Confirm authentication actually engaged — `/versions` stays open by design, +so it cannot tell you whether auth is on. A graph read without credentials +must be rejected: + +```bash +cd docker +# Expect 401 on all three replicas +for port in 8080 8081 8082; do + curl -s -o /dev/null -w "${port}: %{http_code}\n" \ + "http://localhost:${port}/graphs/hugegraph/schema/vertexlabels" +done + +# And a signed-in read must succeed. Compose reads docker/.env by itself, but +# your shell does not — load it first. Passing the credential through +# --config keeps it out of argv, where `ps` would expose it to other users. +set -a; . ./.env; set +a +curl -s -o /dev/null -w '%{http_code}\n' \ + --config <(printf 'user = "admin:%s"\n' "${HUGEGRAPH_ADMIN_PASSWORD}") \ + http://localhost:8080/graphs/hugegraph/schema/vertexlabels +``` + +`200` from the second command with `401` from all three of the first means +authentication is on and working. If the first command returns `200`, the +running image ignored `PASSWORD` and the cluster is **unauthenticated** — +the most likely cause is an image older than the release this integration +needs (see the version note in the quickstart above). + --- ## Hubble for the 3-Node Cluster @@ -291,24 +337,55 @@ cluster's external network (`hugegraph-net` by default, override with starting, stopping, or upgrading Hubble never recreates or restarts PD, Store, or Server containers. Hubble reads the cluster topology from `hugegraph-hubble-3x3.properties`; adjust that file when attaching to a -cluster with different hostnames. +cluster with different hostnames. Its `pd.server` is a single PD address +with no failover, so if that PD is down the operations view goes blind even +though the cluster still has quorum — repoint it at a surviving PD. The +operations view also reports one `SERVER` node, not three: it describes the +replica Hubble is currently talking to, not the whole Server tier. Sign in at `http://localhost:8088` as `admin` with the `HUGEGRAPH_ADMIN_PASSWORD` from `docker/.env`. Hubble binds to host loopback by default (`HUBBLE_PUBLISH_HOST`, same caveats as the single-node setup). +Which flow to use: pick the attach flow when you do not have the cluster's +`docker/.env` — the add-on carries no `:?` guards, so it is the only flow +that runs without those credentials, which is what you want against a +cluster someone else started. Otherwise use the combined flow, including to +add Hubble to an already-running cluster (`up -d --no-deps hubble`). + The two flows below create Hubble in different Compose projects, so manage Hubble with the same flags you started it with: the attach flow always uses `-p hugegraph-hubble -f docker-compose-hubble.yml`, the combined flow always uses both `-f` flags. The explicit `-p` keeps the attach project independent of the directory name and of other Compose projects. +> [!IMPORTANT] +> Do not drop the `-p` from the attach flow. Without it Compose names the +> project after the current directory (`docker`), so Hubble starts in a +> third project that none of the commands below manage: `down` reports +> nothing to remove while `hg-hubble` keeps running, and every later `up` +> in either flow fails on the container name. If that happens, find it with +> `docker ps --filter name=hg-hubble` and remove it with +> `docker rm -f hg-hubble`. + Run one Hubble per host: the single-node stack and both add-on flows all publish `127.0.0.1:8088` and name their container `hg-hubble`. The two add-on flows are therefore mutually exclusive — starting one while the -other's Hubble exists fails with a container-name conflict, so `down` the -flow you are leaving before switching. +other's Hubble exists fails with a container-name conflict, so remove the +Hubble of the flow you are leaving before switching. The two directions are +not symmetric: + +```bash +cd docker +# leaving the attach flow (removes only Hubble) +docker compose -p hugegraph-hubble -f docker-compose-hubble.yml down + +# leaving the combined flow — remove ONLY Hubble; a plain `down` with both +# -f flags would stop all ten containers just to move one. +docker compose -f docker-compose-3pd-3store-3server.yml \ + -f docker-compose-hubble.yml rm -sf hubble +``` ### Attach to a running cluster @@ -555,9 +632,11 @@ The table below reflects the published host ports of the 3-node cluster > [!IMPORTANT] > Cluster ports bind all host interfaces and bypass host firewalls under -> Docker's port publishing; the PD and Store APIs among them are -> unauthenticated. Do not run this file on an untrusted network. Hubble is -> the exception and binds loopback only by default. +> Docker's port publishing. Among them, the Store APIs need no credentials +> at all and the PD APIs accept any password for their internal service +> names, so neither is a real access control — see the trust-boundary note +> in the 3-Node Cluster Quickstart. Do not run this file on an untrusted +> network. Hubble is the exception and binds loopback only by default. The single-node Compose file publishes `8620`, `8520`, `8080`, and Hubble `8088`; Hubble defaults to host loopback. diff --git a/docker/docker-compose-hubble.yml b/docker/docker-compose-hubble.yml index 519cf6e8bf..3e1d96c50a 100644 --- a/docker/docker-compose-hubble.yml +++ b/docker/docker-compose-hubble.yml @@ -25,6 +25,10 @@ networks: external: true name: ${HUGEGRAPH_NETWORK:-hugegraph-net} +volumes: + hg-hubble-db: + hg-hubble-upload-files: + services: hubble: # Pin a release via HUGEGRAPH_VERSION in docker/.env; unset, the image @@ -39,6 +43,8 @@ services: - "${HUBBLE_PUBLISH_HOST:-127.0.0.1}:8088:8088" volumes: - ./hugegraph-hubble-3x3.properties:/hubble/conf/hugegraph-hubble.properties:ro + - hg-hubble-db:/hubble/db + - hg-hubble-upload-files:/hubble/upload-files healthcheck: test: ["CMD-SHELL", "body=$$(curl -fsS http://127.0.0.1:8088/about) && printf '%s' \"$$body\" | grep -q '\"status\":200' && printf '%s' \"$$body\" | grep -q '\"name\":\"hugegraph-hubble\"'"] interval: 10s diff --git a/docker/hugegraph-hubble-3x3.properties b/docker/hugegraph-hubble-3x3.properties index 5184dfc070..af7f4a940b 100644 --- a/docker/hugegraph-hubble-3x3.properties +++ b/docker/hugegraph-hubble-3x3.properties @@ -13,6 +13,15 @@ # See the License for the specific language governing permissions and # limitations under the License. +# This file REPLACES the image's /hubble/conf/hugegraph-hubble.properties +# wholesale — it is bind-mounted over it, not merged with it. Keys absent +# here therefore fall back to the code defaults in HubbleOptions, not to the +# values in the shipped conf file. That is deliberate for this stack (the +# defaults are correct for it), but it means a future Hubble release that +# changes a default, or adds a required option, changes this deployment +# without any diff to this file. Re-check against the shipped conf when +# upgrading Hubble. + server.host=0.0.0.0 server.port=8088 @@ -20,11 +29,15 @@ cluster=hg idc=docker pd.enabled=true -# Bootstrap target; the live topology comes from PD discovery. If this -# replica is down when Hubble starts, point it at another server. +# Unused while pd.enabled=true: in PD mode Hubble picks a Server per request +# from PD-discovered addresses and never reads this value, so editing it has +# no effect on which replica is used. Kept only for the standalone fallback +# (pd.enabled=false). server.direct_url=http://server0:8080 pd.peers=pd0:8686,pd1:8686,pd2:8686 -# PD REST endpoint for the operations view (single address). +# PD REST endpoint for the operations view (single address, no failover). +# If pd0 is down the cluster keeps quorum on pd1/pd2 but this view goes +# blind until you repoint this at a surviving PD. pd.server=pd0:8620 operations.store.allowed_targets=[http://store0:8520,http://store1:8520,http://store2:8520] From 1b4f008902d549fdd86460d3becb92b01ae28c1b Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Wed, 12 Aug 2026 20:55:34 +0530 Subject: [PATCH 04/14] fix(docker): harden credential setup and persist Hubble state --- .github/workflows/server-ci.yml | 14 +++++++++----- docker/README.md | 16 +++++++++++++++- 2 files changed, 24 insertions(+), 6 deletions(-) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index ac31ea9a88..5b140c53a1 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -144,7 +144,12 @@ jobs: local cluster="docker/docker-compose-3pd-3store-3server.yml" local addon="docker/docker-compose-hubble.yml" local rendered + local token_fixture=ci-test-token-secret-32-bytes-long rendered="$(mktemp)" + if [ "${#token_fixture}" -lt 32 ]; then + echo "CI token fixture must be at least 32 bytes" >&2 + return 1 + fi # RETURN only: an EXIT trap would fire after this function's # `local rendered` has gone out of scope, which `set -u` turns @@ -183,10 +188,10 @@ jobs: } ( unset HUGEGRAPH_ADMIN_PASSWORD - export HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-token-secret-32-bytes-long + export HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" assert_guard HUGEGRAPH_ADMIN_PASSWORD "an unset admin password" ) ( export HUGEGRAPH_ADMIN_PASSWORD= - export HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-token-secret-32-bytes-long + export HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" assert_guard HUGEGRAPH_ADMIN_PASSWORD "an empty admin password" ) ( unset HUGEGRAPH_AUTH_TOKEN_SECRET export HUGEGRAPH_ADMIN_PASSWORD=ci-test-password @@ -194,7 +199,6 @@ jobs: ( export HUGEGRAPH_ADMIN_PASSWORD=ci-test-password export HUGEGRAPH_AUTH_TOKEN_SECRET= assert_guard HUGEGRAPH_AUTH_TOKEN_SECRET "an empty token secret" ) - # The add-on alone must define Hubble and nothing else, join the # shared external network with its default name, and need no # credentials or overrides. @@ -224,7 +228,7 @@ jobs: # The combined render below pins an override, so it cannot catch a # drifting default. HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ - HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-token-secret-32-bytes-long \ + HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" \ env -u HUGEGRAPH_NETWORK -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY \ -u HUBBLE_PUBLISH_HOST \ docker compose --env-file /dev/null -f "$cluster" \ @@ -238,7 +242,7 @@ jobs: # CI fails if any file stops honoring the overrides (the add-on # render above covers the defaults). HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ - HUGEGRAPH_AUTH_TOKEN_SECRET=ci-test-token-secret-32-bytes-long \ + HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" \ HUGEGRAPH_NETWORK=ci-test-net \ HUGEGRAPH_VERSION=ci-test-tag \ env -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY -u HUBBLE_PUBLISH_HOST \ diff --git a/docker/README.md b/docker/README.md index 5aa2e4900f..222c144064 100644 --- a/docker/README.md +++ b/docker/README.md @@ -211,6 +211,17 @@ on every Compose subcommand, including `down`. printf "HUGEGRAPH_AUTH_TOKEN_SECRET='%s'\n" "${token_secret}" >> .env unset token_secret fi + admin_value="$(sed -nE "s/${pat}HUGEGRAPH_ADMIN_PASSWORD='([^']*)'[[:space:]]*$/\\1/p" .env | tail -n1)" + token_value="$(sed -nE "s/${pat}HUGEGRAPH_AUTH_TOKEN_SECRET='([^']*)'[[:space:]]*$/\\1/p" .env | tail -n1)" + [ -n "${admin_value}" ] || { + echo "HUGEGRAPH_ADMIN_PASSWORD must be non-empty in docker/.env" >&2 + exit 1 + } + [ "${#token_value}" -ge 32 ] || { + echo "HUGEGRAPH_AUTH_TOKEN_SECRET must be at least 32 bytes in docker/.env" >&2 + exit 1 + } + unset admin_value token_value # The shared cluster network. To override the name, export # HUGEGRAPH_NETWORK in this shell before running the block — a value # in docker/.env is read by Compose, not by this script. @@ -341,7 +352,10 @@ cluster with different hostnames. Its `pd.server` is a single PD address with no failover, so if that PD is down the operations view goes blind even though the cluster still has quorum — repoint it at a surviving PD. The operations view also reports one `SERVER` node, not three: it describes the -replica Hubble is currently talking to, not the whole Server tier. +replica Hubble is currently talking to, not the whole Server tier. The add-on +stores Hubble's H2 database and uploaded files in named volumes +`hg-hubble-db` and `hg-hubble-upload-files`, so recreating the container does +not discard that state. Sign in at `http://localhost:8088` as `admin` with the `HUGEGRAPH_ADMIN_PASSWORD` from `docker/.env`. Hubble binds to host From 950f645f214932c385eb76cab7a733449a643ad1 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Sun, 16 Aug 2026 00:00:44 +0530 Subject: [PATCH 05/14] fix(docker): parse credentials safely and share Hubble volumes Read docker/.env as data instead of sourcing it as shell, and take the quoted value rather than the optional export capture when validating generated credentials. Point Hubble H2 at a file inside the /hubble/db volume, give attach and combined flows the same explicit volume names, and assert PD/auth env on every Server replica plus pd.enabled=true. --- .github/workflows/server-ci.yml | 35 ++++++++++++++++-------- docker/README.md | 47 ++++++++++++++++++++++++++------ docker/docker-compose-hubble.yml | 8 ++++++ 3 files changed, 70 insertions(+), 20 deletions(-) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 5b140c53a1..b75d50e668 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -205,6 +205,7 @@ jobs: env -u HUGEGRAPH_ADMIN_PASSWORD -u HUGEGRAPH_AUTH_TOKEN_SECRET \ -u HUGEGRAPH_NETWORK -u HUGEGRAPH_VERSION \ -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY -u HUBBLE_PUBLISH_HOST \ + -u HUBBLE_DB_VOLUME -u HUBBLE_UPLOAD_VOLUME \ docker compose --env-file /dev/null -f "$addon" config --format json > "$rendered" jq -e ' (.services | keys) == ["hubble"] and @@ -212,6 +213,11 @@ jobs: .networks."hg-net".name == "hugegraph-net" and (.services.hubble.networks | has("hg-net")) and (.services.hubble | has("depends_on") | not) and + .volumes."hg-hubble-db".name == "hugegraph-hubble-db" and + .volumes."hg-hubble-upload-files".name == + "hugegraph-hubble-upload-files" and + .services.hubble.environment.SPRING_DATASOURCE_URL == + "jdbc:h2:file:./db/hubble;DB_CLOSE_ON_EXIT=FALSE" and any(.services.hubble.volumes[]; .target == "/hubble/conf/hugegraph-hubble.properties" and (.source | endswith("hugegraph-hubble-3x3.properties"))) @@ -246,9 +252,11 @@ jobs: HUGEGRAPH_NETWORK=ci-test-net \ HUGEGRAPH_VERSION=ci-test-tag \ env -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY -u HUBBLE_PUBLISH_HOST \ + -u HUBBLE_DB_VOLUME -u HUBBLE_UPLOAD_VOLUME \ docker compose --env-file /dev/null -f "$cluster" -f "$addon" \ config --format json > "$rendered" jq -e ' + . as $root | .name == "hugegraph-3x3" and (.services | keys | length) == 10 and .networks."hg-net".external == true and @@ -258,26 +266,28 @@ jobs: .services.store0.image == "hugegraph/store:ci-test-tag" and .services.server0.image == "hugegraph/server:ci-test-tag" and .services.hubble.image == "hugegraph/hubble:ci-test-tag" and - .services.server0.environment.HG_SERVER_USE_PD == "true" and - .services.server0.environment.HG_SERVER_CLUSTER == "hg" and + all(["server0", "server1", "server2"][]; + $root.services[.].environment.HG_SERVER_USE_PD == "true" and + $root.services[.].environment.HG_SERVER_CLUSTER == "hg" and + $root.services[.].environment.HG_SERVER_INIT_STORE_ENABLED == + "false" and + $root.services[.].environment.PASSWORD == + "ci-test-password" and + $root.services[.].environment.HG_SERVER_AUTH_TOKEN_SECRET == + "ci-test-token-secret-32-bytes-long") and .services.server0.environment.HG_SERVER_REST_URL == "http://server0:8080" and .services.server1.environment.HG_SERVER_REST_URL == "http://server1:8080" and - .services.server1.environment.HG_SERVER_AUTH_TOKEN_SECRET == - "ci-test-token-secret-32-bytes-long" and .services.server2.environment.HG_SERVER_REST_URL == "http://server2:8080" and - .services.server2.environment.HG_SERVER_AUTH_TOKEN_SECRET == - "ci-test-token-secret-32-bytes-long" and - .services.server0.environment.HG_SERVER_INIT_STORE_ENABLED == - "false" and - .services.server0.environment.PASSWORD == - "ci-test-password" and - .services.server0.environment.HG_SERVER_AUTH_TOKEN_SECRET == - "ci-test-token-secret-32-bytes-long" and (.services.hubble | has("depends_on") | not) and .services.hubble.pull_policy == "missing" and + .volumes."hg-hubble-db".name == "hugegraph-hubble-db" and + .volumes."hg-hubble-upload-files".name == + "hugegraph-hubble-upload-files" and + .services.hubble.environment.SPRING_DATASOURCE_URL == + "jdbc:h2:file:./db/hubble;DB_CLOSE_ON_EXIT=FALSE" and (.services.hubble.healthcheck.test[1] | contains("http://127.0.0.1:8088/about") and contains("\"status\":200") and @@ -315,6 +325,7 @@ jobs: | .value.hostname + ":" + .value.environment.HG_PD_REST_PORT' \ "$rendered")" assert_props "cluster=${cluster_name}" "matches HG_SERVER_CLUSTER" + assert_props "pd.enabled=true" "keeps Hubble in PD mode" assert_props "pd.peers=${pd_peers}" "matches HG_SERVER_PD_PEERS" assert_props "pd.server=${pd_rest}" "names a real PD REST endpoint" assert_props "operations.store.allowed_targets=${store_targets}" \ diff --git a/docker/README.md b/docker/README.md index 222c144064..5c32b48d5d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -211,8 +211,24 @@ on every Compose subcommand, including `down`. printf "HUGEGRAPH_AUTH_TOKEN_SECRET='%s'\n" "${token_secret}" >> .env unset token_secret fi - admin_value="$(sed -nE "s/${pat}HUGEGRAPH_ADMIN_PASSWORD='([^']*)'[[:space:]]*$/\\1/p" .env | tail -n1)" - token_value="$(sed -nE "s/${pat}HUGEGRAPH_AUTH_TOKEN_SECRET='([^']*)'[[:space:]]*$/\\1/p" .env | tail -n1)" + # Parse values as data. Do not source .env as shell: a value such as + # HUGEGRAPH_ADMIN_PASSWORD=$(...) would run on the operator host. + # The optional export group is capture 1; the quoted value is capture 2. + read_dotenv_value() { + key="$1" + key_pattern="^[[:space:]]*(export[[:space:]]+)?${key}=" + count="$(grep -Ec "${key_pattern}" .env || true)" + [ "${count}" -eq 1 ] || return 1 + sed -nE "s/${key_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env + } + if ! admin_value="$(read_dotenv_value HUGEGRAPH_ADMIN_PASSWORD)"; then + echo "HUGEGRAPH_ADMIN_PASSWORD must use the documented single-quoted format" >&2 + exit 1 + fi + if ! token_value="$(read_dotenv_value HUGEGRAPH_AUTH_TOKEN_SECRET)"; then + echo "HUGEGRAPH_AUTH_TOKEN_SECRET must use the documented single-quoted format" >&2 + exit 1 + fi [ -n "${admin_value}" ] || { echo "HUGEGRAPH_ADMIN_PASSWORD must be non-empty in docker/.env" >&2 exit 1 @@ -323,13 +339,24 @@ for port in 8080 8081 8082; do "http://localhost:${port}/graphs/hugegraph/schema/vertexlabels" done -# And a signed-in read must succeed. Compose reads docker/.env by itself, but -# your shell does not — load it first. Passing the credential through -# --config keeps it out of argv, where `ps` would expose it to other users. -set -a; . ./.env; set +a +# And a signed-in read must succeed. Parse the generated single-quoted +# value without executing docker/.env as shell. Passing the credential +# through --config keeps it out of argv, where `ps` would expose it. +read_dotenv_value() { + key="$1" + key_pattern="^[[:space:]]*(export[[:space:]]+)?${key}=" + count="$(grep -Ec "${key_pattern}" .env || true)" + [ "${count}" -eq 1 ] || return 1 + sed -nE "s/${key_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env +} +HUGEGRAPH_ADMIN_PASSWORD="$(read_dotenv_value HUGEGRAPH_ADMIN_PASSWORD)" || { + echo "HUGEGRAPH_ADMIN_PASSWORD must use the documented single-quoted format" >&2 + exit 1 +} curl -s -o /dev/null -w '%{http_code}\n' \ --config <(printf 'user = "admin:%s"\n' "${HUGEGRAPH_ADMIN_PASSWORD}") \ http://localhost:8080/graphs/hugegraph/schema/vertexlabels +unset HUGEGRAPH_ADMIN_PASSWORD ``` `200` from the second command with `401` from all three of the first means @@ -354,8 +381,12 @@ though the cluster still has quorum — repoint it at a surviving PD. The operations view also reports one `SERVER` node, not three: it describes the replica Hubble is currently talking to, not the whole Server tier. The add-on stores Hubble's H2 database and uploaded files in named volumes -`hg-hubble-db` and `hg-hubble-upload-files`, so recreating the container does -not discard that state. +`hugegraph-hubble-db` (mounted at `/hubble/db`) and +`hugegraph-hubble-upload-files`. Those names are explicit, so the attach +and combined Compose projects share the same physical volumes. Hubble's +default H2 URL would write `/hubble/db.mv.db` (outside that mount); the +add-on sets `SPRING_DATASOURCE_URL` so the database file lives inside +`/hubble/db`. Recreating the container does not discard that state. Sign in at `http://localhost:8088` as `admin` with the `HUGEGRAPH_ADMIN_PASSWORD` from `docker/.env`. Hubble binds to host diff --git a/docker/docker-compose-hubble.yml b/docker/docker-compose-hubble.yml index 3e1d96c50a..aa32ecc819 100644 --- a/docker/docker-compose-hubble.yml +++ b/docker/docker-compose-hubble.yml @@ -27,7 +27,11 @@ networks: volumes: hg-hubble-db: + # Explicit names so attach (-p hugegraph-hubble) and combined + # (project hugegraph-3x3) share the same physical volumes. + name: ${HUBBLE_DB_VOLUME:-hugegraph-hubble-db} hg-hubble-upload-files: + name: ${HUBBLE_UPLOAD_VOLUME:-hugegraph-hubble-upload-files} services: hubble: @@ -41,6 +45,10 @@ services: networks: [hg-net] ports: - "${HUBBLE_PUBLISH_HOST:-127.0.0.1}:8088:8088" + environment: + # Image default jdbc:h2:file:./db writes /hubble/db.mv.db, outside + # the /hubble/db volume. Point H2 at a file inside the mount. + SPRING_DATASOURCE_URL: jdbc:h2:file:./db/hubble;DB_CLOSE_ON_EXIT=FALSE volumes: - ./hugegraph-hubble-3x3.properties:/hubble/conf/hugegraph-hubble.properties:ro - hg-hubble-db:/hubble/db From 8a0c6b017221c02674857179f2a9a85977ecada4 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 20 Aug 2026 12:05:30 +0530 Subject: [PATCH 06/14] fix(docker): keep the 3-node cluster anonymous for Hubble attach Hubble supports auth.enabled=false, so the cluster file no longer requires PASSWORD. Match Hubble to the anonymous Servers and keep PD registration for discovery. --- .github/workflows/server-ci.yml | 77 ++------- docker/README.md | 159 +++++------------- docker/docker-compose-3pd-3store-3server.yml | 4 - docker/hugegraph-hubble-3x3.properties | 4 + hugegraph-server/README.md | 4 +- .../hugegraph-dist/docker/README.md | 4 +- hugegraph-store/docs/deployment-guide.md | 7 +- 7 files changed, 68 insertions(+), 191 deletions(-) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index b75d50e668..e027c5cdff 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -144,12 +144,7 @@ jobs: local cluster="docker/docker-compose-3pd-3store-3server.yml" local addon="docker/docker-compose-hubble.yml" local rendered - local token_fixture=ci-test-token-secret-32-bytes-long rendered="$(mktemp)" - if [ "${#token_fixture}" -lt 32 ]; then - echo "CI token fixture must be at least 32 bytes" >&2 - return 1 - fi # RETURN only: an EXIT trap would fire after this function's # `local rendered` has gone out of scope, which `set -u` turns @@ -159,46 +154,12 @@ jobs: trap 'rm -f "$rendered"' RETURN # --env-file /dev/null on every invocation: Compose otherwise reads - # docker/.env automatically, and the quickstart tells operators to - # create one holding exactly the credentials these assertions - # control. Without it the checks pass in CI (which has no .env) but - # report false failures for anyone running them locally after - # following the quickstart, and the add-on render below — which - # deliberately unsets the variables to assert their defaults — would - # read whatever HUGEGRAPH_NETWORK/HUGEGRAPH_VERSION that .env holds. - # Pinning an empty env file makes both renders depend only on what - # each invocation sets explicitly. - - # Both cluster credentials are required and may not be empty. - # Each case asserts the guard fired for the *intended* variable: - # a bare non-zero exit would also be produced by a YAML error, a - # renamed file, or a missing docker binary. - assert_guard() { # assert_guard - local var="$1" desc="$2" err - if err="$(docker compose --env-file /dev/null -f "$cluster" \ - config -q 2>&1)"; then - echo "$cluster accepted $desc" >&2 - return 1 - fi - case "$err" in - *"$var"*) : ;; - *) echo "$cluster rejected $desc, but not because of $var: $err" >&2 - return 1 ;; - esac - } + # docker/.env automatically. Without it the add-on render below — + # which deliberately unsets overrides to assert their defaults — + # would read whatever HUGEGRAPH_NETWORK/HUGEGRAPH_VERSION that + # .env holds. Pinning an empty env file makes both renders depend + # only on what each invocation sets explicitly. - ( unset HUGEGRAPH_ADMIN_PASSWORD - export HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" - assert_guard HUGEGRAPH_ADMIN_PASSWORD "an unset admin password" ) - ( export HUGEGRAPH_ADMIN_PASSWORD= - export HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" - assert_guard HUGEGRAPH_ADMIN_PASSWORD "an empty admin password" ) - ( unset HUGEGRAPH_AUTH_TOKEN_SECRET - export HUGEGRAPH_ADMIN_PASSWORD=ci-test-password - assert_guard HUGEGRAPH_AUTH_TOKEN_SECRET "an unset token secret" ) - ( export HUGEGRAPH_ADMIN_PASSWORD=ci-test-password - export HUGEGRAPH_AUTH_TOKEN_SECRET= - assert_guard HUGEGRAPH_AUTH_TOKEN_SECRET "an empty token secret" ) # The add-on alone must define Hubble and nothing else, join the # shared external network with its default name, and need no # credentials or overrides. @@ -233,26 +194,24 @@ jobs: # or the attach flow and the cluster land on different networks. # The combined render below pins an override, so it cannot catch a # drifting default. - HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ - HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" \ - env -u HUGEGRAPH_NETWORK -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY \ - -u HUBBLE_PUBLISH_HOST \ + env -u HUGEGRAPH_NETWORK -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY \ + -u HUBBLE_PUBLISH_HOST -u HUGEGRAPH_ADMIN_PASSWORD \ + -u HUGEGRAPH_AUTH_TOKEN_SECRET \ docker compose --env-file /dev/null -f "$cluster" \ config --format json > "$rendered" jq -e '.networks."hg-net".name == "hugegraph-net"' \ "$rendered" >/dev/null - # The combined render carries the PD-registration and auth - # settings on every server replica and keeps Hubble on loopback. - # Rendered with non-default HUGEGRAPH_NETWORK/HUGEGRAPH_VERSION so - # CI fails if any file stops honoring the overrides (the add-on - # render above covers the defaults). - HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ - HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" \ + # The combined render carries PD-registration on every server + # replica, keeps Server anonymous (no PASSWORD), and keeps Hubble + # on loopback. Rendered with non-default HUGEGRAPH_NETWORK / + # HUGEGRAPH_VERSION so CI fails if any file stops honoring the + # overrides (the add-on render above covers the defaults). HUGEGRAPH_NETWORK=ci-test-net \ HUGEGRAPH_VERSION=ci-test-tag \ env -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY -u HUBBLE_PUBLISH_HOST \ -u HUBBLE_DB_VOLUME -u HUBBLE_UPLOAD_VOLUME \ + -u HUGEGRAPH_ADMIN_PASSWORD -u HUGEGRAPH_AUTH_TOKEN_SECRET \ docker compose --env-file /dev/null -f "$cluster" -f "$addon" \ config --format json > "$rendered" jq -e ' @@ -271,10 +230,9 @@ jobs: $root.services[.].environment.HG_SERVER_CLUSTER == "hg" and $root.services[.].environment.HG_SERVER_INIT_STORE_ENABLED == "false" and - $root.services[.].environment.PASSWORD == - "ci-test-password" and - $root.services[.].environment.HG_SERVER_AUTH_TOKEN_SECRET == - "ci-test-token-secret-32-bytes-long") and + ($root.services[.].environment.PASSWORD == null) and + ($root.services[.].environment.HG_SERVER_AUTH_TOKEN_SECRET == + null)) and .services.server0.environment.HG_SERVER_REST_URL == "http://server0:8080" and .services.server1.environment.HG_SERVER_REST_URL == @@ -326,6 +284,7 @@ jobs: "$rendered")" assert_props "cluster=${cluster_name}" "matches HG_SERVER_CLUSTER" assert_props "pd.enabled=true" "keeps Hubble in PD mode" + assert_props "auth.enabled=false" "matches the anonymous cluster" assert_props "pd.peers=${pd_peers}" "matches HG_SERVER_PD_PEERS" assert_props "pd.server=${pd_rest}" "names a real PD REST endpoint" assert_props "operations.store.allowed_targets=${store_targets}" \ diff --git a/docker/README.md b/docker/README.md index 5c32b48d5d..42976ea7ba 100644 --- a/docker/README.md +++ b/docker/README.md @@ -175,77 +175,32 @@ To validate local images without Compose replacing them with remote `latest`: The cluster and the Hubble add-on share one named Docker network so Hubble can attach to a running cluster without touching it. Treat that network as a -trust boundary: only the Server layer performs real authentication. Store -serves its control APIs with no credentials at all, and while PD's REST -control APIs do require an `Authorization` header, PD only checks that the -Basic-auth *user* is one of its internal service names and never validates -the password — so any client on the network can read and drive them. Note -that this stack depends on that behaviour: Hubble reaches PD as the service -name `hubble` with an empty password, so tightening PD's credential check -would also break Hubble's operations view. Any container on the host can -also join the network by declaring the well-known name. One-time setup: -write the required credentials to a mode-600 `docker/.env` and create the -network. -The cluster file requires both credentials — the admin password enables -authentication, and every Server replica must share one token secret so a -token issued by any server validates on all of them. The `:?` guards fire -on every Compose subcommand, including `down`. +trust boundary: Store serves its control APIs with no credentials at all, and +while PD's REST control APIs do require an `Authorization` header, PD only +checks that the Basic-auth *user* is one of its internal service names and +never validates the password — so any client on the network can read and +drive them. Hubble reaches PD as the service name `hubble` with an empty +password, so tightening PD's credential check would also break Hubble's +operations view. Any container on the host can also join the network by +declaring the well-known name. + +The 3-node Servers stay anonymous (no `PASSWORD`), matching master. Hubble +matches that with `auth.enabled=false` in `hugegraph-hubble-3x3.properties`. +The cluster file still registers every Server with PD (`HG_SERVER_USE_PD`, +`HG_SERVER_CLUSTER`, per-replica `HG_SERVER_REST_URL`) so Hubble can discover +them. One-time setup is only the shared network: ```bash ( set -eu cd docker - command -v openssl >/dev/null 2>&1 || { echo "openssl not found" >&2; exit 1; } - [ -e .env ] || install -m 600 /dev/null .env - chmod 600 .env - # Keep appends on their own lines even if the file was hand-edited. - [ ! -s .env ] || [ -z "$(tail -c1 .env)" ] || printf '\n' >> .env - pat='^[[:space:]]*(export[[:space:]]+)?' - if ! grep -Eq "${pat}HUGEGRAPH_ADMIN_PASSWORD=" .env; then - admin_password="$(openssl rand -base64 12)" - printf "HUGEGRAPH_ADMIN_PASSWORD='%s'\n" "${admin_password}" >> .env - unset admin_password - fi - if ! grep -Eq "${pat}HUGEGRAPH_AUTH_TOKEN_SECRET=" .env; then - token_secret="$(openssl rand -hex 32)" - printf "HUGEGRAPH_AUTH_TOKEN_SECRET='%s'\n" "${token_secret}" >> .env - unset token_secret - fi - # Parse values as data. Do not source .env as shell: a value such as - # HUGEGRAPH_ADMIN_PASSWORD=$(...) would run on the operator host. - # The optional export group is capture 1; the quoted value is capture 2. - read_dotenv_value() { - key="$1" - key_pattern="^[[:space:]]*(export[[:space:]]+)?${key}=" - count="$(grep -Ec "${key_pattern}" .env || true)" - [ "${count}" -eq 1 ] || return 1 - sed -nE "s/${key_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env - } - if ! admin_value="$(read_dotenv_value HUGEGRAPH_ADMIN_PASSWORD)"; then - echo "HUGEGRAPH_ADMIN_PASSWORD must use the documented single-quoted format" >&2 - exit 1 - fi - if ! token_value="$(read_dotenv_value HUGEGRAPH_AUTH_TOKEN_SECRET)"; then - echo "HUGEGRAPH_AUTH_TOKEN_SECRET must use the documented single-quoted format" >&2 - exit 1 - fi - [ -n "${admin_value}" ] || { - echo "HUGEGRAPH_ADMIN_PASSWORD must be non-empty in docker/.env" >&2 - exit 1 - } - [ "${#token_value}" -ge 32 ] || { - echo "HUGEGRAPH_AUTH_TOKEN_SECRET must be at least 32 bytes in docker/.env" >&2 - exit 1 - } - unset admin_value token_value - # The shared cluster network. To override the name, export - # HUGEGRAPH_NETWORK in this shell before running the block — a value - # in docker/.env is read by Compose, not by this script. + # To override the name, export HUGEGRAPH_NETWORK in this shell before + # running the block — a value in docker/.env is read by Compose, not by + # this script. net="${HUGEGRAPH_NETWORK:-hugegraph-net}" docker network inspect "${net}" >/dev/null 2>&1 || docker network create "${net}" - env -u HUGEGRAPH_ADMIN_PASSWORD -u HUGEGRAPH_AUTH_TOKEN_SECRET \ - docker compose -f docker-compose-3pd-3store-3server.yml config --quiet + docker compose -f docker-compose-3pd-3store-3server.yml config --quiet ) ``` @@ -267,30 +222,21 @@ cluster, the Hubble add-on, and the single-node quickstart file all read it from there, so those versions cannot drift apart (`docker-compose.dev.yml` builds PD/Store/Server from source and defaults Hubble to `hugegraph/hubble:latest`; set `HUBBLE_IMAGE` to pin it). -Unpinned, the images default to `latest`; note the authenticated PD/Hubble -integration requires a release newer than `1.7.x`. On an older image the -Server silently ignores `PASSWORD` and `HG_SERVER_AUTH_TOKEN_SECRET`, comes -up healthy, and leaves you an unauthenticated cluster — the 401 check under -"Verify the cluster is healthy" below is what detects this, so run it. -Because the cluster files use `pull_policy: missing`, an already-pulled -`latest` is never refreshed by `up -d` — pull explicitly or pin to pick up -new releases. +Unpinned, the images default to `latest`. The Hubble add-on needs a Hubble +image that understands `auth.enabled=false` (toolchain PR 27 / Apache #758 +and later). Because the cluster files use `pull_policy: missing`, an +already-pulled `latest` is never refreshed by `up -d` — pull explicitly or +pin to pick up new releases. > [!NOTE] > Upgrading an existing 3-node deployment: -> - Create `docker/.env` (block above) before running any Compose command -> against an older stack, `down` included. > - The cluster now joins the pre-created `hugegraph-net` network instead of > a per-project bridge, so the first `up -d` recreates all nine containers. > Named data volumes are unchanged and survive the move; the orphaned > `hugegraph-3x3_hg-net` bridge can be removed with > `docker network rm hugegraph-3x3_hg-net`. -> - Authentication is now enabled: previously unauthenticated clients of the -> graph APIs on ports 8080–8082 will start receiving 401 responses and -> must supply the `admin` credential from `docker/.env` (`/versions` and -> `/openapi.json` stay open, so they cannot serve as an auth smoke test). -> On a cluster whose volumes predate authentication, verify you can sign -> in before decommissioning any existing access path. +> - Graph APIs on ports 8080–8082 stay anonymous, same as master. The +> single-node Compose file still requires `HUGEGRAPH_ADMIN_PASSWORD`. **Startup ordering** is enforced via `depends_on` with `condition: service_healthy`: @@ -327,43 +273,19 @@ curl -H "${pd_auth}" http://localhost:8620/v1/stores curl -H "${pd_auth}" http://localhost:8620/v1/partitions ``` -Confirm authentication actually engaged — `/versions` stays open by design, -so it cannot tell you whether auth is on. A graph read without credentials -must be rejected: +Confirm the graph APIs are still anonymous — a schema read without +credentials must succeed on all three replicas: ```bash -cd docker -# Expect 401 on all three replicas for port in 8080 8081 8082; do curl -s -o /dev/null -w "${port}: %{http_code}\n" \ "http://localhost:${port}/graphs/hugegraph/schema/vertexlabels" done - -# And a signed-in read must succeed. Parse the generated single-quoted -# value without executing docker/.env as shell. Passing the credential -# through --config keeps it out of argv, where `ps` would expose it. -read_dotenv_value() { - key="$1" - key_pattern="^[[:space:]]*(export[[:space:]]+)?${key}=" - count="$(grep -Ec "${key_pattern}" .env || true)" - [ "${count}" -eq 1 ] || return 1 - sed -nE "s/${key_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env -} -HUGEGRAPH_ADMIN_PASSWORD="$(read_dotenv_value HUGEGRAPH_ADMIN_PASSWORD)" || { - echo "HUGEGRAPH_ADMIN_PASSWORD must use the documented single-quoted format" >&2 - exit 1 -} -curl -s -o /dev/null -w '%{http_code}\n' \ - --config <(printf 'user = "admin:%s"\n' "${HUGEGRAPH_ADMIN_PASSWORD}") \ - http://localhost:8080/graphs/hugegraph/schema/vertexlabels -unset HUGEGRAPH_ADMIN_PASSWORD ``` -`200` from the second command with `401` from all three of the first means -authentication is on and working. If the first command returns `200`, the -running image ignored `PASSWORD` and the cluster is **unauthenticated** — -the most likely cause is an image older than the release this integration -needs (see the version note in the quickstart above). +`200` on all three means the cluster is usable without login. `401` means a +Server image enabled auth anyway (usually `PASSWORD` set in the environment +or in `docker/.env`); this Compose file does not set that. --- @@ -388,16 +310,15 @@ default H2 URL would write `/hubble/db.mv.db` (outside that mount); the add-on sets `SPRING_DATASOURCE_URL` so the database file lives inside `/hubble/db`. Recreating the container does not discard that state. -Sign in at `http://localhost:8088` as `admin` with the -`HUGEGRAPH_ADMIN_PASSWORD` from `docker/.env`. Hubble binds to host +Open `http://localhost:8088` — there is no login. Hubble runs with +`auth.enabled=false` against the anonymous cluster. Hubble binds to host loopback by default (`HUBBLE_PUBLISH_HOST`, same caveats as the single-node setup). -Which flow to use: pick the attach flow when you do not have the cluster's -`docker/.env` — the add-on carries no `:?` guards, so it is the only flow -that runs without those credentials, which is what you want against a -cluster someone else started. Otherwise use the combined flow, including to -add Hubble to an already-running cluster (`up -d --no-deps hubble`). +Which flow to use: pick the attach flow when the cluster is already +running. Otherwise use the combined flow, including to add Hubble to an +already-running cluster (`up -d --no-deps hubble`). Neither flow needs +`docker/.env` credentials. The two flows below create Hubble in different Compose projects, so manage Hubble with the same flags you started it with: the attach flow always uses @@ -460,7 +381,7 @@ docker compose -f docker-compose-3pd-3store-3server.yml down -v ### Fresh cluster plus Hubble in one command -After the one-time network and `docker/.env` setup from the quickstart: +After the one-time network setup from the quickstart: ```bash cd docker @@ -470,7 +391,7 @@ docker compose -f docker-compose-3pd-3store-3server.yml \ Hubble has no startup dependency on the cluster, so it reports healthy while PD, Store, and Server are still forming the cluster; wait until every service -shows healthy before signing in: +shows healthy before opening the UI: ```bash cd docker @@ -605,8 +526,8 @@ add-on = `docker-compose-hubble.yml`): | `HUBBLE_PULL_POLICY` | single, add-on | `always` (`missing` for dev and the add-on) | Hubble pull policy | | `HUBBLE_PUBLISH_HOST` | single, add-on | `127.0.0.1` | Hubble host bind address; remote access requires an HTTPS reverse proxy | | `HUGEGRAPH_NETWORK` | cluster, add-on | `hugegraph-net` | Pre-created external Docker network shared by the 3-node cluster and the Hubble add-on; the single-node files use their own project bridge instead | -| `HUGEGRAPH_ADMIN_PASSWORD` | single, cluster | required (`docker/.env`) | Initial admin password; no public default is provided | -| `HUGEGRAPH_AUTH_TOKEN_SECRET` | single, cluster | generated (single); **required** (cluster) | JWT signing secret; explicit values must be at least 32 bytes. The cluster file requires it so all Server replicas validate each other's tokens | +| `HUGEGRAPH_ADMIN_PASSWORD` | single | required (`docker/.env`) | Initial admin password for the single-node file; the 3-node cluster does not set `PASSWORD` | +| `HUGEGRAPH_AUTH_TOKEN_SECRET` | single | generated (single) | JWT signing secret for the single-node file when auth is on; explicit values must be at least 32 bytes | When authentication is enabled and no token secret is supplied, the Server entrypoint generates a random secret and writes it to both authentication diff --git a/docker/docker-compose-3pd-3store-3server.yml b/docker/docker-compose-3pd-3store-3server.yml index 371fbf507a..358eb035a1 100644 --- a/docker/docker-compose-3pd-3store-3server.yml +++ b/docker/docker-compose-3pd-3store-3server.yml @@ -78,10 +78,6 @@ x-server-env: &server-env HG_SERVER_USE_PD: "true" HG_SERVER_MIN_FREE_MEMORY: "0" HG_SERVER_INIT_STORE_ENABLED: "false" - # All replicas must share one token secret so a token issued by any - # server validates on every other server. - HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:?Set a shared auth token secret} - PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:?Set a non-default admin password} x-server-common: &server-common image: hugegraph/server:${HUGEGRAPH_VERSION:-latest} diff --git a/docker/hugegraph-hubble-3x3.properties b/docker/hugegraph-hubble-3x3.properties index af7f4a940b..56cb515a82 100644 --- a/docker/hugegraph-hubble-3x3.properties +++ b/docker/hugegraph-hubble-3x3.properties @@ -29,6 +29,10 @@ cluster=hg idc=docker pd.enabled=true +# Match the anonymous 3-node cluster (no Server PASSWORD). Hubble's code +# default is auth.enabled=true; without this, the UI demands a login +# against a Server that has no users. +auth.enabled=false # Unused while pd.enabled=true: in PD mode Hubble picks a Server per request # from PD-discovered addresses and never reads this value, so editing it has # no effect on which replica is used. Kept only for the standalone fallback diff --git a/hugegraph-server/README.md b/hugegraph-server/README.md index db0cb3ca9e..c57bf4aa0f 100644 --- a/hugegraph-server/README.md +++ b/hugegraph-server/README.md @@ -45,8 +45,8 @@ For a full distributed deployment, use the compose file in the `docker/` directo ```bash cd docker -# One-time setup first: the shared hugegraph-net network and the required -# credentials in docker/.env — see the 3-Node Cluster Quickstart in the +# One-time setup first: the shared hugegraph-net network — see the +# 3-Node Cluster Quickstart in the # guide linked below. docker compose -f docker-compose-3pd-3store-3server.yml up -d ``` diff --git a/hugegraph-server/hugegraph-dist/docker/README.md b/hugegraph-server/hugegraph-dist/docker/README.md index 70ce39b0c8..af13728677 100644 --- a/hugegraph-server/hugegraph-dist/docker/README.md +++ b/hugegraph-server/hugegraph-dist/docker/README.md @@ -125,8 +125,8 @@ For a full distributed HugeGraph cluster with PD, Store, and Server, use the ```bash cd docker -# One-time setup first: the shared hugegraph-net network and the required -# credentials in docker/.env — see the 3-Node Cluster Quickstart in the +# One-time setup first: the shared hugegraph-net network — see the +# 3-Node Cluster Quickstart in the # guide linked below. docker compose -f docker-compose-3pd-3store-3server.yml up -d ``` diff --git a/hugegraph-store/docs/deployment-guide.md b/hugegraph-store/docs/deployment-guide.md index 0de7861a38..8afba1ca0f 100644 --- a/hugegraph-store/docs/deployment-guide.md +++ b/hugegraph-store/docs/deployment-guide.md @@ -678,9 +678,8 @@ For a production-like 3-node distributed deployment, use the compose file at `do ```bash cd docker -# One-time setup first: the shared hugegraph-net network and the required -# credentials in docker/.env — see the 3-Node Cluster Quickstart in -# docker/README.md. +# One-time setup first: the shared hugegraph-net network — see the +# 3-Node Cluster Quickstart in docker/README.md. docker compose -f docker-compose-3pd-3store-3server.yml up -d ``` @@ -724,8 +723,6 @@ environment: HG_SERVER_REST_URL: http://server0:8080 # address registered with PD HG_SERVER_MIN_FREE_MEMORY: "0" # disable free-memory guard locally HG_SERVER_INIT_STORE_ENABLED: "false" # PD/HStore deployments skip init-store - HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:?Set a shared auth token secret} # same value on all replicas, from docker/.env - PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:?} # required; initial admin password, from docker/.env ``` **Startup ordering** is enforced via `depends_on` with `condition: service_healthy`: From 931c258f591ebd3da4375ddfa170a4965589710f Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Thu, 20 Aug 2026 12:10:32 +0530 Subject: [PATCH 07/14] fix(docker): drop stale credential docs from the Hubble add-on The 3-node cluster is anonymous. Remove agent-memory edits that still required docker/.env, and describe Hubble auth.enabled=false without citing toolchain fork PRs. --- .serena/memories/implementation_patterns_and_guidelines.md | 3 +-- .serena/memories/key_file_locations.md | 3 +-- .serena/memories/suggested_commands.md | 6 +----- docker/README.md | 7 +++---- 4 files changed, 6 insertions(+), 13 deletions(-) diff --git a/.serena/memories/implementation_patterns_and_guidelines.md b/.serena/memories/implementation_patterns_and_guidelines.md index e5c5052547..d04e33ce56 100644 --- a/.serena/memories/implementation_patterns_and_guidelines.md +++ b/.serena/memories/implementation_patterns_and_guidelines.md @@ -40,8 +40,7 @@ ## Docker - Single-node: `docker/docker-compose.yml` (bridge network, pd+store+server) -- Cluster: `docker/docker-compose-3pd-3store-3server.yml` (external `hugegraph-net` network + `docker/.env` credentials) -- Hubble add-on for the cluster: `docker/docker-compose-hubble.yml` +- Cluster: `docker/docker-compose-3pd-3store-3server.yml` - Container logs: stdout-based ## CI Pipelines diff --git a/.serena/memories/key_file_locations.md b/.serena/memories/key_file_locations.md index 38fa9cb85b..3f2a60dee0 100644 --- a/.serena/memories/key_file_locations.md +++ b/.serena/memories/key_file_locations.md @@ -17,8 +17,7 @@ ## Docker - `docker/docker-compose.yml` — Single-node (bridge network, pd+store+server) -- `docker/docker-compose-3pd-3store-3server.yml` — 3-node cluster (external `hugegraph-net` network, credentials required) -- `docker/docker-compose-hubble.yml` — Hubble add-on for the 3-node cluster +- `docker/docker-compose-3pd-3store-3server.yml` — 3-node cluster - `docker/docker-compose.dev.yml` — Dev mode ## PD Module diff --git a/.serena/memories/suggested_commands.md b/.serena/memories/suggested_commands.md index 1585fc4b29..346304432f 100644 --- a/.serena/memories/suggested_commands.md +++ b/.serena/memories/suggested_commands.md @@ -47,11 +47,7 @@ bin/enable-auth.sh # Enable auth ## Docker ```bash cd docker && docker compose up -d # Single-node (bridge network) -# Cluster: needs one-time setup first (hugegraph-net network + docker/.env -# credentials) — see docker/README.md "3-Node Cluster Quickstart" -cd docker && docker compose -f docker-compose-3pd-3store-3server.yml up -d -# Hubble add-on for a running cluster -cd docker && docker compose -p hugegraph-hubble -f docker-compose-hubble.yml up -d +cd docker && docker compose -f docker-compose-3pd-3store-3server.yml up -d # Cluster ``` ## Distributed Build (BETA) diff --git a/docker/README.md b/docker/README.md index 42976ea7ba..0c5a85fed0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -223,10 +223,9 @@ it from there, so those versions cannot drift apart (`docker-compose.dev.yml` builds PD/Store/Server from source and defaults Hubble to `hugegraph/hubble:latest`; set `HUBBLE_IMAGE` to pin it). Unpinned, the images default to `latest`. The Hubble add-on needs a Hubble -image that understands `auth.enabled=false` (toolchain PR 27 / Apache #758 -and later). Because the cluster files use `pull_policy: missing`, an -already-pulled `latest` is never refreshed by `up -d` — pull explicitly or -pin to pick up new releases. +image that honors `auth.enabled=false`. Because the cluster files use +`pull_policy: missing`, an already-pulled `latest` is never refreshed by +`up -d` -- pull explicitly or pin to pick up new releases. > [!NOTE] > Upgrading an existing 3-node deployment: From 0d1e67ba853c25ccba7bd8c428e3c324350a67e1 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 24 Aug 2026 02:08:46 +0530 Subject: [PATCH 08/14] Revert "fix(docker): keep the 3-node cluster anonymous for Hubble attach" The 3-node cluster is the quickstart most people copy, so it keeps authentication on by default. Running it anonymously is still supported, but as an explicit opt-in documented alongside the default flow rather than as the default itself. This restores the required admin password and JWT token secret, the credential setup and validation steps in the docker README, and the matching CI assertions. The agent-memory files removed by the follow-up commit stay out of the diff. --- .github/workflows/server-ci.yml | 77 +++++++-- docker/README.md | 158 +++++++++++++----- docker/docker-compose-3pd-3store-3server.yml | 4 + docker/hugegraph-hubble-3x3.properties | 4 - hugegraph-server/README.md | 4 +- .../hugegraph-dist/docker/README.md | 4 +- hugegraph-store/docs/deployment-guide.md | 7 +- 7 files changed, 191 insertions(+), 67 deletions(-) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index e027c5cdff..b75d50e668 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -144,7 +144,12 @@ jobs: local cluster="docker/docker-compose-3pd-3store-3server.yml" local addon="docker/docker-compose-hubble.yml" local rendered + local token_fixture=ci-test-token-secret-32-bytes-long rendered="$(mktemp)" + if [ "${#token_fixture}" -lt 32 ]; then + echo "CI token fixture must be at least 32 bytes" >&2 + return 1 + fi # RETURN only: an EXIT trap would fire after this function's # `local rendered` has gone out of scope, which `set -u` turns @@ -154,12 +159,46 @@ jobs: trap 'rm -f "$rendered"' RETURN # --env-file /dev/null on every invocation: Compose otherwise reads - # docker/.env automatically. Without it the add-on render below — - # which deliberately unsets overrides to assert their defaults — - # would read whatever HUGEGRAPH_NETWORK/HUGEGRAPH_VERSION that - # .env holds. Pinning an empty env file makes both renders depend - # only on what each invocation sets explicitly. + # docker/.env automatically, and the quickstart tells operators to + # create one holding exactly the credentials these assertions + # control. Without it the checks pass in CI (which has no .env) but + # report false failures for anyone running them locally after + # following the quickstart, and the add-on render below — which + # deliberately unsets the variables to assert their defaults — would + # read whatever HUGEGRAPH_NETWORK/HUGEGRAPH_VERSION that .env holds. + # Pinning an empty env file makes both renders depend only on what + # each invocation sets explicitly. + + # Both cluster credentials are required and may not be empty. + # Each case asserts the guard fired for the *intended* variable: + # a bare non-zero exit would also be produced by a YAML error, a + # renamed file, or a missing docker binary. + assert_guard() { # assert_guard + local var="$1" desc="$2" err + if err="$(docker compose --env-file /dev/null -f "$cluster" \ + config -q 2>&1)"; then + echo "$cluster accepted $desc" >&2 + return 1 + fi + case "$err" in + *"$var"*) : ;; + *) echo "$cluster rejected $desc, but not because of $var: $err" >&2 + return 1 ;; + esac + } + ( unset HUGEGRAPH_ADMIN_PASSWORD + export HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" + assert_guard HUGEGRAPH_ADMIN_PASSWORD "an unset admin password" ) + ( export HUGEGRAPH_ADMIN_PASSWORD= + export HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" + assert_guard HUGEGRAPH_ADMIN_PASSWORD "an empty admin password" ) + ( unset HUGEGRAPH_AUTH_TOKEN_SECRET + export HUGEGRAPH_ADMIN_PASSWORD=ci-test-password + assert_guard HUGEGRAPH_AUTH_TOKEN_SECRET "an unset token secret" ) + ( export HUGEGRAPH_ADMIN_PASSWORD=ci-test-password + export HUGEGRAPH_AUTH_TOKEN_SECRET= + assert_guard HUGEGRAPH_AUTH_TOKEN_SECRET "an empty token secret" ) # The add-on alone must define Hubble and nothing else, join the # shared external network with its default name, and need no # credentials or overrides. @@ -194,24 +233,26 @@ jobs: # or the attach flow and the cluster land on different networks. # The combined render below pins an override, so it cannot catch a # drifting default. - env -u HUGEGRAPH_NETWORK -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY \ - -u HUBBLE_PUBLISH_HOST -u HUGEGRAPH_ADMIN_PASSWORD \ - -u HUGEGRAPH_AUTH_TOKEN_SECRET \ + HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ + HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" \ + env -u HUGEGRAPH_NETWORK -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY \ + -u HUBBLE_PUBLISH_HOST \ docker compose --env-file /dev/null -f "$cluster" \ config --format json > "$rendered" jq -e '.networks."hg-net".name == "hugegraph-net"' \ "$rendered" >/dev/null - # The combined render carries PD-registration on every server - # replica, keeps Server anonymous (no PASSWORD), and keeps Hubble - # on loopback. Rendered with non-default HUGEGRAPH_NETWORK / - # HUGEGRAPH_VERSION so CI fails if any file stops honoring the - # overrides (the add-on render above covers the defaults). + # The combined render carries the PD-registration and auth + # settings on every server replica and keeps Hubble on loopback. + # Rendered with non-default HUGEGRAPH_NETWORK/HUGEGRAPH_VERSION so + # CI fails if any file stops honoring the overrides (the add-on + # render above covers the defaults). + HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ + HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" \ HUGEGRAPH_NETWORK=ci-test-net \ HUGEGRAPH_VERSION=ci-test-tag \ env -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY -u HUBBLE_PUBLISH_HOST \ -u HUBBLE_DB_VOLUME -u HUBBLE_UPLOAD_VOLUME \ - -u HUGEGRAPH_ADMIN_PASSWORD -u HUGEGRAPH_AUTH_TOKEN_SECRET \ docker compose --env-file /dev/null -f "$cluster" -f "$addon" \ config --format json > "$rendered" jq -e ' @@ -230,9 +271,10 @@ jobs: $root.services[.].environment.HG_SERVER_CLUSTER == "hg" and $root.services[.].environment.HG_SERVER_INIT_STORE_ENABLED == "false" and - ($root.services[.].environment.PASSWORD == null) and - ($root.services[.].environment.HG_SERVER_AUTH_TOKEN_SECRET == - null)) and + $root.services[.].environment.PASSWORD == + "ci-test-password" and + $root.services[.].environment.HG_SERVER_AUTH_TOKEN_SECRET == + "ci-test-token-secret-32-bytes-long") and .services.server0.environment.HG_SERVER_REST_URL == "http://server0:8080" and .services.server1.environment.HG_SERVER_REST_URL == @@ -284,7 +326,6 @@ jobs: "$rendered")" assert_props "cluster=${cluster_name}" "matches HG_SERVER_CLUSTER" assert_props "pd.enabled=true" "keeps Hubble in PD mode" - assert_props "auth.enabled=false" "matches the anonymous cluster" assert_props "pd.peers=${pd_peers}" "matches HG_SERVER_PD_PEERS" assert_props "pd.server=${pd_rest}" "names a real PD REST endpoint" assert_props "operations.store.allowed_targets=${store_targets}" \ diff --git a/docker/README.md b/docker/README.md index 0c5a85fed0..5c32b48d5d 100644 --- a/docker/README.md +++ b/docker/README.md @@ -175,32 +175,77 @@ To validate local images without Compose replacing them with remote `latest`: The cluster and the Hubble add-on share one named Docker network so Hubble can attach to a running cluster without touching it. Treat that network as a -trust boundary: Store serves its control APIs with no credentials at all, and -while PD's REST control APIs do require an `Authorization` header, PD only -checks that the Basic-auth *user* is one of its internal service names and -never validates the password — so any client on the network can read and -drive them. Hubble reaches PD as the service name `hubble` with an empty -password, so tightening PD's credential check would also break Hubble's -operations view. Any container on the host can also join the network by -declaring the well-known name. - -The 3-node Servers stay anonymous (no `PASSWORD`), matching master. Hubble -matches that with `auth.enabled=false` in `hugegraph-hubble-3x3.properties`. -The cluster file still registers every Server with PD (`HG_SERVER_USE_PD`, -`HG_SERVER_CLUSTER`, per-replica `HG_SERVER_REST_URL`) so Hubble can discover -them. One-time setup is only the shared network: +trust boundary: only the Server layer performs real authentication. Store +serves its control APIs with no credentials at all, and while PD's REST +control APIs do require an `Authorization` header, PD only checks that the +Basic-auth *user* is one of its internal service names and never validates +the password — so any client on the network can read and drive them. Note +that this stack depends on that behaviour: Hubble reaches PD as the service +name `hubble` with an empty password, so tightening PD's credential check +would also break Hubble's operations view. Any container on the host can +also join the network by declaring the well-known name. One-time setup: +write the required credentials to a mode-600 `docker/.env` and create the +network. +The cluster file requires both credentials — the admin password enables +authentication, and every Server replica must share one token secret so a +token issued by any server validates on all of them. The `:?` guards fire +on every Compose subcommand, including `down`. ```bash ( set -eu cd docker - # To override the name, export HUGEGRAPH_NETWORK in this shell before - # running the block — a value in docker/.env is read by Compose, not by - # this script. + command -v openssl >/dev/null 2>&1 || { echo "openssl not found" >&2; exit 1; } + [ -e .env ] || install -m 600 /dev/null .env + chmod 600 .env + # Keep appends on their own lines even if the file was hand-edited. + [ ! -s .env ] || [ -z "$(tail -c1 .env)" ] || printf '\n' >> .env + pat='^[[:space:]]*(export[[:space:]]+)?' + if ! grep -Eq "${pat}HUGEGRAPH_ADMIN_PASSWORD=" .env; then + admin_password="$(openssl rand -base64 12)" + printf "HUGEGRAPH_ADMIN_PASSWORD='%s'\n" "${admin_password}" >> .env + unset admin_password + fi + if ! grep -Eq "${pat}HUGEGRAPH_AUTH_TOKEN_SECRET=" .env; then + token_secret="$(openssl rand -hex 32)" + printf "HUGEGRAPH_AUTH_TOKEN_SECRET='%s'\n" "${token_secret}" >> .env + unset token_secret + fi + # Parse values as data. Do not source .env as shell: a value such as + # HUGEGRAPH_ADMIN_PASSWORD=$(...) would run on the operator host. + # The optional export group is capture 1; the quoted value is capture 2. + read_dotenv_value() { + key="$1" + key_pattern="^[[:space:]]*(export[[:space:]]+)?${key}=" + count="$(grep -Ec "${key_pattern}" .env || true)" + [ "${count}" -eq 1 ] || return 1 + sed -nE "s/${key_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env + } + if ! admin_value="$(read_dotenv_value HUGEGRAPH_ADMIN_PASSWORD)"; then + echo "HUGEGRAPH_ADMIN_PASSWORD must use the documented single-quoted format" >&2 + exit 1 + fi + if ! token_value="$(read_dotenv_value HUGEGRAPH_AUTH_TOKEN_SECRET)"; then + echo "HUGEGRAPH_AUTH_TOKEN_SECRET must use the documented single-quoted format" >&2 + exit 1 + fi + [ -n "${admin_value}" ] || { + echo "HUGEGRAPH_ADMIN_PASSWORD must be non-empty in docker/.env" >&2 + exit 1 + } + [ "${#token_value}" -ge 32 ] || { + echo "HUGEGRAPH_AUTH_TOKEN_SECRET must be at least 32 bytes in docker/.env" >&2 + exit 1 + } + unset admin_value token_value + # The shared cluster network. To override the name, export + # HUGEGRAPH_NETWORK in this shell before running the block — a value + # in docker/.env is read by Compose, not by this script. net="${HUGEGRAPH_NETWORK:-hugegraph-net}" docker network inspect "${net}" >/dev/null 2>&1 || docker network create "${net}" - docker compose -f docker-compose-3pd-3store-3server.yml config --quiet + env -u HUGEGRAPH_ADMIN_PASSWORD -u HUGEGRAPH_AUTH_TOKEN_SECRET \ + docker compose -f docker-compose-3pd-3store-3server.yml config --quiet ) ``` @@ -222,20 +267,30 @@ cluster, the Hubble add-on, and the single-node quickstart file all read it from there, so those versions cannot drift apart (`docker-compose.dev.yml` builds PD/Store/Server from source and defaults Hubble to `hugegraph/hubble:latest`; set `HUBBLE_IMAGE` to pin it). -Unpinned, the images default to `latest`. The Hubble add-on needs a Hubble -image that honors `auth.enabled=false`. Because the cluster files use -`pull_policy: missing`, an already-pulled `latest` is never refreshed by -`up -d` -- pull explicitly or pin to pick up new releases. +Unpinned, the images default to `latest`; note the authenticated PD/Hubble +integration requires a release newer than `1.7.x`. On an older image the +Server silently ignores `PASSWORD` and `HG_SERVER_AUTH_TOKEN_SECRET`, comes +up healthy, and leaves you an unauthenticated cluster — the 401 check under +"Verify the cluster is healthy" below is what detects this, so run it. +Because the cluster files use `pull_policy: missing`, an already-pulled +`latest` is never refreshed by `up -d` — pull explicitly or pin to pick up +new releases. > [!NOTE] > Upgrading an existing 3-node deployment: +> - Create `docker/.env` (block above) before running any Compose command +> against an older stack, `down` included. > - The cluster now joins the pre-created `hugegraph-net` network instead of > a per-project bridge, so the first `up -d` recreates all nine containers. > Named data volumes are unchanged and survive the move; the orphaned > `hugegraph-3x3_hg-net` bridge can be removed with > `docker network rm hugegraph-3x3_hg-net`. -> - Graph APIs on ports 8080–8082 stay anonymous, same as master. The -> single-node Compose file still requires `HUGEGRAPH_ADMIN_PASSWORD`. +> - Authentication is now enabled: previously unauthenticated clients of the +> graph APIs on ports 8080–8082 will start receiving 401 responses and +> must supply the `admin` credential from `docker/.env` (`/versions` and +> `/openapi.json` stay open, so they cannot serve as an auth smoke test). +> On a cluster whose volumes predate authentication, verify you can sign +> in before decommissioning any existing access path. **Startup ordering** is enforced via `depends_on` with `condition: service_healthy`: @@ -272,19 +327,43 @@ curl -H "${pd_auth}" http://localhost:8620/v1/stores curl -H "${pd_auth}" http://localhost:8620/v1/partitions ``` -Confirm the graph APIs are still anonymous — a schema read without -credentials must succeed on all three replicas: +Confirm authentication actually engaged — `/versions` stays open by design, +so it cannot tell you whether auth is on. A graph read without credentials +must be rejected: ```bash +cd docker +# Expect 401 on all three replicas for port in 8080 8081 8082; do curl -s -o /dev/null -w "${port}: %{http_code}\n" \ "http://localhost:${port}/graphs/hugegraph/schema/vertexlabels" done + +# And a signed-in read must succeed. Parse the generated single-quoted +# value without executing docker/.env as shell. Passing the credential +# through --config keeps it out of argv, where `ps` would expose it. +read_dotenv_value() { + key="$1" + key_pattern="^[[:space:]]*(export[[:space:]]+)?${key}=" + count="$(grep -Ec "${key_pattern}" .env || true)" + [ "${count}" -eq 1 ] || return 1 + sed -nE "s/${key_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env +} +HUGEGRAPH_ADMIN_PASSWORD="$(read_dotenv_value HUGEGRAPH_ADMIN_PASSWORD)" || { + echo "HUGEGRAPH_ADMIN_PASSWORD must use the documented single-quoted format" >&2 + exit 1 +} +curl -s -o /dev/null -w '%{http_code}\n' \ + --config <(printf 'user = "admin:%s"\n' "${HUGEGRAPH_ADMIN_PASSWORD}") \ + http://localhost:8080/graphs/hugegraph/schema/vertexlabels +unset HUGEGRAPH_ADMIN_PASSWORD ``` -`200` on all three means the cluster is usable without login. `401` means a -Server image enabled auth anyway (usually `PASSWORD` set in the environment -or in `docker/.env`); this Compose file does not set that. +`200` from the second command with `401` from all three of the first means +authentication is on and working. If the first command returns `200`, the +running image ignored `PASSWORD` and the cluster is **unauthenticated** — +the most likely cause is an image older than the release this integration +needs (see the version note in the quickstart above). --- @@ -309,15 +388,16 @@ default H2 URL would write `/hubble/db.mv.db` (outside that mount); the add-on sets `SPRING_DATASOURCE_URL` so the database file lives inside `/hubble/db`. Recreating the container does not discard that state. -Open `http://localhost:8088` — there is no login. Hubble runs with -`auth.enabled=false` against the anonymous cluster. Hubble binds to host +Sign in at `http://localhost:8088` as `admin` with the +`HUGEGRAPH_ADMIN_PASSWORD` from `docker/.env`. Hubble binds to host loopback by default (`HUBBLE_PUBLISH_HOST`, same caveats as the single-node setup). -Which flow to use: pick the attach flow when the cluster is already -running. Otherwise use the combined flow, including to add Hubble to an -already-running cluster (`up -d --no-deps hubble`). Neither flow needs -`docker/.env` credentials. +Which flow to use: pick the attach flow when you do not have the cluster's +`docker/.env` — the add-on carries no `:?` guards, so it is the only flow +that runs without those credentials, which is what you want against a +cluster someone else started. Otherwise use the combined flow, including to +add Hubble to an already-running cluster (`up -d --no-deps hubble`). The two flows below create Hubble in different Compose projects, so manage Hubble with the same flags you started it with: the attach flow always uses @@ -380,7 +460,7 @@ docker compose -f docker-compose-3pd-3store-3server.yml down -v ### Fresh cluster plus Hubble in one command -After the one-time network setup from the quickstart: +After the one-time network and `docker/.env` setup from the quickstart: ```bash cd docker @@ -390,7 +470,7 @@ docker compose -f docker-compose-3pd-3store-3server.yml \ Hubble has no startup dependency on the cluster, so it reports healthy while PD, Store, and Server are still forming the cluster; wait until every service -shows healthy before opening the UI: +shows healthy before signing in: ```bash cd docker @@ -525,8 +605,8 @@ add-on = `docker-compose-hubble.yml`): | `HUBBLE_PULL_POLICY` | single, add-on | `always` (`missing` for dev and the add-on) | Hubble pull policy | | `HUBBLE_PUBLISH_HOST` | single, add-on | `127.0.0.1` | Hubble host bind address; remote access requires an HTTPS reverse proxy | | `HUGEGRAPH_NETWORK` | cluster, add-on | `hugegraph-net` | Pre-created external Docker network shared by the 3-node cluster and the Hubble add-on; the single-node files use their own project bridge instead | -| `HUGEGRAPH_ADMIN_PASSWORD` | single | required (`docker/.env`) | Initial admin password for the single-node file; the 3-node cluster does not set `PASSWORD` | -| `HUGEGRAPH_AUTH_TOKEN_SECRET` | single | generated (single) | JWT signing secret for the single-node file when auth is on; explicit values must be at least 32 bytes | +| `HUGEGRAPH_ADMIN_PASSWORD` | single, cluster | required (`docker/.env`) | Initial admin password; no public default is provided | +| `HUGEGRAPH_AUTH_TOKEN_SECRET` | single, cluster | generated (single); **required** (cluster) | JWT signing secret; explicit values must be at least 32 bytes. The cluster file requires it so all Server replicas validate each other's tokens | When authentication is enabled and no token secret is supplied, the Server entrypoint generates a random secret and writes it to both authentication diff --git a/docker/docker-compose-3pd-3store-3server.yml b/docker/docker-compose-3pd-3store-3server.yml index 358eb035a1..371fbf507a 100644 --- a/docker/docker-compose-3pd-3store-3server.yml +++ b/docker/docker-compose-3pd-3store-3server.yml @@ -78,6 +78,10 @@ x-server-env: &server-env HG_SERVER_USE_PD: "true" HG_SERVER_MIN_FREE_MEMORY: "0" HG_SERVER_INIT_STORE_ENABLED: "false" + # All replicas must share one token secret so a token issued by any + # server validates on every other server. + HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:?Set a shared auth token secret} + PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:?Set a non-default admin password} x-server-common: &server-common image: hugegraph/server:${HUGEGRAPH_VERSION:-latest} diff --git a/docker/hugegraph-hubble-3x3.properties b/docker/hugegraph-hubble-3x3.properties index 56cb515a82..af7f4a940b 100644 --- a/docker/hugegraph-hubble-3x3.properties +++ b/docker/hugegraph-hubble-3x3.properties @@ -29,10 +29,6 @@ cluster=hg idc=docker pd.enabled=true -# Match the anonymous 3-node cluster (no Server PASSWORD). Hubble's code -# default is auth.enabled=true; without this, the UI demands a login -# against a Server that has no users. -auth.enabled=false # Unused while pd.enabled=true: in PD mode Hubble picks a Server per request # from PD-discovered addresses and never reads this value, so editing it has # no effect on which replica is used. Kept only for the standalone fallback diff --git a/hugegraph-server/README.md b/hugegraph-server/README.md index c57bf4aa0f..db0cb3ca9e 100644 --- a/hugegraph-server/README.md +++ b/hugegraph-server/README.md @@ -45,8 +45,8 @@ For a full distributed deployment, use the compose file in the `docker/` directo ```bash cd docker -# One-time setup first: the shared hugegraph-net network — see the -# 3-Node Cluster Quickstart in the +# One-time setup first: the shared hugegraph-net network and the required +# credentials in docker/.env — see the 3-Node Cluster Quickstart in the # guide linked below. docker compose -f docker-compose-3pd-3store-3server.yml up -d ``` diff --git a/hugegraph-server/hugegraph-dist/docker/README.md b/hugegraph-server/hugegraph-dist/docker/README.md index af13728677..70ce39b0c8 100644 --- a/hugegraph-server/hugegraph-dist/docker/README.md +++ b/hugegraph-server/hugegraph-dist/docker/README.md @@ -125,8 +125,8 @@ For a full distributed HugeGraph cluster with PD, Store, and Server, use the ```bash cd docker -# One-time setup first: the shared hugegraph-net network — see the -# 3-Node Cluster Quickstart in the +# One-time setup first: the shared hugegraph-net network and the required +# credentials in docker/.env — see the 3-Node Cluster Quickstart in the # guide linked below. docker compose -f docker-compose-3pd-3store-3server.yml up -d ``` diff --git a/hugegraph-store/docs/deployment-guide.md b/hugegraph-store/docs/deployment-guide.md index 8afba1ca0f..0de7861a38 100644 --- a/hugegraph-store/docs/deployment-guide.md +++ b/hugegraph-store/docs/deployment-guide.md @@ -678,8 +678,9 @@ For a production-like 3-node distributed deployment, use the compose file at `do ```bash cd docker -# One-time setup first: the shared hugegraph-net network — see the -# 3-Node Cluster Quickstart in docker/README.md. +# One-time setup first: the shared hugegraph-net network and the required +# credentials in docker/.env — see the 3-Node Cluster Quickstart in +# docker/README.md. docker compose -f docker-compose-3pd-3store-3server.yml up -d ``` @@ -723,6 +724,8 @@ environment: HG_SERVER_REST_URL: http://server0:8080 # address registered with PD HG_SERVER_MIN_FREE_MEMORY: "0" # disable free-memory guard locally HG_SERVER_INIT_STORE_ENABLED: "false" # PD/HStore deployments skip init-store + HG_SERVER_AUTH_TOKEN_SECRET: ${HUGEGRAPH_AUTH_TOKEN_SECRET:?Set a shared auth token secret} # same value on all replicas, from docker/.env + PASSWORD: ${HUGEGRAPH_ADMIN_PASSWORD:?} # required; initial admin password, from docker/.env ``` **Startup ordering** is enforced via `depends_on` with `condition: service_healthy`: From c3115579f4f543ecf1b2a4ab0fb0c493b8bf3920 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 24 Aug 2026 02:14:23 +0530 Subject: [PATCH 09/14] fix(docker): fail closed on auth and stop publishing the control plane The 3-node cluster could report healthy while running unauthenticated. The images default to a floating tag with pull_policy: missing, so an older cached image was never refreshed, and that image ignores PASSWORD and the token secret while still answering /versions. Readiness now proves the image enforces authentication: an unauthenticated graph request must return 401 and an authenticated one 200, and all four images pull by default. An incompatible image no longer becomes healthy, so up -d --wait fails instead of handing back a false green. PD and Store publish REST, gRPC and Raft ports, and neither has real authentication. Together with the well-known external network that exposed an unauthenticated control plane on every host interface. Those ports and the Server REST ports now bind to 127.0.0.1, with HUGEGRAPH_CONTROL_PLANE_HOST and HUGEGRAPH_SERVER_PUBLISH_HOST to widen them deliberately. Each Server registers its own REST URL with PD, so a PD-aware client outside Docker received container names it cannot resolve. The registered URLs are configurable through HUGEGRAPH_SERVER0_REST_URL and its siblings, and the README explains when the defaults are wrong. Compose rendering cannot catch any of this, so CI now starts the cluster, attaches Hubble, and asserts the 401/200 pair on all three replicas, PD registration of all three Servers, that attaching Hubble recreates no cluster container, and that the H2 database survives recreation through the shared volumes. Running without authentication stays possible through an explicit opt-in, docker-compose-3x3.non-auth.yml with a matching Hubble properties file, documented next to the default flow along with a prompt for assistants that keeps the authenticated path as the default. --- .github/workflows/server-ci.yml | 109 +++++++++++++++++- docker/README.md | 104 +++++++++++++++-- docker/docker-compose-3pd-3store-3server.yml | 56 ++++++--- docker/docker-compose-3x3.non-auth.yml | 48 ++++++++ docker/docker-compose-hubble.yml | 9 +- .../hugegraph-hubble-3x3.non-auth.properties | 51 ++++++++ 6 files changed, 345 insertions(+), 32 deletions(-) create mode 100644 docker/docker-compose-3x3.non-auth.yml create mode 100644 docker/hugegraph-hubble-3x3.non-auth.properties diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index b75d50e668..fc0220aef8 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -282,7 +282,7 @@ jobs: .services.server2.environment.HG_SERVER_REST_URL == "http://server2:8080" and (.services.hubble | has("depends_on") | not) and - .services.hubble.pull_policy == "missing" and + .services.hubble.pull_policy == "always" and .volumes."hg-hubble-db".name == "hugegraph-hubble-db" and .volumes."hg-hubble-upload-files".name == "hugegraph-hubble-upload-files" and @@ -294,7 +294,10 @@ jobs: contains("\"name\":\"hugegraph-hubble\"")) and any(.services.hubble.ports[]; .target == 8088 and .published == "8088" and - .host_ip == "127.0.0.1") + .host_ip == "127.0.0.1") and + all(["pd0", "pd1", "pd2", "store0", "store1", "store2", + "server0", "server1", "server2"][]; + all($root.services[.].ports[]; .host_ip == "127.0.0.1")) ' "$rendered" >/dev/null # Hubble's properties file is mounted, not rendered, so Compose @@ -330,10 +333,112 @@ jobs: assert_props "pd.server=${pd_rest}" "names a real PD REST endpoint" assert_props "operations.store.allowed_targets=${store_targets}" \ "lists every Store REST endpoint" + + # The documented opt-out must actually opt out: no admin password, + # no shared token secret, and a healthcheck that does not demand + # the 401/200 pair the authenticated stack requires. A silent + # failure here would leave users on an unauthenticated cluster + # that still advertises itself as authenticated. + local non_auth="docker/docker-compose-3x3.non-auth.yml" + HUGEGRAPH_ADMIN_PASSWORD=unused \ + HUGEGRAPH_AUTH_TOKEN_SECRET=unused-token-secret-32-bytes-long \ + docker compose --env-file /dev/null -f "$cluster" -f "$non_auth" \ + config --format json > "$rendered" + jq -e ' + . as $root | + all(["server0", "server1", "server2"][]; + ($root.services[.].environment | has("PASSWORD") | not) and + ($root.services[.].environment + | has("HG_SERVER_AUTH_TOKEN_SECRET") | not) and + ($root.services[.].healthcheck.test[1] + | contains("/versions") and (contains("401") | not))) + ' "$rendered" >/dev/null + grep -Fqx "auth.enabled=false" \ + docker/hugegraph-hubble-3x3.non-auth.properties } check_cluster_compose + - name: Run distributed Compose auth and attach smoke test + if: ${{ env.BACKEND == 'rocksdb' }} + run: | + set -euo pipefail + command -v docker >/dev/null 2>&1 + docker compose version >/dev/null + docker info >/dev/null + + cluster="docker/docker-compose-3pd-3store-3server.yml" + addon="docker/docker-compose-hubble.yml" + run_id="${GITHUB_RUN_ID:-local}" + network="hugegraph-ci-${run_id}" + db_volume="hugegraph-ci-${run_id}-db" + upload_volume="hugegraph-ci-${run_id}-uploads" + admin_password='ci-smoke-admin-password' + token_secret='ci-smoke-token-secret-32-bytes-long' + + # Every Compose call needs the same environment, and cleanup must run + # even when an assertion below fails. + export HUGEGRAPH_NETWORK="$network" + export HUGEGRAPH_ADMIN_PASSWORD="$admin_password" + export HUGEGRAPH_AUTH_TOKEN_SECRET="$token_secret" + export HUBBLE_DB_VOLUME="$db_volume" + export HUBBLE_UPLOAD_VOLUME="$upload_volume" + + cleanup() { + docker compose --env-file /dev/null -f "$cluster" -f "$addon" \ + down -v --remove-orphans >/dev/null 2>&1 || true + docker compose --env-file /dev/null -p "hugegraph-ci-attach-${run_id}" \ + -f "$addon" down -v >/dev/null 2>&1 || true + docker network rm "$network" >/dev/null 2>&1 || true + docker volume rm "$db_volume" "$upload_volume" >/dev/null 2>&1 || true + } + trap cleanup EXIT + docker network create "$network" + + # The cluster must come up healthy on its own. Its Server healthcheck + # already proves 401/200, so --wait failing here means the images do + # not enforce the authentication this stack configures. + docker compose --env-file /dev/null -f "$cluster" up -d --wait + cluster_ids_before="$(docker compose --env-file /dev/null -f "$cluster" \ + ps -q pd0 pd1 pd2 store0 store1 store2 server0 server1 server2 | sort)" + for port in 8080 8081 8082; do + code="$(curl --retry 30 --retry-delay 2 --retry-all-errors -s -o /dev/null \ + -w '%{http_code}' "http://127.0.0.1:${port}/graphs/hugegraph/schema/vertexlabels")" + test "$code" = 401 + code="$(curl --retry 30 --retry-delay 2 --retry-all-errors -s -o /dev/null \ + -w '%{http_code}' -u "admin:${admin_password}" \ + "http://127.0.0.1:${port}/graphs/hugegraph/schema/vertexlabels")" + test "$code" = 200 + done + + # PD must have registered all three Servers, or Hubble's discovery + # would silently see a smaller cluster than the one that is running. + registered="$(curl -fsS "http://127.0.0.1:8620/v1/cluster" \ + | grep -o 'server[0-2]:8080' | sort -u | wc -l)" + test "$registered" -eq 3 + + # Attaching Hubble must not recreate any cluster container. + docker compose --env-file /dev/null -p "hugegraph-ci-attach-${run_id}" \ + -f "$addon" up -d --wait + curl --retry 30 --retry-delay 2 --retry-all-errors -fsS \ + http://127.0.0.1:8088/about | grep -q '"name":"hugegraph-hubble"' + cluster_ids_after="$(docker compose --env-file /dev/null -f "$cluster" \ + ps -q pd0 pd1 pd2 store0 store1 store2 server0 server1 server2 | sort)" + test "$cluster_ids_before" = "$cluster_ids_after" + + # Hubble state must survive recreation, and the combined flow must + # reuse the same physical volumes the attach flow just wrote to. + docker compose --env-file /dev/null -p "hugegraph-ci-attach-${run_id}" \ + -f "$addon" exec -T hubble sh -c 'ls /hubble/db/hubble.mv.db' >/dev/null + docker compose --env-file /dev/null -p "hugegraph-ci-attach-${run_id}" \ + -f "$addon" down + docker compose --env-file /dev/null \ + -f "$cluster" -f "$addon" up -d --wait + curl --retry 30 --retry-delay 2 --retry-all-errors -fsS \ + http://127.0.0.1:8088/about | grep -q '"name":"hugegraph-hubble"' + docker compose --env-file /dev/null -f "$cluster" -f "$addon" \ + exec -T hubble sh -c 'ls /hubble/db/hubble.mv.db' >/dev/null + - name: Run check_port unit tests if: ${{ env.BACKEND == 'rocksdb' }} run: | diff --git a/docker/README.md b/docker/README.md index 5c32b48d5d..c092d59bc0 100644 --- a/docker/README.md +++ b/docker/README.md @@ -182,8 +182,13 @@ Basic-auth *user* is one of its internal service names and never validates the password — so any client on the network can read and drive them. Note that this stack depends on that behaviour: Hubble reaches PD as the service name `hubble` with an empty password, so tightening PD's credential check -would also break Hubble's operations view. Any container on the host can -also join the network by declaring the well-known name. One-time setup: +would also break Hubble's operations view. Because of that, PD and Store +control-plane ports and the three Server REST ports bind to `127.0.0.1` by +default, so an unauthenticated control plane is not reachable from outside the +host. Publish them more widely only by setting `HUGEGRAPH_CONTROL_PLANE_HOST` +or `HUGEGRAPH_SERVER_PUBLISH_HOST`, and only behind a network ACL or TLS +terminator you control. Any container on the host can still join the network +by declaring the well-known name. One-time setup: write the required credentials to a mode-600 `docker/.env` and create the network. The cluster file requires both credentials — the admin password enables @@ -269,12 +274,24 @@ it from there, so those versions cannot drift apart Hubble to `hugegraph/hubble:latest`; set `HUBBLE_IMAGE` to pin it). Unpinned, the images default to `latest`; note the authenticated PD/Hubble integration requires a release newer than `1.7.x`. On an older image the -Server silently ignores `PASSWORD` and `HG_SERVER_AUTH_TOKEN_SECRET`, comes -up healthy, and leaves you an unauthenticated cluster — the 401 check under -"Verify the cluster is healthy" below is what detects this, so run it. -Because the cluster files use `pull_policy: missing`, an already-pulled -`latest` is never refreshed by `up -d` — pull explicitly or pin to pick up -new releases. +Server ignores `PASSWORD` and `HG_SERVER_AUTH_TOKEN_SECRET`, which would +leave you an unauthenticated cluster. Two defaults keep that from passing +unnoticed: the cluster and add-on use `pull_policy: always`, so a stale +cached `latest` is refreshed on every `up -d`, and the Server healthcheck +requires an unauthenticated graph request to return `401` and an +authenticated one to return `200`. An image that does not enforce +authentication therefore never reports healthy, and `up -d --wait` fails +instead of handing you a false green. + +Each Server registers its `HG_SERVER_REST_URL` with PD. The defaults +(`server0`, `server1`, `server2`) are Docker-network names, which is correct +for Hubble and anything else attached to `hugegraph-net`, but they do not +resolve from an unrelated host network. A PD-aware client running outside +Docker will receive names it cannot reach. For that case, set +`HUGEGRAPH_SERVER0_REST_URL`, `HUGEGRAPH_SERVER1_REST_URL`, and +`HUGEGRAPH_SERVER2_REST_URL` to addresses that resolve from both the Server +containers and that client, and publish the matching host ports with +`HUGEGRAPH_SERVER_PUBLISH_HOST`. > [!NOTE] > Upgrading an existing 3-node deployment: @@ -365,6 +382,69 @@ running image ignored `PASSWORD` and the cluster is **unauthenticated** — the most likely cause is an image older than the release this integration needs (see the version note in the quickstart above). +### Running without authentication + +The cluster above authenticates, and that is the right default: it is the +configuration most people copy, and an open graph database on a reachable +port is a bad surprise. For a throwaway local trial on a trusted network you +can opt out with `docker-compose-3x3.non-auth.yml`, which drops the admin +password and the JWT secret from all three Servers. Anyone who can reach the +published ports then has full read and write access to every graph, so do not +use it anywhere else. + +```bash +cd docker + +# No docker/.env and no credential setup are needed for this mode. The two +# throwaway values exist only because Compose evaluates the base file's +# required-variable guards before applying the override, which then drops +# both variables — neither value reaches a container. Needs Compose v2.24+. +HUGEGRAPH_ADMIN_PASSWORD=unused \ +HUGEGRAPH_AUTH_TOKEN_SECRET=unused-token-secret-32-bytes-long \ + docker compose -f docker-compose-3pd-3store-3server.yml \ + -f docker-compose-3x3.non-auth.yml up -d + +# Every graph request now succeeds without credentials: expect 200, not 401. +curl -s -o /dev/null -w '%{http_code}\n' \ + http://localhost:8080/graphs/hugegraph/schema/vertexlabels +``` + +Hubble needs the matching configuration, because its own default is to demand +a login. Point it at the non-auth properties file when you attach it: + +```bash +HUBBLE_PROPERTIES=./hugegraph-hubble-3x3.non-auth.properties \ + docker compose -p hugegraph-hubble -f docker-compose-hubble.yml up -d +``` + +
+Prompt for an AI assistant + +Copy this to an assistant instead of transcribing the commands by hand. + +```markdown +Start the Apache HugeGraph 3-node Docker cluster (3 PD, 3 Store, 3 Server) +with Hubble, from the `docker/` directory of the hugegraph repository. + +Use the authenticated default unless I say otherwise: +1. Follow "3-Node Cluster Quickstart" in docker/README.md to create + docker/.env with a generated admin password and JWT token secret, and to + create the external hugegraph-net network. +2. Start the cluster with docker-compose-3pd-3store-3server.yml, then attach + Hubble with docker-compose-hubble.yml. +3. Verify: an unauthenticated graph request returns 401, an authenticated one + returns 200, and Hubble answers on http://localhost:8088. +4. Tell me the admin password so I can sign in to Hubble. + +Do not weaken authentication to work around an error. If a Server never +becomes healthy, the image is probably older than the release this stack +needs; report that instead of disabling the healthcheck. Only if I explicitly +ask for an unauthenticated cluster, use the "Running without authentication" +section instead. +``` + +
+ --- ## Hubble for the 3-Node Cluster @@ -602,9 +682,15 @@ add-on = `docker-compose-hubble.yml`): | `HUGEGRAPH_SERVER_IMAGE` | single | `hugegraph/server:` | Complete Server image reference | | `HUGEGRAPH_SERVER_PULL_POLICY` | single | `always` (`build` for dev) | Server pull policy | | `HUBBLE_IMAGE` | single, add-on | `hugegraph/hubble:` | Complete Hubble image reference | -| `HUBBLE_PULL_POLICY` | single, add-on | `always` (`missing` for dev and the add-on) | Hubble pull policy | +| `HUBBLE_PULL_POLICY` | single, add-on | `always` (`missing` for dev) | Hubble pull policy | | `HUBBLE_PUBLISH_HOST` | single, add-on | `127.0.0.1` | Hubble host bind address; remote access requires an HTTPS reverse proxy | +| `HUBBLE_PROPERTIES` | add-on | `./hugegraph-hubble-3x3.properties` | Hubble properties file bind-mounted into the container; set it to `./hugegraph-hubble-3x3.non-auth.properties` when the cluster runs without authentication | | `HUGEGRAPH_NETWORK` | cluster, add-on | `hugegraph-net` | Pre-created external Docker network shared by the 3-node cluster and the Hubble add-on; the single-node files use their own project bridge instead | +| `HUGEGRAPH_CONTROL_PLANE_HOST` | cluster | `127.0.0.1` | Host bind address for the PD and Store REST/gRPC/Raft ports; these APIs have no real authentication, so widen this only behind a network ACL or TLS | +| `HUGEGRAPH_SERVER_PUBLISH_HOST` | cluster | `127.0.0.1` | Host bind address for the three Server REST ports | +| `HUGEGRAPH_SERVER0_REST_URL` | cluster | `http://server0:8080` | URL Server 0 registers with PD; must resolve from the Server container and from every PD-aware client | +| `HUGEGRAPH_SERVER1_REST_URL` | cluster | `http://server1:8080` | URL Server 1 registers with PD; same resolution requirement | +| `HUGEGRAPH_SERVER2_REST_URL` | cluster | `http://server2:8080` | URL Server 2 registers with PD; same resolution requirement | | `HUGEGRAPH_ADMIN_PASSWORD` | single, cluster | required (`docker/.env`) | Initial admin password; no public default is provided | | `HUGEGRAPH_AUTH_TOKEN_SECRET` | single, cluster | generated (single); **required** (cluster) | JWT signing secret; explicit values must be at least 32 bytes. The cluster file requires it so all Server replicas validate each other's tokens | diff --git a/docker/docker-compose-3pd-3store-3server.yml b/docker/docker-compose-3pd-3store-3server.yml index 371fbf507a..a69e47b97a 100644 --- a/docker/docker-compose-3pd-3store-3server.yml +++ b/docker/docker-compose-3pd-3store-3server.yml @@ -39,7 +39,7 @@ x-pd-common: &pd-common # Pin a release via HUGEGRAPH_VERSION in docker/.env; unset, the image # tags default to latest. All Compose files here read the same variable. image: hugegraph/pd:${HUGEGRAPH_VERSION:-latest} - pull_policy: missing + pull_policy: always restart: unless-stopped networks: [hg-net] healthcheck: @@ -51,7 +51,7 @@ x-pd-common: &pd-common x-store-common: &store-common image: hugegraph/store:${HUGEGRAPH_VERSION:-latest} - pull_policy: missing + pull_policy: always restart: unless-stopped networks: [hg-net] depends_on: @@ -85,7 +85,7 @@ x-server-env: &server-env x-server-common: &server-common image: hugegraph/server:${HUGEGRAPH_VERSION:-latest} - pull_policy: missing + pull_policy: always restart: unless-stopped networks: [hg-net] depends_on: @@ -93,9 +93,11 @@ x-server-common: &server-common store1: { condition: service_healthy } store2: { condition: service_healthy } healthcheck: - # With HG_SERVER_REST_URL set, the REST server binds that URL's - # hostname rather than localhost, so probe the bound address. - test: ["CMD-SHELL", "curl -fsS $${HG_SERVER_REST_URL}/versions >/dev/null || exit 1"] + # Prove the image actually enforces authentication. Older cached images + # may accept PASSWORD but ignore it while still returning healthy on + # /versions, so readiness requires both 401 without credentials and 200 + # with the configured admin password. + test: ["CMD-SHELL", "base=$${HG_SERVER_REST_URL}; unauth=$$(curl -s -o /dev/null -w '%{http_code}' \"$${base}/graphs/hugegraph/schema/vertexlabels\"); auth=$$(curl -s -o /dev/null -w '%{http_code}' -u \"admin:$${PASSWORD}\" \"$${base}/graphs/hugegraph/schema/vertexlabels\"); test \"$${unauth}\" = 401 && test \"$${auth}\" = 200"] interval: 10s timeout: 5s retries: 30 @@ -118,7 +120,9 @@ services: HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 HG_PD_DATA_PATH: /hugegraph-pd/pd_data HG_PD_INITIAL_STORE_COUNT: 3 - ports: ["8620:8620", "8686:8686"] + ports: + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8620:8620" + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8686:8686" volumes: - hg-pd0-data:/hugegraph-pd/pd_data @@ -135,7 +139,9 @@ services: HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 HG_PD_DATA_PATH: /hugegraph-pd/pd_data HG_PD_INITIAL_STORE_COUNT: 3 - ports: ["8621:8620", "8687:8686"] + ports: + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8621:8620" + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8687:8686" volumes: - hg-pd1-data:/hugegraph-pd/pd_data @@ -152,7 +158,9 @@ services: HG_PD_INITIAL_STORE_LIST: store0:8500,store1:8500,store2:8500 HG_PD_DATA_PATH: /hugegraph-pd/pd_data HG_PD_INITIAL_STORE_COUNT: 3 - ports: ["8622:8620", "8688:8686"] + ports: + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8622:8620" + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8688:8686" volumes: - hg-pd2-data:/hugegraph-pd/pd_data @@ -168,7 +176,10 @@ services: HG_STORE_REST_PORT: "8520" HG_STORE_RAFT_ADDRESS: store0:8510 HG_STORE_DATA_PATH: /hugegraph-store/storage - ports: ["8500:8500", "8510:8510", "8520:8520"] + ports: + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8500:8500" + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8510:8510" + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8520:8520" volumes: - hg-store0-data:/hugegraph-store/storage @@ -183,7 +194,10 @@ services: HG_STORE_REST_PORT: "8520" HG_STORE_RAFT_ADDRESS: store1:8510 HG_STORE_DATA_PATH: /hugegraph-store/storage - ports: ["8501:8500", "8511:8510", "8521:8520"] + ports: + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8501:8500" + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8511:8510" + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8521:8520" volumes: - hg-store1-data:/hugegraph-store/storage @@ -198,7 +212,10 @@ services: HG_STORE_REST_PORT: "8520" HG_STORE_RAFT_ADDRESS: store2:8510 HG_STORE_DATA_PATH: /hugegraph-store/storage - ports: ["8502:8500", "8512:8510", "8522:8520"] + ports: + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8502:8500" + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8512:8510" + - "${HUGEGRAPH_CONTROL_PLANE_HOST:-127.0.0.1}:8522:8520" volumes: - hg-store2-data:/hugegraph-store/storage @@ -209,8 +226,9 @@ services: hostname: server0 environment: <<: *server-env - HG_SERVER_REST_URL: http://server0:8080 - ports: ["8080:8080"] + HG_SERVER_REST_URL: ${HUGEGRAPH_SERVER0_REST_URL:-http://server0:8080} + ports: + - "${HUGEGRAPH_SERVER_PUBLISH_HOST:-127.0.0.1}:8080:8080" server1: <<: *server-common @@ -218,8 +236,9 @@ services: hostname: server1 environment: <<: *server-env - HG_SERVER_REST_URL: http://server1:8080 - ports: ["8081:8080"] + HG_SERVER_REST_URL: ${HUGEGRAPH_SERVER1_REST_URL:-http://server1:8080} + ports: + - "${HUGEGRAPH_SERVER_PUBLISH_HOST:-127.0.0.1}:8081:8080" server2: <<: *server-common @@ -227,5 +246,6 @@ services: hostname: server2 environment: <<: *server-env - HG_SERVER_REST_URL: http://server2:8080 - ports: ["8082:8080"] + HG_SERVER_REST_URL: ${HUGEGRAPH_SERVER2_REST_URL:-http://server2:8080} + ports: + - "${HUGEGRAPH_SERVER_PUBLISH_HOST:-127.0.0.1}:8082:8080" diff --git a/docker/docker-compose-3x3.non-auth.yml b/docker/docker-compose-3x3.non-auth.yml new file mode 100644 index 0000000000..cad51aafcd --- /dev/null +++ b/docker/docker-compose-3x3.non-auth.yml @@ -0,0 +1,48 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Opt-in override that runs the 3-node cluster WITHOUT authentication. +# +# The default stack authenticates, and that is the configuration to use unless +# you have a reason not to. This override exists for throwaway local trials on +# a trusted network: no admin password, no JWT secret, no login in Hubble. +# Anyone who can reach the published ports can read and modify every graph. +# +# Usage (see "Running without authentication" in README.md): +# +# HUGEGRAPH_ADMIN_PASSWORD=unused \ +# HUGEGRAPH_AUTH_TOKEN_SECRET=unused-token-secret-32-bytes-long \ +# docker compose -f docker-compose-3pd-3store-3server.yml \ +# -f docker-compose-3x3.non-auth.yml up -d +# +# The two throwaway values are required only because Compose interpolates the +# base file's `:?` guards before it applies this override; `!reset` below drops +# both variables, so neither value ever reaches a container. `!reset` needs +# Docker Compose v2.24 or newer. + +services: + server0: &non-auth-server + environment: + PASSWORD: !reset null + HG_SERVER_AUTH_TOKEN_SECRET: !reset null + healthcheck: + # The default healthcheck proves the image enforces authentication + # (401 without credentials, 200 with). That check cannot pass here, so + # fall back to probing the bound REST address. + test: ["CMD-SHELL", "curl -fsS $${HG_SERVER_REST_URL}/versions >/dev/null || exit 1"] + + server1: *non-auth-server + + server2: *non-auth-server diff --git a/docker/docker-compose-hubble.yml b/docker/docker-compose-hubble.yml index aa32ecc819..67b1b6c4cb 100644 --- a/docker/docker-compose-hubble.yml +++ b/docker/docker-compose-hubble.yml @@ -36,9 +36,10 @@ volumes: services: hubble: # Pin a release via HUGEGRAPH_VERSION in docker/.env; unset, the image - # tag defaults to latest. + # tag defaults to latest. Pull by default so an older cached image cannot + # silently satisfy an auth-sensitive deployment. image: ${HUBBLE_IMAGE:-hugegraph/hubble:${HUGEGRAPH_VERSION:-latest}} - pull_policy: ${HUBBLE_PULL_POLICY:-missing} + pull_policy: ${HUBBLE_PULL_POLICY:-always} container_name: hg-hubble hostname: hubble restart: unless-stopped @@ -50,7 +51,9 @@ services: # the /hubble/db volume. Point H2 at a file inside the mount. SPRING_DATASOURCE_URL: jdbc:h2:file:./db/hubble;DB_CLOSE_ON_EXIT=FALSE volumes: - - ./hugegraph-hubble-3x3.properties:/hubble/conf/hugegraph-hubble.properties:ro + # Point HUBBLE_PROPERTIES at the non-auth variant when the cluster runs + # without authentication; see "Running without authentication". + - ${HUBBLE_PROPERTIES:-./hugegraph-hubble-3x3.properties}:/hubble/conf/hugegraph-hubble.properties:ro - hg-hubble-db:/hubble/db - hg-hubble-upload-files:/hubble/upload-files healthcheck: diff --git a/docker/hugegraph-hubble-3x3.non-auth.properties b/docker/hugegraph-hubble-3x3.non-auth.properties new file mode 100644 index 0000000000..d4544350d2 --- /dev/null +++ b/docker/hugegraph-hubble-3x3.non-auth.properties @@ -0,0 +1,51 @@ +# Licensed to the Apache Software Foundation (ASF) under one or more +# contributor license agreements. See the NOTICE file distributed with +# this work for additional information regarding copyright ownership. +# The ASF licenses this file to You under the Apache License, Version 2.0 +# (the "License"); you may not use this file except in compliance with +# the License. You may obtain a copy of the License at +# +# http://www.apache.org/licenses/LICENSE-2.0 +# +# Unless required by applicable law or agreed to in writing, software +# distributed under the License is distributed on an "AS IS" BASIS, +# WITHOUT WARRANTIES OR CONDITIONS OF ANY KIND, either express or implied. +# See the License for the specific language governing permissions and +# limitations under the License. + +# Hubble configuration for the 3-node cluster running WITHOUT authentication, +# i.e. started with docker-compose-3x3.non-auth.yml. It is identical to +# hugegraph-hubble-3x3.properties except for auth.enabled below. Use it only +# on a trusted local network; the default stack keeps authentication on. +# +# This file REPLACES the image's /hubble/conf/hugegraph-hubble.properties +# wholesale — it is bind-mounted over it, not merged with it. Keys absent +# here therefore fall back to the code defaults in HubbleOptions, not to the +# values in the shipped conf file. Re-check against the shipped conf when +# upgrading Hubble. + +server.host=0.0.0.0 +server.port=8088 + +cluster=hg +idc=docker + +pd.enabled=true +# Hubble's code default is auth.enabled=true. Without this line the UI would +# demand a login against Servers that have no users configured. +auth.enabled=false +# Unused while pd.enabled=true: in PD mode Hubble picks a Server per request +# from PD-discovered addresses and never reads this value, so editing it has +# no effect on which replica is used. Kept only for the standalone fallback +# (pd.enabled=false). +server.direct_url=http://server0:8080 +pd.peers=pd0:8686,pd1:8686,pd2:8686 +# PD REST endpoint for the operations view (single address, no failover). +# If pd0 is down the cluster keeps quorum on pd1/pd2 but this view goes +# blind until you repoint this at a surviving PD. +pd.server=pd0:8620 + +operations.store.allowed_targets=[http://store0:8520,http://store1:8520,http://store2:8520] + +# Dashboard is not part of this Compose stack. +dashboard.address= From 10c6c6b55bac56a36b7b22a7dbe7802f03a30c13 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 24 Aug 2026 02:26:13 +0530 Subject: [PATCH 10/14] ci(docker): assert the topology and shared Hubble state the smoke test proves Running the smoke test against a real cluster showed two of its assertions were wrong. PD's /v1/cluster returns the PD peers, not the graph servers, so grepping it for registered Server addresses always found nothing; /v1/registry answers 405 to GET and 500 to POST, so there is no simple REST proof of Server registration. Assert what PD does report and what actually matters for a distributed deployment: three PD peers and three Stores in state Up. The persistence check only listed the H2 database file, which passes even if the attach and combined flows use different volumes. Write a marker through the attach flow and read it back through the combined flow after Hubble has been recreated, which is the property the explicit volume names exist for. --- .github/workflows/server-ci.yml | 33 +++++++++++++++++++++++---------- 1 file changed, 23 insertions(+), 10 deletions(-) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index fc0220aef8..bd217be330 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -411,11 +411,17 @@ jobs: test "$code" = 200 done - # PD must have registered all three Servers, or Hubble's discovery - # would silently see a smaller cluster than the one that is running. - registered="$(curl -fsS "http://127.0.0.1:8620/v1/cluster" \ - | grep -o 'server[0-2]:8080' | sort -u | wc -l)" - test "$registered" -eq 3 + # The distributed topology must really be distributed: three PD peers + # and three Stores registered and Up. A partial cluster otherwise + # still passes a container-level health check. PD requires an + # Authorization header whose user is an internal service name; it + # never validates the password, hence the empty one. + pd_nodes="$(curl -fsS -u 'hubble:' "http://127.0.0.1:8620/v1/cluster" \ + | grep -o '"restUrl":"pd[0-2]:8620"' | sort -u | wc -l)" + test "$pd_nodes" -eq 3 + stores_up="$(curl -fsS -u 'hubble:' "http://127.0.0.1:8620/v1/stores" \ + | grep -o '"state":"Up"' | wc -l)" + test "$stores_up" -eq 3 # Attaching Hubble must not recreate any cluster container. docker compose --env-file /dev/null -p "hugegraph-ci-attach-${run_id}" \ @@ -426,18 +432,25 @@ jobs: ps -q pd0 pd1 pd2 store0 store1 store2 server0 server1 server2 | sort)" test "$cluster_ids_before" = "$cluster_ids_after" - # Hubble state must survive recreation, and the combined flow must - # reuse the same physical volumes the attach flow just wrote to. + # The H2 database must live inside the mounted volume, not beside it, + # and the attach and combined flows must share the same physical + # volumes even though they run under different Compose projects. + # Write a marker through the attach flow, then read it back through + # the combined flow after the container has been recreated. docker compose --env-file /dev/null -p "hugegraph-ci-attach-${run_id}" \ - -f "$addon" exec -T hubble sh -c 'ls /hubble/db/hubble.mv.db' >/dev/null + -f "$addon" exec -T hubble sh -c \ + 'ls /hubble/db/hubble.mv.db && echo ci-marker > /hubble/upload-files/ci-marker' \ + >/dev/null docker compose --env-file /dev/null -p "hugegraph-ci-attach-${run_id}" \ -f "$addon" down docker compose --env-file /dev/null \ -f "$cluster" -f "$addon" up -d --wait curl --retry 30 --retry-delay 2 --retry-all-errors -fsS \ http://127.0.0.1:8088/about | grep -q '"name":"hugegraph-hubble"' - docker compose --env-file /dev/null -f "$cluster" -f "$addon" \ - exec -T hubble sh -c 'ls /hubble/db/hubble.mv.db' >/dev/null + marker="$(docker compose --env-file /dev/null -f "$cluster" -f "$addon" \ + exec -T hubble sh -c \ + 'ls /hubble/db/hubble.mv.db >/dev/null && cat /hubble/upload-files/ci-marker')" + test "$(printf '%s' "$marker" | tr -d '\r\n')" = "ci-marker" - name: Run check_port unit tests if: ${{ env.BACKEND == 'rocksdb' }} From 7ea0efe28a038351316d521e8f111f1656c5fb02 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 24 Aug 2026 02:32:39 +0530 Subject: [PATCH 11/14] docs(docker): correct the port reference and harden the non-auth placeholders The Port Reference section still said cluster ports bind every host interface, which contradicted both the quickstart section and the compose file after the loopback change. It now describes the loopback default and what widening it actually exposes. The non-auth override documented a placeholder token secret that was long enough to be accepted as a real one. Anyone who copied the two placeholder lines into docker/.env and then started the default stack would have got a cluster that looked authenticated while signing tokens with a key published in this repository. Both placeholders are now obvious non-values and the token is deliberately shorter than the 32 bytes the Server requires, so it aborts startup instead. The README and the override header both say to pass them inline and never store them. Also lists the two new files in the file table and documents HUBBLE_DB_VOLUME and HUBBLE_UPLOAD_VOLUME, which were usable but undocumented. --- .github/workflows/server-ci.yml | 14 +++++--- docker/README.md | 32 +++++++++++++------ docker/docker-compose-3x3.non-auth.yml | 18 +++++++---- .../hugegraph-hubble-3x3.non-auth.properties | 2 +- 4 files changed, 44 insertions(+), 22 deletions(-) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index bd217be330..8bb8252b94 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -339,9 +339,12 @@ jobs: # the 401/200 pair the authenticated stack requires. A silent # failure here would leave users on an unauthenticated cluster # that still advertises itself as authenticated. + # Use the exact placeholders the README documents, including the + # deliberately short token: it must never reach a container, and + # the assertions below are what prove the override drops it. local non_auth="docker/docker-compose-3x3.non-auth.yml" - HUGEGRAPH_ADMIN_PASSWORD=unused \ - HUGEGRAPH_AUTH_TOKEN_SECRET=unused-token-secret-32-bytes-long \ + HUGEGRAPH_ADMIN_PASSWORD=non-auth-placeholder \ + HUGEGRAPH_AUTH_TOKEN_SECRET=non-auth-placeholder \ docker compose --env-file /dev/null -f "$cluster" -f "$non_auth" \ config --format json > "$rendered" jq -e ' @@ -398,7 +401,8 @@ jobs: # The cluster must come up healthy on its own. Its Server healthcheck # already proves 401/200, so --wait failing here means the images do # not enforce the authentication this stack configures. - docker compose --env-file /dev/null -f "$cluster" up -d --wait + docker compose --env-file /dev/null -f "$cluster" up -d --wait \ + --wait-timeout 420 cluster_ids_before="$(docker compose --env-file /dev/null -f "$cluster" \ ps -q pd0 pd1 pd2 store0 store1 store2 server0 server1 server2 | sort)" for port in 8080 8081 8082; do @@ -425,7 +429,7 @@ jobs: # Attaching Hubble must not recreate any cluster container. docker compose --env-file /dev/null -p "hugegraph-ci-attach-${run_id}" \ - -f "$addon" up -d --wait + -f "$addon" up -d --wait --wait-timeout 180 curl --retry 30 --retry-delay 2 --retry-all-errors -fsS \ http://127.0.0.1:8088/about | grep -q '"name":"hugegraph-hubble"' cluster_ids_after="$(docker compose --env-file /dev/null -f "$cluster" \ @@ -444,7 +448,7 @@ jobs: docker compose --env-file /dev/null -p "hugegraph-ci-attach-${run_id}" \ -f "$addon" down docker compose --env-file /dev/null \ - -f "$cluster" -f "$addon" up -d --wait + -f "$cluster" -f "$addon" up -d --wait --wait-timeout 420 curl --retry 30 --retry-delay 2 --retry-all-errors -fsS \ http://127.0.0.1:8088/about | grep -q '"name":"hugegraph-hubble"' marker="$(docker compose --env-file /dev/null -f "$cluster" -f "$addon" \ diff --git a/docker/README.md b/docker/README.md index c092d59bc0..ad6c479ed7 100644 --- a/docker/README.md +++ b/docker/README.md @@ -9,8 +9,10 @@ running HugeGraph: | `docker-compose.dev.yml` | PD, Store, and Server built from source, plus Hubble | | `docker-compose-3pd-3store-3server.yml` | 3-node distributed cluster (PD + Store + Server) | | `docker-compose-hubble.yml` | Hubble add-on for the 3-node cluster (attachable to a running cluster) | +| `docker-compose-3x3.non-auth.yml` | Opt-in override that runs the 3-node cluster without authentication | | `hugegraph-hubble.properties` | Hubble configuration mounted by the single-node files | | `hugegraph-hubble-3x3.properties` | Hubble configuration mounted by the add-on; edit when attaching to a cluster with different hostnames | +| `hugegraph-hubble-3x3.non-auth.properties` | Hubble configuration for the cluster when it runs without authentication | ## Prerequisites @@ -396,11 +398,17 @@ use it anywhere else. cd docker # No docker/.env and no credential setup are needed for this mode. The two -# throwaway values exist only because Compose evaluates the base file's +# placeholders exist only because Compose evaluates the base file's # required-variable guards before applying the override, which then drops -# both variables — neither value reaches a container. Needs Compose v2.24+. -HUGEGRAPH_ADMIN_PASSWORD=unused \ -HUGEGRAPH_AUTH_TOKEN_SECRET=unused-token-secret-32-bytes-long \ +# both variables, so neither value reaches a container. Needs Compose v2.24+. +# +# Keep them inline on this command; do not write them into docker/.env. They +# are published values rather than secrets, and the token placeholder is +# deliberately shorter than the 32 bytes the Server requires, so if it ever +# reaches an authenticated Server that Server refuses to start instead of +# signing tokens with a key anyone can read here. +HUGEGRAPH_ADMIN_PASSWORD=non-auth-placeholder \ +HUGEGRAPH_AUTH_TOKEN_SECRET=non-auth-placeholder \ docker compose -f docker-compose-3pd-3store-3server.yml \ -f docker-compose-3x3.non-auth.yml up -d @@ -685,6 +693,8 @@ add-on = `docker-compose-hubble.yml`): | `HUBBLE_PULL_POLICY` | single, add-on | `always` (`missing` for dev) | Hubble pull policy | | `HUBBLE_PUBLISH_HOST` | single, add-on | `127.0.0.1` | Hubble host bind address; remote access requires an HTTPS reverse proxy | | `HUBBLE_PROPERTIES` | add-on | `./hugegraph-hubble-3x3.properties` | Hubble properties file bind-mounted into the container; set it to `./hugegraph-hubble-3x3.non-auth.properties` when the cluster runs without authentication | +| `HUBBLE_DB_VOLUME` | add-on | `hugegraph-hubble-db` | Volume holding Hubble's H2 database; the explicit name is what lets the attach and combined flows share state, so change it only to run a second independent Hubble | +| `HUBBLE_UPLOAD_VOLUME` | add-on | `hugegraph-hubble-upload-files` | Volume holding Hubble's uploaded files; same naming caveat as `HUBBLE_DB_VOLUME` | | `HUGEGRAPH_NETWORK` | cluster, add-on | `hugegraph-net` | Pre-created external Docker network shared by the 3-node cluster and the Hubble add-on; the single-node files use their own project bridge instead | | `HUGEGRAPH_CONTROL_PLANE_HOST` | cluster | `127.0.0.1` | Host bind address for the PD and Store REST/gRPC/Raft ports; these APIs have no real authentication, so widen this only behind a network ACL or TLS | | `HUGEGRAPH_SERVER_PUBLISH_HOST` | cluster | `127.0.0.1` | Host bind address for the three Server REST ports | @@ -762,12 +772,14 @@ The table below reflects the published host ports of the 3-node cluster (`docker-compose-hubble.yml`). > [!IMPORTANT] -> Cluster ports bind all host interfaces and bypass host firewalls under -> Docker's port publishing. Among them, the Store APIs need no credentials -> at all and the PD APIs accept any password for their internal service -> names, so neither is a real access control — see the trust-boundary note -> in the 3-Node Cluster Quickstart. Do not run this file on an untrusted -> network. Hubble is the exception and binds loopback only by default. +> Every port in this file binds to `127.0.0.1` by default. That matters +> because the Store APIs need no credentials at all and the PD APIs accept +> any password for their internal service names, so neither is a real access +> control (see the trust-boundary note in the 3-Node Cluster Quickstart). +> Widening the bind with `HUGEGRAPH_CONTROL_PLANE_HOST` or +> `HUGEGRAPH_SERVER_PUBLISH_HOST` publishes an unauthenticated control plane, +> and Docker's port publishing bypasses host firewalls, so do that only +> behind a network ACL or TLS terminator you control. The single-node Compose file publishes `8620`, `8520`, `8080`, and Hubble `8088`; Hubble defaults to host loopback. diff --git a/docker/docker-compose-3x3.non-auth.yml b/docker/docker-compose-3x3.non-auth.yml index cad51aafcd..e44b39ac0e 100644 --- a/docker/docker-compose-3x3.non-auth.yml +++ b/docker/docker-compose-3x3.non-auth.yml @@ -22,15 +22,21 @@ # # Usage (see "Running without authentication" in README.md): # -# HUGEGRAPH_ADMIN_PASSWORD=unused \ -# HUGEGRAPH_AUTH_TOKEN_SECRET=unused-token-secret-32-bytes-long \ +# HUGEGRAPH_ADMIN_PASSWORD=non-auth-placeholder \ +# HUGEGRAPH_AUTH_TOKEN_SECRET=non-auth-placeholder \ # docker compose -f docker-compose-3pd-3store-3server.yml \ # -f docker-compose-3x3.non-auth.yml up -d # -# The two throwaway values are required only because Compose interpolates the -# base file's `:?` guards before it applies this override; `!reset` below drops -# both variables, so neither value ever reaches a container. `!reset` needs -# Docker Compose v2.24 or newer. +# The two placeholders are required only because Compose interpolates the base +# file's `:?` guards before it applies this override; `!reset` below drops both +# variables, so neither value ever reaches a container. `!reset` needs Docker +# Compose v2.24 or newer. +# +# Pass them inline on this command only. Do not put them in docker/.env. They +# are published values, so they are not credentials, and the token placeholder +# is deliberately shorter than the 32 bytes the Server entrypoint requires: if +# it ever reaches an authenticated Server it aborts startup instead of signing +# tokens with a key that is printed in this repository. services: server0: &non-auth-server diff --git a/docker/hugegraph-hubble-3x3.non-auth.properties b/docker/hugegraph-hubble-3x3.non-auth.properties index d4544350d2..0cbd51ff70 100644 --- a/docker/hugegraph-hubble-3x3.non-auth.properties +++ b/docker/hugegraph-hubble-3x3.non-auth.properties @@ -19,7 +19,7 @@ # on a trusted local network; the default stack keeps authentication on. # # This file REPLACES the image's /hubble/conf/hugegraph-hubble.properties -# wholesale — it is bind-mounted over it, not merged with it. Keys absent +# wholesale: it is bind-mounted over it, not merged with it. Keys absent # here therefore fall back to the code defaults in HubbleOptions, not to the # values in the shipped conf file. Re-check against the shipped conf when # upgrading Hubble. From 6b6c3a3cca0815f08706e8e5489b709054c73b8e Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Mon, 24 Aug 2026 02:40:28 +0530 Subject: [PATCH 12/14] fix(docker): apply the non-auth reset to every Server, not just the first The override shared one YAML anchor across the three Server services. The plain healthcheck override propagates through the alias, but the !reset tags do not on every Compose version: on 5.1.2 only the anchored service loses PASSWORD and HG_SERVER_AUTH_TOKEN_SECRET, so server1 and server2 keep them and come up authenticated while server0 does not. The result is a cluster that is half authenticated, which is worse than either mode on its own, and readiness still passes on all three because the healthcheck override does propagate. The same file renders correctly on 5.1.4, so this depends on the Compose version rather than failing everywhere, which is how it survived a live run. Spell the three services out instead of aliasing them; the CI render check already asserts all three, so a regression fails there. --- docker/docker-compose-3x3.non-auth.yml | 22 +++++++++++++++++++--- 1 file changed, 19 insertions(+), 3 deletions(-) diff --git a/docker/docker-compose-3x3.non-auth.yml b/docker/docker-compose-3x3.non-auth.yml index e44b39ac0e..bc15bb170f 100644 --- a/docker/docker-compose-3x3.non-auth.yml +++ b/docker/docker-compose-3x3.non-auth.yml @@ -38,8 +38,14 @@ # it ever reaches an authenticated Server it aborts startup instead of signing # tokens with a key that is printed in this repository. +# Each service is spelled out instead of sharing a YAML anchor. `!reset` does +# not survive an anchor/alias reference on every Compose version: on 5.1.2 the +# reset applies only to the anchored service, leaving server1 and server2 with +# a password and token secret, so the cluster would be half authenticated. The +# plain healthcheck override in the same block does propagate, which makes the +# failure easy to miss. Keep these three blocks identical. services: - server0: &non-auth-server + server0: environment: PASSWORD: !reset null HG_SERVER_AUTH_TOKEN_SECRET: !reset null @@ -49,6 +55,16 @@ services: # fall back to probing the bound REST address. test: ["CMD-SHELL", "curl -fsS $${HG_SERVER_REST_URL}/versions >/dev/null || exit 1"] - server1: *non-auth-server + server1: + environment: + PASSWORD: !reset null + HG_SERVER_AUTH_TOKEN_SECRET: !reset null + healthcheck: + test: ["CMD-SHELL", "curl -fsS $${HG_SERVER_REST_URL}/versions >/dev/null || exit 1"] - server2: *non-auth-server + server2: + environment: + PASSWORD: !reset null + HG_SERVER_AUTH_TOKEN_SECRET: !reset null + healthcheck: + test: ["CMD-SHELL", "curl -fsS $${HG_SERVER_REST_URL}/versions >/dev/null || exit 1"] From 96160375497e48a8a9389418576c45ec25f87de6 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Tue, 25 Aug 2026 17:32:51 +0530 Subject: [PATCH 13/14] fix(docker): repair the seams around the cluster contract Self-review found the advertised-address option cannot work. HG_SERVER_REST_URL becomes restserver.url, which is the address the REST server binds, not only the one registered with PD. A Server given an externally reachable address exits with java.net.BindException and restarts forever, confirmed by running the cluster with the documented override: that replica crash-looped while the two defaults stayed healthy. Separating bind from advertise needs a Server-side setting, so the three overrides are removed and the limitation is documented instead. The Server healthcheck now asserts only that an unauthenticated graph request returns 401. That is what proves the image enforces authentication, and unlike the authenticated half it stays correct after an operator rotates the admin password through the API, which would otherwise leave every replica permanently unhealthy on a working cluster. The shared Hubble volumes are external now. A fixed name is not enough: Compose removes a fixed-name non-external volume on `down -v` from any project that declares the name, so leaving the attach flow destroyed the combined flow's H2 database. The setup block creates them alongside the network, and the H2 path is absolute so it cannot drift outside the mount. Also: the image pull policy is overridable, since `always` with no escape made the cluster unusable offline even with every image cached; the non-auth flow documents the network, volumes and its own teardown, which the guards demand on `down` too; the dotenv reader rejects unquoted values instead of returning an empty password, and its block runs in a subshell so a failure cannot close an interactive shell; CI unsets the operator-facing variables it asserts defaults for, covers the non-auth properties with the same topology contract, and runs the live smoke test only when this contract changes, since it boots published images and should not redden unrelated pull requests. --- .github/workflows/server-ci.yml | 58 ++++++++-- docker/README.md | 111 ++++++++++++++----- docker/docker-compose-3pd-3store-3server.yml | 24 ++-- docker/docker-compose-hubble.yml | 14 ++- docker/hugegraph-hubble-3x3.properties | 4 + 5 files changed, 164 insertions(+), 47 deletions(-) diff --git a/.github/workflows/server-ci.yml b/.github/workflows/server-ci.yml index 8bb8252b94..9728849d47 100644 --- a/.github/workflows/server-ci.yml +++ b/.github/workflows/server-ci.yml @@ -214,10 +214,12 @@ jobs: (.services.hubble.networks | has("hg-net")) and (.services.hubble | has("depends_on") | not) and .volumes."hg-hubble-db".name == "hugegraph-hubble-db" and + .volumes."hg-hubble-db".external == true and .volumes."hg-hubble-upload-files".name == "hugegraph-hubble-upload-files" and + .volumes."hg-hubble-upload-files".external == true and .services.hubble.environment.SPRING_DATASOURCE_URL == - "jdbc:h2:file:./db/hubble;DB_CLOSE_ON_EXIT=FALSE" and + "jdbc:h2:file:/hubble/db/hubble;DB_CLOSE_ON_EXIT=FALSE" and any(.services.hubble.volumes[]; .target == "/hubble/conf/hugegraph-hubble.properties" and (.source | endswith("hugegraph-hubble-3x3.properties"))) @@ -236,6 +238,8 @@ jobs: HUGEGRAPH_ADMIN_PASSWORD=ci-test-password \ HUGEGRAPH_AUTH_TOKEN_SECRET="${token_fixture}" \ env -u HUGEGRAPH_NETWORK -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY \ + -u HUGEGRAPH_PULL_POLICY -u HUGEGRAPH_CONTROL_PLANE_HOST \ + -u HUGEGRAPH_SERVER_PUBLISH_HOST -u HUBBLE_PROPERTIES \ -u HUBBLE_PUBLISH_HOST \ docker compose --env-file /dev/null -f "$cluster" \ config --format json > "$rendered" @@ -253,6 +257,8 @@ jobs: HUGEGRAPH_VERSION=ci-test-tag \ env -u HUBBLE_IMAGE -u HUBBLE_PULL_POLICY -u HUBBLE_PUBLISH_HOST \ -u HUBBLE_DB_VOLUME -u HUBBLE_UPLOAD_VOLUME \ + -u HUGEGRAPH_PULL_POLICY -u HUGEGRAPH_CONTROL_PLANE_HOST \ + -u HUGEGRAPH_SERVER_PUBLISH_HOST -u HUBBLE_PROPERTIES \ docker compose --env-file /dev/null -f "$cluster" -f "$addon" \ config --format json > "$rendered" jq -e ' @@ -275,6 +281,8 @@ jobs: "ci-test-password" and $root.services[.].environment.HG_SERVER_AUTH_TOKEN_SECRET == "ci-test-token-secret-32-bytes-long") and + (.services.server0.healthcheck.test[1] | + contains("= 401") and (contains("PASSWORD") | not)) and .services.server0.environment.HG_SERVER_REST_URL == "http://server0:8080" and .services.server1.environment.HG_SERVER_REST_URL == @@ -284,10 +292,12 @@ jobs: (.services.hubble | has("depends_on") | not) and .services.hubble.pull_policy == "always" and .volumes."hg-hubble-db".name == "hugegraph-hubble-db" and + .volumes."hg-hubble-db".external == true and .volumes."hg-hubble-upload-files".name == "hugegraph-hubble-upload-files" and + .volumes."hg-hubble-upload-files".external == true and .services.hubble.environment.SPRING_DATASOURCE_URL == - "jdbc:h2:file:./db/hubble;DB_CLOSE_ON_EXIT=FALSE" and + "jdbc:h2:file:/hubble/db/hubble;DB_CLOSE_ON_EXIT=FALSE" and (.services.hubble.healthcheck.test[1] | contains("http://127.0.0.1:8088/about") and contains("\"status\":200") and @@ -329,6 +339,7 @@ jobs: "$rendered")" assert_props "cluster=${cluster_name}" "matches HG_SERVER_CLUSTER" assert_props "pd.enabled=true" "keeps Hubble in PD mode" + assert_props "auth.enabled=true" "keeps Hubble in authenticated mode" assert_props "pd.peers=${pd_peers}" "matches HG_SERVER_PD_PEERS" assert_props "pd.server=${pd_rest}" "names a real PD REST endpoint" assert_props "operations.store.allowed_targets=${store_targets}" \ @@ -356,14 +367,42 @@ jobs: ($root.services[.].healthcheck.test[1] | contains("/versions") and (contains("401") | not))) ' "$rendered" >/dev/null - grep -Fqx "auth.enabled=false" \ - docker/hugegraph-hubble-3x3.non-auth.properties + # The non-auth properties file is never booted by CI, so without + # this its topology can drift from the compose files unnoticed and + # users become the detection mechanism. Same contract, same source + # of truth, only the auth mode differs. + props="docker/hugegraph-hubble-3x3.non-auth.properties" + assert_props "auth.enabled=false" "runs Hubble without a login" + assert_props "cluster=${cluster_name}" "matches HG_SERVER_CLUSTER" + assert_props "pd.enabled=true" "keeps Hubble in PD mode" + assert_props "pd.peers=${pd_peers}" "matches HG_SERVER_PD_PEERS" + assert_props "pd.server=${pd_rest}" "names a real PD REST endpoint" + assert_props "operations.store.allowed_targets=${store_targets}" \ + "lists every Store REST endpoint" } check_cluster_compose - - name: Run distributed Compose auth and attach smoke test + # Only runs when this contract changes. The step boots the published + # images to prove the Compose wiring end to end, so leaving it on every + # build would turn a slow or rate-limited registry into a red run for + # pull requests that touch none of these files. + - name: Check whether the Compose contract changed if: ${{ env.BACKEND == 'rocksdb' }} + id: compose_changed + run: | + set -euo pipefail + base="${{ github.event.pull_request.base.sha }}" + changed=true + if [ -n "$base" ] && git cat-file -e "$base^{commit}" 2>/dev/null; then + if git diff --quiet "$base" HEAD -- docker/ .github/workflows/server-ci.yml; then + changed=false + fi + fi + echo "changed=$changed" >> "$GITHUB_OUTPUT" + + - name: Run distributed Compose auth and attach smoke test + if: ${{ env.BACKEND == 'rocksdb' && steps.compose_changed.outputs.changed == 'true' }} run: | set -euo pipefail command -v docker >/dev/null 2>&1 @@ -397,10 +436,15 @@ jobs: } trap cleanup EXIT docker network create "$network" + # Both Hubble volumes are external, so nothing creates them implicitly. + docker volume create "$db_volume" >/dev/null + docker volume create "$upload_volume" >/dev/null # The cluster must come up healthy on its own. Its Server healthcheck - # already proves 401/200, so --wait failing here means the images do - # not enforce the authentication this stack configures. + # already requires an unauthenticated graph request to return 401, so + # --wait failing here means the images do not enforce the + # authentication this stack configures. The authenticated half is + # asserted below, where the credentials are known to be current. docker compose --env-file /dev/null -f "$cluster" up -d --wait \ --wait-timeout 420 cluster_ids_before="$(docker compose --env-file /dev/null -f "$cluster" \ diff --git a/docker/README.md b/docker/README.md index ad6c479ed7..17d7673551 100644 --- a/docker/README.md +++ b/docker/README.md @@ -226,7 +226,11 @@ on every Compose subcommand, including `down`. key_pattern="^[[:space:]]*(export[[:space:]]+)?${key}=" count="$(grep -Ec "${key_pattern}" .env || true)" [ "${count}" -eq 1 ] || return 1 - sed -nE "s/${key_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env + value="$(sed -nE "s/${key_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env)" + # sed prints nothing and still exits 0 when the value is unquoted or + # double-quoted, which would otherwise hand back an empty credential. + [ -n "${value}" ] || return 1 + printf '%s\n' "${value}" } if ! admin_value="$(read_dotenv_value HUGEGRAPH_ADMIN_PASSWORD)"; then echo "HUGEGRAPH_ADMIN_PASSWORD must use the documented single-quoted format" >&2 @@ -251,6 +255,14 @@ on every Compose subcommand, including `down`. net="${HUGEGRAPH_NETWORK:-hugegraph-net}" docker network inspect "${net}" >/dev/null 2>&1 || docker network create "${net}" + # Hubble's volumes are external for the same reason the network is: both + # documented flows share them, and an external volume is not destroyed by a + # `down -v` from either one. + for vol in "${HUBBLE_DB_VOLUME:-hugegraph-hubble-db}" \ + "${HUBBLE_UPLOAD_VOLUME:-hugegraph-hubble-upload-files}"; do + docker volume inspect "${vol}" >/dev/null 2>&1 || + docker volume create "${vol}" >/dev/null + done env -u HUGEGRAPH_ADMIN_PASSWORD -u HUGEGRAPH_AUTH_TOKEN_SECRET \ docker compose -f docker-compose-3pd-3store-3server.yml config --quiet ) @@ -285,15 +297,20 @@ authenticated one to return `200`. An image that does not enforce authentication therefore never reports healthy, and `up -d --wait` fails instead of handing you a false green. -Each Server registers its `HG_SERVER_REST_URL` with PD. The defaults -(`server0`, `server1`, `server2`) are Docker-network names, which is correct -for Hubble and anything else attached to `hugegraph-net`, but they do not -resolve from an unrelated host network. A PD-aware client running outside -Docker will receive names it cannot reach. For that case, set -`HUGEGRAPH_SERVER0_REST_URL`, `HUGEGRAPH_SERVER1_REST_URL`, and -`HUGEGRAPH_SERVER2_REST_URL` to addresses that resolve from both the Server -containers and that client, and publish the matching host ports with -`HUGEGRAPH_SERVER_PUBLISH_HOST`. +Each Server registers its `HG_SERVER_REST_URL` with PD, and the defaults +(`server0`, `server1`, `server2`) are Docker-network names. That is correct +for Hubble and anything else attached to `hugegraph-net`, and it is a known +limitation for anything that is not: a PD-aware client running outside Docker +discovers names it cannot resolve. + +Pointing `HG_SERVER_REST_URL` at an externally reachable address does not fix +this, so this stack does not offer it as an option. The same value is also the +address the REST server binds, so a Server given an address that is not local +to its container exits with `java.net.BindException: Cannot assign requested +address` and restarts forever. Separating the two needs an advertised-address +setting in the Server itself, which is outside the scope of these Compose +files. Until then, reach the cluster from outside Docker through the published +Server ports directly rather than through PD discovery. > [!NOTE] > Upgrading an existing 3-node deployment: @@ -304,10 +321,16 @@ containers and that client, and publish the matching host ports with > Named data volumes are unchanged and survive the move; the orphaned > `hugegraph-3x3_hg-net` bridge can be removed with > `docker network rm hugegraph-3x3_hg-net`. +> - The published ports now bind `127.0.0.1` instead of every interface, so a +> client on another machine loses the connection outright rather than seeing +> a 401. Set `HUGEGRAPH_SERVER_PUBLISH_HOST` (and +> `HUGEGRAPH_CONTROL_PLANE_HOST` for PD and Store) to restore reachability, +> behind a network ACL or TLS terminator. > - Authentication is now enabled: previously unauthenticated clients of the -> graph APIs on ports 8080–8082 will start receiving 401 responses and -> must supply the `admin` credential from `docker/.env` (`/versions` and -> `/openapi.json` stay open, so they cannot serve as an auth smoke test). +> graph APIs on ports 8080-8082 that can still reach the host will start +> receiving 401 responses and must supply the `admin` credential from +> `docker/.env` (`/versions` and `/openapi.json` stay open, so they cannot +> serve as an auth smoke test). > On a cluster whose volumes predate authentication, verify you can sign > in before decommissioning any existing access path. @@ -361,12 +384,19 @@ done # And a signed-in read must succeed. Parse the generated single-quoted # value without executing docker/.env as shell. Passing the credential # through --config keeps it out of argv, where `ps` would expose it. +# The subshell keeps the `exit 1` below from closing an interactive shell. +( +set -eu read_dotenv_value() { key="$1" key_pattern="^[[:space:]]*(export[[:space:]]+)?${key}=" count="$(grep -Ec "${key_pattern}" .env || true)" [ "${count}" -eq 1 ] || return 1 - sed -nE "s/${key_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env + value="$(sed -nE "s/${key_pattern}'([^']*)'[[:space:]]*$/\\2/p" .env)" + # sed prints nothing and still exits 0 when the value is unquoted or + # double-quoted, which would otherwise hand back an empty credential. + [ -n "${value}" ] || return 1 + printf '%s\n' "${value}" } HUGEGRAPH_ADMIN_PASSWORD="$(read_dotenv_value HUGEGRAPH_ADMIN_PASSWORD)" || { echo "HUGEGRAPH_ADMIN_PASSWORD must use the documented single-quoted format" >&2 @@ -375,7 +405,7 @@ HUGEGRAPH_ADMIN_PASSWORD="$(read_dotenv_value HUGEGRAPH_ADMIN_PASSWORD)" || { curl -s -o /dev/null -w '%{http_code}\n' \ --config <(printf 'user = "admin:%s"\n' "${HUGEGRAPH_ADMIN_PASSWORD}") \ http://localhost:8080/graphs/hugegraph/schema/vertexlabels -unset HUGEGRAPH_ADMIN_PASSWORD +) ``` `200` from the second command with `401` from all three of the first means @@ -397,12 +427,21 @@ use it anywhere else. ```bash cd docker -# No docker/.env and no credential setup are needed for this mode. The two -# placeholders exist only because Compose evaluates the base file's -# required-variable guards before applying the override, which then drops -# both variables, so neither value reaches a container. Needs Compose v2.24+. +# The network and Hubble's volumes are external in every flow, so create them +# once even here. This mode skips only the credential half of the setup block. +docker network inspect hugegraph-net >/dev/null 2>&1 || + docker network create hugegraph-net +for vol in hugegraph-hubble-db hugegraph-hubble-upload-files; do + docker volume inspect "${vol}" >/dev/null 2>&1 || + docker volume create "${vol}" >/dev/null +done + +# No docker/.env is needed. The two placeholders exist only because Compose +# evaluates the base file's required-variable guards before applying the +# override, which then drops both variables, so neither value reaches a +# container. Needs Compose v2.24+. # -# Keep them inline on this command; do not write them into docker/.env. They +# Keep them inline on every command; do not write them into docker/.env. They # are published values rather than secrets, and the token placeholder is # deliberately shorter than the 32 bytes the Server requires, so if it ever # reaches an authenticated Server that Server refuses to start instead of @@ -415,6 +454,13 @@ HUGEGRAPH_AUTH_TOKEN_SECRET=non-auth-placeholder \ # Every graph request now succeeds without credentials: expect 200, not 401. curl -s -o /dev/null -w '%{http_code}\n' \ http://localhost:8080/graphs/hugegraph/schema/vertexlabels + +# Teardown. The base file's `:?` guards fire on every subcommand, `down` +# included, so the same two placeholders are required here. +HUGEGRAPH_ADMIN_PASSWORD=non-auth-placeholder \ +HUGEGRAPH_AUTH_TOKEN_SECRET=non-auth-placeholder \ + docker compose -f docker-compose-3pd-3store-3server.yml \ + -f docker-compose-3x3.non-auth.yml down -v ``` Hubble needs the matching configuration, because its own default is to demand @@ -425,6 +471,13 @@ HUBBLE_PROPERTIES=./hugegraph-hubble-3x3.non-auth.properties \ docker compose -p hugegraph-hubble -f docker-compose-hubble.yml up -d ``` +`HUBBLE_PROPERTIES` is not remembered between commands. It selects a bind +mount, so any later Compose command for Hubble that omits it silently +recreates the container against the authenticated properties file, and Hubble +comes back demanding a login the Servers cannot serve. Repeat the variable on +every Hubble command in this mode, including `up -d --no-deps hubble` and the +local-image commands below. +
Prompt for an AI assistant @@ -538,12 +591,21 @@ docker compose -p hugegraph-hubble -f docker-compose-hubble.yml ps docker compose -p hugegraph-hubble -f docker-compose-hubble.yml down ``` -To remove everything in this flow, take down Hubble first, then the cluster: +To remove everything in this flow, take down Hubble first, then the cluster. +The cluster file's `:?` guards fire on `down` too, so this needs `docker/.env` +or the same two variables in the environment. If you attached to a cluster +someone else started and never had those credentials, remove only Hubble with +the command above and leave the cluster to whoever owns it. ```bash cd docker docker compose -p hugegraph-hubble -f docker-compose-hubble.yml down docker compose -f docker-compose-3pd-3store-3server.yml down -v + +# Hubble's volumes are external, so no `down` removes them. Drop them +# explicitly when you want the H2 database and uploaded files gone, or a +# later deployment reattaches the old state. +docker volume rm hugegraph-hubble-db hugegraph-hubble-upload-files ``` ### Fresh cluster plus Hubble in one command @@ -696,11 +758,9 @@ add-on = `docker-compose-hubble.yml`): | `HUBBLE_DB_VOLUME` | add-on | `hugegraph-hubble-db` | Volume holding Hubble's H2 database; the explicit name is what lets the attach and combined flows share state, so change it only to run a second independent Hubble | | `HUBBLE_UPLOAD_VOLUME` | add-on | `hugegraph-hubble-upload-files` | Volume holding Hubble's uploaded files; same naming caveat as `HUBBLE_DB_VOLUME` | | `HUGEGRAPH_NETWORK` | cluster, add-on | `hugegraph-net` | Pre-created external Docker network shared by the 3-node cluster and the Hubble add-on; the single-node files use their own project bridge instead | +| `HUGEGRAPH_PULL_POLICY` | cluster | `always` | Pull policy for the PD, Store, and Server images. The default refreshes a stale cached `latest`; set it to `missing` to run offline or against locally built tags | | `HUGEGRAPH_CONTROL_PLANE_HOST` | cluster | `127.0.0.1` | Host bind address for the PD and Store REST/gRPC/Raft ports; these APIs have no real authentication, so widen this only behind a network ACL or TLS | | `HUGEGRAPH_SERVER_PUBLISH_HOST` | cluster | `127.0.0.1` | Host bind address for the three Server REST ports | -| `HUGEGRAPH_SERVER0_REST_URL` | cluster | `http://server0:8080` | URL Server 0 registers with PD; must resolve from the Server container and from every PD-aware client | -| `HUGEGRAPH_SERVER1_REST_URL` | cluster | `http://server1:8080` | URL Server 1 registers with PD; same resolution requirement | -| `HUGEGRAPH_SERVER2_REST_URL` | cluster | `http://server2:8080` | URL Server 2 registers with PD; same resolution requirement | | `HUGEGRAPH_ADMIN_PASSWORD` | single, cluster | required (`docker/.env`) | Initial admin password; no public default is provided | | `HUGEGRAPH_AUTH_TOKEN_SECRET` | single, cluster | generated (single); **required** (cluster) | JWT signing secret; explicit values must be at least 32 bytes. The cluster file requires it so all Server replicas validate each other's tokens | @@ -815,7 +875,8 @@ The single-node Compose file publishes `8620`, `8520`, `8080`, and Hubble |---------|----------|----------| | PD | `GET /v1/health` | `200 OK` | | Store | `GET /v1/health` | `200 OK` | -| Server | `GET /versions` | `200 OK` with version JSON | +| Server (single-node files) | `GET /versions` | `200 OK` with version JSON | +| Server (3-node cluster) | `GET /graphs/hugegraph/schema/vertexlabels`, unauthenticated | `401`, which proves the image enforces authentication; `/versions` is open by design and cannot | | Hubble | `GET /about` | `200` JSON with Hubble name and version | --- diff --git a/docker/docker-compose-3pd-3store-3server.yml b/docker/docker-compose-3pd-3store-3server.yml index a69e47b97a..0197a79d7a 100644 --- a/docker/docker-compose-3pd-3store-3server.yml +++ b/docker/docker-compose-3pd-3store-3server.yml @@ -39,7 +39,7 @@ x-pd-common: &pd-common # Pin a release via HUGEGRAPH_VERSION in docker/.env; unset, the image # tags default to latest. All Compose files here read the same variable. image: hugegraph/pd:${HUGEGRAPH_VERSION:-latest} - pull_policy: always + pull_policy: ${HUGEGRAPH_PULL_POLICY:-always} restart: unless-stopped networks: [hg-net] healthcheck: @@ -51,7 +51,7 @@ x-pd-common: &pd-common x-store-common: &store-common image: hugegraph/store:${HUGEGRAPH_VERSION:-latest} - pull_policy: always + pull_policy: ${HUGEGRAPH_PULL_POLICY:-always} restart: unless-stopped networks: [hg-net] depends_on: @@ -85,7 +85,7 @@ x-server-env: &server-env x-server-common: &server-common image: hugegraph/server:${HUGEGRAPH_VERSION:-latest} - pull_policy: always + pull_policy: ${HUGEGRAPH_PULL_POLICY:-always} restart: unless-stopped networks: [hg-net] depends_on: @@ -93,11 +93,13 @@ x-server-common: &server-common store1: { condition: service_healthy } store2: { condition: service_healthy } healthcheck: - # Prove the image actually enforces authentication. Older cached images - # may accept PASSWORD but ignore it while still returning healthy on - # /versions, so readiness requires both 401 without credentials and 200 - # with the configured admin password. - test: ["CMD-SHELL", "base=$${HG_SERVER_REST_URL}; unauth=$$(curl -s -o /dev/null -w '%{http_code}' \"$${base}/graphs/hugegraph/schema/vertexlabels\"); auth=$$(curl -s -o /dev/null -w '%{http_code}' -u \"admin:$${PASSWORD}\" \"$${base}/graphs/hugegraph/schema/vertexlabels\"); test \"$${unauth}\" = 401 && test \"$${auth}\" = 200"] + # Prove the image actually enforces authentication. /versions is open by + # design, so an image that ignores PASSWORD still answers it and looks + # healthy while the graph APIs are wide open. Requiring 401 on a graph + # request catches exactly that, and it stays correct after an operator + # rotates the admin password through the API, which an authenticated + # probe pinned to the seeded PASSWORD would not. + test: ["CMD-SHELL", "test \"$$(curl -s -o /dev/null -w '%{http_code}' \"$${HG_SERVER_REST_URL}/graphs/hugegraph/schema/vertexlabels\")\" = 401"] interval: 10s timeout: 5s retries: 30 @@ -226,7 +228,7 @@ services: hostname: server0 environment: <<: *server-env - HG_SERVER_REST_URL: ${HUGEGRAPH_SERVER0_REST_URL:-http://server0:8080} + HG_SERVER_REST_URL: http://server0:8080 ports: - "${HUGEGRAPH_SERVER_PUBLISH_HOST:-127.0.0.1}:8080:8080" @@ -236,7 +238,7 @@ services: hostname: server1 environment: <<: *server-env - HG_SERVER_REST_URL: ${HUGEGRAPH_SERVER1_REST_URL:-http://server1:8080} + HG_SERVER_REST_URL: http://server1:8080 ports: - "${HUGEGRAPH_SERVER_PUBLISH_HOST:-127.0.0.1}:8081:8080" @@ -246,6 +248,6 @@ services: hostname: server2 environment: <<: *server-env - HG_SERVER_REST_URL: ${HUGEGRAPH_SERVER2_REST_URL:-http://server2:8080} + HG_SERVER_REST_URL: http://server2:8080 ports: - "${HUGEGRAPH_SERVER_PUBLISH_HOST:-127.0.0.1}:8082:8080" diff --git a/docker/docker-compose-hubble.yml b/docker/docker-compose-hubble.yml index 67b1b6c4cb..9e821c2ddb 100644 --- a/docker/docker-compose-hubble.yml +++ b/docker/docker-compose-hubble.yml @@ -26,12 +26,17 @@ networks: name: ${HUGEGRAPH_NETWORK:-hugegraph-net} volumes: + # Pre-created and external, like the shared network above. A fixed name + # alone is not enough: Compose deletes a fixed-name non-external volume on + # `down -v` from ANY project that declares the name, so tearing down the + # attach flow would silently destroy the combined flow's Hubble database. + # `external` makes both flows share the state and survive either teardown. hg-hubble-db: - # Explicit names so attach (-p hugegraph-hubble) and combined - # (project hugegraph-3x3) share the same physical volumes. name: ${HUBBLE_DB_VOLUME:-hugegraph-hubble-db} + external: true hg-hubble-upload-files: name: ${HUBBLE_UPLOAD_VOLUME:-hugegraph-hubble-upload-files} + external: true services: hubble: @@ -48,8 +53,9 @@ services: - "${HUBBLE_PUBLISH_HOST:-127.0.0.1}:8088:8088" environment: # Image default jdbc:h2:file:./db writes /hubble/db.mv.db, outside - # the /hubble/db volume. Point H2 at a file inside the mount. - SPRING_DATASOURCE_URL: jdbc:h2:file:./db/hubble;DB_CLOSE_ON_EXIT=FALSE + # the /hubble/db volume. Use an absolute path so the database stays in + # the mount even if a future image changes its working directory. + SPRING_DATASOURCE_URL: jdbc:h2:file:/hubble/db/hubble;DB_CLOSE_ON_EXIT=FALSE volumes: # Point HUBBLE_PROPERTIES at the non-auth variant when the cluster runs # without authentication; see "Running without authentication". diff --git a/docker/hugegraph-hubble-3x3.properties b/docker/hugegraph-hubble-3x3.properties index af7f4a940b..424cf11c94 100644 --- a/docker/hugegraph-hubble-3x3.properties +++ b/docker/hugegraph-hubble-3x3.properties @@ -29,6 +29,10 @@ cluster=hg idc=docker pd.enabled=true +# Pin the mode explicitly. Absent keys fall back to the code defaults, which a +# future Hubble release can change, and the non-auth twin of this file sets the +# opposite value. +auth.enabled=true # Unused while pd.enabled=true: in PD mode Hubble picks a Server per request # from PD-discovered addresses and never reads this value, so editing it has # no effect on which replica is used. Kept only for the standalone fallback From 05715714378d3af2d66678ac71dc7fe276ab5781 Mon Sep 17 00:00:00 2001 From: Himanshu Verma Date: Tue, 25 Aug 2026 18:01:23 +0530 Subject: [PATCH 14/14] docs(docker): scope the auth safety net to the cluster that has it The version-pinning paragraph opened by naming the cluster, the add-on and the single-node quickstart, then concluded that an image which ignores PASSWORD never reports healthy. Only the 3-node cluster has that check. The single-node files probe /versions, which stays open whether authentication works or not, so a reader who pinned a version exactly as instructed got no protection there and no warning about it. The same paragraph also still described the old healthcheck, which required an authenticated 200 alongside the 401. It asserts only the 401 now. --- docker/README.md | 17 +++++++++++------ 1 file changed, 11 insertions(+), 6 deletions(-) diff --git a/docker/README.md b/docker/README.md index 17d7673551..dd66a1159e 100644 --- a/docker/README.md +++ b/docker/README.md @@ -290,12 +290,17 @@ Unpinned, the images default to `latest`; note the authenticated PD/Hubble integration requires a release newer than `1.7.x`. On an older image the Server ignores `PASSWORD` and `HG_SERVER_AUTH_TOKEN_SECRET`, which would leave you an unauthenticated cluster. Two defaults keep that from passing -unnoticed: the cluster and add-on use `pull_policy: always`, so a stale -cached `latest` is refreshed on every `up -d`, and the Server healthcheck -requires an unauthenticated graph request to return `401` and an -authenticated one to return `200`. An image that does not enforce -authentication therefore never reports healthy, and `up -d --wait` fails -instead of handing you a false green. +unnoticed **in the 3-node cluster**: it pulls on every `up -d`, so a stale +cached `latest` is refreshed, and its Server healthcheck requires an +unauthenticated graph request to return `401`. An image that does not enforce +authentication therefore never reports healthy there, and `up -d --wait` +fails instead of handing you a false green. + +The single-node files have no such net. Their Server healthcheck probes +`/versions`, which stays open whether authentication works or not, so an +incompatible image reports healthy while the graph APIs are unauthenticated. +Run the 401 check under "Verify the cluster is healthy" by hand against +`localhost:8080` after starting them. Each Server registers its `HG_SERVER_REST_URL` with PD, and the defaults (`server0`, `server1`, `server2`) are Docker-network names. That is correct