From 4ad3bf37754f39307517f2da7f82723c68ab760f Mon Sep 17 00:00:00 2001 From: Christopher Meng Date: Sat, 13 Jun 2026 09:41:05 -0400 Subject: [PATCH 1/3] Add IDN support rsync can now connect to IDN (internationalized domain name) hosts, and IDN names are recognized in a daemon's hosts allow/deny. --- .github/workflows/almalinux-8-build.yml | 3 +- .github/workflows/android-static-build.yml | 2 +- .github/workflows/asan-build.yml | 2 +- .github/workflows/coverage.yml | 2 +- .github/workflows/cygwin-build.yml | 2 +- .github/workflows/fleettest.yml | 2 +- .github/workflows/freebsd-build.yml | 2 +- .github/workflows/macos-build.yml | 2 +- .github/workflows/netbsd-build.yml | 2 +- .github/workflows/openbsd-build.yml | 2 +- .github/workflows/scan-build.yml | 2 +- .github/workflows/solaris-build.yml | 2 +- .github/workflows/ubuntu-22.04-build.yml | 2 +- .github/workflows/ubuntu-build.yml | 2 +- .github/workflows/ubuntu-version-mix.yml | 2 +- .github/workflows/valgrind.yml | 2 +- access.c | 14 ++ configure.ac | 23 +++- main.c | 15 +++ socket.c | 12 ++ testsuite/idn_test.py | 148 +++++++++++++++++++++ usage.c | 5 + 22 files changed, 233 insertions(+), 17 deletions(-) create mode 100644 testsuite/idn_test.py diff --git a/.github/workflows/almalinux-8-build.yml b/.github/workflows/almalinux-8-build.yml index 7a9ab7761..fb8718728 100644 --- a/.github/workflows/almalinux-8-build.yml +++ b/.github/workflows/almalinux-8-build.yml @@ -51,7 +51,8 @@ jobs: attr libattr-devel acl libacl-devel \ zstd libzstd-devel \ lz4 lz4-devel \ - xxhash xxhash-devel + xxhash xxhash-devel \ + libidn2 libidn2-devel alternatives --set python3 /usr/bin/python3.9 pip3 install commonmark - name: configure diff --git a/.github/workflows/android-static-build.yml b/.github/workflows/android-static-build.yml index 5d4216568..32cdc6467 100644 --- a/.github/workflows/android-static-build.yml +++ b/.github/workflows/android-static-build.yml @@ -79,7 +79,7 @@ jobs: # checksums and its bundled zlib. ./configure --host=${{ matrix.triple }} --build=x86_64-pc-linux-gnu \ --enable-ipv6 \ - --disable-zstd --disable-lz4 --disable-xxhash --disable-openssl \ + --disable-zstd --disable-lz4 --disable-xxhash --disable-openssl --disable-idn \ --disable-iconv --disable-iconv-open \ --disable-acl-support --disable-xattr-support \ --disable-md2man --disable-roll-simd \ diff --git a/.github/workflows/asan-build.yml b/.github/workflows/asan-build.yml index 4f4b6b5ce..a1dacadf1 100644 --- a/.github/workflows/asan-build.yml +++ b/.github/workflows/asan-build.yml @@ -44,7 +44,7 @@ jobs: - name: prep run: | sudo apt-get update - sudo apt-get install -y clang acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev openssl + sudo apt-get install -y clang acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev openssl echo "/usr/local/bin" >>"$GITHUB_PATH" - name: configure # -DNDEBUG builds as a shipped release does (assert() compiled out), so diff --git a/.github/workflows/coverage.yml b/.github/workflows/coverage.yml index 88ffde508..3e6feacd1 100644 --- a/.github/workflows/coverage.yml +++ b/.github/workflows/coverage.yml @@ -29,7 +29,7 @@ jobs: - name: prep run: | sudo apt-get update - sudo apt-get install -y acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl gcovr + sudo apt-get install -y acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl gcovr echo "/usr/local/bin" >>"$GITHUB_PATH" - name: configure run: ./configure --enable-coverage --with-rrsync diff --git a/.github/workflows/cygwin-build.yml b/.github/workflows/cygwin-build.yml index 1a72ad4fc..ab5d9b8a7 100644 --- a/.github/workflows/cygwin-build.yml +++ b/.github/workflows/cygwin-build.yml @@ -29,7 +29,7 @@ jobs: run: choco install -y --no-progress cygwin cyg-get - name: prep run: | - cyg-get make autoconf automake gcc-core attr libattr-devel python39 python39-pip libzstd-devel liblz4-devel libssl-devel libxxhash0 libxxhash-devel + cyg-get make autoconf automake gcc-core attr libattr-devel python39 python39-pip libzstd-devel liblz4-devel libssl-devel libxxhash0 libxxhash-devel libidn2-devel echo "C:/tools/cygwin/bin" >>$Env:GITHUB_PATH - name: commonmark run: bash -c 'python3 -mpip install --user commonmark' diff --git a/.github/workflows/fleettest.yml b/.github/workflows/fleettest.yml index d79b6e804..1e8539623 100644 --- a/.github/workflows/fleettest.yml +++ b/.github/workflows/fleettest.yml @@ -42,7 +42,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y gcc g++ gawk autoconf automake \ - acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev \ + acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev \ python3-cmarkgfm openssl rsync openssh-server - name: set up ssh to localhost run: | diff --git a/.github/workflows/freebsd-build.yml b/.github/workflows/freebsd-build.yml index c4039cc7c..d4b8f1f8f 100644 --- a/.github/workflows/freebsd-build.yml +++ b/.github/workflows/freebsd-build.yml @@ -34,7 +34,7 @@ jobs: pkg install -y bash autotools m4 devel/xxhash zstd liblz4 python3 archivers/liblz4 git run: | freebsd-version - ./configure --with-rrsync -disable-zstd --disable-md2man --disable-xxhash --disable-lz4 + ./configure --with-rrsync -disable-zstd --disable-md2man --disable-xxhash --disable-lz4 --disable-idn make ./rsync --version make check diff --git a/.github/workflows/macos-build.yml b/.github/workflows/macos-build.yml index 88080e97a..2190f7784 100644 --- a/.github/workflows/macos-build.yml +++ b/.github/workflows/macos-build.yml @@ -27,7 +27,7 @@ jobs: fetch-depth: 0 - name: prep run: | - brew install automake openssl xxhash zstd lz4 + brew install automake openssl xxhash zstd lz4 libidn2 pip3 install --user --break-system-packages commonmark echo "$(brew --prefix)/bin" >>"$GITHUB_PATH" - name: configure diff --git a/.github/workflows/netbsd-build.yml b/.github/workflows/netbsd-build.yml index 02fa387b2..cb16fef8e 100644 --- a/.github/workflows/netbsd-build.yml +++ b/.github/workflows/netbsd-build.yml @@ -35,7 +35,7 @@ jobs: ln -sf /usr/pkg/bin/python3.12 /usr/pkg/bin/python3 run: | uname -a - ./configure --with-rrsync --disable-zstd --disable-md2man --disable-xxhash --disable-lz4 + ./configure --with-rrsync --disable-zstd --disable-md2man --disable-xxhash --disable-lz4 --disable-idn make ./rsync --version make check diff --git a/.github/workflows/openbsd-build.yml b/.github/workflows/openbsd-build.yml index 89d06ad15..7f43657b5 100644 --- a/.github/workflows/openbsd-build.yml +++ b/.github/workflows/openbsd-build.yml @@ -36,7 +36,7 @@ jobs: uname -a export AUTOCONF_VERSION=2.71 export AUTOMAKE_VERSION=1.16 - ./configure --with-rrsync --disable-zstd --disable-md2man --disable-xxhash --disable-lz4 + ./configure --with-rrsync --disable-zstd --disable-md2man --disable-xxhash --disable-lz4 --disable-idn make ./rsync --version # The flipper (symlink-race) tests are excluded on OpenBSD, as on the diff --git a/.github/workflows/scan-build.yml b/.github/workflows/scan-build.yml index 42f02cc3c..e30e078ed 100644 --- a/.github/workflows/scan-build.yml +++ b/.github/workflows/scan-build.yml @@ -32,7 +32,7 @@ jobs: - name: prep run: | sudo apt-get update - sudo apt-get install -y clang-18 clang-tools-18 acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libpopt-dev openssl + sudo apt-get install -y clang-18 clang-tools-18 acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev libpopt-dev openssl - name: configure (under scan-build) # Run configure under scan-build so its analyzer compiler-wrapper is baked # into the Makefile's $(CC); --disable-md2man avoids the doc toolchain. diff --git a/.github/workflows/solaris-build.yml b/.github/workflows/solaris-build.yml index b2bea760f..8896a156a 100644 --- a/.github/workflows/solaris-build.yml +++ b/.github/workflows/solaris-build.yml @@ -34,7 +34,7 @@ jobs: pkg install bash automake gnu-m4 pkg://solaris/runtime/python-35 autoconf gcc git run: | uname -a - ./configure --with-rrsync -disable-zstd --disable-md2man --disable-xxhash --disable-lz4 + ./configure --with-rrsync -disable-zstd --disable-md2man --disable-xxhash --disable-lz4 --disable-idn make ./rsync --version make check diff --git a/.github/workflows/ubuntu-22.04-build.yml b/.github/workflows/ubuntu-22.04-build.yml index 8c110d942..df0142621 100644 --- a/.github/workflows/ubuntu-22.04-build.yml +++ b/.github/workflows/ubuntu-22.04-build.yml @@ -31,7 +31,7 @@ jobs: fetch-depth: 0 - name: prep run: | - sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl + sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl echo "/usr/local/bin" >>"$GITHUB_PATH" - name: configure run: ./configure --with-rrsync diff --git a/.github/workflows/ubuntu-build.yml b/.github/workflows/ubuntu-build.yml index 5fec4c884..3b76b68b1 100644 --- a/.github/workflows/ubuntu-build.yml +++ b/.github/workflows/ubuntu-build.yml @@ -27,7 +27,7 @@ jobs: fetch-depth: 0 - name: prep run: | - sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl + sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl echo "/usr/local/bin" >>"$GITHUB_PATH" - name: configure run: ./configure --with-rrsync diff --git a/.github/workflows/ubuntu-version-mix.yml b/.github/workflows/ubuntu-version-mix.yml index ee6c45e29..ef4fb8e77 100644 --- a/.github/workflows/ubuntu-version-mix.yml +++ b/.github/workflows/ubuntu-version-mix.yml @@ -48,7 +48,7 @@ jobs: fetch-depth: 0 - name: prep run: | - sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl + sudo apt-get install acl libacl1-dev attr libattr1-dev liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl echo "/usr/local/bin" >>"$GITHUB_PATH" - name: configure run: ./configure --with-rrsync diff --git a/.github/workflows/valgrind.yml b/.github/workflows/valgrind.yml index 49afaa020..55ed499be 100644 --- a/.github/workflows/valgrind.yml +++ b/.github/workflows/valgrind.yml @@ -36,7 +36,7 @@ jobs: run: | sudo apt-get update sudo apt-get install -y valgrind acl libacl1-dev attr libattr1-dev \ - liblz4-dev libzstd-dev libxxhash-dev python3-cmarkgfm openssl + liblz4-dev libzstd-dev libxxhash-dev libidn2-dev python3-cmarkgfm openssl echo "/usr/local/bin" >>"$GITHUB_PATH" - name: configure run: ./configure --with-rrsync --enable-debug diff --git a/access.c b/access.c index 059fa7399..529eae42b 100644 --- a/access.c +++ b/access.c @@ -23,6 +23,9 @@ #ifdef HAVE_NETGROUP_H #include #endif +#ifdef SUPPORT_IDN +#include +#endif static int allow_forward_dns; @@ -33,6 +36,9 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t struct hostent *hp; unsigned int i; const char *host = *host_ptr; +#ifdef SUPPORT_IDN + char idn_tok[1024], *idn; +#endif if (!host || !*host) return 0; @@ -42,6 +48,14 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t return innetgr(tok + 1, host, NULL, NULL); #endif +#ifdef SUPPORT_IDN + if (idn2_to_ascii_8z(tok, &idn, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL) == IDN2_OK) { + strlcpy(idn_tok, idn, sizeof idn_tok); + idn2_free(idn); + tok = idn_tok; + } +#endif + /* First check if the reverse-DNS-determined hostname matches. */ if (iwildmatch(tok, host)) return 1; diff --git a/configure.ac b/configure.ac index 57cf2828d..a2c8066a7 100644 --- a/configure.ac +++ b/configure.ac @@ -13,7 +13,7 @@ AC_CHECK_HEADERS(poll.h sys/fcntl.h sys/select.h fcntl.h sys/time.h sys/unistd.h sys/acl.h acl/libacl.h attr/xattr.h sys/xattr.h sys/extattr.h dl.h \ popt.h popt/popt.h linux/falloc.h netinet/in_systm.h netgroup.h \ zlib.h xxhash.h openssl/md4.h openssl/md5.h zstd.h lz4.h sys/file.h \ - sys/resource.h bsd/string.h) + sys/resource.h bsd/string.h idn2.h) AC_CHECK_HEADERS([netinet/ip.h], [], [], [[#include ]]) AC_HEADER_MAJOR_FIXED @@ -626,6 +626,27 @@ else AC_MSG_RESULT(no) fi +AC_MSG_CHECKING([whether to enable IDN support]) +AC_ARG_ENABLE([idn], + AS_HELP_STRING([--disable-idn], [disable to omit IDN (Internationalized Domain Name) support])) +AH_TEMPLATE([SUPPORT_IDN], +[Undefine if you do not want IDN support. By default this is defined.]) +if test x"$enable_idn" != x"no"; then + if test x"$ac_cv_header_idn2_h" = x"yes"; then + AC_MSG_RESULT(yes) + AC_SEARCH_LIBS(idn2_lookup_ul, idn2, + [AC_DEFINE(SUPPORT_IDN)], + [err_msg="$err_msg$nl- Failed to find idn2_lookup_ul function in idn2 lib."; + no_lib="$no_lib idn"]) + else + AC_MSG_RESULT(no) + err_msg="$err_msg$nl- Failed to find idn2.h for IDN support." + no_lib="$no_lib idn" + fi +else + AC_MSG_RESULT(no) +fi + if test x"$no_lib" != x; then echo "" echo "Configure found the following issues:" diff --git a/main.c b/main.c index 6050d0e5b..1c7e43ac9 100644 --- a/main.c +++ b/main.c @@ -34,6 +34,9 @@ #ifdef HAVE_SYS_RESOURCE_H #include #endif +#ifdef SUPPORT_IDN +#include +#endif extern int dry_run; extern int list_only; @@ -521,6 +524,18 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in char *args[MAX_ARGS], *need_to_free = NULL; pid_t pid; int dash_l_set = 0; +#ifdef SUPPORT_IDN + char idn_machine[1024]; + + if (machine && daemon_connection > 0) { + char *idn; + if (idn2_lookup_ul(machine, &idn, IDN2_NONTRANSITIONAL) == IDN2_OK) { + strlcpy(idn_machine, idn, sizeof idn_machine); + idn2_free(idn); + machine = idn_machine; + } + } +#endif if (!read_batch && !local_server) { char *t, *f, in_quote = '\0'; diff --git a/socket.c b/socket.c index c08cbe52a..675e00c44 100644 --- a/socket.c +++ b/socket.c @@ -35,6 +35,9 @@ #include #endif #include +#ifdef SUPPORT_IDN +#include +#endif extern char *bind_address; extern char *sockopts; @@ -343,6 +346,15 @@ int open_socket_out(char *host, int port, const char *bind_addr, int af_hint) int proxied = 0; char buffer[1024]; char *proxy_user = NULL, *proxy_pass = NULL; +#ifdef SUPPORT_IDN + char *idn, idn_host[1024]; + + if (idn2_lookup_ul(host, &idn, IDN2_NONTRANSITIONAL) == IDN2_OK) { + strlcpy(idn_host, idn, sizeof idn_host); + idn2_free(idn); + host = idn_host; + } +#endif /* if we have a RSYNC_PROXY env variable then redirect our * connection via a web proxy at the given address. */ diff --git a/testsuite/idn_test.py b/testsuite/idn_test.py new file mode 100644 index 000000000..2a9246073 --- /dev/null +++ b/testsuite/idn_test.py @@ -0,0 +1,148 @@ +#!/usr/bin/env python3 +# Verify that rsync converts an IDN (internationalized domain name) host to +# its IDNA A-label (Punycode) form. +# +# Two daemon connection methods carry the host name out of rsync, so both are +# checked: +# * daemon over a remote shell (what rsync-ssl does): the host is handed to +# the --rsh helper. +# * direct daemon socket: observed through a dummy HTTP proxy (RSYNC_PROXY) on +# loopback, so this part only runs under --use-tcp. +# A plain remote-shell transfer (host:path) is intentionally left alone, since +# that name belongs to the user's ssh. + +import os +import shlex +import socket +import subprocess +import sys +import threading + +from rsyncfns import ( + RSYNC, SCRATCHDIR, USE_TCP, claim_ports, run_rsync, + test_fail, test_skipped, +) + + +if '"IDN": true' not in run_rsync('-VV', check=True, capture_output=True).stdout: + test_skipped("rsync built without IDN support") + + +def find_utf8_locale(): + try: + out = subprocess.check_output(['locale', '-a'], text=True, + stderr=subprocess.DEVNULL) + except (OSError, subprocess.CalledProcessError): + return None + avail = out.split() + for want in ('C.UTF-8', 'C.utf8', 'en_US.UTF-8', 'en_US.utf8'): + if want in avail: + return want + for loc in avail: + if loc.lower().replace('-', '').endswith('utf8'): + return loc + return None + + +utf8_locale = find_utf8_locale() +if not utf8_locale: + test_skipped("no UTF-8 locale available to encode the IDN host") + +idn_host = "\u010ci\u010dku.example" +ascii_host = "xn--iku-eqab.example" + +env = os.environ.copy() +env['LC_ALL'] = utf8_locale +out_dir = (str(SCRATCHDIR / 'out') + '/').encode() + + +def run_idn(url, *extra, extra_env=None): + # A bytes argv keeps the UTF-8 host intact regardless of Python's + # filesystem encoding. + e = dict(env) + if extra_env: + e.update(extra_env) + argv = [a.encode() for a in shlex.split(RSYNC)] + argv += [a.encode() for a in extra] + argv += [url.encode('utf-8'), out_dir] + return subprocess.run(argv, capture_output=True, env=e, timeout=30) + + +# --- daemon over a remote shell (the rsync-ssl mechanism) ------------------ +helper = SCRATCHDIR / 'idn-rsh.sh' +helper.write_text('#!/bin/sh\nprintf %s "$1" > "$IDN_RSH_OUT"\nexit 1\n') +helper.chmod(0o755) + +hostfile = SCRATCHDIR / 'idn-rsh-host' +if hostfile.exists(): + hostfile.unlink() + +run_idn(f"rsync://{idn_host}/module/", f"--rsh={helper}", + extra_env={'IDN_RSH_OUT': str(hostfile)}) + +got = hostfile.read_text() if hostfile.exists() else '' +if got != ascii_host: + test_fail(f"daemon-over-rsh sent host {got!r}, expected A-label {ascii_host!r}") +print(f"OK: daemon-over-rsh (rsync-ssl style) host sent as {ascii_host}") + + +# --- direct daemon socket, observed via a dummy proxy ----------------------- +if not USE_TCP: + print("direct-socket proxy check needs --use-tcp; skipping that part") + sys.exit(0) + +PROXY_PORT = 13335 +claim_ports(PROXY_PORT) + +listener = socket.socket(socket.AF_INET, socket.SOCK_STREAM) +listener.setsockopt(socket.SOL_SOCKET, socket.SO_REUSEADDR, 1) +listener.bind(('127.0.0.1', PROXY_PORT)) +listener.listen(1) + +captured = {} + + +def serve_one(): + conn, _ = listener.accept() + conn.settimeout(5) + data = b"" + try: + while b"\r\n\r\n" not in data and len(data) < 65536: + chunk = conn.recv(8192) + if not chunk: + break + data += chunk + except socket.timeout: + pass + captured['request'] = data + try: + conn.sendall(b"HTTP/1.0 403 Forbidden\r\n\r\n") + conn.shutdown(socket.SHUT_RDWR) + except OSError: + pass + conn.close() + + +t = threading.Thread(target=serve_one) +t.daemon = True +t.start() + +proc = run_idn(f"rsync://{idn_host}:873/whatever/", + extra_env={'RSYNC_PROXY': f'127.0.0.1:{PROXY_PORT}'}) + +t.join(timeout=15) +listener.close() + +if proc.returncode >= 128: + sys.stderr.write(proc.stderr.decode('latin1')) + test_fail(f"rsync killed by signal (status={proc.returncode})") + +request = captured.get('request', b'') +if not request: + test_fail("dummy proxy received no CONNECT request from rsync") + +if ascii_host.encode() not in request: + sys.stderr.write("proxy received: %r\n" % request.split(b"\r\n", 1)[0]) + test_fail(f"expected A-label {ascii_host} in the proxy CONNECT request") + +print(f"OK: direct-socket CONNECT host sent as {ascii_host}") diff --git a/usage.c b/usage.c index ab6063a00..de586839c 100644 --- a/usage.c +++ b/usage.c @@ -156,6 +156,11 @@ static void print_info_flags(enum logcode f) #endif "crtimes", +#ifndef SUPPORT_IDN + "no " +#endif + "IDN", + "*Optimizations", #ifndef USE_ROLL_SIMD From 878db59b4f0cea0e6341bbdab1412b1252db537e Mon Sep 17 00:00:00 2001 From: Christopher Meng Date: Mon, 10 Aug 2026 00:30:07 -0400 Subject: [PATCH 2/3] idn: convert host names label by label MIME-Version: 1.0 Content-Type: text/plain; charset=UTF-8 Content-Transfer-Encoding: 8bit The IDNA mapping folds some non-ASCII characters onto ASCII ones, so running a whole hosts allow/deny token through idn2_to_ascii_8z() could hand back a pattern the admin never wrote: a "＊" (U+FF0A FULLWIDTH ASTERISK) entry came back as "*" and let every host in. Convert label by label instead, keeping an ASCII label byte for byte and using a converted label only when it comes back as a bare A-label. An ASCII-only config now behaves as it did before there was IDN support, and a token that cannot be converted is left alone and so matches nothing. The client side shares the same helper, and neither side truncates a name at its 1024-byte buffer any more. strlower() folds only ASCII now, since its one caller is the hosts allow/deny list, which can hold UTF-8. Adds testsuite/daemon-access-idn and extends testsuite/idn to cover Unicode, punycode, mixed-case and invalid input on both sides. --- INSTALL.md | 16 +++ access.c | 13 +-- main.c | 17 ++- rsync.1.md | 13 +++ rsyncd.conf.5.md | 12 +++ socket.c | 12 +-- support/install_deps_ubuntu.sh | 1 + testsuite/daemon-access-idn_test.py | 162 ++++++++++++++++++++++++++++ testsuite/idn_test.py | 48 +++++++-- testsuite/skiplist/common.txt | 1 + util1.c | 95 +++++++++++++++- 11 files changed, 353 insertions(+), 37 deletions(-) create mode 100755 testsuite/daemon-access-idn_test.py diff --git a/INSTALL.md b/INSTALL.md index 8beb68c0d..e9b70aa10 100644 --- a/INSTALL.md +++ b/INSTALL.md @@ -114,6 +114,16 @@ checksums. [4]: https://www.openssl.org/docs/man1.0.2/man3/crypto.html +## libidn2 + +The [libidn2 library][5] converts an internationalized domain name into the +IDNA A-label ("Punycode") form that a resolver understands. Installing this +development library lets rsync connect to a daemon whose name has non-ASCII +characters in it, and lets a daemon's "hosts allow" & "hosts deny" settings be +written the same way. + +[5]: https://www.gnu.org/software/libidn/#libidn2 + ## Package summary To help you get the libraries installed, here are some package install commands @@ -131,6 +141,7 @@ like. > sudo apt install -y libzstd-dev > sudo apt install -y liblz4-dev > sudo apt install -y libssl-dev + > sudo apt install -y libidn2-dev Or run support/install_deps_ubuntu.sh @@ -144,6 +155,7 @@ Or run support/install_deps_ubuntu.sh > sudo yum -y install libzstd-devel > sudo yum -y install lz4-devel > sudo yum -y install openssl-devel + > sudo yum -y install libidn2-devel > python3 -mpip install --user commonmark - For Fedora 33: @@ -154,6 +166,7 @@ Or run support/install_deps_ubuntu.sh > sudo dnf -y install libzstd-devel > sudo dnf -y install lz4-devel > sudo dnf -y install openssl-devel + > sudo dnf -y install libidn2-devel - For FreeBSD (this assumes that the python3 version is 3.7): @@ -161,6 +174,7 @@ Or run support/install_deps_ubuntu.sh > sudo pkg install -y xxhash > sudo pkg install -y zstd > sudo pkg install -y liblz4 + > sudo pkg install -y libidn2 - For macOS: @@ -169,6 +183,7 @@ Or run support/install_deps_ubuntu.sh > brew install zstd > brew install lz4 > brew install openssl + > brew install libidn2 - For Cygwin (with all cygwin programs stopped, run the appropriate setup program from a cmd shell): @@ -177,6 +192,7 @@ Or run support/install_deps_ubuntu.sh > setup-x86_64 --quiet-mode -P libzstd-devel > setup-x86_64 --quiet-mode -P liblz4-devel > setup-x86_64 --quiet-mode -P libssl-devel + > setup-x86_64 --quiet-mode -P libidn2-devel Sometimes cygwin has commonmark packaged and sometimes it doesn't. Now that its python38 has stabilized, you could install python38-commonmark. Or just diff --git a/access.c b/access.c index 529eae42b..a9b6149c8 100644 --- a/access.c +++ b/access.c @@ -23,9 +23,6 @@ #ifdef HAVE_NETGROUP_H #include #endif -#ifdef SUPPORT_IDN -#include -#endif static int allow_forward_dns; @@ -37,7 +34,7 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t unsigned int i; const char *host = *host_ptr; #ifdef SUPPORT_IDN - char idn_tok[1024], *idn; + char idn_tok[1024]; #endif if (!host || !*host) @@ -49,11 +46,11 @@ static int match_hostname(const char **host_ptr, const char *addr, const char *t #endif #ifdef SUPPORT_IDN - if (idn2_to_ascii_8z(tok, &idn, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL) == IDN2_OK) { - strlcpy(idn_tok, idn, sizeof idn_tok); - idn2_free(idn); + /* A hostname reaches us from DNS as ASCII, so fold an IDN token to its + * A-label form before comparing. An all-ASCII token, and a token we + * can't fold, are both left as they are. */ + if (idn_to_ascii(tok, 0, idn_tok, sizeof idn_tok)) tok = idn_tok; - } #endif /* First check if the reverse-DNS-determined hostname matches. */ diff --git a/main.c b/main.c index 1c7e43ac9..8381935cf 100644 --- a/main.c +++ b/main.c @@ -34,9 +34,6 @@ #ifdef HAVE_SYS_RESOURCE_H #include #endif -#ifdef SUPPORT_IDN -#include -#endif extern int dry_run; extern int list_only; @@ -527,14 +524,12 @@ static pid_t do_cmd(char *cmd, char *machine, char *user, char **remote_argv, in #ifdef SUPPORT_IDN char idn_machine[1024]; - if (machine && daemon_connection > 0) { - char *idn; - if (idn2_lookup_ul(machine, &idn, IDN2_NONTRANSITIONAL) == IDN2_OK) { - strlcpy(idn_machine, idn, sizeof idn_machine); - idn2_free(idn); - machine = idn_machine; - } - } + /* A daemon-over-remote-shell host is ours to resolve, so give the helper + * the A-label form. A "host:path" transfer is left alone because that + * name belongs to the user's ssh, which may be matching it against an + * ssh_config Host pattern. */ + if (machine && daemon_connection > 0 && idn_to_ascii(machine, 1, idn_machine, sizeof idn_machine)) + machine = idn_machine; #endif if (!read_batch && !local_server) { diff --git a/rsync.1.md b/rsync.1.md index 01017afdf..370721eca 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -412,6 +412,19 @@ when scripting rsync. WARNING: On some systems, environment variables are visible to all users. On those systems using [`--password-file`](#opt) is recommended. +If rsync was built with IDN support (look for "IDN" in `rsync --version`), the +daemon host may contain non-ASCII characters: those labels are converted to +their IDNA A-label ("Punycode") form before the name is looked up. The name is +read using your locale's character encoding, so be sure your locale is set +correctly. Only the non-ASCII labels change, so an address literal, a name you +punycoded yourself, and a name that is not a valid IDN are all looked up just +as you typed them. + +This applies to the host of a daemon connection only. The host of a plain +remote-shell transfer (the single-colon syntax) is passed to your remote-shell +program unchanged, since that name may well be an ssh_config "Host" alias +rather than a DNS name. + You may establish the connection via a web proxy by setting the environment variable [`RSYNC_PROXY`](#) to a hostname:port pair pointing to your web proxy. Note that your web proxy's configuration must support proxy connections to port diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md index 338abe71b..5a384d472 100644 --- a/rsyncd.conf.5.md +++ b/rsyncd.conf.5.md @@ -853,6 +853,18 @@ in the values of parameters. See that section for details. - an '@' followed by a netgroup name, which will match if the reverse DNS of the connecting IP is in the specified netgroup. + If rsync was built with IDN support (look for "IDN" in `rsync --version`), + a hostname pattern may contain non-ASCII characters: this file is read as + UTF-8, and each non-ASCII label is converted to its IDNA A-label + ("Punycode") form before matching, since the name the daemon has for a + client always reaches it from DNS as ASCII. Only those labels change, so + an address, a mask, an already-punycoded name, and the wildcard characters + are all matched exactly as written. A pattern that cannot be converted + that way is matched as it stands, and thus matches nothing. That includes + a pattern whose conversion would have introduced a character it was not + written with, such as the U+FF0A FULLWIDTH ASTERISK that the IDNA mapping + turns into a "*". + Note IPv6 link-local addresses can have a scope in the address specification: diff --git a/socket.c b/socket.c index 675e00c44..1449055f5 100644 --- a/socket.c +++ b/socket.c @@ -35,9 +35,6 @@ #include #endif #include -#ifdef SUPPORT_IDN -#include -#endif extern char *bind_address; extern char *sockopts; @@ -347,13 +344,12 @@ int open_socket_out(char *host, int port, const char *bind_addr, int af_hint) char buffer[1024]; char *proxy_user = NULL, *proxy_pass = NULL; #ifdef SUPPORT_IDN - char *idn, idn_host[1024]; + char idn_host[1024]; - if (idn2_lookup_ul(host, &idn, IDN2_NONTRANSITIONAL) == IDN2_OK) { - strlcpy(idn_host, idn, sizeof idn_host); - idn2_free(idn); + /* The resolver only speaks ASCII, so an IDN host goes out as A-labels. + * An all-ASCII host is passed along untouched. */ + if (idn_to_ascii(host, 1, idn_host, sizeof idn_host)) host = idn_host; - } #endif /* if we have a RSYNC_PROXY env variable then redirect our diff --git a/support/install_deps_ubuntu.sh b/support/install_deps_ubuntu.sh index ac49055bb..619e3d78e 100755 --- a/support/install_deps_ubuntu.sh +++ b/support/install_deps_ubuntu.sh @@ -9,3 +9,4 @@ sudo apt install -y libxxhash-dev sudo apt install -y libzstd-dev sudo apt install -y liblz4-dev sudo apt install -y libssl-dev +sudo apt install -y libidn2-dev diff --git a/testsuite/daemon-access-idn_test.py b/testsuite/daemon-access-idn_test.py new file mode 100755 index 000000000..a265e5ac4 --- /dev/null +++ b/testsuite/daemon-access-idn_test.py @@ -0,0 +1,162 @@ +#!/usr/bin/env python3 +"""Daemon coverage: IDN hosts allow / hosts deny matching (access.c). + +A daemon gets its peer's name from DNS as ASCII, so an rsyncd.conf entry +written with non-ASCII characters is folded to its IDNA A-label form before +being matched. The IDNA mapping also folds some non-ASCII characters onto +ASCII ones (U+FF0A FULLWIDTH ASTERISK becomes '*'), so the checks below cover +both directions: a Unicode name that has to match, and the tokens that must +stay denied rather than turn into a wildcard or an address/mask. Punycode, +mixed case and an unconvertible name are covered too. + +The peer name isn't assumed: a throwaway daemon comes up first to log the name +this host's resolver gives it, and the real config is written around that. +"forward lookup" is off throughout, which pins the match on the reverse-DNS +name and keeps a denied module from waiting out a resolver timeout on a name +that deliberately doesn't exist. + +Like daemon-access-ip_test.py this needs a real TCP peer (--use-tcp), and the +config sets no global hosts allow so each module's own patterns decide. +""" + +import re +import subprocess + +from rsyncfns import ( + FROMDIR, SCRATCHDIR, + claim_ports, make_tree, require_tcp, rmtree, rsync_argv, start_rsyncd, + start_test_daemon, test_fail, test_skipped, +) + +PROBE_PORT = 12896 +DAEMON_PORT = 12898 +require_tcp("hosts allow/deny hostname matching needs a real TCP peer") + +if '"IDN": true' not in subprocess.run(rsync_argv('-VV'), capture_output=True, + text=True).stdout: + test_skipped("rsync built without IDN support") + +src = FROMDIR +rmtree(src) +make_tree(src, depth=2) + + +def write_conf(path, modules, log, pidfile): + lines = [ + '# autogenerated by daemon-access-idn_test.py', + f'pid file = {pidfile}', + 'use chroot = no', + 'forward lookup = no', + f'log file = {log}', + '', + ] + for mod, params in modules: + lines.append(f'[{mod}]') + lines.append(f'\tpath = {src}') + lines.append('\tread only = yes') + lines += [f'\t{k} = {v}' for k, v in params.items()] + lines.append('') + # rsyncd.conf is read as UTF-8 by the daemon whatever the test's locale is. + path.write_text('\n'.join(lines) + '\n', encoding='utf-8') + return path + + +# --- find out what this host's resolver calls the loopback peer ------------- +# A throwaway daemon with one wide-open module: connect once, read the name it +# logged for us, then shut it down before the real config goes up. +probe_log = SCRATCHDIR / 'rsyncd-idn-probe.log' +probe_conf = write_conf(SCRATCHDIR / 'access-idn-probe.conf', [('probe', {})], + probe_log, SCRATCHDIR / 'rsyncd-idn-probe.pid') +claim_ports(PROBE_PORT) +probe = start_rsyncd(probe_conf, PROBE_PORT) +try: + subprocess.run(rsync_argv('-r', f'rsync://localhost:{PROBE_PORT}/probe/'), + stdout=subprocess.DEVNULL, stderr=subprocess.DEVNULL) +finally: + probe.terminate() + probe.wait(timeout=5) + +m = re.search(r'connect from (\S+) \(', probe_log.read_text(errors='replace')) +if not m: + test_fail(f"no 'connect from' line in the probe daemon log {probe_log}") +peer = m.group(1) +print(f"daemon sees its peer as {peer!r}") + + +def fullwidth(name): + """The fullwidth-forms spelling of an ASCII name. + + IDNA (UTS #46) maps U+FF01..U+FF5E back onto ASCII, so this gives us a + genuinely non-ASCII name that folds to a peer name we can predict without + controlling DNS. Label separators stay ASCII dots. + """ + return ''.join(chr(ord(c) - 0x21 + 0xFF01) if '!' <= c <= '~' and c != '.' + else c for c in name) + + +if fullwidth(peer) == peer: + test_skipped(f"peer name {peer!r} has no ASCII to respell in fullwidth") + +ZWSP = '​' # maps to nothing, leaving no label at all +FW_STAR = '＊' # FULLWIDTH ASTERISK, which IDNA maps to '*' +FW_SLASH = '／' # FULLWIDTH SOLIDUS, which IDNA maps to '/' + +modules = [ + ('ascii-name', {'hosts allow': peer}), + ('ascii-upper', {'hosts allow': peer.upper()}), + ('ascii-wild', {'hosts allow': peer[:1] + '*'}), + # The same name in fullwidth forms, which only matches once the token has + # been folded to A-labels. + ('idn-name', {'hosts allow': fullwidth(peer)}), + ('idn-mixedcase', {'hosts allow': fullwidth(peer.upper())}), + ('idn-deny', {'hosts deny': fullwidth(peer)}), + # A real IDN that is not the peer, plus its punycode spelling: both must + # stay denied, and neither may be mistaken for a wildcard. + ('idn-other', {'hosts allow': 'čičku.example'}), + ('idn-puny', {'hosts allow': 'xn--iku-eqab.example'}), + # Tokens whose IDNA mapping yields ASCII the author never wrote. Left + # unconverted they match nothing; converted blindly, the first two would + # allow every host and the third would read as an address/mask. + ('wide-star', {'hosts allow': FW_STAR}), + ('wide-star-dom', {'hosts allow': FW_STAR + '.example'}), + ('wide-mask', {'hosts allow': '127.0.0.0' + FW_SLASH + '8'}), + # An IDN that can't be converted at all (its label maps to nothing). + ('bad-idn', {'hosts allow': ZWSP + '.example'}), +] + +conf = write_conf(SCRATCHDIR / 'access-idn.conf', modules, + SCRATCHDIR / 'rsyncd.log', SCRATCHDIR / 'rsyncd.pid') +url = start_test_daemon(conf, DAEMON_PORT) + + +def connect(mod): + """Return rsync's exit code for listing the module over the daemon.""" + return subprocess.run(rsync_argv('-r', f'{url}{mod}/'), + stdout=subprocess.DEVNULL, stderr=subprocess.PIPE, + text=True).returncode + + +def allowed(mod, why): + if connect(mod) != 0: + test_fail(f"connection to {mod} should be ALLOWED ({why}) but was refused") + + +def denied(mod, why): + if connect(mod) == 0: + test_fail(f"connection to {mod} should be DENIED ({why}) but succeeded") + + +allowed('ascii-name', "the peer's own name in a hosts allow") +allowed('ascii-upper', "hostname matching is case-insensitive") +allowed('ascii-wild', "an ASCII wildcard still matches") +allowed('idn-name', f"fullwidth {peer!r} folds to the peer's name") +allowed('idn-mixedcase', "IDNA case-folds the token") +denied('idn-deny', "hosts deny sees the folded token too") +denied('idn-other', "a different IDN must not match the peer") +denied('idn-puny', "an A-label for a different host must not match the peer") +denied('wide-star', "U+FF0A must not become a '*' that allows every host") +denied('wide-star-dom', "U+FF0A must not become a wildcard label") +denied('wide-mask', "U+FF0F must not become an address/mask separator") +denied('bad-idn', "an unconvertible IDN must not match anything") + +print("daemon-access-idn: IDN hosts allow/deny matching + no wildcard widening") diff --git a/testsuite/idn_test.py b/testsuite/idn_test.py index 2a9246073..326deae30 100644 --- a/testsuite/idn_test.py +++ b/testsuite/idn_test.py @@ -1,6 +1,9 @@ #!/usr/bin/env python3 # Verify that rsync converts an IDN (internationalized domain name) host to -# its IDNA A-label (Punycode) form. +# its IDNA A-label (Punycode) form, and that it leaves an ASCII host name +# alone. Only the labels that are not ASCII get rewritten, so an address +# literal, an already-punycoded name, and a name that isn't a valid IDN all +# reach the resolver as typed. # # Two daemon connection methods carry the host name out of rsync, so both are # checked: @@ -10,6 +13,8 @@ # loopback, so this part only runs under --use-tcp. # A plain remote-shell transfer (host:path) is intentionally left alone, since # that name belongs to the user's ssh. +# +# The daemon side of IDN -- hosts allow/deny matching -- is daemon-access-idn. import os import shlex @@ -74,16 +79,41 @@ def run_idn(url, *extra, extra_env=None): helper.chmod(0o755) hostfile = SCRATCHDIR / 'idn-rsh-host' -if hostfile.exists(): - hostfile.unlink() -run_idn(f"rsync://{idn_host}/module/", f"--rsh={helper}", - extra_env={'IDN_RSH_OUT': str(hostfile)}) -got = hostfile.read_text() if hostfile.exists() else '' -if got != ascii_host: - test_fail(f"daemon-over-rsh sent host {got!r}, expected A-label {ascii_host!r}") -print(f"OK: daemon-over-rsh (rsync-ssl style) host sent as {ascii_host}") +def rsh_host(url_host): + """The host name rsync hands the --rsh helper for rsync:///.""" + if hostfile.exists(): + hostfile.unlink() + run_idn(f"rsync://{url_host}/module/", f"--rsh={helper}", + extra_env={'IDN_RSH_OUT': str(hostfile)}) + if not hostfile.exists(): + test_fail(f"the --rsh helper never ran for {url_host!r}") + return hostfile.read_bytes().decode('utf-8', 'surrogateescape') + + +def check_rsh(url_host, want, what): + got = rsh_host(url_host) + if got != want: + test_fail(f"daemon-over-rsh sent host {got!r} for {what} " + f"({url_host!r}), expected {want!r}") + print(f"OK: {what} -> {got}") + + +# A U-label becomes its A-label, case-folded by the IDNA mapping. An ASCII +# label is handed on byte for byte, case included, since DNS doesn't care. +check_rsh(idn_host, ascii_host, "a Unicode host") +check_rsh("ČIČKU.Example", "xn--iku-eqab.Example", + "a mixed-case Unicode host") +check_rsh(ascii_host, ascii_host, "an already-punycoded host") +check_rsh("XN--IKU-EQAB.Example", "XN--IKU-EQAB.Example", + "a mixed-case punycoded host") +# A name that isn't a valid IDN goes out as-is instead of being rewritten into +# some other name (the U+200B one would map to ".example"), so the resolver +# fails on it just as it did before. +check_rsh("xn--0.example", "xn--0.example", "an undecodable A-label") +check_rsh("ـx.example", "ـx.example", "a label with a disallowed character") +check_rsh("​.example", "​.example", "a label that maps to nothing") # --- direct daemon socket, observed via a dummy proxy ----------------------- diff --git a/testsuite/skiplist/common.txt b/testsuite/skiplist/common.txt index 67f4b4ee6..cb09e33e8 100644 --- a/testsuite/skiplist/common.txt +++ b/testsuite/skiplist/common.txt @@ -11,6 +11,7 @@ checksum-zero-blocklen # the pure-Python receiver needs a real TCP socket; run with --use-tcp chroot-basis-forge-inner-module # the pure-Python sender needs a real TCP daemon; run with --use-tcp +daemon-access-idn # hosts allow/deny hostname matching needs a real TCP peer daemon-access-ip # hosts allow/deny address matching needs a real TCP peer daemon-argv-limit # raw malicious daemon client needs a real TCP daemon; run with --use-tcp daemon-chroot # daemon chroot path needs the real start_daemon socket flow diff --git a/util1.c b/util1.c index 87ff22595..6b896bb05 100644 --- a/util1.c +++ b/util1.c @@ -24,6 +24,9 @@ #include "ifuncs.h" #include "itypes.h" #include "inums.h" +#ifdef SUPPORT_IDN +#include +#endif extern int dry_run; extern int module_id; @@ -915,16 +918,106 @@ void glob_expand_module(char *base1, char *arg, char ***argv_p, int *argc_p, int /** * Convert a string to lower case + * + * Only ASCII is folded. The hosts allow/deny list that calls this can hold + * UTF-8, and a per-byte fold via the locale's ctype would mangle it (in + * ISO-8859-1 the 0xC4 lead byte of "č" is an upper-case 'Ä'). **/ void strlower(char *s) { while (*s) { - if (isUpper(s)) + if (!(*(unsigned char *)s & 0x80) && isUpper(s)) *s = toLower(s); s++; } } +#ifdef SUPPORT_IDN +/* Does this label hold nothing but the [-a-z0-9] of an A-label? */ +static int is_a_label(const char *s) +{ + if (!*s) + return 0; + + for ( ; *s; s++) { + if (!(*s >= 'a' && *s <= 'z') && !(*s >= '0' && *s <= '9') && *s != '-') + return 0; + } + + return 1; +} + +/** + * Convert the non-ASCII labels of a host name into their IDNA A-label + * (Punycode) form, putting the result in buf. Returns 1 if buf was filled in, + * or 0 to tell the caller to keep the name it has. + * + * A label that is already ASCII is copied verbatim, so an address, a mask, an + * xn-- name, and any wildmatch characters come out just as they went in. A + * converted label is only used if it comes back as a bare A-label: the IDNA + * mapping folds some non-ASCII characters onto ASCII ones (U+FF0A FULLWIDTH + * ASTERISK becomes '*'), and a hosts allow/deny entry must not pick up a + * wildcard that its author never typed. Anything else leaves the name alone, + * which fails to match instead of matching too much. + * + * Set from_locale for a name that came from the command line, which is in the + * user's locale encoding; the daemon's config file is read as UTF-8. + **/ +int idn_to_ascii(const char *name, int from_locale, char *buf, size_t buflen) +{ + const char *lab, *end; + size_t len = 0; + int converted = 0; + + for (lab = name; ; lab = end + 1) { + char label[256], *idn; + size_t lablen, alen; + int is_ascii = 1; + + for (end = lab; *end && *end != '.'; end++) { + if (*(unsigned char *)end & 0x80) + is_ascii = 0; + } + lablen = end - lab; + + if (is_ascii) { + if (len + lablen + 2 > buflen) + return 0; + memcpy(buf + len, lab, lablen); + len += lablen; + } else { + int rc; + if (lablen >= sizeof label) + return 0; + memcpy(label, lab, lablen); + label[lablen] = '\0'; + rc = from_locale + ? idn2_lookup_ul(label, &idn, IDN2_NONTRANSITIONAL) + : idn2_to_ascii_8z(label, &idn, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL); + if (rc != IDN2_OK) + return 0; + alen = strlen(idn); + if (!is_a_label(idn) || len + alen + 2 > buflen) { + idn2_free(idn); + return 0; + } + memcpy(buf + len, idn, alen); + len += alen; + idn2_free(idn); + converted = 1; + } + + if (!*end) + break; + buf[len++] = '.'; + } + + buf[len] = '\0'; + + return converted; +} +#endif + /** * Split a string into tokens based (usually) on whitespace & commas. If the * string starts with a comma (after skipping any leading whitespace), then From 8254de44de423eaf1a25a5b32a75aa4937abaa38 Mon Sep 17 00:00:00 2001 From: Christopher Meng Date: Tue, 11 Aug 2026 12:51:59 -0400 Subject: [PATCH 3/3] idn: hand libidn2 the same flags on both sides The client path called idn2_lookup_ul() without IDN2_NFC_INPUT while the daemon path passed it to idn2_to_ascii_8z(). Both normalize either way -- idn2_lookup_ul() ors the flag in itself, and TR46 normalizes as it maps -- but there is no reason for the two calls to read differently, so pass one set of flags from one place. The flag asks libidn2 to normalize the label rather than promising that it already is: it gates the u32_normalize() call, and without it a decomposed label comes back IDN2_NOT_NFC. Adds composed/decomposed cases to testsuite/idn, which sees the exact host name rsync hands out, and a decomposed hosts allow token to testsuite/daemon-access-idn. --- rsync.1.md | 7 ++++--- rsyncd.conf.5.md | 12 +++++++----- testsuite/daemon-access-idn_test.py | 7 +++++++ testsuite/idn_test.py | 10 +++++++++- util1.c | 10 +++++++--- 5 files changed, 34 insertions(+), 12 deletions(-) diff --git a/rsync.1.md b/rsync.1.md index 370721eca..7725a01e7 100644 --- a/rsync.1.md +++ b/rsync.1.md @@ -416,9 +416,10 @@ If rsync was built with IDN support (look for "IDN" in `rsync --version`), the daemon host may contain non-ASCII characters: those labels are converted to their IDNA A-label ("Punycode") form before the name is looked up. The name is read using your locale's character encoding, so be sure your locale is set -correctly. Only the non-ASCII labels change, so an address literal, a name you -punycoded yourself, and a name that is not a valid IDN are all looked up just -as you typed them. +correctly. A name typed with combining characters is normalized on the way, so +it is looked up the same as its precomposed spelling. Only the non-ASCII +labels change, so an address literal, a name you punycoded yourself, and a name +that is not a valid IDN are all looked up just as you typed them. This applies to the host of a daemon connection only. The host of a plain remote-shell transfer (the single-colon syntax) is passed to your remote-shell diff --git a/rsyncd.conf.5.md b/rsyncd.conf.5.md index 5a384d472..d16d6f3b4 100644 --- a/rsyncd.conf.5.md +++ b/rsyncd.conf.5.md @@ -857,11 +857,13 @@ in the values of parameters. See that section for details. a hostname pattern may contain non-ASCII characters: this file is read as UTF-8, and each non-ASCII label is converted to its IDNA A-label ("Punycode") form before matching, since the name the daemon has for a - client always reaches it from DNS as ASCII. Only those labels change, so - an address, a mask, an already-punycoded name, and the wildcard characters - are all matched exactly as written. A pattern that cannot be converted - that way is matched as it stands, and thus matches nothing. That includes - a pattern whose conversion would have introduced a character it was not + client always reaches it from DNS as ASCII. A pattern written with + combining characters is normalized on the way, so it matches the same as + its precomposed spelling. Only those labels change, so an address, a + mask, an already-punycoded name, and the wildcard characters are all + matched exactly as written. A pattern that cannot be converted that way + is matched as it stands, and thus matches nothing. That includes a + pattern whose conversion would have introduced a character it was not written with, such as the U+FF0A FULLWIDTH ASTERISK that the IDNA mapping turns into a "*". diff --git a/testsuite/daemon-access-idn_test.py b/testsuite/daemon-access-idn_test.py index a265e5ac4..05fea678a 100755 --- a/testsuite/daemon-access-idn_test.py +++ b/testsuite/daemon-access-idn_test.py @@ -114,6 +114,12 @@ def fullwidth(name): # stay denied, and neither may be mistaken for a wildcard. ('idn-other', {'hosts allow': 'čičku.example'}), ('idn-puny', {'hosts allow': 'xn--iku-eqab.example'}), + # idn-other's token respelled with combining carons: an equivalent name + # under Unicode, so it has to be treated the same way. (A decomposed + # token that *matches* isn't constructible here -- the peer name comes + # from DNS and is ASCII -- so idn_test checks the two spellings convert + # alike, and this checks the daemon agrees they don't match.) + ('idn-nfd', {'hosts allow': 'c\u030ci' 'c\u030cku.example'}), # Tokens whose IDNA mapping yields ASCII the author never wrote. Left # unconverted they match nothing; converted blindly, the first two would # allow every host and the third would read as an address/mask. @@ -154,6 +160,7 @@ def denied(mod, why): denied('idn-deny', "hosts deny sees the folded token too") denied('idn-other', "a different IDN must not match the peer") denied('idn-puny', "an A-label for a different host must not match the peer") +denied('idn-nfd', "a decomposed spelling of that name must not match either") denied('wide-star', "U+FF0A must not become a '*' that allows every host") denied('wide-star-dom', "U+FF0A must not become a wildcard label") denied('wide-mask', "U+FF0F must not become an address/mask separator") diff --git a/testsuite/idn_test.py b/testsuite/idn_test.py index 326deae30..a4e914f8f 100644 --- a/testsuite/idn_test.py +++ b/testsuite/idn_test.py @@ -3,7 +3,8 @@ # its IDNA A-label (Punycode) form, and that it leaves an ASCII host name # alone. Only the labels that are not ASCII get rewritten, so an address # literal, an already-punycoded name, and a name that isn't a valid IDN all -# reach the resolver as typed. +# reach the resolver as typed. A name typed with combining marks is normalized +# on the way, so it converts the same as its precomposed spelling. # # Two daemon connection methods carry the host name out of rsync, so both are # checked: @@ -55,6 +56,10 @@ def find_utf8_locale(): idn_host = "\u010ci\u010dku.example" ascii_host = "xn--iku-eqab.example" +# The same name with each caron letter spelled as a plain "c" plus a combining +# caron (U+030C). Unicode calls the two spellings equivalent, so both have to +# come out as the same A-label; libidn2 is what normalizes them. +nfd_host = "c\u030ci" "c\u030cku.example" env = os.environ.copy() env['LC_ALL'] = utf8_locale @@ -103,6 +108,9 @@ def check_rsh(url_host, want, what): # A U-label becomes its A-label, case-folded by the IDNA mapping. An ASCII # label is handed on byte for byte, case included, since DNS doesn't care. check_rsh(idn_host, ascii_host, "a Unicode host") +check_rsh(nfd_host, ascii_host, "a decomposed Unicode host") +check_rsh("C\u030cI" "C\u030cKU.Example", "xn--iku-eqab.Example", + "a decomposed mixed-case Unicode host") check_rsh("ČIČKU.Example", "xn--iku-eqab.Example", "a mixed-case Unicode host") check_rsh(ascii_host, ascii_host, "an already-punycoded host") diff --git a/util1.c b/util1.c index 6b896bb05..fc136b12b 100644 --- a/util1.c +++ b/util1.c @@ -986,14 +986,18 @@ int idn_to_ascii(const char *name, int from_locale, char *buf, size_t buflen) memcpy(buf + len, lab, lablen); len += lablen; } else { + /* IDN2_NFC_INPUT has libidn2 normalize the label, so a name + * typed with combining marks folds to the same A-label as + * its composed spelling. IDN2_NONTRANSITIONAL asks for the + * TR46 processing that everything else does these days. */ + int flags = IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL; int rc; if (lablen >= sizeof label) return 0; memcpy(label, lab, lablen); label[lablen] = '\0'; - rc = from_locale - ? idn2_lookup_ul(label, &idn, IDN2_NONTRANSITIONAL) - : idn2_to_ascii_8z(label, &idn, IDN2_NFC_INPUT | IDN2_NONTRANSITIONAL); + rc = from_locale ? idn2_lookup_ul(label, &idn, flags) + : idn2_to_ascii_8z(label, &idn, flags); if (rc != IDN2_OK) return 0; alen = strlen(idn);