From: Dirk Brenken Date: Wed, 16 Sep 2026 17:28:58 +0000 (+0200) Subject: shunt: update to 0.1.8 X-Git-Url: http://git.99rst.org/?a=commitdiff_plain;h=93a34793be4416febf6dd835ad3aa97c26b27dd7;p=openwrt-packages.git shunt: update to 0.1.8 - new per-policy `domain_file` list: domain patterns from a file, one per line, for community-maintained lists too large for UCI. Entries are learned by snoop only, never polled - a bare name lives under /tmp/shunt (tmpfs, no flash wear), an absolute path is taken as given; a missing file is an issue, not a fatal error, the policy stays rendered with empty sets - new `shunt refresh` (ubus method `refresh`): re-reads the files and swaps the matcher in place, no restart, learned addresses stay - matcher: regex fast path, 4x faster on 100k patterns - routes and rules are written over rtnetlink via ucode-mod-rtnl instead of one `ip` process per command; route.uc renders the netlink operations, the new src/rt.uc applies them and reads the verdict from rtnl.error() - `rp_filter_manage` writes /proc/sys directly, `nft` is the last remaining child process - `+ip` dropped from DEPENDS - README update Co-authored-by: Claude Signed-off-by: Dirk Brenken --- diff --git a/net/shunt/Makefile b/net/shunt/Makefile index a51bedbbd..b8d200cf6 100644 --- a/net/shunt/Makefile +++ b/net/shunt/Makefile @@ -5,7 +5,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=shunt -PKG_VERSION:=0.1.6 +PKG_VERSION:=0.1.8 PKG_RELEASE:=1 PKG_LICENSE:=GPL-3.0-or-later PKG_LICENSE_FILES:= @@ -19,7 +19,7 @@ define Package/shunt TITLE:=Policy based routing via passive DNS observation DEPENDS:=+ucode +ucode-mod-fs +ucode-mod-socket +ucode-mod-uci \ +ucode-mod-uloop +ucode-mod-resolv +ucode-mod-ubus \ - +ucode-mod-rtnl +ucode-mod-log +rpcd-mod-ucode +nftables-json +ip + +ucode-mod-rtnl +ucode-mod-log +rpcd-mod-ucode +nftables-json PKGARCH:=all endef @@ -29,8 +29,8 @@ define Package/shunt/description configured names and a passive AF_PACKET observer picks every DNS answer off the wire, so shunt works unchanged with dnsmasq, unbound or any other backend. Marks are applied via an own nftables table with per-element - counters, routing via fwmark rules and per-policy tables, with gateway - discovery and interface events from netifd. + counters, routing via fwmark rules and per-policy tables written over + rtnetlink, with gateway discovery and interface events from netifd. endef define Package/shunt/conffiles diff --git a/net/shunt/README.md b/net/shunt/README.md index 60a643146..1a4b45501 100644 --- a/net/shunt/README.md +++ b/net/shunt/README.md @@ -50,6 +50,7 @@ shunt check * Routes by client address, client MAC, destination CIDR and domain, in any combination * Resolver independent: works with any DNS backend, and with an encrypted upstream, because it reads the plaintext leg between client and resolver * Wildcard domains (`*.example.com`), learned passively as clients use them +* Domain lists from a file - a community-maintained set of thousands of names, without a single UCI entry per name * Per-policy killswitch: hold the traffic when the interface drops, instead of leaking it out of the normal uplink * Per-policy action: `route` marks the traffic for the policy interface, `bypass` exempts it from every policy below * Your own networks stay reachable from a policy client, without listing them anywhere @@ -62,7 +63,7 @@ shunt check ## Prerequisites * OpenWrt with fw4/nftables -* `ucode` plus `ucode-mod-fs`, `ucode-mod-socket`, `ucode-mod-uci`, `ucode-mod-uloop`, `ucode-mod-resolv`, `ucode-mod-ubus`, `ucode-mod-rtnl`, `ucode-mod-log` and `rpcd-mod-ucode` - all pulled in by the package +* `ucode` plus `ucode-mod-fs`, `ucode-mod-socket`, `ucode-mod-uci`, `ucode-mod-uloop`, `ucode-mod-resolv`, `ucode-mod-ubus`, `ucode-mod-rtnl`, `ucode-mod-log` and `rpcd-mod-ucode` - all pulled in by the package; `ip` is not needed, routes and rules are written over rtnetlink `ucode-mod-resolv` and `ucode-mod-ubus` are soft at runtime: without resolv, poll is skipped and the observer carries the service alone; without ubus, gateway discovery and interface events are skipped and the config's own values are used. Both cost one warning in the log, not a failed start. @@ -147,6 +148,7 @@ All functions are available from the command line, and the config file can be ed shunt check # render everything, print marks and issues, change nothing shunt run # foreground, the procd service entry point shunt flush # tear down table, rules, routes and the mapping file +shunt refresh # make the running daemon re-read its domain files shunt -v # echo every message to the terminal as well ``` @@ -158,6 +160,8 @@ Logging goes to syslog under the tag `shunt`, so `logread -e shunt` shows everyt `shunt flush` is the escape hatch if the daemon ever dies without tearing down. It is idempotent and safe on a box that never ran shunt. +`shunt refresh` asks the daemon over ubus to re-read every `domain_file` and reports what it found; it is the one change that does not need a restart. Without a running daemon it says so and exits 1 - nothing is lost, the next start reads the files anyway. Everything else in the config still takes `/etc/init.d/shunt restart`. + ## shunt config options @@ -193,8 +197,9 @@ Each `config policy` section is one routing policy. **The section must be named, | dport | destination ports, single or a range like `8000-8080` | | dst | destination addresses or CIDRs | | domain | domain patterns, see below | +| domain_file | files with domain patterns, one per line, see below | -`src`, `src_mac`, `dst` and `domain` are lists and may repeat. +`src`, `src_mac`, `dst`, `domain` and `domain_file` are lists and may repeat. Interfaces are resolved through netifd: a logical name resolves to its `l3_device`, a raw netdev is adopted if netifd knows it, and a device netifd knows nothing about passes through as given - which on OpenWrt means a tunnel started outside netifd, since wireguard and the other tunnel types have netifd protocols of their own. Gateways are discovered from the same dump, merged across sibling entries, because netifd splits families. `gw4`/`gw6` override discovery and always win; on a point to point interface no gateway is needed at all. @@ -241,6 +246,36 @@ Rule 3 is what makes one domain usable by two client groups over two different u Matching is label aligned, never string suffix: `evilexample.com` does not match `*.example.com`. A bad pattern is collected as an issue, never fatal. + +### Domain lists from a file + +A `list domain` entry is right for a handful of names. A community-maintained list has thousands, and the place for those is a file, not UCI: + +``` +config policy 'vpn' + option interface 'wg0' + list src '192.168.1.0/24' + list domain 'my-own-exception.example' + list domain_file 'community.txt' +``` + +A bare name lives under `/tmp/shunt/`, which shunt creates at start; an absolute path is taken as given, for a list kept on a mounted drive. The default is tmpfs on purpose: a list of that size rewritten every day belongs in RAM, not on the flash the overlay sits on. The price is that after a reboot the file is not there until whatever produces it has run again, and that is handled, not worked around - see below. + +The file holds one pattern per line in the same two shapes as `list domain` - `example.com` or `*.example.com` - with `#` starting a comment and blank lines ignored. It is read at start, validated by the same rules, and its patterns join the policy's own; `domain` and `domain_file` are additive, and a file may be named by more than one policy. A file that is not there, is unreadable or holds no usable pattern is reported as an issue against the policy and contributes nothing; the policy itself is still rendered, so with `domain_file` as its only domain selector its learned sets simply stay empty until the file arrives. Bad lines are counted and the first one is named, rather than logging every one of them. + +Two things distinguish a file entry from a `list domain` one, both deliberate: + +* **File entries are learned by snoop only.** poll resolves the names you typed, not the ones from a file - ten thousand names every `poll_interval` is a DNS storm, not a service. A name from a file reaches its set when a client on the observed LAN segment asks for it, which for a large list is exactly when it matters and never before. The [first-connection caveat](#known-limitations) applies to every one of them. +* **shunt reads the file, it does not fetch it.** Where the list comes from and how often it changes is a job for cron, a script or a package of its own, and whatever writes the file runs `shunt refresh` afterwards: the daemon re-reads every domain file and swaps its matcher, and nothing else moves - no restart, the nftables table, the routes and the learned addresses all stay as they are. Write the new list to a temporary name and rename it over the old one, so a half-written file is never read: + +```sh +uclient-fetch -q -O /tmp/shunt/community.tmp https://example.org/domains.txt && mv /tmp/shunt/community.tmp /tmp/shunt/community.txt && shunt refresh +``` + +Run from cron that covers the reboot case as well: the first run after boot creates the file and the refresh loads it. That keeps shunt independent of the network, of any download tool and of the format a list publisher happens to choose - converting `hosts` or dnsmasq syntax into one pattern per line is one `sed` away and stays outside the daemon. + +What a large list costs: 100k patterns take about two seconds to load and roughly 40 MB of memory on an x86 test box, so expect several seconds and a proportionally smaller footprint on a router, at start and on every refresh. A file above 4 MiB is refused outright as an issue. `shunt check` prints how many files were read and how many patterns they contributed, `shunt refresh` does the same for the running daemon, and `ubus call shunt status` shows the count under `files` together with the time of the last refresh. + ### Actions: route or bypass `action 'route'` (default) is the policy shape everything above describes: matching traffic is marked and looked up in the policy's own table. @@ -294,7 +329,7 @@ Note that domain precedence is resolved *before* this, at the matcher: the most ## How addresses are learned Two sources feed the same nftables sets, union with an element timeout. They are complementary, not alternative modes. -* **poll** resolves the configured names through whatever system resolver exists, on a fixed interval. It warms the sets before the first client packet, so first contact does not race. Wildcards are not names and cannot be polled. +* **poll** resolves the configured names through whatever system resolver exists, on a fixed interval. It warms the sets before the first client packet, so first contact does not race. Wildcards are not names and cannot be polled, and names from a `domain_file` are not polled by design - see [Domain lists from a file](#domain-lists-from-a-file). * **snoop** passively observes DNS responses on the LAN side via AF_PACKET with a BPF filter matching **UDP source port 53** - answers, not questions - including one level of VLAN tagging. It covers CDN variance and wildcards, which poll cannot. It reads; it never writes anything back onto the wire and never sits between a client and its resolver. If it dies, DNS keeps working and only the policy stops applying. @@ -403,14 +438,15 @@ config policy 'office' Almost nothing. The daemon and the rpcd backend work through ucode's native bindings - `fs`, `socket` for the AF_PACKET observer, `uci`, `uloop`, `ubus`, `resolv`, `rtnl` and `log` - and rpcd carries the LuCI side, so there is no shell glue, no `awk`, no temporary state files. Logging goes to syslog through the binding, not through a `logger` process per line. -Two external commands remain: +One external command remains: | Command | Why | | :--- | :--- | | `nft` | the ruleset is applied and read as one atomic batch; ucode has no nftables binding | -| `ip` | routes and rules are written this way, although `rtnl` already reads them - replaceable | -Nothing is ever handed to a shell for parsing: `system()` takes an argument array, and where stderr has to be captured the wrapper is `sh -c 'exec "$0" "$@"'`, which passes arguments through untouched. `popen()` only ever runs fixed command lines - `nft -f -` in the daemon and `nft -j list table` in the rpcd backend - so no configuration value or captured data reaches a command line. +Routes and rules go over rtnetlink through `ucode-mod-rtnl`, the same binding the rpcd backend reads them with, so there is no `ip` process per command and no `ip` dependency. A route on a device that does not exist yet is reported by the kernel's own reason rather than an exit code, and `rp_filter_manage` writes the `/proc/sys` value directly instead of calling `sysctl`. + +Nothing is ever handed to a shell for parsing: `system()` takes an argument array and only ever runs `nft`, and `popen()` only ever runs fixed command lines - `nft -f -` in the daemon and `nft -j list table` in the rpcd backend - so no configuration value or captured data reaches a command line. ## What shunt creates on the system @@ -428,7 +464,7 @@ fwmark << 24, mask 0xff000000 ip rule pref 31000 + keep_local's main lookup 31500 + the policy table routing table 8000 + -/etc/iproute2/rt_tables.d/shunt.conf the table name mapping +/etc/iproute2/rt_tables.d/shunt.conf the table name mapping, for `ip route show` only ``` A `bypass` policy is only a rule in the prerouting and output chains: no mark, no table, no ip rule, and it does not count against the 255. The mark mask is fixed at `0xff000000`, which allows 255 policies. The `output` chain is `type route` so the router's own marked traffic is re-routed after the mark is set. @@ -492,7 +528,7 @@ ip route get The first answer must name the policy table, the second the normal uplink. With `keep_local` on - the default - pick a destination **outside** your own networks for this: an address in an attached subnet or behind a static route deliberately answers with `main` in both lines, and that is the feature working, not the policy failing. -**The first line needs iproute2's `ip`**, because BusyBox's `route get` does not understand `mark` - one build rejects it outright, the OpenWrt one sends an incomplete netlink request that the kernel answers with `EINVAL`. shunt's `ip` dependency (`ip-tiny`) covers it: `route` and `rule` are complete there, the tiny build only strips exotic objects. The second line, without a mark, works with BusyBox too. +**The first line needs iproute2's `ip`**, because BusyBox's `route get` does not understand `mark` - one build rejects it outright, the OpenWrt one sends an incomplete netlink request that the kernel answers with `EINVAL`. shunt itself no longer depends on `ip`, so install `ip-tiny` for this check: `route` and `rule` are complete there, the tiny build only strips exotic objects. The second line, without a mark, works with BusyBox too. Adding `from iif br-lan` makes the lookup more precise, with one further catch worth a confused test session: **`iif` is not optional** there. Without it, `from` a non-local address makes the kernel validate a locally originated lookup and answer `ENETUNREACH` regardless of any table's content, which reads like broken routing and is not. diff --git a/net/shunt/files/shunt.rpcd b/net/shunt/files/shunt.rpcd index 16411b8b8..aaf0d7c53 100644 --- a/net/shunt/files/shunt.rpcd +++ b/net/shunt/files/shunt.rpcd @@ -19,6 +19,7 @@ const RT = rtnl.const; import { load as config_load, parse as config_parse } from 'shunt.config'; import { resolve as netifd_resolve } from 'shunt.netifd'; import { compile as match_compile } from 'shunt.match'; +import { load as files_load } from 'shunt.domain_file'; import { action_name, compile as nft_compile } from 'shunt.nft'; import { compile as route_compile } from 'shunt.route'; import { names as poll_names } from 'shunt.poll'; @@ -58,11 +59,15 @@ function render() { cfg.policies = netifd_resolve(cfg.policies, dump); + let f = files_load(cfg.policies); + + cfg.policies = f.policies; + let m = match_compile(cfg.policies); let n = nft_compile(cfg.policies); let r = route_compile(cfg.policies, n.marks); - return { cfg, matcher: m, nft: n, route: r }; + return { cfg, files: f, matcher: m, nft: n, route: r }; } // Policy devices whose marked traffic the kernel would drop: max(all, ), @@ -265,6 +270,7 @@ return { interface: (m.action == 'route') ? p?.interface : null, fallback: (m.action == 'route') ? p?.fallback : null, domains: length(p?.domains ?? []), + file_domains: length(p?.file_domains ?? []), rules: (m.action == 'route' && rules) ? length(rules[m.name] ?? []) : null, routes: (m.action == 'route') ? routes[m.name] : null @@ -282,6 +288,7 @@ return { rp_filter_strict: rp_filter_strict(), issues: [ ...st.cfg.issues, + ...st.files.issues, ...st.matcher.issues, ...st.nft.issues, ...st.route.issues diff --git a/net/shunt/files/shunt.uc b/net/shunt/files/shunt.uc index abb65f043..62d15da00 100755 --- a/net/shunt/files/shunt.uc +++ b/net/shunt/files/shunt.uc @@ -8,13 +8,15 @@ // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (c) 2026 Dirk Brenken -import { popen, writefile, readfile, unlink, mkdir, lstat, error as fs_error } from 'fs'; +import { popen, writefile, readfile, unlink, mkdir, error as fs_error } from 'fs'; import { openlog, syslog, LOG_PID, LOG_DAEMON, LOG_ERR, LOG_WARNING, LOG_NOTICE, LOG_INFO, LOG_DEBUG } from 'log'; import { load as cfg_load, parse as cfg_parse } from 'shunt.config'; import { compile as match_compile } from 'shunt.match'; +import { load as files_load, DIR as FILES_DIR } from 'shunt.domain_file'; import { action_name, compile as nft_compile, refresh, teardown, TABLE } from 'shunt.nft'; import { compile as route_compile } from 'shunt.route'; +import { open as rt_open, exec as rt_exec, describe as rt_describe } from 'shunt.rt'; import { open as snoop_open, observe, RECV_LEN } from 'shunt.snoop'; import { names as poll_names, plan as poll_plan, addresses as poll_addresses, @@ -89,39 +91,40 @@ function report(kind, issues) { log('warn', sprintf('%s: %J', kind, i)); } -const RUN_DIR = '/tmp/.shunt'; -const RUN_ERR = RUN_DIR + '/cmd.err'; - -function capture_ok() { - mkdir(RUN_DIR, 0o700); - - let st = lstat(RUN_DIR); - - return st != null && st.type == 'directory' && st.uid == 0 && - !st.perm.group_write && !st.perm.other_write && - !st.perm.group_read && !st.perm.other_read; +// The one remaining child process besides `nft -f -`: nft's own existence +// check for the table. Not called `run` - that name is the daemon's own +// entry point. +function quiet(argv) { + return system([ '/bin/sh', '-c', 'exec "$0" "$@" 2>/dev/null', ...argv ]); } -function loud(argv) { - if (!capture_ok()) - return { rc: quiet(argv), err: '' }; +let rtnl_mod = null; - let rc = system([ '/bin/sh', '-c', - sprintf('exec "$0" "$@" 2>%s', RUN_ERR), ...argv ]); - let err = ''; +// Routes and rules go over rtnetlink through ucode-mod-rtnl. Without the +// module nothing can be routed, so a missing one is an error, once. +function rtnl() { + if (rtnl_mod != null) + return rtnl_mod; - if (rc != 0) - err = replace(trim(readfile(RUN_ERR) ?? ''), /\s*\n\s*/g, '; '); + rtnl_mod = rt_open() ?? false; - unlink(RUN_ERR); + if (!rtnl_mod) + log('err', 'ucode-mod-rtnl missing - routes and rules cannot be applied'); - return { rc, err }; + return rtnl_mod; } -// quiet() drops the child's stderr, loud() keeps it for the warning. Neither -// may be called `run` - that name is the daemon's own entry point. -function quiet(argv) { - return system([ '/bin/sh', '-c', 'exec "$0" "$@" 2>/dev/null', ...argv ]); +// Fire and forget, the keeper's and the teardown's way: an EEXIST on a +// rule that is already there or an ESRCH on one that is already gone is +// the expected answer, not a fault. +function rt_quiet(ops) { + let rt = rtnl(); + + if (!rt) + return; + + for (let op in ops) + rt_exec(rt, op); } function nft_pipe(batch, what) { @@ -181,26 +184,24 @@ function apply(state) { log('warn', sprintf('cannot write %s: %s', RT_TABLES, fs_error())); } - for (let argv in state.route.del) - quiet(argv); + rt_quiet(state.route.del); + let rt = rtnl(); let failed = 0; let reasons = {}; - for (let argv in state.route.add) { - let r = loud(argv); - - if (r.rc != 0) { - let why = length(r.err) ? r.err : sprintf('exit %d', r.rc); + for (let op in state.route.add) { + let why = rt ? rt_exec(rt, op) : 'ucode-mod-rtnl missing'; + if (why != null) { failed++; reasons[why] = (reasons[why] ?? 0) + 1; - debug(sprintf('not applied: %s - %s', join(' ', argv), why)); + debug(sprintf('not applied: %s - %s', rt_describe(op), why)); } } for (let why in reasons) - log('warn', sprintf('%d of %d route/rule command(s) not applied - %s', + log('warn', sprintf('%d of %d route/rule operation(s) not applied - %s', reasons[why], length(state.route.add), why)); if (failed) @@ -211,8 +212,7 @@ function apply(state) { function flush(state) { if (state) - for (let argv in state.route.del) - quiet(argv); + rt_quiet(state.route.del); nft_pipe(teardown(), 'teardown'); @@ -274,8 +274,9 @@ function rp_filter_blocked(policies) { // device that exists. Off by default: changing a security setting is opt-in. function rp_filter_apply(policies) { for (let dev in policy_devices(policies)) - if (rp_read(dev) != '' && rp_read(dev) != '2') - loud([ 'sysctl', '-w', sprintf('net.ipv4.conf.%s.rp_filter=2', dev) ]); + if (rp_read(dev) != '' && rp_read(dev) != '2' && + !writefile(`/proc/sys/net/ipv4/conf/${dev}/rp_filter`, '2\n')) + log('warn', sprintf('cannot set rp_filter on %s: %s', dev, fs_error())); } // Reads the live /proc value, so when rp_filter_apply has done its job the @@ -298,9 +299,15 @@ function build_state(silent) { cfg.policies = netifd_resolve(cfg.policies, netifd_dump()); - let matcher = null; + let matcher = null, files = null; + // The files are read only when the matcher is built from them - a + // teardown renders the policy from its config alone, see nft.uc. if (!silent) { + files = files_load(cfg.policies); + cfg.policies = files.policies; + report('file', files.issues); + matcher = match_compile(cfg.policies); report('domain', matcher.issues); } @@ -313,7 +320,7 @@ function build_state(silent) { if (!silent) report('route', r.issues); - return { cfg, matcher, nft: n, route: r }; + return { cfg, matcher, files, nft: n, route: r }; } // nft -f reads the entire ruleset before resolving a single name, so on a box @@ -408,11 +415,36 @@ function run() { return 1; } + mkdir(FILES_DIR, 0o755); + let cache = dedupe_create(state.cfg.global.entry_ttl); let targets = poll_names(state.cfg.policies); let stats = { started: time(), resolv: false, snoop: [], - matched: 0, drops: {}, reapplied: 0 }; + matched: 0, drops: {}, reapplied: 0, + files: { count: state.files.files, entries: state.files.entries, + refreshed: null } }; + + // Re-reads the domain files and swaps the matcher - the one thing a + // changed list needs. Nothing else moves: the sets exist whether the + // file does, poll never had the file names, routes and rules are not + // touched, and the learned addresses stay put. Runs on the uloop + // thread between callbacks, so snoop never sees a half-built matcher. + function refresh_files() { + let f = files_load(state.cfg.policies); + + report('file', f.issues); + + state.cfg.policies = f.policies; + state.matcher = match_compile(state.cfg.policies); + + stats.files = { count: f.files, entries: f.entries, refreshed: time() }; + + log('info', sprintf('refresh: %d file(s), %d pattern(s), %d issue(s)', + f.files, f.entries, length(f.issues))); + + return { files: f.files, entries: f.entries, issues: f.issues }; + } try { resolv = require('resolv'); @@ -469,8 +501,7 @@ function run() { // and on a failed query, while snoop keeps feeding queue_writes() in // all three cases. function tick() { - for (let argv in state.route.add) - quiet(argv); + rt_quiet(state.route.add); ensure_table(wq); @@ -510,10 +541,8 @@ function run() { check_rp_filter(resolved); - for (let argv in state.route.del) - quiet(argv); - for (let argv in r.add) - quiet(argv); + rt_quiet(state.route.del); + rt_quiet(r.add); state.route = r; } @@ -553,11 +582,25 @@ function run() { drops: stats.drops }, reapplied: stats.reapplied, - dedupe: cache.size() + dedupe: cache.size(), + files: stats.files }; } - let obj = c.publish('shunt', { status: { call: () => status_reply() } }); + function refresh_reply() { + try { + return refresh_files(); + } + catch (e) { + log('err', sprintf('refresh failed: %s', e)); + return { error: `${e}` }; + } + } + + let obj = c.publish('shunt', { + status: { call: () => status_reply() }, + refresh: { call: () => refresh_reply() } + }); if (!obj) log('warn', sprintf('cannot publish ubus object: %s', @@ -639,6 +682,43 @@ function run() { return 0; } +// Asks the running daemon to re-read its domain files. Without a daemon +// there is nothing to refresh and nothing lost: the next start reads them. +function refresh() { + let c = null; + + try { + c = require('ubus').connect(); + } + catch (e) { + c = null; + } + + if (!c) { + warn('ubus unavailable - cannot reach the daemon\n'); + return 1; + } + + let r = c.call('shunt', 'refresh'); + + if (r == null) { + warn('shunt is not running - the files are read at start\n'); + return 1; + } + + if (r.error) { + warn(sprintf('refresh failed: %s\n', r.error)); + return 1; + } + + printf('files: %d file(s), %d pattern(s)\n', r.files, r.entries); + + for (let i in r.issues) + printf('issue: %J\n', i); + + return 0; +} + function check() { verbose = true; @@ -659,10 +739,12 @@ function check() { printf(' %-16s mark 0x%08x table %d pref %d\n', m.name, m.mark, m.rt_table, m.rt_prio); - let total = length(state.matcher.issues) + length(state.nft.issues) + - length(state.route.issues); + let total = length(state.files.issues) + length(state.matcher.issues) + + length(state.nft.issues) + length(state.route.issues); printf('issues: %d (see above)\n', total); + printf('files: %d file(s), %d pattern(s)\n', state.files.files, + state.files.entries); printf('poll: %d name(s)\n', length(poll_names(state.cfg.policies))); return length(state.nft.marks) ? 0 : 2; @@ -685,7 +767,9 @@ case 'check': case 'flush': flush(build_state(true)); exit(0); +case 'refresh': + exit(refresh()); default: - warn('usage: shunt [-v] run|check|flush\n'); + warn('usage: shunt [-v] run|check|flush|refresh\n'); exit(2); } diff --git a/net/shunt/src/config.uc b/net/shunt/src/config.uc index 11e16c4b5..8ab9d7ee7 100644 --- a/net/shunt/src/config.uc +++ b/net/shunt/src/config.uc @@ -171,7 +171,8 @@ export function parse(sections) { dport: to_list(v.dport), proto: to_list(v.proto), dst: to_list(v.dst), - domains: to_list(v.domain) + domains: to_list(v.domain), + domain_files: to_list(v.domain_file) }); } diff --git a/net/shunt/src/domain_file.uc b/net/shunt/src/domain_file.uc new file mode 100644 index 000000000..6700c2ccf --- /dev/null +++ b/net/shunt/src/domain_file.uc @@ -0,0 +1,175 @@ +// shunt - domain patterns from files +// +// Reads the files a policy names in `domain_file` and attaches their +// patterns as `file_domains`, kept apart from the hand-entered `domains`: +// the matcher takes both, poll takes only the latter. A file is the shape +// a community list comes in, and resolving ten thousand names every poll +// interval is not a service, it is a DNS storm - so file entries are +// learned passively, by snoop, when a client asks for them. +// +// Where the file comes from is not shunt's business. cron, a script, a +// package of its own - whatever writes it runs `shunt refresh`, and the +// daemon re-reads. A bare name lives under DIR, which is tmpfs: a list of +// that size has no business on flash, and a file that is not there yet +// after a reboot is the expected state, not a fault. +// +// load() is pure and takes the reader as an argument; read() is the one +// backed by the file system. +// +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (c) 2026 Dirk Brenken + +import { readfile, stat, error as fs_error } from 'fs'; +import { pattern } from 'shunt.match'; + +// Hard limit on one file: a list of that size is the wrong tool on a +// router, and reading it into a ucode array would show why. +export const LIM = { + size: 4 * 1024 * 1024 +}; + +export const DIR = '/tmp/shunt'; + +// A bare name resolves under DIR; an absolute path is taken as given, for +// a list that lives on a mounted drive or in an overlay of the user's +// choosing. Anything else - empty, or a relative path with directories - +// is not a file name. +export function resolve(name) { + if (type(name) != 'string' || !length(name)) + return null; + if (substr(name, 0, 1) == '/') + return name; + if (index(name, '/') >= 0) + return null; + + return `${DIR}/${name}`; +}; + +export function read(path) { + let st = stat(path); + + if (st == null) + return { error: 'not found - `shunt refresh` loads it once it is there', missing: true }; + + if (st.type != 'file') + return { error: `not a regular file (${st.type})` }; + + if (st.size > LIM.size) + return { error: sprintf('too large (%d bytes, limit %d)', st.size, LIM.size) }; + + let text = readfile(path); + + if (text == null) + return { error: fs_error() ?? 'unreadable' }; + + return { text }; +}; + +// One pattern per line, `#` starts a comment, blanks are skipped. Only the +// count and the first offender are reported, not every bad line - a list +// with a thousand broken entries must not turn into a thousand log lines. +export function parse(text) { + let patterns = [], seen = {}; + let lines = 0, rejected = 0, first = null; + + for (let line in split(text ?? '', '\n')) { + lines++; + + let hash = index(line, '#'); + if (hash >= 0) + line = substr(line, 0, hash); + + line = trim(line); + if (!length(line)) + continue; + + let r = pattern(line); + + if (r.error) { + rejected++; + if (first == null) + first = sprintf('line %d: %s', lines, r.error); + continue; + } + + let key = (r.wild ? '*.' : '') + r.pat; + + if (seen[key]) + continue; + + seen[key] = true; + push(patterns, key); + } + + return { patterns, rejected, first }; +}; + +export function load(policies, reader) { + let rd = reader ?? read; + let cache = {}, issues = [], out = []; + let files = 0, entries = 0; + + function reject(policy, entry, reason) { + push(issues, { policy, entry, reason }); + } + + function fetch(path) { + if (cache[path] != null) + return cache[path]; + + let file = resolve(path); + let res = { file: file ?? path, patterns: [], error: null }; + + if (file == null) { + res.error = 'not a file name - a bare name or an absolute path'; + } + else { + let f = rd(file); + + if (f.error) { + res.error = f.missing ? f.error : `cannot read: ${f.error}`; + } + else { + let p = parse(f.text); + + res.patterns = p.patterns; + + if (!length(p.patterns)) + res.error = p.rejected + ? sprintf('no usable pattern, %d line(s) rejected (%s)', + p.rejected, p.first) + : 'no usable pattern'; + else if (p.rejected) + res.warn = sprintf('%d line(s) rejected (%s), %d pattern(s) kept', + p.rejected, p.first, length(p.patterns)); + + files++; + entries += length(p.patterns); + } + } + + cache[path] = res; + + return res; + } + + for (let p in (policies ?? [])) { + let file_domains = []; + + for (let path in (p?.domain_files ?? [])) { + let res = fetch(path); + + if (res.error) + reject(p?.name, res.file, res.error); + else if (res.warn) + reject(p?.name, res.file, res.warn); + + for (let pat in res.patterns) + push(file_domains, pat); + } + + push(out, { ...p, file_domains }); + } + + return { policies: out, issues, files, entries }; +}; diff --git a/net/shunt/src/match.uc b/net/shunt/src/match.uc index 165c8b6bb..c3b252ced 100644 --- a/net/shunt/src/match.uc +++ b/net/shunt/src/match.uc @@ -35,19 +35,49 @@ export function normalize(s) { return s; }; +// The regex is the fast path and says the same thing ok_label() does; the +// label walk runs only on failure, to name the offender. Measured on 100k +// patterns from a file: four times faster than walking every label. +const RE_NAME = /^[a-z0-9_-]{1,63}(\.[a-z0-9_-]{1,63})*$/; + function validate(name) { if (!length(name)) return 'empty'; if (length(name) > LIM.name) return 'too long'; + if (match(name, RE_NAME)) + return null; + for (let l in split(name, '.')) if (!ok_label(l)) return `bad label '${l}'`; - return null; + return 'invalid'; } +// One pattern, checked the way compile() checks it: `{ pat, wild }` for a +// usable one, `{ error }` otherwise. Exported so a domain file is validated +// by the same rules as a `list domain` entry. +export function pattern(raw) { + let pat = normalize(raw); + let wild = false; + + if (substr(pat, 0, 2) == '*.') { + wild = true; + pat = substr(pat, 2); + } + + if (index(pat, '*') >= 0) + return { error: 'wildcard only allowed as leading *. label' }; + + let bad = validate(pat); + if (bad) + return { error: bad }; + + return { pat, wild }; +}; + export function compile(policies) { let exact = {}, wild = {}, issues = []; @@ -55,43 +85,46 @@ export function compile(policies) { push(issues, { policy, pattern, reason }); } + function insert(pname, pat, is_wild) { + let map = is_wild ? wild : exact; + + if (!map[pat]) + map[pat] = []; + + let dup = false; + + for (let owner in map[pat]) + if (owner == pname) + dup = true; + + if (!dup) + push(map[pat], pname); + } + for (let pi = 0; pi < length(policies ?? []); pi++) { let p = policies[pi]; let pname = p?.name ?? `#${pi}`; for (let raw in (p?.domains ?? [])) { - let pat = normalize(raw); - let is_wild = false; - - if (substr(pat, 0, 2) == '*.') { - is_wild = true; - pat = substr(pat, 2); - } - - if (index(pat, '*') >= 0) { - reject(pname, raw, 'wildcard only allowed as leading *. label'); - continue; - } + let r = pattern(raw); - let bad = validate(pat); - if (bad) { - reject(pname, raw, bad); + if (r.error) { + reject(pname, raw, r.error); continue; } - let map = is_wild ? wild : exact; - - if (!map[pat]) - map[pat] = []; - - let dup = false; - - for (let owner in map[pat]) - if (owner == pname) - dup = true; + insert(pname, r.pat, r.wild); + } - if (!dup) - push(map[pat], pname); + // file_domains arrive canonical from domain_file.uc - validated, + // lower case, deduplicated - so they are not checked a second + // time: with a list of a hundred thousand names that is the + // difference between a start and a stall. + for (let pat in (p?.file_domains ?? [])) { + if (substr(pat, 0, 2) == '*.') + insert(pname, substr(pat, 2), true); + else + insert(pname, pat, false); } } diff --git a/net/shunt/src/nft.uc b/net/shunt/src/nft.uc index a964d5592..a5ec3f909 100644 --- a/net/shunt/src/nft.uc +++ b/net/shunt/src/nft.uc @@ -209,7 +209,11 @@ export function compile(policies, opts) { if (length(ports) && !length(protos)) protos = [ 'tcp', 'udp' ]; - let has_dom = length(p.domains ?? []) > 0; + // A named domain file counts as a domain selector whether or not it + // was readable at this moment: the policy's sets exist and stay + // empty until the file is, and `shunt flush` tears the policy down + // without having to read anything. + let has_dom = length(p.domains ?? []) + length(p.domain_files ?? []) > 0; let has_dst_any = length(dst['4']) || length(dst['6']); // Ports and protocols were asked for and none survived validation. diff --git a/net/shunt/src/route.uc b/net/shunt/src/route.uc index b3b71bba6..c4bf1eecb 100644 --- a/net/shunt/src/route.uc +++ b/net/shunt/src/route.uc @@ -1,7 +1,9 @@ -// shunt - ip rule and route renderer +// shunt - rule and route renderer // -// Renders the argv arrays for the policy routing tables and their rules. -// Pure, like nft.uc - nothing here talks to the kernel. +// Renders the netlink operations for the policy routing tables and their +// rules, as rt.uc sends them. Pure, like nft.uc - nothing here talks to +// the kernel, which is why the handful of kernel constants it needs are +// spelled out below rather than read from the rtnl module. // // SPDX-License-Identifier: GPL-3.0-or-later // Copyright (c) 2026 Dirk Brenken @@ -12,6 +14,19 @@ const RE_IFACE = /^[A-Za-z0-9_][A-Za-z0-9_.-]{0,14}$/; const BLACKHOLE_METRIC = 9999; +// Kernel ABI, from linux/rtnetlink.h and linux/fib_rules.h. Fixed for as +// long as netlink exists, so a literal here costs nothing and keeps the +// renderer importable without ucode-mod-rtnl. +const AF = { '4': 2, '6': 10 }; +const ANY = { '4': '0.0.0.0/0', '6': '::/0' }; +const RTN_UNICAST = 1; +const RTN_BLACKHOLE = 6; +const RTPROT_BOOT = 3; +const RT_SCOPE_UNIVERSE = 0; +const RT_SCOPE_LINK = 253; +const RT_TABLE_MAIN = 254; +const FR_ACT_TO_TBL = 1; + // Policy options arrive as UCI strings - config.uc only collects them - so the // one boolean among them is read here, with the rest of the routing checks. function to_bool(v, dflt) { @@ -85,27 +100,28 @@ export function compile(policies, marks, opts) { keep = true; } - let fwmark = sprintf('0x%x/0x%x', m.mark, mask); - let table = sprintf('%d', m.rt_table); - let pref = sprintf('%d', m.rt_prio); - let pref_local = sprintf('%d', m.rt_prio_local); + let table = m.rt_table; push(tables, sprintf('%d\tshunt_%s', m.rt_table, m.name)); for (let fam in [ '4', '6' ]) { - let v = `-${fam}`; - - let route = [ 'ip', v, 'route', 'replace', 'default' ]; + let family = AF[fam]; + + // `ip route replace default [via gw] dev iface table n`: a + // gateway makes it a global-scope route, without one it is + // the point to point form with link scope, as ip renders it. + let route = { family, table, dst: ANY[fam], oif: iface, + type: RTN_UNICAST, protocol: RTPROT_BOOT, + scope: gw[fam] ? RT_SCOPE_UNIVERSE : RT_SCOPE_LINK }; if (gw[fam]) - push(route, 'via', gw[fam]); - push(route, 'dev', iface, 'table', table); - push(add, route); + route.gateway = gw[fam]; + push(add, { cmd: 'newroute', msg: route }); if (fb == 'block') - push(add, [ 'ip', v, 'route', 'replace', 'blackhole', - 'default', 'metric', - sprintf('%d', BLACKHOLE_METRIC), - 'table', table ]); + push(add, { cmd: 'newroute', msg: { family, table, + dst: ANY[fam], type: RTN_BLACKHOLE, + protocol: RTPROT_BOOT, scope: RT_SCOPE_UNIVERSE, + priority: BLACKHOLE_METRIC } }); // Ahead of the policy rule and on the same mark: main is // consulted with its default route suppressed, so marked traffic @@ -113,18 +129,20 @@ export function compile(policies, marks, opts) { // subnet, every static route - keeps taking it, and only what // would have used the default route reaches the policy table. if (keep) - push(add, [ 'ip', v, 'rule', 'add', 'pref', pref_local, - 'fwmark', fwmark, 'lookup', 'main', - 'suppress_prefixlength', '0' ]); + push(add, { cmd: 'newrule', msg: { family, + action: FR_ACT_TO_TBL, priority: m.rt_prio_local, + fwmark: m.mark, fwmask: mask, table: RT_TABLE_MAIN, + suppress_prefixlen: 0 } }); - push(add, [ 'ip', v, 'rule', 'add', 'pref', pref, - 'fwmark', fwmark, 'lookup', table ]); + push(add, { cmd: 'newrule', msg: { family, + action: FR_ACT_TO_TBL, priority: m.rt_prio, + fwmark: m.mark, fwmask: mask, table } }); - unshift(del, [ 'ip', v, 'route', 'flush', 'table', table ]); - unshift(del, [ 'ip', v, 'rule', 'del', 'pref', pref ]); + unshift(del, { cmd: 'flush', msg: { family, table } }); + unshift(del, { cmd: 'delrule', msg: { family, priority: m.rt_prio } }); // Deleted whether or not it is rendered now: keep_local may have // been on when the running ruleset was applied. - unshift(del, [ 'ip', v, 'rule', 'del', 'pref', pref_local ]); + unshift(del, { cmd: 'delrule', msg: { family, priority: m.rt_prio_local } }); } } diff --git a/net/shunt/src/rt.uc b/net/shunt/src/rt.uc new file mode 100644 index 000000000..53c49f6be --- /dev/null +++ b/net/shunt/src/rt.uc @@ -0,0 +1,110 @@ +// shunt - route and rule application +// +// Sends the operations route.uc renders to the kernel over rtnetlink, +// through ucode-mod-rtnl, where an `ip` process per command used to be. +// +// rtnl.request() answers null on success, false when the kernel refused +// and null again when an attribute did not encode - an `oif` naming a +// device that is not there yet, the boot-time case. So the return value +// is not the verdict; rtnl.error() is, and it is null after a success. +// +// SPDX-License-Identifier: GPL-3.0-or-later +// Copyright (c) 2026 Dirk Brenken + +// require() inside so the module stays importable without rtnl, and the +// caller decides what a missing module means for it. +export function open() { + let rtnl; + + try { + rtnl = require('rtnl'); + } + catch (e) { + return null; + } + + return rtnl; +}; + +function call(rtnl, cmd, flags, msg) { + rtnl.request(cmd, flags, msg); + + return rtnl.error(); +} + +// The kernel does not filter a route dump by table on its own - the whole +// forwarding base comes back - so the table is picked out here. A default +// route carries no dst attribute in the dump but needs one to be deleted, +// and the rest of the entry is echoed so the match is unambiguous. +function flush(rtnl, C, msg) { + let routes = rtnl.request(C.RTM_GETROUTE, C.NLM_F_DUMP, + { family: msg.family, table: msg.table }); + + if (type(routes) != 'array') + return rtnl.error(); + + let err = null; + + for (let r in routes) { + if (r?.table != msg.table) + continue; + + let del = { family: r.family, table: r.table, + dst: r.dst ?? (r.family == C.AF_INET6 ? '::/0' : '0.0.0.0/0'), + type: r.type, scope: r.scope, protocol: r.protocol }; + + for (let k in [ 'oif', 'gateway', 'priority' ]) + if (r[k] != null) + del[k] = r[k]; + + err = call(rtnl, C.RTM_DELROUTE, 0, del) ?? err; + } + + return err; +} + +// One rendered operation. Returns null when the kernel took it, otherwise +// the reason as text - with no exit code left to interpret. +export function exec(rtnl, op) { + let C = rtnl.const; + + switch (op?.cmd) { + case 'newroute': + return call(rtnl, C.RTM_NEWROUTE, C.NLM_F_CREATE | C.NLM_F_REPLACE, op.msg); + case 'newrule': + return call(rtnl, C.RTM_NEWRULE, C.NLM_F_CREATE | C.NLM_F_EXCL, op.msg); + case 'delrule': + return call(rtnl, C.RTM_DELRULE, 0, op.msg); + case 'flush': + return flush(rtnl, C, op.msg); + default: + return sprintf('unknown operation %J', op?.cmd); + } +}; + +// For the log: the operation in the shape a reader knows from `ip`. +export function describe(op) { + let m = op?.msg ?? {}; + let v = (m.family == 10) ? '-6' : '-4'; + + switch (op?.cmd) { + case 'newroute': + return sprintf('ip %s route replace %sdefault%s%s%s table %d', v, + m.type == 6 ? 'blackhole ' : '', + m.gateway ? ` via ${m.gateway}` : '', + m.oif ? ` dev ${m.oif}` : '', + m.priority != null ? ` metric ${m.priority}` : '', m.table); + case 'newrule': + return sprintf('ip %s rule add pref %d fwmark 0x%x/0x%x lookup %s%s', v, + m.priority, m.fwmark, m.fwmask, + m.table == 254 ? 'main' : sprintf('%d', m.table), + m.suppress_prefixlen != null + ? sprintf(' suppress_prefixlength %d', m.suppress_prefixlen) : ''); + case 'delrule': + return sprintf('ip %s rule del pref %d', v, m.priority); + case 'flush': + return sprintf('ip %s route flush table %d', v, m.table); + default: + return sprintf('%J', op); + } +}; diff --git a/net/shunt/test.sh b/net/shunt/test.sh index 394ae21e3..56cc96700 100755 --- a/net/shunt/test.sh +++ b/net/shunt/test.sh @@ -6,5 +6,7 @@ shunt 2>&1 | grep 'usage: shunt' || exit 1 ucode -e 'import * as a from "shunt.config"; import * as b from "shunt.nft"; import * as c from "shunt.dns"; import * as d from "shunt.frame"; - import * as e from "shunt.match"; print("modules-ok\n");' | + import * as e from "shunt.match"; import * as f from "shunt.domain_file"; + import * as g from "shunt.route"; import * as h from "shunt.rt"; + print("modules-ok\n");' | grep 'modules-ok'