From: Stan Grishin Date: Mon, 15 Jun 2026 01:42:31 +0000 (+0000) Subject: adblock-fast: update to 1.2.4-2 X-Git-Url: http://git.99rst.org/?a=commitdiff_plain;h=ad7bc019e28d6b7d33e22ac4b5c5515d54ccd307;p=openwrt-packages.git adblock-fast: update to 1.2.4-2 Maintainer: me Compile tested: x86_64, Dell EMC Edge620, OpenWrt 25.12.4 Run tested: x86_64, Dell EMC Edge620, OpenWrt 25.12.4 Description: Update to 1.2.4 - Update PKG_VERSION to 1.2.4 and PKG_RELEASE to 2. - Update documentation URL from melmac.ca to mossdef.org. README.md: - Update documentation URL from melmac.ca to mossdef.org. files/etc/config/adblock-fast: - Add default option `download_connect_timeout '10'`. - Add default option `download_allow_insecure '1'`. - Set default `parallel_downloads` to `8`. - Remove commented `download_max_time` line. files/etc/init.d/adblock-fast: - Increment `initCompat` to 17. - Add `download_connect_timeout`, `download_max_time`, `download_allow_insecure` to config validation schema. files/etc/uci-defaults/90-adblock-fast: - Use `initCompat` from init script to stamp `config_compat`. - Migrate `parallel_downloads` from boolean to numeric cap for compat < 15. - Seed new `download_connect_timeout` from old `download_timeout` for compat < 16. - Add new `config_compat` and `config_version` stamping. files/lib/adblock-fast/adblock-fast.uc: - Increment package `compat` to 17. - Introduce `task_slot_ram` for per-downloader memory budgeting. - Add `download_connect_timeout`, `download_max_time`, `download_allow_insecure` to `reload` triggers. - Store `downloader` kind and track `_last_dl_timeout` status. - Refine downloader detection: prefer curl, then uclient-fetch, then GNU wget, finally generic wget. - Update `curl` command flags for `--insecure`, `--connect-timeout`, `--speed-limit`/`--speed-time`, `--max-time`. - Update `uclient-fetch` flags for `--no-check-certificate`. - Update `wget` flags for `--no-check-certificate`, `--connect-timeout`, `--read-timeout`. - Improve SSL support detection for uclient-fetch. - Capture `curl` exit code 28 as a distinct timeout failure. - Expand `get_text` cases for `errorDetectingFileType`, `warningMissingRecommendedPackages` (args), `warningParallelDownloadsThrottled`, `warningDownloadTimeout`. - Allow `download_allow_insecure` for `get_url_filesize` calls. - Set `download_allow_insecure` default to true in `config_schema`. - Change `parallel_downloads` from boolean to integer `8` default in `config_schema`. - Add `download_connect_timeout` and `download_max_time` to `config_schema`. - Read up to 4KB for `detect_file_type`. - Improve `warningMissingRecommendedPackages` output: list missing packages and use `apk` or `opkg` install command. - Split `process_file_url` into parallelizable `prepare_file_url` and serial `apply_result`, plus `emit_dl_line`. - Implement memory-aware throttling for `parallel_downloads`. - Revert to single `process_file_url` for serial uses and test runner. - Correct `get_mem_available` to return current free memory only. - Rename `get_mem_total` to `get_mem_available`. Signed-off-by: Stan Grishin --- diff --git a/net/adblock-fast/Makefile b/net/adblock-fast/Makefile index 2b4385d51..01d58c0fe 100644 --- a/net/adblock-fast/Makefile +++ b/net/adblock-fast/Makefile @@ -4,8 +4,8 @@ include $(TOPDIR)/rules.mk PKG_NAME:=adblock-fast -PKG_VERSION:=1.2.2 -PKG_RELEASE:=18 +PKG_VERSION:=1.2.4 +PKG_RELEASE:=2 PKG_MAINTAINER:=Stan Grishin PKG_LICENSE:=AGPL-3.0-or-later @@ -34,7 +34,7 @@ endef define Package/adblock-fast/description Fast AdBlocking script to block ad or abuse/malware domains with Dnsmasq, SmartDNS or Unbound. Script supports local/remote list of domains and hosts-files for both block-listing and allow-listing. -Please see https://docs.openwrt.melmac.ca/adblock-fast/ for more information. +Please see https://docs.mossdef.org/adblock-fast/ for more information. endef define Package/adblock-fast/conffiles diff --git a/net/adblock-fast/README.md b/net/adblock-fast/README.md index df2669414..f3fd7f695 100644 --- a/net/adblock-fast/README.md +++ b/net/adblock-fast/README.md @@ -1,7 +1,7 @@ # adblock-fast [![OpenWrt](https://img.shields.io/badge/OpenWrt-Compatible-blueviolet)](https://openwrt.org) -[![Web UI](https://img.shields.io/badge/Web_UI-Available-blue)](https://docs.openwrt.melmac.ca/adblock-fast/) +[![Web UI](https://img.shields.io/badge/Web_UI-Available-blue)](https://docs.mossdef.org/adblock-fast/) [![Lightweight](https://img.shields.io/badge/Size-Lightweight-brightgreen)](https://openwrt.org/packages/pkgdata/adblock-fast) [![License](https://img.shields.io/badge/License-AGPL--3.0--or--later-lightgrey)](https://github.com/stangri/adblock-fast/blob/master/LICENSE) @@ -17,4 +17,4 @@ It runs once to process and install blocklists, then exits — keeping memory us - Reverts if DNS resolution fails after restart 📚 **Full documentation:** -[https://docs.openwrt.melmac.ca/adblock-fast/](https://docs.openwrt.melmac.ca/adblock-fast/) +[https://docs.mossdef.org/adblock-fast/](https://docs.mossdef.org/adblock-fast/) diff --git a/net/adblock-fast/files/etc/config/adblock-fast b/net/adblock-fast/files/etc/config/adblock-fast index e82639b0a..6ef939b87 100644 --- a/net/adblock-fast/files/etc/config/adblock-fast +++ b/net/adblock-fast/files/etc/config/adblock-fast @@ -16,7 +16,10 @@ config adblock-fast 'config' option dns 'dnsmasq.servers' list dnsmasq_instance '*' # option dnsmasq_config_file_url 'https://small.oisd.nl/dnsmasq2' + option download_connect_timeout '10' option download_timeout '10' +# option download_max_time '' + option download_allow_insecure '1' option force_dns '1' list force_dns_port '53' list force_dns_port '853' @@ -28,13 +31,13 @@ config adblock-fast 'config' # list force_dns_port '4434' # list force_dns_port '5443' # list force_dns_port '8443' - option parallel_downloads '1' + option parallel_downloads '8' option pause_timeout '60' option rpcd_token '' option procd_trigger_wan6 '0' option procd_boot_wan_timeout '60' option verbosity '2' - option heartbeat_domain 'heartbeat.melmac.ca' + option heartbeat_domain 'heartbeat.mossdef.org' option heartbeat_sleep_timeout '10' option sanity_check '1' option update_config_sizes '1' diff --git a/net/adblock-fast/files/etc/init.d/adblock-fast b/net/adblock-fast/files/etc/init.d/adblock-fast index 115295a4b..ecbd00488 100644 --- a/net/adblock-fast/files/etc/init.d/adblock-fast +++ b/net/adblock-fast/files/etc/init.d/adblock-fast @@ -22,6 +22,7 @@ if type extra_command 1>/dev/null 2>&1; then fi readonly packageName='adblock-fast' +readonly initCompat='17' readonly _ucode="ucode -S -L /lib/${packageName} /lib/${packageName}/cli.uc --" _procd_svc_data= _fw4_restart= @@ -108,6 +109,9 @@ load_validate_config() { 'config_update_enabled:bool:0' \ 'config_update_url:string:https://cdn.jsdelivr.net/gh/openwrt/packages/net/adblock-fast/files/adblock-fast.config.update' \ 'download_timeout:range(1,60):20' \ + 'download_connect_timeout:range(1,60):10' \ + 'download_max_time:or("", uinteger)' \ + 'download_allow_insecure:bool:1' \ 'pause_timeout:range(1,600):20' \ 'rpcd_token:string' \ 'curl_additional_param:or("", string)' \ @@ -120,7 +124,7 @@ load_validate_config() { 'dns:or("dnsmasq.addnhosts", "dnsmasq.conf", "dnsmasq.ipset", "dnsmasq.nftset", "dnsmasq.servers", "smartdns.domainset", "smartdns.ipset", "smartdns.nftset", "unbound.adb_list"):dnsmasq.servers' \ 'dnsmasq_instance:list(or("*", "-", uinteger, uci("dhcp", "@dnsmasq"))):*' \ 'smartdns_instance:list(or("*", "-", uinteger, uci("smartdns", "@smartdns"))):*' \ - 'heartbeat_domain:or("-", string):heartbeat.melmac.ca' \ + 'heartbeat_domain:or("-", string):heartbeat.mossdef.org' \ 'heartbeat_sleep_timeout:range(1,60):10' \ 'dnsmasq_sanity_check:bool:1' \ 'dnsmasq_validity_check:bool:0' \ diff --git a/net/adblock-fast/files/etc/uci-defaults/90-adblock-fast b/net/adblock-fast/files/etc/uci-defaults/90-adblock-fast index 0e2cd0331..f5c489ffc 100644 --- a/net/adblock-fast/files/etc/uci-defaults/90-adblock-fast +++ b/net/adblock-fast/files/etc/uci-defaults/90-adblock-fast @@ -4,6 +4,15 @@ readonly pkg='adblock-fast' +# Source the init script for $initCompat (compat-era stamp) and $_ucode. +readonly initFile="/etc/init.d/${pkg}" +# shellcheck disable=SC1091 +[ -s "$initFile" ] && . "$initFile" + +# Capture the previously-stamped config_compat before we (re)stamp it at +# the end of this script — used to gate one-shot migrations. +old_compat="$(uci -q get "${pkg}.config.config_compat")" + # ── Transition to list names ───────────────────────────────────────── # Adds 'name' to file_url sections that lack one, using the pristine default config @@ -58,7 +67,34 @@ if [ -z "$(uci -q get "${pkg}.config.dnsmasq_sanity_check")" ] \ uci -q delete "${pkg}.config.sanity_check" fi -# ── Commit if anything changed ─────────────────────────────────────── +# ── Migrate parallel_downloads bool → numeric cap (one-shot) ───────── +# Pre-1.2.4 (compat < 15) treated parallel_downloads as a bool — '1' meant +# "parallel on". 1.2.4 made it a numeric cap. Convert the legacy '1' to 8. +# Once a 1.2.4+ run stamps config_compat below, this never re-fires. + +if [ "${old_compat:-14}" -lt 15 ] 2>/dev/null; then + [ "$(uci -q get "${pkg}.config.parallel_downloads")" = "1" ] && \ + uci set "${pkg}.config.parallel_downloads=8" +fi + +# ── Split download_timeout → download_connect_timeout (compat < 16) ── +# Pre-compat-16, download_timeout doubled as the connection-phase timeout +# (curl --connect-timeout, wget --timeout). It now means a transfer-stall +# timeout, and the connection phase has its own download_connect_timeout. +# Seed the new option from the user's existing download_timeout so their +# old connect tolerance is preserved. Skip if download_connect_timeout is +# already set (fresh installs ship one); never re-fires once compat >= 16. + +if [ "${old_compat:-14}" -lt 16 ] 2>/dev/null; then + if [ -z "$(uci -q get "${pkg}.config.download_connect_timeout")" ] \ + && [ -n "$(uci -q get "${pkg}.config.download_timeout")" ]; then + uci set "${pkg}.config.download_connect_timeout=$(uci -q get "${pkg}.config.download_timeout")" + fi +fi + +# ── Version tracking ────────────────────────── +[ -n "$initCompat" ] && uci set "${pkg}.config.config_compat=$initCompat" +uci set "${pkg}.config.config_version=$($_ucode version 2>/dev/null)" [ -n "$(uci -q changes "$pkg" 2>/dev/null)" ] && uci commit "$pkg" diff --git a/net/adblock-fast/files/lib/adblock-fast/adblock-fast.uc b/net/adblock-fast/files/lib/adblock-fast/adblock-fast.uc index 5c4e6779e..2309cb519 100644 --- a/net/adblock-fast/files/lib/adblock-fast/adblock-fast.uc +++ b/net/adblock-fast/files/lib/adblock-fast/adblock-fast.uc @@ -15,13 +15,17 @@ import * as uloop from 'uloop'; const pkg = { name: 'adblock-fast', version: 'dev-test', - compat: '14', + compat: '17', memory_threshold: 33554432, + // Per parallel-download task slot: ucode task child (~1 MB) + the + // downloader's RSS. Measured: curl ~2.8 MB, uclient-fetch ~2.5 MB, + // GNU wget ~8.5 MB. Conservative round-ups, in bytes. + task_slot_ram: { curl: 4194304, 'uclient-fetch': 4194304, wget: 10485760, 'default': 10485760 }, config_file: '/etc/config/adblock-fast', dnsmasq_file: '/var/run/adblock-fast/adblock-fast.dnsmasq', run_file: '/dev/shm/adblock-fast', triggers: { - reload: 'parallel_downloads debug download_timeout allowed_domain blocked_domain allowed_url blocked_url dns config_update_enabled config_update_url dnsmasq_config_file_url curl_additional_param curl_max_file_size curl_retry', + reload: 'parallel_downloads debug download_timeout download_connect_timeout download_max_time download_allow_insecure allowed_domain blocked_domain allowed_url blocked_url dns config_update_enabled config_update_url dnsmasq_config_file_url curl_additional_param curl_max_file_size curl_retry', restart: 'compressed_cache compressed_cache_dir force_dns led force_dns_port', }, }; @@ -179,6 +183,9 @@ let env = { // Downloader (set lazily by env.get_downloader()) _dl_cache: null, + // Set by download(): true when the last transfer was aborted by a timeout + // (only curl reports this distinctly — see download()). + _last_dl_timeout: false, // Guard flags _detected: false, @@ -319,30 +326,68 @@ env.detect = function() { env.get_downloader = function() { if (env._dl_cache) return env._dl_cache; - let command, flag, ssl_supported; + let command, flag, ssl_supported, kind; + // Preference: curl, then uclient-fetch (both ~2.5-2.8 MB RSS); GNU wget + // is last — it is ~8.5 MB RSS, ~3x heavier, costly when run in parallel. + // Timeout semantics differ per downloader (see README support matrix): + // download_connect_timeout → connection phase only (curl/GNU wget). + // download_timeout → abort a stalled transfer: curl has no read-timeout, + // so emulate it with --speed-limit 1 --speed-time (abort if avg speed + // stays below 1 B/s for that many seconds); GNU wget uses --read-timeout; + // uclient-fetch / BusyBox wget only have --timeout (inactivity-based). + // download_max_time → hard cap on the whole transfer; generically named + // so any downloader can adopt it, though only curl (--max-time) + // implements it today. if (is_present('curl')) { - command = 'curl -f --silent --insecure'; - if (cfg.curl_additional_param) command += ' ' + shell_quote(cfg.curl_additional_param); + kind = 'curl'; + command = 'curl -f --silent'; + if (cfg.download_allow_insecure) command += ' --insecure'; + if (cfg.curl_additional_param) command += ' ' + cfg.curl_additional_param; if (cfg.curl_max_file_size) command += ' --max-filesize ' + cfg.curl_max_file_size; if (cfg.curl_retry) command += ' --retry ' + cfg.curl_retry; - if (cfg.download_timeout) command += ' --connect-timeout ' + cfg.download_timeout; + if (cfg.download_connect_timeout) command += ' --connect-timeout ' + cfg.download_connect_timeout; + if (cfg.download_timeout) command += ' --speed-limit 1 --speed-time ' + cfg.download_timeout; + if (cfg.download_max_time) command += ' --max-time ' + cfg.download_max_time; flag = '-o'; - } else if (is_present('/usr/libexec/wget-ssl')) { - command = '/usr/libexec/wget-ssl --no-check-certificate -q'; + } else if (is_present('uclient-fetch')) { + kind = 'uclient-fetch'; + command = 'uclient-fetch -q'; + if (cfg.download_allow_insecure) command += ' --no-check-certificate'; if (cfg.download_timeout) command += ' --timeout ' + cfg.download_timeout; flag = '-O'; + } else if (is_present('/usr/libexec/wget-ssl')) { + kind = 'wget'; + command = '/usr/libexec/wget-ssl -q'; + if (cfg.download_allow_insecure) command += ' --no-check-certificate'; + if (cfg.download_connect_timeout) command += ' --connect-timeout ' + cfg.download_connect_timeout; + if (cfg.download_timeout) command += ' --read-timeout ' + cfg.download_timeout; + flag = '-O'; } else if (is_present('wget') && cmd_rc("wget --version 2>/dev/null | grep -q '+https'") == 0) { - command = 'wget --no-check-certificate -q'; - if (cfg.download_timeout) command += ' --timeout ' + cfg.download_timeout; + kind = 'wget'; + command = 'wget -q'; + if (cfg.download_allow_insecure) command += ' --no-check-certificate'; + if (cfg.download_connect_timeout) command += ' --connect-timeout ' + cfg.download_connect_timeout; + if (cfg.download_timeout) command += ' --read-timeout ' + cfg.download_timeout; flag = '-O'; } else { - command = 'uclient-fetch --no-check-certificate -q'; + // Last-ditch: nothing detected — use the /usr/bin/wget ALTERNATIVES + // alias (GNU wget or uclient-fetch), the downloader name most likely + // to exist on any OpenWrt system. Only --timeout is portable across + // BusyBox wget / uclient-fetch, so download_connect_timeout is omitted here. + kind = 'wget'; + command = 'wget -q'; + if (cfg.download_allow_insecure) command += ' --no-check-certificate'; if (cfg.download_timeout) command += ' --timeout ' + cfg.download_timeout; flag = '-O'; } + // uclient-fetch is built with TLS support but loads it at runtime from a + // libustream-ssl provider — so its HTTPS capability is gated on that .so, + // which curl/wget --version greps cannot see. ssl_supported = cmd_rc("curl --version 2>/dev/null | grep -q 'Protocols: .*https.*'") == 0 || - cmd_rc("wget --version 2>/dev/null | grep -q '+ssl'") == 0; - env._dl_cache = { command, flag, ssl_supported }; + cmd_rc("wget --version 2>/dev/null | grep -q '+ssl'") == 0 || + stat('/lib/libustream-ssl.so') != null || + stat('/usr/lib/libustream-ssl.so') != null; + env._dl_cache = { command, flag, ssl_supported, kind }; return env._dl_cache; }; @@ -422,8 +467,15 @@ function awk_dedup_subdomains(input, output) { function download(url, dest) { let dlt = env.get_downloader(); - return system(sprintf('%s %s %s %s 2>/dev/null', - dlt.command, shell_quote(url), dlt.flag, shell_quote(dest))) == 0; + let rc = system(sprintf('%s %s %s %s 2>/dev/null', + dlt.command, shell_quote(url), dlt.flag, shell_quote(dest))); + // curl exits 28 for ANY timeout (--connect-timeout / --speed-time / + // --max-time). No other downloader exposes a distinct timeout exit code — + // a wget/uclient-fetch stall collapses into a generic network-failure code + // indistinguishable from DNS/refused/reset — so a timeout-aborted transfer + // is only reliably separable from other failures when curl is in use. + env._last_dl_timeout = (dlt.kind == 'curl' && rc == 28); + return rc == 0; } function service_restart(name) { @@ -640,7 +692,7 @@ function get_text(r, ...args) { case 'errorParsingList': return "Failed to parse"; case 'errorNoSSLSupport': return "No HTTPS/SSL support on device"; case 'errorCreatingDirectory': return "Failed to create output/cache/gzip file directory"; - case 'errorDetectingFileType': return "Failed to detect format"; + case 'errorDetectingFileType': return sprintf("Failed to detect format for %s", a); case 'errorNothingToDo': return "No blocked list URLs nor blocked-domains enabled"; case 'errorTooLittleRam': return sprintf("Free ram (%s) is not enough to process all enabled block-lists", a); case 'errorCreatingBackupFile': return sprintf("Failed to create backup file %s", a); @@ -660,9 +712,11 @@ function get_text(r, ...args) { case 'statusTriggerBootWait': return "waiting for trigger (on_boot)"; case 'statusTriggerStartWait': return "waiting for trigger (on_start)"; case 'warningExternalDnsmasqConfig': return "Use of external dnsmasq config file detected, please set 'dns' option to 'dnsmasq.conf'"; - case 'warningMissingRecommendedPackages': return "Some recommended packages are missing"; + case 'warningMissingRecommendedPackages': return sprintf("Recommended packages are missing: %s", a); case 'warningInvalidCompressedCacheDir': return sprintf("Invalid compressed cache directory '%s'", a); case 'warningFreeRamCheckFail': return "Can't detect free RAM"; + case 'warningParallelDownloadsThrottled': return sprintf("Parallel downloads reduced to %s due to low free memory", a); + case 'warningDownloadTimeout': return sprintf("Download of %s timed out; the server may be too slow — consider increasing download_timeout, download_connect_timeout or download_max_time", a); case 'warningSanityCheckTLD': return sprintf("Sanity check discovered TLDs in %s", a); case 'warningSanityCheckLeadingDot': return sprintf("Sanity check discovered leading dots in %s", a); case 'warningInvalidDomainsRemoved': return sprintf("Removed %s invalid domain entries from block-list (domains starting with -/./numbers or containing invalid patterns)", a); @@ -738,10 +792,12 @@ function get_url_filesize(url) { // ucode-lsp disable if (!url) return null; let size = ''; if (is_present('curl')) { - size = cmd_output(sprintf("curl --silent --insecure --fail --head --request GET --connect-timeout 2 %s | awk -F': ' '{IGNORECASE=1}/content-length/ {gsub(/\\r/, \"\"); print $2}'", shell_quote(url))); + let insecure = cfg.download_allow_insecure ? '--insecure ' : ''; + size = cmd_output(sprintf("curl --silent %s--fail --head --request GET --connect-timeout 2 %s | awk -F': ' '{IGNORECASE=1}/content-length/ {gsub(/\\r/, \"\"); print $2}'", insecure, shell_quote(url))); } if (!size && is_present('uclient-fetch')) { - size = cmd_output(sprintf("uclient-fetch --spider --timeout 2 %s -O /dev/null 2>&1 | sed -n '/^Download/ s/.*\\(\\([0-9]*\\) bytes\\).*/\\1/p'", shell_quote(url))); + let insecure = cfg.download_allow_insecure ? '--no-check-certificate ' : ''; + size = cmd_output(sprintf("uclient-fetch --spider %s--timeout 2 %s -O /dev/null 2>&1 | sed -n '/^Download/ s/.*\\(\\([0-9]*\\) bytes\\).*/\\1/p'", insecure, shell_quote(url))); } return size ? size : null; } @@ -839,10 +895,11 @@ const config_schema = { // ucode-lsp disable debug_performance: ['bool', false], dnsmasq_sanity_check: ['bool', true], dnsmasq_validity_check: ['bool', false], + download_allow_insecure: ['bool', true], enabled: ['bool', false], force_dns: ['bool', true], ipv6_enabled: ['bool', false], - parallel_downloads: ['bool', true], + parallel_downloads: ['int', 8], procd_trigger_wan6: ['bool', false], update_config_sizes: ['bool', true], // Strings @@ -852,6 +909,8 @@ const config_schema = { // ucode-lsp disable curl_retry: ['string', '3'], dns: ['string', 'dnsmasq.servers'], dnsmasq_config_file_url: ['string'], + download_connect_timeout: ['string', '10'], + download_max_time: ['string'], download_timeout: ['string', '20'], heartbeat_sleep_timeout: ['string', '10'], led: ['string'], @@ -868,7 +927,7 @@ const config_schema = { // ucode-lsp disable force_dns_port: ['list', '53 853'], smartdns_instance: ['list', '*'], // Domain (sanitized, '-' means disabled) - heartbeat_domain: ['domain', 'heartbeat.melmac.ca'], + heartbeat_domain: ['domain', 'heartbeat.mossdef.org'], // Directory (validated via realpath) compressed_cache_dir: ['dir', '/etc'], }; @@ -924,7 +983,12 @@ function load_dl_command() { env.get_downloader(); } // ── detect_file_type ──────────────────────────────────────────────── function detect_file_type(file) { - let first_line = split(readfile(file) || '', '\n')[0]; + let first_line = ''; + let fh = open(file, 'r'); + if (fh) { + first_line = split(fh.read(4096) || '', '\n')[0]; + fh.close(); + } for (let name in keys(list_formats)) { let fmt = list_formats[name]; if (fmt.first_line && first_line == fmt.first_line) return name; @@ -1145,13 +1209,18 @@ env.load = function(param, validation_result) { let missing = []; for (let key in bins) { if (!is_present(bins[key][0])) { - push(status_data.warnings, { code: 'warningMissingRecommendedPackages', info: bins[key][1] }); push(missing, bins[key][1]); } } + if (length(missing)) + push(status_data.warnings, { code: 'warningMissingRecommendedPackages', info: join(', ', missing) }); + if (length(missing) && param != 'quiet') { - output.warning(get_text('warningMissingRecommendedPackages') + ', install them by running:'); - output.print('opkg update; opkg --force-overwrite install ' + join(' ', missing) + ';'); + output.warning(get_text('warningMissingRecommendedPackages', join(', ', missing)) + '; install them by running:'); + if (is_present('apk')) + output.print('apk update; apk add ' + join(' ', missing) + ';'); + else + output.print('opkg update; opkg --force-overwrite install ' + join(' ', missing) + ';'); } }; @@ -1565,107 +1634,129 @@ function resolver(action) { // ── process_file_url ──────────────────────────────────────────────── -function process_file_url(section, url_override, action_override, predownloaded) { - let url, file_action, name, size_val; +// prepare_file_url(): parallelizable half — resolve config, download into +// out_file, detect format, filter in place. Returns a result object with +// everything callers need; no printing, no shared state — safe in a +// uloop.task child. The [ DL ] line is emitted by emit_dl_line() (below), +// called by both serial callers and the parallel uloop.task callback so +// the line shows in the parent's visible stderr. +function prepare_file_url(section, url_override, action_override, out_file) { + let url, file_action, name; + let res = { section: section, url: null, name: null, action: 'block', + out_file: out_file, size: null, format: null, ok: false, code: null, skip: false }; if (section && !url_override) { let sec_cur = cursor(); sec_cur.load(pkg.name); - let en = sec_cur.get(pkg.name, section, 'enabled'); - if (en == '0') return true; + if (sec_cur.get(pkg.name, section, 'enabled') == '0') { res.ok = true; res.skip = true; return res; } url = sec_cur.get(pkg.name, section, 'url'); file_action = sec_cur.get(pkg.name, section, 'action') || 'block'; name = sec_cur.get(pkg.name, section, 'name'); - size_val = sec_cur.get(pkg.name, section, 'size'); } else { url = url_override; file_action = action_override || 'block'; } + res.url = url; res.name = name; res.action = file_action; - if (!cfg.enabled) return true; - if (!url) return false; + if (!cfg.enabled) { res.ok = true; res.skip = true; return res; } + if (!url) { res.skip = true; return res; } - let label = replace(url, /^[a-z]+:\/\//, ''); - label = replace(label, /\/.*$/, ''); - label = name || label; - label = 'List: ' + label; - - let type_name, d_tmp; - switch (file_action) { - case 'allow': type_name = 'Allowed'; d_tmp = tmp.allowed; break; - case 'block': type_name = 'Blocked'; d_tmp = tmp.b; break; - case 'file': type_name = 'File'; d_tmp = tmp.b; break; + if (is_https_url(url) && !env.get_downloader().ssl_supported) { + res.code = 'errorNoSSLSupport'; + return res; } - if (!predownloaded && is_https_url(url) && !env.get_downloader().ssl_supported) { - output.info(sym.fail[0]); - output.verbose('[ DL ] ' + type_name + ' ' + label + ' ' + sym.fail[1] + '\\n'); - push(status_data.errors, { code: 'errorNoSSLSupport', info: name || url }); - return true; + if (!download(url, out_file) || !(stat(out_file)?.size > 0)) { + res.code = 'errorDownloadingList'; + res.timed_out = env._last_dl_timeout; + return res; } - let r_tmp = predownloaded || trim(cmd_output('mktemp -q -t "' + pkg.name + '_tmp.XXXXXXXX"')); - if (predownloaded && !(stat(r_tmp)?.size > 0)) { - output.info(sym.fail[0]); - output.verbose('[ DL ] ' + type_name + ' ' + label + ' ' + sym.fail[1] + '\\n'); - push(status_data.errors, { code: 'errorDownloadingList', info: name || url }); - unlink(r_tmp); - return true; - } - if (!predownloaded && (!url || !download(url, r_tmp) || !(stat(r_tmp)?.size > 0))) { - output.info(sym.fail[0]); - output.verbose('[ DL ] ' + type_name + ' ' + label + ' ' + sym.fail[1] + '\\n'); - push(status_data.errors, { code: 'errorDownloadingList', info: name || url }); - } else { - // Ensure newline at end - ensure_trailing_newline(r_tmp); + ensure_trailing_newline(out_file); + res.size = get_local_filesize(out_file); - // Update size in config or RAM mirror - if (section) { - let new_size = get_local_filesize(r_tmp); - if (new_size != null && ('' + size_val) != ('' + new_size)) { - let c = cfg.update_config_sizes ? uci(pkg.name) : ram_uci(pkg.name); - c.set(pkg.name, section, 'size', '' + new_size); - c.save(pkg.name); - } - } + let format = detect_file_type(out_file); + let filter = list_formats[format]?.filter; + if (!filter) { + res.code = 'errorDetectingFileType'; + return res; + } + res.format = format; + if (format == 'hosts') + sed_inplace('/# Title: StevenBlack/,/# Custom host records are listed here/d', out_file); + if (filter && file_action != 'file') + sed_inplace(filter, out_file); - let format = detect_file_type(r_tmp); - let filter = list_formats[format]?.filter; - if (!filter) { - output.info(sym.fail[0]); - output.verbose('[ DL ] ' + type_name + ' ' + label + ' ' + sym.fail[1] + '\\n'); - push(status_data.errors, { code: 'errorDetectingFileType', info: name || url }); - unlink(r_tmp); - return true; - } - if (format == 'hosts') - sed_inplace('/# Title: StevenBlack/,/# Custom host records are listed here/d', r_tmp); + if (!(stat(out_file)?.size > 0)) { + res.code = 'errorParsingList'; + return res; + } - if (filter && file_action != 'file') - sed_inplace(filter, r_tmp); + ensure_trailing_newline(out_file); + res.ok = true; + return res; +} - if (!(stat(r_tmp)?.size > 0)) { - output.info(sym.fail[0]); - output.verbose('[ DL ] ' + type_name + ' ' + label + ' (' + format + ') ' + sym.fail[1] + '\\n'); - push(status_data.errors, { code: 'errorParsingList', info: name || url }); - } else { - // Ensure file ends with newline, then append to accumulator - ensure_trailing_newline(r_tmp); - let inp = open(r_tmp, 'r'); - let out = open(d_tmp, 'a'); - if (inp && out) { - let chunk; - while ((chunk = inp.read(65536)) && length(chunk)) - out.write(chunk); - } - if (inp) inp.close(); - if (out) out.close(); - output.info(sym.ok[0]); - output.verbose('[ DL ] ' + type_name + ' ' + label + ' (' + format + ') ' + sym.ok[1] + '\\n'); - } +// emit_dl_line(): print the live [ DL ] outcome line for one result. +// Called from serial callers AND from the parallel download_lists +// callback as each task reports. Output goes to the PARENT'S stderr (and +// syslog) so it's visible in the install terminal — uloop.task does not +// propagate the child's stderr, which is why prepare_file_url is silent. +function emit_dl_line(res) { + if (!res || res.skip) return; + let type_name = (res.action == 'allow') ? 'Allowed' : + (res.action == 'file') ? 'File' : 'Blocked'; + let url = res.url || ''; + let label = res.name; + if (!label) { + label = replace(url, /^[a-z]+:\/\//, ''); + label = replace(label, /\/.*$/, ''); } - unlink(r_tmp); + label = 'List: ' + label; + let outcome = res.ok ? sym.ok : sym.fail; + let fmtsuffix = (res.ok && res.format) ? ' (' + res.format + ')' : ''; + output.info(outcome[0]); + output.verbose('[ DL ] ' + type_name + ' ' + label + fmtsuffix + ' ' + outcome[1] + '\\n'); +} + +// apply_result(): serial half — append to accumulator, record errors, +// stage the size update. Parent-side only; never call from a task child. +function apply_result(res) { + if (!res || res.skip) return; + if (res.code) { + push(status_data.errors, { code: res.code, info: res.name || res.url }); + if (res.timed_out) + push(status_data.warnings, { code: 'warningDownloadTimeout', info: res.name || res.url }); + return; + } + if (!res.ok) return; + if (res.section && res.size != null) { + let c = cfg.update_config_sizes ? uci(pkg.name) : ram_uci(pkg.name); + if (('' + c.get(pkg.name, res.section, 'size')) != ('' + res.size)) { + c.set(pkg.name, res.section, 'size', '' + res.size); + c.save(pkg.name); + } + } + let d_tmp = (res.action == 'allow') ? tmp.allowed : tmp.b; + let inp = open(res.out_file, 'r'); + let out = open(d_tmp, 'a'); + if (inp && out) { + let chunk; + while ((chunk = inp.read(65536)) && length(chunk)) + out.write(chunk); + } + if (inp) inp.close(); + if (out) out.close(); +} + +// process_file_url(): thin wrapper for serial callers (dnsmasq file, +// non-parallel mode). Kept exported — also the test-runner's anchor. +function process_file_url(section, url_override, action_override) { + let out_file = trim(cmd_output('mktemp -q -t "' + pkg.name + '_tmp.XXXXXXXX"')); + let res = prepare_file_url(section, url_override, action_override, out_file); + emit_dl_line(res); + apply_result(res); + unlink(out_file); return true; } @@ -1696,25 +1787,45 @@ function download_dnsmasq_file() { // ── download_lists ────────────────────────────────────────────────── function download_lists() { - // RAM check let free_mem = get_mem_available(); - if (!free_mem) { - push(status_data.warnings, { code: 'warningFreeRamCheckFail', info: '' }); - output.warning(get_text('warningFreeRamCheckFail')); - } else { - let total_sizes = 0; - let c = cfg.update_config_sizes ? uci(pkg.name) : ram_uci(pkg.name); - uci(pkg.name).foreach(pkg.name, 'file_url', (s) => { - if (s.enabled == '0') return; - let sz = c.get(pkg.name, s['.name'], 'size'); - if (!sz && s.url) sz = get_url_filesize(s.url); - if (sz) total_sizes += int('' + sz); - }); - if (free_mem < total_sizes * 2) { + + // Enumerate enabled lists and sum their sizes in one pass. + let download_cfgs = []; + let total_sizes = 0; + let szc = cfg.update_config_sizes ? uci(pkg.name) : ram_uci(pkg.name); + uci(pkg.name).foreach(pkg.name, 'file_url', (s) => { + if (s.enabled == '0') return; + push(download_cfgs, s['.name']); + let sz = szc.get(pkg.name, s['.name'], 'size'); + if (!sz && s.url) sz = get_url_filesize(s.url); + if (sz) total_sizes += int('' + sz); + }); + let n_lists = length(download_cfgs); + + // RAM budget: list data needs ~total_sizes*2; each parallel task slot + // needs the ucode child + its downloader (task_slot_ram, keyed by kind). + // Effective cap = min(configured parallel_downloads, list count, what + // free RAM affords) — so a tight router throttles itself down. + let base = total_sizes * 2; + let dlt = env.get_downloader(); + let slot_ram = pkg.task_slot_ram[dlt.kind] || pkg.task_slot_ram['default']; + let task_cap = +cfg.parallel_downloads; + if (free_mem) { + if (free_mem < base + slot_ram) { push(status_data.errors, { code: 'errorTooLittleRam', info: '' + free_mem }); return false; } + let affordable = int((free_mem - base) / slot_ram); + if (task_cap > affordable) { + task_cap = affordable; + push(status_data.warnings, { code: 'warningParallelDownloadsThrottled', info: '' + task_cap }); + } + } else { + push(status_data.warnings, { code: 'warningFreeRamCheckFail', info: '' }); + output.warning(get_text('warningFreeRamCheckFail')); } + if (task_cap > n_lists) task_cap = n_lists; + if (task_cap < 1) task_cap = 1; status_data.message = get_text('statusDownloading') + '...'; status_data.status = 'statusDownloading'; @@ -1728,44 +1839,77 @@ function download_lists() { output.info('Downloading lists '); - // Process each file_url section - let download_cfgs = []; - uci(pkg.name).foreach(pkg.name, 'file_url', (s) => push(download_cfgs, s['.name'])); - - if (cfg.parallel_downloads && uloop && length(download_cfgs) > 1) { - // Parallel mode: download all files first, then process each - let dlt = env.get_downloader(); - let jobs = []; - for (let cfg_name in download_cfgs) { - let sec_cur = cursor(); - sec_cur.load(pkg.name); - if (sec_cur.get(pkg.name, cfg_name, 'enabled') == '0') continue; - let url = sec_cur.get(pkg.name, cfg_name, 'url'); - if (!url) continue; - if (is_https_url(url) && !dlt.ssl_supported) { - let name = sec_cur.get(pkg.name, cfg_name, 'name'); - push(status_data.errors, { code: 'errorNoSSLSupport', info: name || url }); - output.info(sym.fail[0]); - continue; + if (cfg.parallel_downloads && uloop && task_cap > 1 && n_lists > 1) { + // Parallel: a bounded uloop.task pool, <= task_cap tasks in flight. + // Each task forks; the child runs prepare_file_url (download + detect + // + filter) and pipe.sends the result; the parent applies serially + // after the loop. popen/system are safe inside the child — it is not + // running the event loop. + if (cfg.debug_performance) + logger_debug(sprintf('[PERF] download_lists: %d lists, cap %d, downloader %s (~%d KB/slot), MemAvailable %d KB', + n_lists, task_cap, dlt.kind, slot_ram / 1024, (free_mem || 0) / 1024)); + uloop.init(); + let next = 0, running = 0, results = {}, tmps = []; + let spawn; + spawn = function() { + while (running < task_cap && next < n_lists) { + let cfg_name = download_cfgs[next++]; + let out_file = trim(cmd_output('mktemp -q -t "' + pkg.name + '_tmp.XXXXXXXX"')); + push(tmps, out_file); + running++; + let t = uloop.task( + function(pipe) { + let res = prepare_file_url(cfg_name, null, null, out_file); + if (cfg.debug_performance) { + let m = match(readfile('/proc/self/status') || '', /VmHWM:[ \t]+([0-9]+)/); + res.peak_rss = m ? int(m[1]) : 0; + } + pipe.send(res); + }, + function(res) { + // uloop.task may fire this callback twice per child + // (data send + child-exit EOF). Count completed sections, + // not callback firings, so null/duplicate fires are no-ops. + if (!res || !res.section || results[res.section]) return; + results[res.section] = res; + running--; + emit_dl_line(res); + if (cfg.debug_performance) + logger_debug(sprintf('[PERF] task %s: peak RSS %d KB, list size %d KB', + res.name || res.url || '?', +(res.peak_rss || 0), int((res.size || 0) / 1024))); + if (length(keys(results)) >= n_lists) uloop.end(); + else spawn(); + } + ); + if (t == null) { + // Fork failed — no callback will ever fire for this section; + // record a sentinel so the completion gate accounts for it. + results[cfg_name] = false; + running--; + if (length(keys(results)) >= n_lists) uloop.end(); + } } - let r_tmp = trim(cmd_output('mktemp -q -t "' + pkg.name + '_tmp.XXXXXXXX"')); - push(jobs, { cfg_name, url, r_tmp }); - } - if (length(jobs) > 0) { - uloop.init(); - let pending = length(jobs); - for (let i = 0; i < length(jobs); i++) { - let job = jobs[i]; - let dl_cmd = sprintf('%s %s %s %s 2>/dev/null', - dlt.command, shell_quote(job.url), dlt.flag, shell_quote(job.r_tmp)); - uloop.process('/bin/sh', ['-c', dl_cmd], {}, () => { - process_file_url(job.cfg_name, null, null, job.r_tmp); - if (--pending == 0) uloop.end(); - }); + }; + spawn(); + // Watchdog: never hang if a task child dies without reporting. + uloop.timer(600000, function() { uloop.end(); }); + uloop.run(); + uloop.done(); + for (let cfg_name in download_cfgs) { + let res = results[cfg_name]; + if (res) { + apply_result(res); + } else { + // Task didn't report (crash/timeout/spawn fail) — surface a + // user-friendly label, not the anonymous UCI section name. + let sec_cur = cursor(); + sec_cur.load(pkg.name); + let label = sec_cur.get(pkg.name, cfg_name, 'name') || + sec_cur.get(pkg.name, cfg_name, 'url') || cfg_name; + push(status_data.errors, { code: 'errorDownloadingList', info: label }); } - uloop.run(); - uloop.done(); } + for (let f in tmps) unlink(f); } else { for (let cfg_name in download_cfgs) process_file_url(cfg_name); @@ -2034,6 +2178,8 @@ function adb_config_update(param) { if (!download(cfg.config_update_url, r_tmp) || !(stat(r_tmp)?.size > 0)) { output.failn(); push(status_data.errors, { code: 'errorDownloadingConfigUpdate', info: '' }); + if (env._last_dl_timeout) + push(status_data.warnings, { code: 'warningDownloadTimeout', info: 'Config Update file' }); } else { if (system(sprintf("sed -f %s -i %s 2>/dev/null", shell_quote(r_tmp), shell_quote(pkg.config_file))) == 0) output.okn(); diff --git a/net/adblock-fast/tests/01_pipeline/10_servers_mode_allow_list b/net/adblock-fast/tests/01_pipeline/10_servers_mode_allow_list index 949bc9ff3..485630354 100644 --- a/net/adblock-fast/tests/01_pipeline/10_servers_mode_allow_list +++ b/net/adblock-fast/tests/01_pipeline/10_servers_mode_allow_list @@ -23,6 +23,8 @@ for domains from allowed list files, not just config-defined allowed_domain. "update_config_sizes": "0", "heartbeat_domain": "-", "download_timeout": "10", + "download_connect_timeout": "10", + "download_allow_insecure": "1", "pause_timeout": "20", "curl_retry": "1", "compressed_cache_dir": "TESTDIR/cache", diff --git a/net/adblock-fast/tests/mocks/uci/adblock-fast.json b/net/adblock-fast/tests/mocks/uci/adblock-fast.json index 26aa18085..4cc911955 100644 --- a/net/adblock-fast/tests/mocks/uci/adblock-fast.json +++ b/net/adblock-fast/tests/mocks/uci/adblock-fast.json @@ -19,6 +19,8 @@ "update_config_sizes": "0", "heartbeat_domain": "-", "download_timeout": "10", + "download_connect_timeout": "10", + "download_allow_insecure": "1", "pause_timeout": "20", "curl_retry": "1", "compressed_cache_dir": "TESTDIR/cache" diff --git a/net/adblock-fast/tests/run_tests.sh b/net/adblock-fast/tests/run_tests.sh index 3037c5ac0..8d03b0f7f 100644 --- a/net/adblock-fast/tests/run_tests.sh +++ b/net/adblock-fast/tests/run_tests.sh @@ -8,7 +8,7 @@ # - Exports internal functions for test access # - Uses real shell commands (sed/sort/grep/awk) with mock UCI/UBus # -# Usage: cd source.openwrt.melmac.ca/adblock-fast && bash tests/run_tests.sh [test_file...] +# Usage: cd source.mossdef.org/adblock-fast && bash tests/run_tests.sh [test_file...] set -o pipefail