From: Dirk Brenken Date: Thu, 10 Sep 2026 18:02:03 +0000 (+0200) Subject: banip: update 1.8.13-2 X-Git-Url: http://git.99rst.org/?a=commitdiff_plain;h=b9e0cefee0866381c146b32da2e1e81dbeffb368;p=openwrt-packages.git banip: update 1.8.13-2 - cap the GeoIP lookup at 15 batch requests per report run (ip-api.com rate limit) - reuse the map data for 60 seconds instead of querying the service again - log batch requests without geo data - limit the parallel background jobs exactly to `ban_cores (same as adblock) - LuCI: fix the GeoIP map description - readme update Signed-off-by: Dirk Brenken --- diff --git a/net/banip/Makefile b/net/banip/Makefile index 30ce292bc..68ca3410c 100644 --- a/net/banip/Makefile +++ b/net/banip/Makefile @@ -6,7 +6,7 @@ include $(TOPDIR)/rules.mk PKG_NAME:=banip PKG_VERSION:=1.8.13 -PKG_RELEASE:=1 +PKG_RELEASE:=2 PKG_LICENSE:=GPL-3.0-or-later PKG_MAINTAINER:=Dirk Brenken diff --git a/net/banip/README.md b/net/banip/README.md index 373033fab..537f2e06b 100644 --- a/net/banip/README.md +++ b/net/banip/README.md @@ -535,7 +535,7 @@ To make this work, banIP uses the following external components: * [Leaflet](https://leafletjs.com/) is a lightweight open-source JavaScript library for interactive maps * The free and quite fast [IP Geolocation API](https://ip-api.com/) to resolve the required IP/geolocation information -The basemap is no longer pulled from a tile service. CARTO started to require an API key for the raster basemaps at basemaps.cartocdn.com and watermarks every unauthenticated tile request, and a key is bound to a single customer, so it cannot be shipped with a package that lands on every installation. banIP therefore draws the basemap from country outlines that come with `luci-app-banip`: [Natural Earth](https://www.naturalearthdata.com) 1:110m, public domain, stripped of all attributes and simplified to 36 kB. The map page issues no request to a third party, works without a WAN connection and leaks no part of the admin session to a CDN. The outlines are enough to locate an IP, so the map does not zoom in beyond level 6 and labels the continents rather than the countries. +The basemap is no longer pulled from a tile service. CARTO started to require an API key for the raster basemaps at basemaps.cartocdn.com and watermarks every unauthenticated tile request, and a key is bound to a single customer, so it cannot be shipped with a package that lands on every installation. banIP therefore draws the basemap from country outlines that come with `luci-app-banip`: [Natural Earth](https://www.naturalearthdata.com) 1:110m, public domain, stripped of all attributes and simplified to 36 kB. No tile service is contacted anymore, but the map page still loads the Leaflet library from unpkg.com (integrity-checked via SRI and requested without cookies), so it needs a WAN connection. The outlines are enough to locate an IP, so the map does not zoom in beyond level 6 and labels the continents rather than the countries. **Optional: a higher detail basemap** @@ -549,7 +549,7 @@ scp world-50m.json root@openwrt:/www/luci-static/resources/view/banip/ The result is roughly 105 kB, about three times the shipped file. Please note: this file is not part of any package, so it is removed on sysupgrade unless you add its path to `/etc/sysupgrade.conf`, and it stays behind when `luci-app-banip` is uninstalled. -Please note: the free ip-api.com batch endpoint is rate limited to 15 requests per minute per source IP. Requests beyond that limit are throttled with HTTP 429, and constantly exceeding the limit gets the IP banned for an hour — in both cases the map stays empty and banIP logs an info message. To stay below the limit banIP collects the top listed elements of all Sets, deduplicates them and resolves them in as few batch requests as possible (100 IPs each, the maximum the endpoint accepts). A setup with up to ~1500 mapped elements therefore needs no more than 15 requests per report run. If you run a large number of Sets and regenerate the report frequently, lower `ban_map` to `0` or reduce the number of active feeds. +Please note: the free ip-api.com batch endpoint is rate limited to 15 requests per minute per source IP. Requests beyond that limit are throttled with HTTP 429, and constantly exceeding the limit gets the IP banned for an hour — in both cases the affected part of the map stays empty and banIP logs an info message. To stay below the limit banIP collects the top listed elements of all Sets, deduplicates them and resolves them in batch requests of 100 IPs each (the maximum the endpoint accepts), capped at 15 requests per report run. Elements beyond ~1500 are left out of the map and banIP logs an info message. A report regenerated within 60 seconds of the last geo lookup reuses its map data instead of querying the service again. Please note that other clients behind the same public IP count against the same limit. **CGI interface to receive remote logging events** banIP ships a basic cgi interface in `/www/cgi-bin/banip` to receive remote logging events (disabled by default). The cgi interface evaluates logging events via GET or POST request (see examples below). To enable the cgi interface set the following options: diff --git a/net/banip/files/banip-functions.sh b/net/banip/files/banip-functions.sh index 2565612ae..6f139bd03 100644 --- a/net/banip/files/banip-functions.sh +++ b/net/banip/files/banip-functions.sh @@ -2184,7 +2184,7 @@ f_lookup() { fi done >"${tmp_dir}/${cnt}" ) & - [ "${cnt}" -gt "${ban_cores}" ] && wait -n + [ "${cnt}" -ge "${ban_cores}" ] && wait -n cnt_domain="${cnt}" cnt="$((cnt + 1))" done @@ -2241,19 +2241,29 @@ f_lookup() { f_report() { local report_jsn report_txt tmp_val table_json item sep table_sets set_cnt set_inbound set_outbound set_cntinbound set_cntoutbound set_proto set_dport set_details local cnt ip expr detail jsnval timestamp autoadd_allow autoadd_block sum_sets sum_setinbound sum_setoutbound sum_cntelements sum_cntinbound sum_cntoutbound - local jsn table_jsn set_jsn map_jsn map_lookup chunk_no chain set_elements uplink_ip sum_setelements sum_synflood sum_udpflood sum_icmpflood sum_ctinvalid sum_tcpinvalid sum_setports sum_bcp38 output="${1}" + local jsn table_jsn set_jsn map_jsn map_ts map_lookup geo_ts geo_now geo_skip chunk_no chunk_skip rsp_no chain set_elements uplink_ip sum_setelements sum_synflood sum_udpflood sum_icmpflood sum_ctinvalid sum_tcpinvalid sum_setports sum_bcp38 output="${1}" f_conf f_mkdir "${ban_reportdir}" report_jsn="${ban_reportdir}/ban_report.jsn" report_txt="${ban_reportdir}/ban_report.txt" map_jsn="${ban_reportdir}/ban_map.jsn" + map_ts="${ban_reportdir}/ban_map.ts" if [ "${output}" != "json" ]; then # json output preparation # - : >"${report_txt}" >"${report_jsn}" >"${map_jsn}" + : >"${report_txt}" >"${report_jsn}" + read -r geo_now _ <"/proc/uptime" + geo_now="${geo_now%%.*}" + geo_ts="" + [ -s "${map_ts}" ] && read -r geo_ts <"${map_ts}" + case "${geo_ts}" in + "" | *[!0-9]*) ;; + *) [ "${geo_ts}" -le "${geo_now}" ] && [ "$((geo_now - geo_ts))" -lt "60" ] && geo_skip="1" ;; + esac + [ "${geo_skip}" = "1" ] || : >"${map_jsn}" [ "${output}" = "gen" ] && printf '%s\n' "0" >"${ban_rundir}/banIP.report" table_jsn="${ban_rundir}/report.table.jsn" "${ban_nftcmd}" -tj list table inet banIP 2>>"${ban_errorlog}" >"${table_jsn}" @@ -2341,7 +2351,7 @@ f_report() { }" >"${report_jsn}.${item}" "${ban_rmcmd}" -f "${set_jsn}"* ) & - [ "${cnt}" -gt "${ban_cores}" ] && wait -n + [ "${cnt}" -ge "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" done wait @@ -2446,7 +2456,7 @@ f_report() { # retrieve/prepare map data # - if [ "${ban_nftcount}" = "1" ] && [ "${ban_map}" = "1" ] && [ -s "${report_jsn}" ]; then + if [ "${ban_nftcount}" = "1" ] && [ "${ban_map}" = "1" ] && [ -s "${report_jsn}" ] && [ "${geo_skip}" != "1" ]; then f_getdl map_lookup="${ban_rundir}/report.map.lookup" : >"${map_lookup}" @@ -2483,12 +2493,15 @@ f_report() { fi # split the deduplicated IPs into batch requests of 100 IPs each, - # the maximum the geo service accepts + # the maximum the geo service accepts, capped at 15 requests per run # if [ -s "${map_lookup}" ]; then - "${ban_awkcmd}" -F '\t' -v file="${map_jsn}" -v size="100" \ - '!seen[$1]++{no=int(cnt++/size)+1;printf "%s\"%s\"",(chunk[no]++?", ":""),$1 >(file ".req." no)}END{for(i=1;i<=no;i++)close(file ".req." i);printf "%s\n",no+0 >(file ".num")}' "${map_lookup}" + "${ban_awkcmd}" -F '\t' -v file="${map_jsn}" -v size="100" -v max="15" \ + '!seen[$1]++{if(cnt>=size*max){skip++;next};no=int(cnt++/size)+1;printf "%s\"%s\"",(chunk[no]++?", ":""),$1 >(file ".req." no)}END{for(i=1;i<=no;i++)close(file ".req." i);printf "%s %s\n",no+0,skip+0 >(file ".num")}' "${map_lookup}" chunk_no="$("${ban_catcmd}" "${map_jsn}.num" 2>>"${ban_errorlog}")" + chunk_skip="${chunk_no#* }" + chunk_no="${chunk_no%% *}" + [ "${chunk_skip:-0}" -gt "0" ] && f_log "info" "geo lookup capped at 15 requests, ${chunk_skip} IPs left out of the map" cnt="1" while [ "${cnt}" -le "${chunk_no:-0}" ]; do ( @@ -2498,25 +2511,31 @@ f_report() { match($0,/"query"[ \t]*:[ \t]*"[^"]+"/){query=substr($0,RSTART,RLENGTH);sub(/^"query"[ \t]*:[ \t]*"/,"",query);sub(/"$/,"",query);if(query in feed)printf ",{\"%s\": %s}\n",feed[query],$0}' \ "${map_lookup}" - >"${map_jsn}.rsp.${cnt}" ) & - [ "${cnt}" -gt "${ban_cores}" ] && wait -n + [ "${cnt}" -ge "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" done wait + read -r geo_now _ <"/proc/uptime" + printf '%s\n' "${geo_now%%.*}" >"${map_ts}" # assemble map data from the batch fragments # cnt="1" + rsp_no="0" while [ "${cnt}" -le "${chunk_no:-0}" ]; do if [ -s "${map_jsn}.rsp.${cnt}" ]; then + rsp_no="$((rsp_no + 1))" [ -s "${map_jsn}" ] || printf '%s' ",[{}" >>"${map_jsn}" "${ban_catcmd}" "${map_jsn}.rsp.${cnt}" >>"${map_jsn}" fi cnt="$((cnt + 1))" done - [ -s "${map_jsn}" ] || f_log "info" "no geo data received, the rate limit of '${ban_geourl}' has probably been exceeded" - f_log "debug" "f_report ::: geo requests: ${chunk_no:-0}, map data: $([ -s "${map_jsn}" ] && printf '%s' "yes" || printf '%s' "no")" + [ "${rsp_no}" -lt "${chunk_no:-0}" ] && f_log "info" "$((chunk_no - rsp_no)) of ${chunk_no} geo requests returned no data" + f_log "debug" "f_report ::: geo requests: ${chunk_no:-0}, skipped IPs: ${chunk_skip:-0}, map data: $([ -s "${map_jsn}" ] && printf '%s' "yes" || printf '%s' "no")" fi "${ban_rmcmd}" -f "${map_lookup}" "${map_jsn}".req.* "${map_jsn}".rsp.* "${map_jsn}.num" + elif [ "${ban_map}" = "1" ] && [ "${geo_skip}" = "1" ]; then + f_log "debug" "f_report ::: geo requests: 0, map data: reused" fi # text output preparation @@ -2609,7 +2628,7 @@ f_report() { [ -s "${report_txt}" ] && "${ban_catcmd}" "${report_txt}" ;; "json") - if [ "${ban_nftcount}" = "1" ] && [ "${ban_map}" = "1" ] && [ -s "${map_jsn}" ]; then + if [ "${ban_nftcount}" = "1" ] && [ "${ban_map}" = "1" ] && [ -s "${report_jsn}" ] && [ -s "${map_jsn}" ]; then jsn="$("${ban_catcmd}" "${report_jsn}" "${map_jsn}" 2>>"${ban_errorlog}")" [ -n "${jsn}" ] && printf '[%s]]\n' "${jsn}" else @@ -2692,7 +2711,7 @@ f_search() { printf ' %s\n' "IP found in Set '${item}'" >"${tmp_result}.${item}" fi ) & - [ "${cnt}" -gt "${ban_cores}" ] && wait -n + [ "${cnt}" -ge "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" done wait diff --git a/net/banip/files/banip-service.sh b/net/banip/files/banip-service.sh index c53d2a45c..df74e45de 100755 --- a/net/banip/files/banip-service.sh +++ b/net/banip/files/banip-service.sh @@ -100,13 +100,13 @@ for feed in allowlist ${ban_monitorfeed} ${ban_feed} blocklist; do if [ "${feed}" = "country" ] && [ "${ban_countrysplit}" = "1" ]; then for country in ${ban_country}; do (f_down "${feed}.${country}" "${feed_ipv}" "${feed_url_4}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") & - [ "${cnt}" -gt "${ban_cores}" ] && wait -n + [ "${cnt}" -ge "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" done elif [ "${feed}" = "asn" ] && [ "${ban_asnsplit}" = "1" ]; then for asn in ${ban_asn}; do (f_down "${feed}.${asn}" "${feed_ipv}" "${feed_url_4}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") & - [ "${cnt}" -gt "${ban_cores}" ] && wait -n + [ "${cnt}" -ge "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" done else @@ -115,7 +115,7 @@ for feed in allowlist ${ban_monitorfeed} ${ban_feed} blocklist; do f_down "${feed}" "${feed_ipv}" "${feed_url_4}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}" else (f_down "${feed}" "${feed_ipv}" "${feed_url_4}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") & - [ "${cnt}" -gt "${ban_cores}" ] && wait -n + [ "${cnt}" -ge "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" fi fi @@ -128,18 +128,18 @@ for feed in allowlist ${ban_monitorfeed} ${ban_feed} blocklist; do if [ "${feed}" = "country" ] && [ "${ban_countrysplit}" = "1" ]; then for country in ${ban_country}; do (f_down "${feed}.${country}" "${feed_ipv}" "${feed_url_6}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") & - [ "${cnt}" -gt "${ban_cores}" ] && wait -n + [ "${cnt}" -ge "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" done elif [ "${feed}" = "asn" ] && [ "${ban_asnsplit}" = "1" ]; then for asn in ${ban_asn}; do (f_down "${feed}.${asn}" "${feed_ipv}" "${feed_url_6}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") & - [ "${cnt}" -gt "${ban_cores}" ] && wait -n + [ "${cnt}" -ge "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" done else (f_down "${feed}" "${feed_ipv}" "${feed_url_6}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") & - [ "${cnt}" -gt "${ban_cores}" ] && wait -n + [ "${cnt}" -ge "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" fi fi @@ -154,7 +154,7 @@ f_log "info" "start banIP domain lookup" cnt="1" for list in allowlist blocklist; do (f_lookup "${list}") & - [ "${cnt}" -gt "${ban_cores}" ] && wait -n + [ "${cnt}" -ge "${ban_cores}" ] && wait -n cnt="$((cnt + 1))" done wait