* [Leaflet](https://leafletjs.com/) is a lightweight open-source JavaScript library for interactive maps
* The free and quite fast [IP Geolocation API](https://ip-api.com/) to resolve the required IP/geolocation information
-The basemap is no longer pulled from a tile service. CARTO started to require an API key for the raster basemaps at basemaps.cartocdn.com and watermarks every unauthenticated tile request, and a key is bound to a single customer, so it cannot be shipped with a package that lands on every installation. banIP therefore draws the basemap from country outlines that come with `luci-app-banip`: [Natural Earth](https://www.naturalearthdata.com) 1:110m, public domain, stripped of all attributes and simplified to 36 kB. The map page issues no request to a third party, works without a WAN connection and leaks no part of the admin session to a CDN. The outlines are enough to locate an IP, so the map does not zoom in beyond level 6 and labels the continents rather than the countries.
+The basemap is no longer pulled from a tile service. CARTO started to require an API key for the raster basemaps at basemaps.cartocdn.com and watermarks every unauthenticated tile request, and a key is bound to a single customer, so it cannot be shipped with a package that lands on every installation. banIP therefore draws the basemap from country outlines that come with `luci-app-banip`: [Natural Earth](https://www.naturalearthdata.com) 1:110m, public domain, stripped of all attributes and simplified to 36 kB. No tile service is contacted anymore, but the map page still loads the Leaflet library from unpkg.com (integrity-checked via SRI and requested without cookies), so it needs a WAN connection. The outlines are enough to locate an IP, so the map does not zoom in beyond level 6 and labels the continents rather than the countries.
**Optional: a higher detail basemap**
The result is roughly 105 kB, about three times the shipped file. Please note: this file is not part of any package, so it is removed on sysupgrade unless you add its path to `/etc/sysupgrade.conf`, and it stays behind when `luci-app-banip` is uninstalled.
-Please note: the free ip-api.com batch endpoint is rate limited to 15 requests per minute per source IP. Requests beyond that limit are throttled with HTTP 429, and constantly exceeding the limit gets the IP banned for an hour — in both cases the map stays empty and banIP logs an info message. To stay below the limit banIP collects the top listed elements of all Sets, deduplicates them and resolves them in as few batch requests as possible (100 IPs each, the maximum the endpoint accepts). A setup with up to ~1500 mapped elements therefore needs no more than 15 requests per report run. If you run a large number of Sets and regenerate the report frequently, lower `ban_map` to `0` or reduce the number of active feeds.
+Please note: the free ip-api.com batch endpoint is rate limited to 15 requests per minute per source IP. Requests beyond that limit are throttled with HTTP 429, and constantly exceeding the limit gets the IP banned for an hour — in both cases the affected part of the map stays empty and banIP logs an info message. To stay below the limit banIP collects the top listed elements of all Sets, deduplicates them and resolves them in batch requests of 100 IPs each (the maximum the endpoint accepts), capped at 15 requests per report run. Elements beyond ~1500 are left out of the map and banIP logs an info message. A report regenerated within 60 seconds of the last geo lookup reuses its map data instead of querying the service again. Please note that other clients behind the same public IP count against the same limit.
**CGI interface to receive remote logging events**
banIP ships a basic cgi interface in `/www/cgi-bin/banip` to receive remote logging events (disabled by default). The cgi interface evaluates logging events via GET or POST request (see examples below). To enable the cgi interface set the following options:
fi
done >"${tmp_dir}/${cnt}"
) &
- [ "${cnt}" -gt "${ban_cores}" ] && wait -n
+ [ "${cnt}" -ge "${ban_cores}" ] && wait -n
cnt_domain="${cnt}"
cnt="$((cnt + 1))"
done
f_report() {
local report_jsn report_txt tmp_val table_json item sep table_sets set_cnt set_inbound set_outbound set_cntinbound set_cntoutbound set_proto set_dport set_details
local cnt ip expr detail jsnval timestamp autoadd_allow autoadd_block sum_sets sum_setinbound sum_setoutbound sum_cntelements sum_cntinbound sum_cntoutbound
- local jsn table_jsn set_jsn map_jsn map_lookup chunk_no chain set_elements uplink_ip sum_setelements sum_synflood sum_udpflood sum_icmpflood sum_ctinvalid sum_tcpinvalid sum_setports sum_bcp38 output="${1}"
+ local jsn table_jsn set_jsn map_jsn map_ts map_lookup geo_ts geo_now geo_skip chunk_no chunk_skip rsp_no chain set_elements uplink_ip sum_setelements sum_synflood sum_udpflood sum_icmpflood sum_ctinvalid sum_tcpinvalid sum_setports sum_bcp38 output="${1}"
f_conf
f_mkdir "${ban_reportdir}"
report_jsn="${ban_reportdir}/ban_report.jsn"
report_txt="${ban_reportdir}/ban_report.txt"
map_jsn="${ban_reportdir}/ban_map.jsn"
+ map_ts="${ban_reportdir}/ban_map.ts"
if [ "${output}" != "json" ]; then
# json output preparation
#
- : >"${report_txt}" >"${report_jsn}" >"${map_jsn}"
+ : >"${report_txt}" >"${report_jsn}"
+ read -r geo_now _ <"/proc/uptime"
+ geo_now="${geo_now%%.*}"
+ geo_ts=""
+ [ -s "${map_ts}" ] && read -r geo_ts <"${map_ts}"
+ case "${geo_ts}" in
+ "" | *[!0-9]*) ;;
+ *) [ "${geo_ts}" -le "${geo_now}" ] && [ "$((geo_now - geo_ts))" -lt "60" ] && geo_skip="1" ;;
+ esac
+ [ "${geo_skip}" = "1" ] || : >"${map_jsn}"
[ "${output}" = "gen" ] && printf '%s\n' "0" >"${ban_rundir}/banIP.report"
table_jsn="${ban_rundir}/report.table.jsn"
"${ban_nftcmd}" -tj list table inet banIP 2>>"${ban_errorlog}" >"${table_jsn}"
}" >"${report_jsn}.${item}"
"${ban_rmcmd}" -f "${set_jsn}"*
) &
- [ "${cnt}" -gt "${ban_cores}" ] && wait -n
+ [ "${cnt}" -ge "${ban_cores}" ] && wait -n
cnt="$((cnt + 1))"
done
wait
# retrieve/prepare map data
#
- if [ "${ban_nftcount}" = "1" ] && [ "${ban_map}" = "1" ] && [ -s "${report_jsn}" ]; then
+ if [ "${ban_nftcount}" = "1" ] && [ "${ban_map}" = "1" ] && [ -s "${report_jsn}" ] && [ "${geo_skip}" != "1" ]; then
f_getdl
map_lookup="${ban_rundir}/report.map.lookup"
: >"${map_lookup}"
fi
# split the deduplicated IPs into batch requests of 100 IPs each,
- # the maximum the geo service accepts
+ # the maximum the geo service accepts, capped at 15 requests per run
#
if [ -s "${map_lookup}" ]; then
- "${ban_awkcmd}" -F '\t' -v file="${map_jsn}" -v size="100" \
- '!seen[$1]++{no=int(cnt++/size)+1;printf "%s\"%s\"",(chunk[no]++?", ":""),$1 >(file ".req." no)}END{for(i=1;i<=no;i++)close(file ".req." i);printf "%s\n",no+0 >(file ".num")}' "${map_lookup}"
+ "${ban_awkcmd}" -F '\t' -v file="${map_jsn}" -v size="100" -v max="15" \
+ '!seen[$1]++{if(cnt>=size*max){skip++;next};no=int(cnt++/size)+1;printf "%s\"%s\"",(chunk[no]++?", ":""),$1 >(file ".req." no)}END{for(i=1;i<=no;i++)close(file ".req." i);printf "%s %s\n",no+0,skip+0 >(file ".num")}' "${map_lookup}"
chunk_no="$("${ban_catcmd}" "${map_jsn}.num" 2>>"${ban_errorlog}")"
+ chunk_skip="${chunk_no#* }"
+ chunk_no="${chunk_no%% *}"
+ [ "${chunk_skip:-0}" -gt "0" ] && f_log "info" "geo lookup capped at 15 requests, ${chunk_skip} IPs left out of the map"
cnt="1"
while [ "${cnt}" -le "${chunk_no:-0}" ]; do
(
match($0,/"query"[ \t]*:[ \t]*"[^"]+"/){query=substr($0,RSTART,RLENGTH);sub(/^"query"[ \t]*:[ \t]*"/,"",query);sub(/"$/,"",query);if(query in feed)printf ",{\"%s\": %s}\n",feed[query],$0}' \
"${map_lookup}" - >"${map_jsn}.rsp.${cnt}"
) &
- [ "${cnt}" -gt "${ban_cores}" ] && wait -n
+ [ "${cnt}" -ge "${ban_cores}" ] && wait -n
cnt="$((cnt + 1))"
done
wait
+ read -r geo_now _ <"/proc/uptime"
+ printf '%s\n' "${geo_now%%.*}" >"${map_ts}"
# assemble map data from the batch fragments
#
cnt="1"
+ rsp_no="0"
while [ "${cnt}" -le "${chunk_no:-0}" ]; do
if [ -s "${map_jsn}.rsp.${cnt}" ]; then
+ rsp_no="$((rsp_no + 1))"
[ -s "${map_jsn}" ] || printf '%s' ",[{}" >>"${map_jsn}"
"${ban_catcmd}" "${map_jsn}.rsp.${cnt}" >>"${map_jsn}"
fi
cnt="$((cnt + 1))"
done
- [ -s "${map_jsn}" ] || f_log "info" "no geo data received, the rate limit of '${ban_geourl}' has probably been exceeded"
- f_log "debug" "f_report ::: geo requests: ${chunk_no:-0}, map data: $([ -s "${map_jsn}" ] && printf '%s' "yes" || printf '%s' "no")"
+ [ "${rsp_no}" -lt "${chunk_no:-0}" ] && f_log "info" "$((chunk_no - rsp_no)) of ${chunk_no} geo requests returned no data"
+ f_log "debug" "f_report ::: geo requests: ${chunk_no:-0}, skipped IPs: ${chunk_skip:-0}, map data: $([ -s "${map_jsn}" ] && printf '%s' "yes" || printf '%s' "no")"
fi
"${ban_rmcmd}" -f "${map_lookup}" "${map_jsn}".req.* "${map_jsn}".rsp.* "${map_jsn}.num"
+ elif [ "${ban_map}" = "1" ] && [ "${geo_skip}" = "1" ]; then
+ f_log "debug" "f_report ::: geo requests: 0, map data: reused"
fi
# text output preparation
[ -s "${report_txt}" ] && "${ban_catcmd}" "${report_txt}"
;;
"json")
- if [ "${ban_nftcount}" = "1" ] && [ "${ban_map}" = "1" ] && [ -s "${map_jsn}" ]; then
+ if [ "${ban_nftcount}" = "1" ] && [ "${ban_map}" = "1" ] && [ -s "${report_jsn}" ] && [ -s "${map_jsn}" ]; then
jsn="$("${ban_catcmd}" "${report_jsn}" "${map_jsn}" 2>>"${ban_errorlog}")"
[ -n "${jsn}" ] && printf '[%s]]\n' "${jsn}"
else
printf ' %s\n' "IP found in Set '${item}'" >"${tmp_result}.${item}"
fi
) &
- [ "${cnt}" -gt "${ban_cores}" ] && wait -n
+ [ "${cnt}" -ge "${ban_cores}" ] && wait -n
cnt="$((cnt + 1))"
done
wait
if [ "${feed}" = "country" ] && [ "${ban_countrysplit}" = "1" ]; then
for country in ${ban_country}; do
(f_down "${feed}.${country}" "${feed_ipv}" "${feed_url_4}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") &
- [ "${cnt}" -gt "${ban_cores}" ] && wait -n
+ [ "${cnt}" -ge "${ban_cores}" ] && wait -n
cnt="$((cnt + 1))"
done
elif [ "${feed}" = "asn" ] && [ "${ban_asnsplit}" = "1" ]; then
for asn in ${ban_asn}; do
(f_down "${feed}.${asn}" "${feed_ipv}" "${feed_url_4}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") &
- [ "${cnt}" -gt "${ban_cores}" ] && wait -n
+ [ "${cnt}" -ge "${ban_cores}" ] && wait -n
cnt="$((cnt + 1))"
done
else
f_down "${feed}" "${feed_ipv}" "${feed_url_4}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}"
else
(f_down "${feed}" "${feed_ipv}" "${feed_url_4}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") &
- [ "${cnt}" -gt "${ban_cores}" ] && wait -n
+ [ "${cnt}" -ge "${ban_cores}" ] && wait -n
cnt="$((cnt + 1))"
fi
fi
if [ "${feed}" = "country" ] && [ "${ban_countrysplit}" = "1" ]; then
for country in ${ban_country}; do
(f_down "${feed}.${country}" "${feed_ipv}" "${feed_url_6}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") &
- [ "${cnt}" -gt "${ban_cores}" ] && wait -n
+ [ "${cnt}" -ge "${ban_cores}" ] && wait -n
cnt="$((cnt + 1))"
done
elif [ "${feed}" = "asn" ] && [ "${ban_asnsplit}" = "1" ]; then
for asn in ${ban_asn}; do
(f_down "${feed}.${asn}" "${feed_ipv}" "${feed_url_6}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") &
- [ "${cnt}" -gt "${ban_cores}" ] && wait -n
+ [ "${cnt}" -ge "${ban_cores}" ] && wait -n
cnt="$((cnt + 1))"
done
else
(f_down "${feed}" "${feed_ipv}" "${feed_url_6}" "${feed_rule}" "${feed_chain:-"in"}" "${feed_flag}") &
- [ "${cnt}" -gt "${ban_cores}" ] && wait -n
+ [ "${cnt}" -ge "${ban_cores}" ] && wait -n
cnt="$((cnt + 1))"
fi
fi
cnt="1"
for list in allowlist blocklist; do
(f_lookup "${list}") &
- [ "${cnt}" -gt "${ban_cores}" ] && wait -n
+ [ "${cnt}" -ge "${ban_cores}" ] && wait -n
cnt="$((cnt + 1))"
done
wait
git.99rst.org / openwrt-packages.git / commitdiff