diff --git a/Cargo.lock b/Cargo.lock index 147f7eed..437ef87d 100644 --- a/Cargo.lock +++ b/Cargo.lock @@ -176,6 +176,7 @@ dependencies = [ "dashmap", "futures", "http 1.4.0", + "hyper-util", "rcgen", "reqwest 0.12.28", "rustls", diff --git a/config.example.yaml b/config.example.yaml index ee7185b9..04a85056 100644 --- a/config.example.yaml +++ b/config.example.yaml @@ -531,22 +531,17 @@ upstream: # gateway downloads it and sends the bytes inline. An upstream that # reads the URL itself still gets the URL. # - # The download only reaches publicly routable addresses: loopback, - # private (RFC 1918), CGNAT, link-local (including the 169.254.169.254 - # metadata endpoint), IPv6 unique-local / link-local, multicast and - # reserved ranges are refused with a 400, for every address a name - # resolves to and for every redirect hop (at most 10). It follows the - # request's own upstream timeout, retries a timeout or connection - # failure up to 3 attempts in all, and never forwards the caller's - # headers. + # The URL is the caller's, so which addresses the download may reach is + # outbound.ssrf_protection (below). Otherwise it is an ordinary upstream + # request: it goes through HTTPS_PROXY / HTTP_PROXY / ALL_PROXY unless + # NO_PROXY covers the host, and verifies certificates per upstream.tls + # (ca_file, verify) — but never presents the upstream.tls client + # certificate. It follows at most 10 redirects, the request's own + # upstream timeout, retries a timeout or connection failure up to 3 + # attempts in all, and never forwards the caller's headers. # media_fetch: # # false refuses such parts with a 400 instead of downloading them. # enabled: true - # # Hosts that may be reached although they resolve to a non-public - # # address, as host (any port) or host:port; IPv6 with a port as - # # [addr]:port. There is no switch that turns the address check off. - # allowed_hosts: [] - # # e.g. ["media.internal", "10.0.0.5:9000"] # # Largest download accepted per URL, in bytes (default 50 MiB). # # All downloads of one request also count, with its inbound body, # # against proxy.request_body_limit_bytes when that is set (413 past it). @@ -562,6 +557,36 @@ upstream: # # GovCloud. One value serves every Bedrock provider key. # mantle_endpoint: null +# Outbound requests to URLs an API caller supplied — today, the media +# download (upstream.media_fetch). URLs an operator configured (a provider +# key's api_base, guardrail, MCP and A2A endpoints, JWKS, exporters) are +# not affected. +# outbound: +# # Server-side request forgery protection. On (the default), such a +# # request only reaches publicly routable addresses: loopback, private +# # (RFC 1918), CGNAT, link-local (including the 169.254.169.254 metadata +# # endpoint), benchmarking (198.18.0.0/15), IPv6 unique-local / +# # link-local, multicast and reserved ranges are refused with a 400, for +# # every address a name resolves to and for every redirect hop. A direct +# # connection goes only to the addresses that were checked. When a +# # proxy applies to the URL, the name is resolved and checked locally +# # first and the proxy then resolves it again itself. +# # +# # Behind a fake-ip DNS or a transparent proxy (Clash, sing-box, +# # Surge, …) public names resolve into 198.18.0.0/15, so every +# # download is refused: list those hosts in allowed_hosts, or turn the +# # check off. +# ssrf_protection: +# # false turns the address check off entirely — no address is +# # refused and allowed_hosts is ignored. +# enabled: true +# # Hosts that may be reached although they resolve to a non-public +# # address, as host (any port) or host:port; IPv6 with a port as +# # [addr]:port. Env: AISIX_OUTBOUND__SSRF_PROTECTION__ALLOWED_HOSTS +# # takes a comma-separated list. +# allowed_hosts: [] +# # e.g. ["media.internal", "10.0.0.5:9000"] + # Connection-layer settings for the inbound side — the clients, or the # gateway in front of this one, talking to the listeners above. The # mirror image of `upstream:`. diff --git a/config.managed.yaml b/config.managed.yaml index bf10ffc2..35a68a0b 100644 --- a/config.managed.yaml +++ b/config.managed.yaml @@ -281,14 +281,13 @@ upstream: # # client_cert_file / client_key_file for mutual TLS # # verify: true # false accepts ANY certificate — test use only # Media URLs an upstream cannot read itself (Bedrock, Claude on Vertex, - # http:// toward Anthropic / Gemini) are downloaded and sent inline. - # Only publicly routable addresses are reached; allowed_hosts (host or - # host:port) is the only exception. A request's downloads count, with - # its inbound body, against proxy.request_body_limit_bytes when set. - # See config.example.yaml. + # http:// toward Anthropic / Gemini) are downloaded and sent inline, + # through HTTP(S)_PROXY like any upstream call and verified per + # upstream.tls. Which addresses it may reach is outbound.ssrf_protection. + # A request's downloads count, with its inbound body, against + # proxy.request_body_limit_bytes when set. See config.example.yaml. # media_fetch: # enabled: true - # allowed_hosts: [] # max_bytes: 52428800 # Bedrock Mantle, where count_tokens for a Claude model bedrock-runtime # cannot count is retried. null derives https://bedrock-mantle.{region}.api.aws @@ -296,6 +295,16 @@ upstream: # bedrock: # mantle_endpoint: null +# URLs an API caller supplied (the media download) only reach publicly +# routable addresses; allowed_hosts (host or host:port) is the exception +# and enabled: false turns the check off. Behind a fake-ip DNS that maps +# public names into 198.18.0.0/15, list those hosts or turn it off. +# See config.example.yaml. +# outbound: +# ssrf_protection: +# enabled: true +# allowed_hosts: [] + # Connection-layer settings for the inbound side. `idle_timeout_secs: 0` # (the default) never closes an idle client connection, leaving that to # the peer — set it only when a node in front pools connections for a diff --git a/config.reference.json b/config.reference.json index 91dbb420..980b4c71 100644 --- a/config.reference.json +++ b/config.reference.json @@ -77,6 +77,12 @@ }, "service_name": "aisix" }, + "outbound": { + "ssrf_protection": { + "allowed_hosts": [], + "enabled": true + } + }, "proxy": { "addr": "0.0.0.0:3000", "listeners": [], @@ -111,7 +117,6 @@ }, "connect_timeout_ms": 5000, "media_fetch": { - "allowed_hosts": [], "enabled": true, "max_bytes": 52428800 }, diff --git a/crates/aisix-core/src/config.rs b/crates/aisix-core/src/config.rs index 8e27607f..0a4e263f 100644 --- a/crates/aisix-core/src/config.rs +++ b/crates/aisix-core/src/config.rs @@ -69,6 +69,10 @@ pub struct Config { /// see [`UpstreamConfig`]. #[serde(default)] pub upstream: UpstreamConfig, + /// Rules for outbound requests to URLs an API caller supplied — see + /// [`OutboundConfig`]. + #[serde(default)] + pub outbound: OutboundConfig, /// Connection-layer tuning for the inbound side: how long an idle /// client connection is held, and how often a stalled SSE response /// emits a heartbeat — see [`DownstreamConfig`]. @@ -1869,9 +1873,8 @@ pub struct BedrockUpstreamConfig { /// downloads it and sends the bytes inline. An upstream that reads the URL /// itself still gets the URL. /// -/// The download refuses any address that is not publicly routable -/// (loopback, private, link-local, cloud metadata, …); `allowed_hosts` is -/// the only way past that check. +/// Which addresses the download may reach is +/// [`OutboundConfig::ssrf_protection`]. #[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] #[cfg_attr(test, derive(schemars::JsonSchema))] #[serde(deny_unknown_fields, default)] @@ -1879,12 +1882,6 @@ pub struct MediaFetchConfig { /// `false` refuses a media URL the target upstream cannot read with a /// 400 instead of downloading it. pub enabled: bool, - /// Hosts the download may reach even though they resolve to a - /// non-public address, as `host` (any port) or `host:port`; an IPv6 - /// address is written in brackets when it carries a port - /// (`[fd00::1]:8080`). The host is still resolved once and the - /// connection pinned to that answer. - pub allowed_hosts: Vec, /// Largest body accepted per URL, in bytes. A larger download is /// abandoned and the request refused with a 400. pub max_bytes: u64, @@ -1894,12 +1891,58 @@ impl Default for MediaFetchConfig { fn default() -> Self { Self { enabled: true, - allowed_hosts: Vec::new(), max_bytes: 50 * 1024 * 1024, } } } +/// Outbound requests to a URL an API caller supplied, as opposed to one an +/// operator configured. Today that is the media download +/// (`upstream.media_fetch`). Configured URLs — a provider key's +/// `api_base`, guardrail, MCP and A2A endpoints, JWKS, exporters — are not +/// governed by it. +#[derive(Debug, Clone, Default, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields, default)] +pub struct OutboundConfig { + /// Server-side request forgery protection — see + /// [`SsrfProtectionConfig`]. + pub ssrf_protection: SsrfProtectionConfig, +} + +/// Keeps a caller-supplied URL from reaching the operator's own network. +/// +/// On, every address such a request connects to must be publicly routable +/// (not loopback, private, link-local, CGNAT, cloud metadata, benchmarking +/// `198.18.0.0/15`, …). A host name is resolved and every address in the +/// answer checked; a direct connection goes only to those addresses, and +/// each redirect hop is checked the same way. When an `HTTP(S)_PROXY` +/// applies to the URL, the name is resolved and checked the same way +/// before the request is handed to the proxy, which then resolves it +/// again itself. +#[derive(Debug, Clone, PartialEq, Eq, Serialize, Deserialize)] +#[cfg_attr(test, derive(schemars::JsonSchema))] +#[serde(deny_unknown_fields, default)] +pub struct SsrfProtectionConfig { + /// `false` turns the address check off entirely: the request is an + /// ordinary fetch and `allowed_hosts` is ignored. + pub enabled: bool, + /// Hosts a request may reach even though they resolve to a + /// non-public address, as `host` (any port) or `host:port`; an IPv6 + /// address is written in brackets when it carries a port + /// (`[fd00::1]:8080`). + pub allowed_hosts: Vec, +} + +impl Default for SsrfProtectionConfig { + fn default() -> Self { + Self { + enabled: true, + allowed_hosts: Vec::new(), + } + } +} + impl Default for UpstreamConfig { fn default() -> Self { Self { @@ -2095,7 +2138,7 @@ impl Default for ShutdownConfig { /// than handed to a root struct that rejects unknown fields. /// `config_top_level_keys_match_the_struct` fails the build when a field /// is added or renamed without this list following. -const TOP_LEVEL_ENV_KEYS: [&str; 12] = [ +const TOP_LEVEL_ENV_KEYS: [&str; 13] = [ "admin", "bedrock_endpoint_url", "cache", @@ -2103,6 +2146,7 @@ const TOP_LEVEL_ENV_KEYS: [&str; 12] = [ "etcd", "managed", "observability", + "outbound", "proxy", "ratelimit", "resources_file", @@ -2231,7 +2275,7 @@ const ENV_LIST_PARSE_KEYS: &[&str] = &[ "cache.redis.sentinels", "ratelimit.redis.nodes", "ratelimit.redis.sentinels", - "upstream.media_fetch.allowed_hosts", + "outbound.ssrf_protection.allowed_hosts", ]; impl Config { @@ -5170,6 +5214,7 @@ observability: "cache": {}, "ratelimit": {}, "upstream": {"tls": {}}, + "outbound": {}, "downstream": {}, "shutdown": {}, "managed": {}, diff --git a/crates/aisix-gateway/Cargo.toml b/crates/aisix-gateway/Cargo.toml index acf06e53..9dc2fdf3 100644 --- a/crates/aisix-gateway/Cargo.toml +++ b/crates/aisix-gateway/Cargo.toml @@ -24,6 +24,9 @@ dashmap.workspace = true tracing.workspace = true base64.workspace = true url.workspace = true +# The environment-proxy matcher reqwest itself uses, so `ssrf` knows which +# caller-supplied URLs a proxy will take. +hyper-util = { version = "0.1", default-features = false, features = ["client-proxy"] } # Outbound TLS trust (upstream_tls): the non-reqwest outbound stacks need # the root store assembled directly, so the rustls side is a direct dep. rustls.workspace = true diff --git a/crates/aisix-gateway/src/lib.rs b/crates/aisix-gateway/src/lib.rs index c8bb2545..b1788c21 100644 --- a/crates/aisix-gateway/src/lib.rs +++ b/crates/aisix-gateway/src/lib.rs @@ -38,6 +38,7 @@ pub mod hub; pub mod media_fetch; pub mod response_tap; pub mod sse; +pub mod ssrf; pub mod structured_output; pub mod tool_calls; pub mod upstream_headers; diff --git a/crates/aisix-gateway/src/media_fetch.rs b/crates/aisix-gateway/src/media_fetch.rs index 752fa62b..d6b977d4 100644 --- a/crates/aisix-gateway/src/media_fetch.rs +++ b/crates/aisix-gateway/src/media_fetch.rs @@ -2,26 +2,22 @@ //! the URL itself, so the bytes can travel inline instead. //! //! The URL is the caller's, and the download runs from inside the -//! operator's network, so every address it connects to must be publicly -//! routable. The name is resolved here, every address the answer holds is -//! checked, and the connection goes to exactly those addresses — never to -//! a second lookup that a rebinding name could answer differently. That -//! holds for `http` and `https` alike, because the check lives in the -//! client's resolver; the host name still rides `Host` and SNI. Redirects -//! are followed by hand so each hop passes the same check. -//! `upstream.media_fetch.allowed_hosts` is the only way past it. +//! operator's network, so each hop goes through `outbound.ssrf_protection` +//! ([`crate::ssrf`]); redirects are followed by hand so every hop passes +//! the same check. Otherwise it is an ordinary upstream request: the +//! environment proxies and `upstream.tls` certificate verification apply, +//! but never the upstream client identity. use std::borrow::Cow; use std::collections::hash_map::Entry; use std::collections::HashMap; -use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::net::IpAddr; use std::sync::atomic::{AtomicUsize, Ordering}; -use std::sync::{Arc, Mutex, OnceLock, PoisonError}; +use std::sync::{Mutex, OnceLock, PoisonError}; use std::time::Duration; use base64::Engine as _; use futures::StreamExt; -use reqwest::dns::{Addrs, Name, Resolve, Resolving}; use reqwest::{header, Url}; use tokio::time::Instant; @@ -31,6 +27,7 @@ use crate::content_part::{ display_uri, mime_type_from_path, normalize_mime_type, parse_part, ContentPart, Media, MediaSource, }; +use crate::ssrf::{Refusal, Route, SsrfProtection}; /// Attempts per URL on a timeout or transport failure. An HTTP error /// status is an answer and is not retried. @@ -44,90 +41,34 @@ const MAX_CONCURRENT: usize = 8; #[derive(Debug, Clone)] pub struct MediaFetchSettings { enabled: bool, - allowed_hosts: Vec, max_bytes: u64, + /// `outbound.ssrf_protection`, which every hop goes through. + ssrf: SsrfProtection, /// `proxy.request_body_limit_bytes`: the inbound body and everything /// downloaded for the request share it. `0` is no limit. request_body_limit_bytes: usize, } -#[derive(Debug, Clone, PartialEq, Eq)] -struct AllowedHost { - host: String, - port: Option, -} - impl MediaFetchSettings { - pub fn from_config(cfg: &aisix_core::config::MediaFetchConfig) -> Result { - let allowed_hosts = cfg - .allowed_hosts - .iter() - .map(|entry| { - parse_allowed_host(entry).ok_or_else(|| { - format!( - "upstream.media_fetch.allowed_hosts: {entry:?} is not a host or \ - host:port (write an IPv6 address with a port as [addr]:port)" - ) - }) - }) - .collect::>()?; - Ok(Self { + pub fn new(cfg: &aisix_core::config::MediaFetchConfig, ssrf: SsrfProtection) -> Self { + Self { enabled: cfg.enabled, - allowed_hosts, max_bytes: cfg.max_bytes, + ssrf, request_body_limit_bytes: 0, - }) - } - - fn allows(&self, host: &str, port: u16) -> bool { - self.allowed_hosts - .iter() - .any(|a| a.host == host && a.port.is_none_or(|p| p == port)) - } -} - -fn parse_allowed_host(entry: &str) -> Option { - let entry = entry.trim(); - if entry.is_empty() || entry.contains(['/', '@', '?', '#']) { - return None; - } - let port = |p: &str| p.parse::().ok(); - if let Some(rest) = entry.strip_prefix('[') { - let (addr, tail) = rest.split_once(']')?; - let host = addr.parse::().ok()?.to_string(); - let port = match tail { - "" => None, - tail => Some(port(tail.strip_prefix(':')?)?), - }; - return Some(AllowedHost { host, port }); - } - if let Ok(v6) = entry.parse::() { - return Some(AllowedHost { - host: v6.to_string(), - port: None, - }); + } } - let (host, port) = match entry.rsplit_once(':') { - Some((host, p)) => (host, Some(port(p)?)), - None => (entry, None), - }; - let host = host.trim_end_matches('.').to_ascii_lowercase(); - (!host.is_empty()).then_some(AllowedHost { host, port }) } static SETTINGS: OnceLock = OnceLock::new(); /// Install `upstream.media_fetch`, with `proxy.request_body_limit_bytes` -/// as the per-request total. Called once during boot; fails on an -/// `allowed_hosts` entry that does not parse. -pub fn init( - cfg: &aisix_core::config::MediaFetchConfig, - request_body_limit_bytes: usize, -) -> Result<(), String> { - let mut settings = MediaFetchSettings::from_config(cfg)?; +/// as the per-request total. Called once during boot, after +/// [`crate::ssrf::init`]. +pub fn init(cfg: &aisix_core::config::MediaFetchConfig, request_body_limit_bytes: usize) { + let mut settings = MediaFetchSettings::new(cfg, crate::ssrf::settings().clone()); settings.request_body_limit_bytes = request_body_limit_bytes; let _ = SETTINGS.set(settings); - Ok(()) } /// What one request may still hold: its inbound body plus every byte @@ -169,7 +110,7 @@ impl RequestBudget { fn settings() -> &'static MediaFetchSettings { SETTINGS.get_or_init(|| { - MediaFetchSettings::from_config(&Default::default()).expect("the default config parses") + MediaFetchSettings::new(&Default::default(), crate::ssrf::settings().clone()) }) } @@ -498,8 +439,8 @@ async fn download( ) -> Result<(Vec, Option, Url), Failure> { let mut current = url.clone(); for hop in 0..=MAX_REDIRECTS { - let guarded = check_target(¤t, settings)?; - let resp = client(guarded)? + let route = settings.ssrf.route(¤t).await?; + let resp = client(route)? .get(current.clone()) .send() .await @@ -577,94 +518,36 @@ async fn read_capped( } } -/// Whether `url` may be requested, and through which client: `true` for -/// the one whose resolver refuses non-public addresses, `false` for an -/// allow-listed host. -fn check_target(url: &Url, settings: &MediaFetchSettings) -> Result { - let (host, ip) = match url.host() { - Some(url::Host::Domain(d)) => (d.trim_end_matches('.').to_ascii_lowercase(), None), - Some(url::Host::Ipv4(a)) => (a.to_string(), Some(IpAddr::V4(a))), - Some(url::Host::Ipv6(a)) => (a.to_string(), Some(IpAddr::V6(a))), - None => return Err(Failure::BadUrl), +fn client(route: Route) -> Result<&'static reqwest::Client, Failure> { + static PINNED: OnceLock> = OnceLock::new(); + static DIRECT: OnceLock> = OnceLock::new(); + static PROXIED: OnceLock> = OnceLock::new(); + let cell = match route { + Route::Pinned => &PINNED, + Route::Direct => &DIRECT, + Route::Proxied => &PROXIED, }; - let port = url.port_or_known_default().ok_or(Failure::BadUrl)?; - if settings.allows(&host, port) { - return Ok(false); - } - // The resolver never sees an address literal, so it is checked here. - if let Some(ip) = ip.filter(|ip| !is_public(*ip)) { - return Err(Failure::Blocked(ip)); - } - Ok(true) -} - -fn client(guarded: bool) -> Result<&'static reqwest::Client, Failure> { - static GUARDED: OnceLock> = OnceLock::new(); - static ALLOWED: OnceLock> = OnceLock::new(); - let cell = if guarded { &GUARDED } else { &ALLOWED }; cell.get_or_init(|| { - crate::upstream_http::caller_chosen_peer_client_builder() - .dns_resolver(Arc::new(MediaResolver { guarded })) - .redirect(reqwest::redirect::Policy::none()) - // A proxy would resolve the name itself, past the check. - .no_proxy() - .build() - .map_err(|e| e.to_string()) + let b = crate::upstream_http::caller_chosen_peer_client_builder() + .redirect(reqwest::redirect::Policy::none()); + // The route already decided whether a proxy takes the URL. The + // proxied client keeps the shared resolver: it only resolves the + // proxy's own host. + let b = match route { + Route::Pinned => b + .dns_resolver(crate::ssrf::direct_resolver(true)) + .no_proxy(), + Route::Direct => b + .dns_resolver(crate::ssrf::direct_resolver(false)) + .no_proxy(), + Route::Proxied => b, + }; + b.build().map_err(|e| e.to_string()) }) .as_ref() .map_err(|_| Failure::Transport) } -/// Resolves a name and, when `guarded`, refuses the whole answer if any -/// address in it is not publicly routable. The client connects only to -/// the addresses returned here. -struct MediaResolver { - guarded: bool, -} - -impl Resolve for MediaResolver { - fn resolve(&self, name: Name) -> Resolving { - let guarded = self.guarded; - Box::pin(async move { - let addrs: Vec = tokio::net::lookup_host((name.as_str(), 0)) - .await - .map_err(|_| Box::new(Unresolvable) as Box)? - .collect(); - if addrs.is_empty() { - return Err(Box::new(Unresolvable) as _); - } - if guarded { - if let Some(ip) = addrs.iter().map(SocketAddr::ip).find(|ip| !is_public(*ip)) { - return Err(Box::new(BlockedAddress(ip)) as _); - } - } - Ok(Box::new(addrs.into_iter()) as Addrs) - }) - } -} - -#[derive(Debug)] -struct BlockedAddress(IpAddr); - -impl std::fmt::Display for BlockedAddress { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - write!(f, "{} is not a publicly routable address", self.0) - } -} - -impl std::error::Error for BlockedAddress {} - -#[derive(Debug)] -struct Unresolvable; - -impl std::fmt::Display for Unresolvable { - fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { - f.write_str("the host does not resolve") - } -} - -impl std::error::Error for Unresolvable {} - #[derive(Debug, Clone, PartialEq, Eq)] enum Failure { BadUrl, @@ -693,8 +576,8 @@ impl Failure { Self::Blocked(_) => { return format!( "the gateway does not download the media URL {shown}: its host is not \ - a publicly routable address. To allow it, an operator adds the host to \ - upstream.media_fetch.allowed_hosts" + a publicly routable address. {}", + Refusal::blocked_hint() ); } Self::UnknownType => { @@ -725,16 +608,19 @@ impl Failure { } } -fn classify(err: reqwest::Error) -> Failure { - let mut source: Option<&(dyn std::error::Error + 'static)> = Some(&err); - while let Some(e) = source { - if let Some(BlockedAddress(ip)) = e.downcast_ref::() { - return Failure::Blocked(*ip); +impl From for Failure { + fn from(r: Refusal) -> Self { + match r { + Refusal::BadUrl => Self::BadUrl, + Refusal::Blocked(ip) => Self::Blocked(ip), + Refusal::Unresolvable => Self::Unresolvable, } - if e.downcast_ref::().is_some() { - return Failure::Unresolvable; - } - source = e.source(); + } +} + +fn classify(err: reqwest::Error) -> Failure { + if let Some(refusal) = Refusal::from_error(&err) { + return refusal.into(); } if err.is_timeout() { Failure::Timeout @@ -743,69 +629,6 @@ fn classify(err: reqwest::Error) -> Failure { } } -/// Whether an address is one the download may reach without an -/// allow-list entry: globally routable unicast, and not a cloud -/// platform's own service address. -fn is_public(ip: IpAddr) -> bool { - match ip { - IpAddr::V4(v4) => is_public_v4(v4), - IpAddr::V6(v6) => match v6.to_ipv4_mapped() { - Some(v4) => is_public_v4(v4), - None => is_public_v6(v6), - }, - } -} - -fn is_public_v4(a: Ipv4Addr) -> bool { - let [o0, o1, o2, _] = a.octets(); - let blocked = o0 == 0 // "this network", incl. 0.0.0.0 - || a.is_loopback() - || a.is_private() - || (o0 == 100 && (o1 & 0xc0) == 64) // CGNAT 100.64.0.0/10 - || a.is_link_local() // 169.254.0.0/16, incl. instance metadata - || (o0 == 192 && o1 == 0 && o2 == 0) // IETF protocol assignments - || a.is_documentation() - || (o0 == 198 && (o1 & 0xfe) == 18) // benchmarking 198.18.0.0/15 - || (o0 == 192 && o1 == 88 && o2 == 99) // 6to4 relay anycast - || a.is_multicast() - || o0 >= 240 // reserved, incl. broadcast - || a == Ipv4Addr::new(168, 63, 129, 16); // Azure platform endpoint - !blocked -} - -fn is_public_v6(a: Ipv6Addr) -> bool { - let s = a.segments(); - let embedded = |hi: u16, lo: u16| Ipv4Addr::from(((hi as u32) << 16) | lo as u32); - if a.is_unspecified() || a.is_loopback() || a.is_multicast() { - return false; - } - match s { - // IPv4-compatible (deprecated) ::/96 - [0, 0, 0, 0, 0, 0, _, _] => false, - // NAT64 64:ff9b::/96 reaches the IPv4 address it embeds. - [0x64, 0xff9b, 0, 0, 0, 0, hi, lo] => is_public_v4(embedded(hi, lo)), - // Local-use NAT64 64:ff9b:1::/48 - [0x64, 0xff9b, 1, ..] => false, - // Discard-only 100::/64 - [0x100, 0, 0, 0, ..] => false, - // IETF protocol assignments 2001::/23 (Teredo included), documentation 2001:db8::/32 - [0x2001, b, ..] if b < 0x200 || b == 0xdb8 => false, - // 6to4 2002::/16 reaches the IPv4 address it embeds. - [0x2002, hi, lo, ..] => is_public_v4(embedded(hi, lo)), - // Documentation 3fff::/20 - [0x3fff, b, ..] if b < 0x1000 => false, - // Unique-local fc00::/7, link-local fe80::/10, site-local fec0::/10 - [first, ..] - if (first & 0xfe00) == 0xfc00 - || (first & 0xffc0) == 0xfe80 - || (first & 0xffc0) == 0xfec0 => - { - false - } - _ => true, - } -} - /// The media type a body's first bytes announce, for the formats the /// multimodal upstreams accept. fn sniff(b: &[u8]) -> Option<&'static str> { @@ -847,6 +670,8 @@ fn sniff(b: &[u8]) -> Option<&'static str> { #[cfg(test)] mod tests { + use std::sync::Arc; + use super::*; async fn dl( @@ -875,72 +700,6 @@ mod tests { assert_eq!(roomy.used.load(Ordering::Relaxed), 15); } - #[test] - fn only_globally_routable_addresses_are_public() { - for blocked in [ - "0.0.0.0", - "127.0.0.1", - "10.1.2.3", - "172.16.0.1", - "192.168.1.1", - "100.64.0.1", - "100.127.255.255", - "169.254.169.254", - "192.0.0.8", - "192.0.2.1", - "198.18.0.1", - "198.51.100.1", - "203.0.113.1", - "224.0.0.1", - "255.255.255.255", - "168.63.129.16", - "::", - "::1", - "::ffff:127.0.0.1", - "::ffff:169.254.169.254", - "64:ff9b::a00:1", - "2002:a00:1::", - "2001:db8::1", - "fd00::1", - "fe80::1", - "ff02::1", - ] { - let ip: IpAddr = blocked.parse().unwrap(); - assert!(!is_public(ip), "{blocked} must be refused"); - } - for public in [ - "1.1.1.1", - "8.8.8.8", - "100.128.0.1", - "::ffff:8.8.8.8", - "2606:4700:4700::1111", - "64:ff9b::808:808", - ] { - let ip: IpAddr = public.parse().unwrap(); - assert!(is_public(ip), "{public} must be allowed"); - } - } - - #[test] - fn allowed_hosts_match_host_and_optional_port() { - let settings = MediaFetchSettings::from_config(&aisix_core::config::MediaFetchConfig { - allowed_hosts: vec![ - "Media.Internal".into(), - "127.0.0.1:9000".into(), - "[fd00::1]:8080".into(), - ], - ..Default::default() - }) - .unwrap(); - assert!(settings.allows("media.internal", 443)); - assert!(settings.allows("127.0.0.1", 9000)); - assert!(!settings.allows("127.0.0.1", 9001)); - assert!(settings.allows("fd00::1", 8080)); - for bad in ["", "http://x", "a:b", "[::1", "user@host"] { - assert!(parse_allowed_host(bad).is_none(), "{bad:?}"); - } - } - /// A one-shot HTTP server on loopback answering every request with /// `response`. async fn serve(response: &'static [u8]) -> u16 { @@ -958,12 +717,22 @@ mod tests { } fn with_allowed(hosts: &[&str]) -> MediaFetchSettings { - MediaFetchSettings::from_config(&aisix_core::config::MediaFetchConfig { + MediaFetchSettings::new( + &aisix_core::config::MediaFetchConfig { + max_bytes: 16, + ..Default::default() + }, + ssrf(true, hosts), + ) + } + + fn ssrf(enabled: bool, hosts: &[&str]) -> SsrfProtection { + SsrfProtection::from_config(&aisix_core::config::SsrfProtectionConfig { + enabled, allowed_hosts: hosts.iter().map(|h| h.to_string()).collect(), - max_bytes: 16, - ..Default::default() }) .unwrap() + .without_proxies() } #[tokio::test] @@ -996,6 +765,45 @@ mod tests { assert!(matches!(got, Err(Failure::Blocked(_))), "{got:?}"); } + #[tokio::test] + async fn with_protection_off_a_private_address_is_fetched_and_redirects_stay_capped() { + let port = + serve(b"HTTP/1.1 200 OK\r\ncontent-length: 4\r\nconnection: close\r\n\r\nPNG!").await; + let off = MediaFetchSettings::new(&Default::default(), ssrf(false, &[])); + for url in [ + format!("http://localhost:{port}/a.png"), + format!("http://127.0.0.1:{port}/a.png"), + ] { + let (body, ..) = dl(&Url::parse(&url).unwrap(), &off).await.unwrap(); + assert_eq!(body, b"PNG!", "{url}"); + } + let redirector = serve_self_redirect().await; + let got = dl( + &Url::parse(&format!("http://127.0.0.1:{redirector}/a")).unwrap(), + &off, + ) + .await; + assert_eq!(got.map(|_| ()), Err(Failure::TooManyRedirects)); + } + + /// A loopback server that redirects every request back to itself. + async fn serve_self_redirect() -> u16 { + use tokio::io::{AsyncReadExt, AsyncWriteExt}; + let listener = tokio::net::TcpListener::bind("127.0.0.1:0").await.unwrap(); + let port = listener.local_addr().unwrap().port(); + let response = format!( + "HTTP/1.1 302 Found\r\nlocation: http://127.0.0.1:{port}/again\r\ncontent-length: 0\r\nconnection: close\r\n\r\n" + ); + tokio::spawn(async move { + while let Ok((mut sock, _)) = listener.accept().await { + let mut buf = [0u8; 4096]; + let _ = sock.read(&mut buf).await; + let _ = sock.write_all(response.as_bytes()).await; + } + }); + port + } + #[tokio::test] async fn a_body_past_the_cap_is_refused_whether_declared_or_not() { let declared = serve( @@ -1071,14 +879,10 @@ mod tests { /// charged to the end-to-end budget only. #[tokio::test] async fn every_part_naming_a_downloaded_url_gets_it_and_the_time_is_charged() { - init( - &aisix_core::config::MediaFetchConfig { - allowed_hosts: vec!["127.0.0.1".into()], - ..Default::default() - }, - 0, - ) - .unwrap(); + let _ = SETTINGS.set(MediaFetchSettings::new( + &Default::default(), + ssrf(true, &["127.0.0.1"]), + )); let port = serve(b"HTTP/1.1 200 OK\r\ncontent-type: image/png\r\ncontent-length: 4\r\nconnection: close\r\n\r\nPNG!").await; let url = format!("http://127.0.0.1:{port}/img"); let req: ChatFormat = serde_json::from_value(serde_json::json!({ diff --git a/crates/aisix-gateway/src/ssrf.rs b/crates/aisix-gateway/src/ssrf.rs new file mode 100644 index 00000000..531c7b46 --- /dev/null +++ b/crates/aisix-gateway/src/ssrf.rs @@ -0,0 +1,450 @@ +//! `outbound.ssrf_protection`: whether a request to a URL an API caller +//! supplied may reach a given address. +//! +//! On, every address the request connects to must be publicly routable. +//! A direct connection resolves the name here, checks every address in +//! the answer, and connects to exactly those addresses — never to a +//! second lookup that a rebinding name could answer differently; that +//! holds for `http` and `https` alike because the check lives in the +//! client's resolver, while the host name still rides `Host` and SNI. +//! When an environment proxy applies to the URL, the name is resolved and +//! checked here first and the proxy then resolves it again itself, which +//! cannot be pinned. `allowed_hosts` is the only way past the check, and +//! `enabled: false` removes it. + +use std::net::{IpAddr, Ipv4Addr, Ipv6Addr, SocketAddr}; +use std::sync::{Arc, OnceLock}; + +use hyper_util::client::proxy::matcher::Matcher; +use reqwest::dns::{Addrs, Name, Resolve, Resolving}; +use reqwest::Url; + +/// `outbound.ssrf_protection`, parsed once at boot. +#[derive(Clone)] +pub struct SsrfProtection { + enabled: bool, + allowed_hosts: Vec, + /// Which URLs an environment proxy takes: the matcher the HTTP clients + /// themselves use, read from the same environment. + proxies: Arc, +} + +impl std::fmt::Debug for SsrfProtection { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.debug_struct("SsrfProtection") + .field("enabled", &self.enabled) + .field("allowed_hosts", &self.allowed_hosts) + .finish_non_exhaustive() + } +} + +#[derive(Debug, Clone, PartialEq, Eq)] +struct AllowedHost { + host: String, + port: Option, +} + +impl SsrfProtection { + pub fn from_config(cfg: &aisix_core::config::SsrfProtectionConfig) -> Result { + let allowed_hosts = cfg + .allowed_hosts + .iter() + .map(|entry| { + parse_allowed_host(entry).ok_or_else(|| { + format!( + "outbound.ssrf_protection.allowed_hosts: {entry:?} is not a host or \ + host:port (write an IPv6 address with a port as [addr]:port)" + ) + }) + }) + .collect::>()?; + Ok(Self { + enabled: cfg.enabled, + allowed_hosts, + proxies: Arc::new(Matcher::from_system()), + }) + } + + /// As if no proxy were configured, whatever the environment says. + #[cfg(test)] + pub(crate) fn without_proxies(mut self) -> Self { + self.proxies = Arc::new(Matcher::builder().build()); + self + } + + fn proxied(&self, url: &Url) -> bool { + url.as_str() + .parse::() + .is_ok_and(|uri| self.proxies.intercept(&uri).is_some()) + } + + fn allows(&self, host: &str, port: u16) -> bool { + self.allowed_hosts + .iter() + .any(|a| a.host == host && a.port.is_none_or(|p| p == port)) + } + + /// How a request to `url` is sent, or why it may not be. Async because + /// a URL an environment proxy takes is resolved and checked here. + pub async fn route(&self, url: &Url) -> Result { + let (host, ip) = match url.host() { + Some(url::Host::Domain(d)) => (d.trim_end_matches('.').to_ascii_lowercase(), None), + Some(url::Host::Ipv4(a)) => (a.to_string(), Some(IpAddr::V4(a))), + Some(url::Host::Ipv6(a)) => (a.to_string(), Some(IpAddr::V6(a))), + None => return Err(Refusal::BadUrl), + }; + let port = url.port_or_known_default().ok_or(Refusal::BadUrl)?; + let proxied = self.proxied(url); + let unchecked = if proxied { + Route::Proxied + } else { + Route::Direct + }; + if !self.enabled || self.allows(&host, port) { + return Ok(unchecked); + } + // The resolver never sees an address literal, so it is checked here. + if let Some(ip) = ip { + return if is_public(ip) { + Ok(unchecked) + } else { + Err(Refusal::Blocked(ip)) + }; + } + if !proxied { + return Ok(Route::Pinned); + } + check_all(resolve(&host).await?)?; + Ok(Route::Proxied) + } +} + +/// The client a permitted request goes out on. +#[derive(Debug, Clone, Copy, PartialEq, Eq)] +pub enum Route { + /// Direct, through the resolver that refuses non-public addresses. + Pinned, + /// Direct with nothing to check: the target is allowed or protection + /// is off. + Direct, + /// Through the environment proxy that takes the URL; the target is + /// allowed, already checked, or protection is off. + Proxied, +} + +#[derive(Debug, Clone, PartialEq, Eq)] +pub enum Refusal { + BadUrl, + Blocked(IpAddr), + Unresolvable, +} + +impl Refusal { + /// The error a caller gets for a blocked host, naming the settings an + /// operator would change. + pub fn blocked_hint() -> &'static str { + "To allow it, an operator adds the host to outbound.ssrf_protection.allowed_hosts, \ + or turns the check off with outbound.ssrf_protection.enabled: false" + } + + /// The refusal a resolver error carries, if any. + pub fn from_error(err: &(dyn std::error::Error + 'static)) -> Option { + let mut source = Some(err); + while let Some(e) = source { + if let Some(BlockedAddress(ip)) = e.downcast_ref::() { + return Some(Self::Blocked(*ip)); + } + if e.downcast_ref::().is_some() { + return Some(Self::Unresolvable); + } + source = e.source(); + } + None + } +} + +static SETTINGS: OnceLock = OnceLock::new(); + +/// Install `outbound.ssrf_protection`. Called once during boot; fails on +/// an `allowed_hosts` entry that does not parse. +pub fn init(cfg: &aisix_core::config::SsrfProtectionConfig) -> Result<(), String> { + let _ = SETTINGS.set(SsrfProtection::from_config(cfg)?); + Ok(()) +} + +pub fn settings() -> &'static SsrfProtection { + SETTINGS.get_or_init(|| { + SsrfProtection::from_config(&Default::default()).expect("the default config parses") + }) +} + +/// The resolver for a direct client: for [`Route::Pinned`] (`guarded`) +/// it refuses the whole answer if any address in it is not publicly +/// routable, and the client connects only to the addresses it returns. +/// Caller-chosen names never enter the shared DNS cache, whose capacity +/// assumes configured upstream hosts. +pub fn direct_resolver(guarded: bool) -> Arc { + Arc::new(DirectResolver { guarded }) +} + +struct DirectResolver { + guarded: bool, +} + +impl Resolve for DirectResolver { + fn resolve(&self, name: Name) -> Resolving { + let guarded = self.guarded; + Box::pin(async move { + let addrs = resolve(name.as_str()) + .await + .map_err(|_| Box::new(Unresolvable) as Box)?; + if guarded { + if let Err(Refusal::Blocked(ip)) = check_all(addrs.iter().copied()) { + return Err(Box::new(BlockedAddress(ip)) as _); + } + } + Ok(Box::new(addrs.into_iter()) as Addrs) + }) + } +} + +async fn resolve(host: &str) -> Result, Refusal> { + let addrs: Vec = tokio::net::lookup_host((host, 0)) + .await + .map_err(|_| Refusal::Unresolvable)? + .collect(); + if addrs.is_empty() { + return Err(Refusal::Unresolvable); + } + Ok(addrs) +} + +fn check_all(addrs: impl IntoIterator) -> Result<(), Refusal> { + match addrs.into_iter().map(|a| a.ip()).find(|ip| !is_public(*ip)) { + Some(ip) => Err(Refusal::Blocked(ip)), + None => Ok(()), + } +} + +#[derive(Debug)] +struct BlockedAddress(IpAddr); + +impl std::fmt::Display for BlockedAddress { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + write!(f, "{} is not a publicly routable address", self.0) + } +} + +impl std::error::Error for BlockedAddress {} + +#[derive(Debug)] +struct Unresolvable; + +impl std::fmt::Display for Unresolvable { + fn fmt(&self, f: &mut std::fmt::Formatter<'_>) -> std::fmt::Result { + f.write_str("the host does not resolve") + } +} + +impl std::error::Error for Unresolvable {} + +fn parse_allowed_host(entry: &str) -> Option { + let entry = entry.trim(); + if entry.is_empty() || entry.contains(['/', '@', '?', '#']) { + return None; + } + let port = |p: &str| p.parse::().ok(); + if let Some(rest) = entry.strip_prefix('[') { + let (addr, tail) = rest.split_once(']')?; + let host = addr.parse::().ok()?.to_string(); + let port = match tail { + "" => None, + tail => Some(port(tail.strip_prefix(':')?)?), + }; + return Some(AllowedHost { host, port }); + } + if let Ok(v6) = entry.parse::() { + return Some(AllowedHost { + host: v6.to_string(), + port: None, + }); + } + let (host, port) = match entry.rsplit_once(':') { + Some((host, p)) => (host, Some(port(p)?)), + None => (entry, None), + }; + let host = host.trim_end_matches('.').to_ascii_lowercase(); + (!host.is_empty()).then_some(AllowedHost { host, port }) +} + +/// Whether an address is one a request may reach without an allow-list +/// entry: globally routable unicast, and not a cloud platform's own +/// service address. +fn is_public(ip: IpAddr) -> bool { + match ip { + IpAddr::V4(v4) => is_public_v4(v4), + IpAddr::V6(v6) => match v6.to_ipv4_mapped() { + Some(v4) => is_public_v4(v4), + None => is_public_v6(v6), + }, + } +} + +fn is_public_v4(a: Ipv4Addr) -> bool { + let [o0, o1, o2, _] = a.octets(); + let blocked = o0 == 0 // "this network", incl. 0.0.0.0 + || a.is_loopback() + || a.is_private() + || (o0 == 100 && (o1 & 0xc0) == 64) // CGNAT 100.64.0.0/10 + || a.is_link_local() // 169.254.0.0/16, incl. instance metadata + || (o0 == 192 && o1 == 0 && o2 == 0) // IETF protocol assignments + || a.is_documentation() + || (o0 == 198 && (o1 & 0xfe) == 18) // benchmarking 198.18.0.0/15 + || (o0 == 192 && o1 == 88 && o2 == 99) // 6to4 relay anycast + || a.is_multicast() + || o0 >= 240 // reserved, incl. broadcast + || a == Ipv4Addr::new(168, 63, 129, 16); // Azure platform endpoint + !blocked +} + +fn is_public_v6(a: Ipv6Addr) -> bool { + let s = a.segments(); + let embedded = |hi: u16, lo: u16| Ipv4Addr::from(((hi as u32) << 16) | lo as u32); + if a.is_unspecified() || a.is_loopback() || a.is_multicast() { + return false; + } + match s { + // IPv4-compatible (deprecated) ::/96 + [0, 0, 0, 0, 0, 0, _, _] => false, + // IPv4-translated (SIIT) ::ffff:0:0:0/96 reaches the IPv4 address it embeds. + [0, 0, 0, 0, 0xffff, 0, hi, lo] => is_public_v4(embedded(hi, lo)), + // NAT64 64:ff9b::/96 reaches the IPv4 address it embeds. + [0x64, 0xff9b, 0, 0, 0, 0, hi, lo] => is_public_v4(embedded(hi, lo)), + // Local-use NAT64 64:ff9b:1::/48 + [0x64, 0xff9b, 1, ..] => false, + // Discard-only 100::/64 + [0x100, 0, 0, 0, ..] => false, + // IETF protocol assignments 2001::/23 (Teredo included), documentation 2001:db8::/32 + [0x2001, b, ..] if b < 0x200 || b == 0xdb8 => false, + // 6to4 2002::/16 reaches the IPv4 address it embeds. + [0x2002, hi, lo, ..] => is_public_v4(embedded(hi, lo)), + // Documentation 3fff::/20 + [0x3fff, b, ..] if b < 0x1000 => false, + // Unique-local fc00::/7, link-local fe80::/10, site-local fec0::/10 + [first, ..] + if (first & 0xfe00) == 0xfc00 + || (first & 0xffc0) == 0xfe80 + || (first & 0xffc0) == 0xfec0 => + { + false + } + _ => true, + } +} + +#[cfg(test)] +mod tests { + use super::*; + + fn with_allowed(hosts: &[&str]) -> SsrfProtection { + SsrfProtection::from_config(&aisix_core::config::SsrfProtectionConfig { + allowed_hosts: hosts.iter().map(|h| h.to_string()).collect(), + ..Default::default() + }) + .unwrap() + .without_proxies() + } + + #[test] + fn only_globally_routable_addresses_are_public() { + for blocked in [ + "0.0.0.0", + "127.0.0.1", + "10.1.2.3", + "172.16.0.1", + "192.168.1.1", + "100.64.0.1", + "100.127.255.255", + "169.254.169.254", + "192.0.0.8", + "192.0.2.1", + "198.18.0.1", + "198.51.100.1", + "203.0.113.1", + "224.0.0.1", + "255.255.255.255", + "168.63.129.16", + "::", + "::1", + "::ffff:127.0.0.1", + "::ffff:169.254.169.254", + "::ffff:0:7f00:1", + "::ffff:0:a9fe:a9fe", + "64:ff9b::a00:1", + "2002:a00:1::", + "2001:db8::1", + "fd00::1", + "fe80::1", + "ff02::1", + ] { + let ip: IpAddr = blocked.parse().unwrap(); + assert!(!is_public(ip), "{blocked} must be refused"); + } + for public in [ + "1.1.1.1", + "8.8.8.8", + "100.128.0.1", + "::ffff:8.8.8.8", + "::ffff:0:808:808", + "2606:4700:4700::1111", + "64:ff9b::808:808", + ] { + let ip: IpAddr = public.parse().unwrap(); + assert!(is_public(ip), "{public} must be allowed"); + } + } + + #[test] + fn allowed_hosts_match_host_and_optional_port() { + let settings = with_allowed(&["Media.Internal", "127.0.0.1:9000", "[fd00::1]:8080"]); + assert!(settings.allows("media.internal", 443)); + assert!(settings.allows("127.0.0.1", 9000)); + assert!(!settings.allows("127.0.0.1", 9001)); + assert!(settings.allows("fd00::1", 8080)); + for bad in ["", "http://x", "a:b", "[::1", "user@host"] { + assert!(parse_allowed_host(bad).is_none(), "{bad:?}"); + } + let err = SsrfProtection::from_config(&aisix_core::config::SsrfProtectionConfig { + allowed_hosts: vec!["http://x".into()], + ..Default::default() + }) + .unwrap_err(); + assert!( + err.contains("outbound.ssrf_protection.allowed_hosts"), + "{err}" + ); + } + + #[tokio::test] + async fn disabled_protection_checks_nothing() { + let off = SsrfProtection::from_config(&aisix_core::config::SsrfProtectionConfig { + enabled: false, + allowed_hosts: vec![], + }) + .unwrap() + .without_proxies(); + for url in ["http://127.0.0.1/a", "http://localhost/a", "http://[::1]/a"] { + let url = Url::parse(url).unwrap(); + assert_eq!(off.route(&url).await, Ok(Route::Direct), "{url}"); + } + let on = with_allowed(&[]); + assert_eq!( + on.route(&Url::parse("http://127.0.0.1/a").unwrap()).await, + Err(Refusal::Blocked("127.0.0.1".parse().unwrap())) + ); + assert_eq!( + on.route(&Url::parse("http://localhost/a").unwrap()).await, + Ok(Route::Pinned) + ); + } +} diff --git a/crates/aisix-gateway/src/upstream_http.rs b/crates/aisix-gateway/src/upstream_http.rs index 5d8e019e..6748c1ec 100644 --- a/crates/aisix-gateway/src/upstream_http.rs +++ b/crates/aisix-gateway/src/upstream_http.rs @@ -141,15 +141,17 @@ pub fn client_builder() -> reqwest::ClientBuilder { } /// [`client_builder`] for a peer the caller, not the operator, chose (a -/// media URL): the same connection settings and trust roots, but never -/// the upstream mTLS client identity, which only upstreams may be shown, -/// and never `upstream.tls.verify: false`, which an operator sets for the -/// upstreams they configured, not for any host a caller names. +/// media URL): the same connection settings and certificate verification +/// (`upstream.tls.ca_file` and `verify`), but never the upstream mTLS +/// client identity, which only upstreams may be shown. pub fn caller_chosen_peer_client_builder() -> reqwest::ClientBuilder { let mut b = base_client_builder(); for root in &crate::upstream_tls::reqwest_material().roots { b = b.add_root_certificate(root.clone()); } + if !config().tls.verify { + b = b.danger_accept_invalid_certs(true); + } b } diff --git a/crates/aisix-server/src/main.rs b/crates/aisix-server/src/main.rs index cb3ad7b8..e71d0678 100644 --- a/crates/aisix-server/src/main.rs +++ b/crates/aisix-server/src/main.rs @@ -270,11 +270,12 @@ async fn async_main(cfg: Config) -> anyhow::Result<()> { // pools are constructed once and can't be reconfigured afterwards. aisix_gateway::upstream_http::init(upstream_http_config(&cfg.upstream)?) .map_err(|e| anyhow::anyhow!("upstream TLS init failed: {e}"))?; + aisix_gateway::ssrf::init(&cfg.outbound.ssrf_protection) + .map_err(|e| anyhow::anyhow!("{e}"))?; aisix_gateway::media_fetch::init( &cfg.upstream.media_fetch, cfg.proxy.request_body_limit_bytes, - ) - .map_err(|e| anyhow::anyhow!("{e}"))?; + ); aisix_provider_bedrock::init_mantle_endpoint(cfg.upstream.bedrock.mantle_endpoint.clone()); run(cfg).await } diff --git a/tests/e2e/src/cases/chat-media-parts-native-providers-e2e.test.ts b/tests/e2e/src/cases/chat-media-parts-native-providers-e2e.test.ts index 08b68a5f..efe80261 100644 --- a/tests/e2e/src/cases/chat-media-parts-native-providers-e2e.test.ts +++ b/tests/e2e/src/cases/chat-media-parts-native-providers-e2e.test.ts @@ -1,5 +1,11 @@ +import { execFileSync } from "node:child_process"; import { createHash } from "node:crypto"; -import { createServer, type Server } from "node:http"; +import { mkdtempSync, readFileSync } from "node:fs"; +import { createServer, request as httpRequest, type Server } from "node:http"; +import { createServer as createHttpsServer } from "node:https"; +import { connect as netConnect } from "node:net"; +import { tmpdir } from "node:os"; +import { join } from "node:path"; import { afterAll, beforeAll, describe, expect, test } from "vitest"; import { EtcdClient, @@ -17,9 +23,11 @@ import { // reference where the upstream reads that URL itself; otherwise the gateway // downloads it and sends the bytes inline. The download reaches only // publicly routable addresses unless the host is in -// `upstream.media_fetch.allowed_hosts`: the media server below listens on -// loopback, so `localhost:` is allow-listed and `127.0.0.1:`, -// the same server, is not. +// `outbound.ssrf_protection.allowed_hosts` or that protection is off: the +// media server below listens on loopback, so `localhost:` is +// allow-listed and `127.0.0.1:`, the same server, is not. Like any +// other upstream request, the download goes through an environment proxy +// and follows `upstream.tls`. // // Reference: // - @@ -184,9 +192,8 @@ describe("chat media content parts → native provider media shapes", () => { media = await startMediaServer(); app = await spawnApp({ extra: { - upstream: { - media_fetch: { allowed_hosts: [`localhost:${media.port}`], max_bytes: MAX_BYTES }, - }, + upstream: { media_fetch: { max_bytes: MAX_BYTES } }, + outbound: { ssrf_protection: { allowed_hosts: [`localhost:${media.port}`] } }, }, }); const seed = new SeedClient(etcd, app.etcdPrefix); @@ -606,13 +613,14 @@ describe("chat media content parts → native provider media shapes", () => { for (const model of ["media-nova", "media-bedrock-claude", "media-vertex-claude", "media-claude", "media-gemini"]) { const message = await refused(model, user(unlisted("/cat.png"))); expect(message).toMatch(/^messages\[0\]\.content\[1\]: /); - expect(message).toContain("upstream.media_fetch.allowed_hosts"); + expect(message).toContain("outbound.ssrf_protection.allowed_hosts"); + expect(message).toContain("outbound.ssrf_protection.enabled"); } // Refused before any request left the gateway. expect(hits("/cat.png")).toBe(before); expect(await refused("media-nova", user(allowed("/to-private.png")))).toContain( - "upstream.media_fetch.allowed_hosts", + "outbound.ssrf_protection.allowed_hosts", ); const followed = await chat("media-nova", [text("hi"), image(allowed("/to-allowed.png"))]); expect(followed.messages[0].content).toContainEqual({ @@ -661,9 +669,8 @@ describe("chat media URLs with upstream.media_fetch.enabled: false", () => { media = await startMediaServer(); app = await spawnApp({ extra: { - upstream: { - media_fetch: { enabled: false, allowed_hosts: [`localhost:${media.port}`] }, - }, + upstream: { media_fetch: { enabled: false } }, + outbound: { ssrf_protection: { allowed_hosts: [`localhost:${media.port}`] } }, }, }); const seed = new SeedClient(etcd, app.etcdPrefix); @@ -742,7 +749,7 @@ describe("chat media downloads count against proxy.request_body_limit_bytes", () media = await startMediaServer(); app = await spawnApp({ requestBodyLimitBytes: LIMIT, - extra: { upstream: { media_fetch: { allowed_hosts: [`localhost:${media.port}`] } } }, + extra: { outbound: { ssrf_protection: { allowed_hosts: [`localhost:${media.port}`] } } }, }); const seed = new SeedClient(etcd, app.etcdPrefix); const pk = await seed.createProviderKey({ @@ -831,3 +838,300 @@ describe("chat media downloads count against proxy.request_body_limit_bytes", () expect(JSON.parse(viaMessages.text).error.type).toBe("request_too_large"); }); }); + +/** A forward proxy on loopback: absolute-form `http://` requests are + * relayed, `CONNECT` is tunnelled. `seen` records each request line. */ +async function startForwardProxy(): Promise<{ + url: string; + seen: string[]; + close(): Promise; +}> { + const seen: string[] = []; + const server: Server = createServer((req, res) => { + seen.push(`${req.method} ${req.url}`); + const target = new URL(req.url ?? ""); + const out = httpRequest( + { + host: target.hostname, + port: target.port, + path: `${target.pathname}${target.search}`, + method: req.method, + headers: req.headers, + }, + (up) => { + res.writeHead(up.statusCode ?? 502, up.headers); + up.pipe(res); + }, + ); + out.on("error", () => res.destroy()); + req.pipe(out); + }); + server.on("connect", (req, client, head) => { + seen.push(`CONNECT ${req.url}`); + const [host, port] = (req.url ?? "").split(":"); + const up = netConnect(Number(port), host, () => { + client.write("HTTP/1.1 200 Connection Established\r\n\r\n"); + up.write(head); + up.pipe(client); + client.pipe(up); + }); + up.on("error", () => client.destroy()); + client.on("error", () => up.destroy()); + }); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const addr = server.address(); + if (addr === null || typeof addr === "string") throw new Error("no port"); + return { + url: `http://127.0.0.1:${addr.port}`, + seen, + close: () => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }), + }; +} + +/** Serves `/cat.png` over https with a self-signed certificate for + * `localhost`, or `undefined` without openssl. */ +async function startHttpsMediaServer(): Promise< + { port: number; hits: () => number; close(): Promise } | undefined +> { + try { + execFileSync("openssl", ["version"], { stdio: "ignore" }); + } catch { + return undefined; + } + const dir = mkdtempSync(join(tmpdir(), "aisix-media-tls-")); + execFileSync( + "openssl", + [ + "req", "-x509", "-newkey", "rsa:2048", "-nodes", "-days", "1", + "-keyout", join(dir, "key.pem"), "-out", join(dir, "cert.pem"), + "-subj", "/CN=localhost", "-addext", "subjectAltName=DNS:localhost", + ], + { stdio: "ignore" }, + ); + let hits = 0; + const server = createHttpsServer( + { key: readFileSync(join(dir, "key.pem")), cert: readFileSync(join(dir, "cert.pem")) }, + (req, res) => { + hits += 1; + res.statusCode = req.url === "/cat.png" ? 200 : 404; + res.setHeader("content-type", "image/png"); + res.end(PNG); + }, + ); + await new Promise((resolve) => server.listen(0, "127.0.0.1", resolve)); + const addr = server.address(); + if (addr === null || typeof addr === "string") throw new Error("no port"); + return { + port: addr.port, + hits: () => hits, + close: () => + new Promise((resolve) => { + server.closeAllConnections(); + server.close(() => resolve()); + }), + }; +} + +/** A gateway with one Bedrock Nova model (which cannot read a media URL, + * so every URL is downloaded) and a caller allowed to use it. */ +async function spawnNovaGateway( + caller: string, + upstreamBase: string, + options: Parameters[0], +): Promise { + const etcd = new EtcdClient(); + const app = await spawnApp(options); + const seed = new SeedClient(etcd, app.etcdPrefix); + const pk = await seed.createProviderKey({ + display_name: `${caller}-bedrock-pk`, + provider: "bedrock", + adapter: "bedrock", + secret: JSON.stringify({ + access_key_id: "AKIA-media-e2e", + secret_access_key: "sk-media-e2e", + region: "us-west-2", + }), + api_base: upstreamBase, + }); + await seed.createModel({ + display_name: `${caller}-nova`, + provider: "bedrock", + model_name: "amazon.nova-pro-v1:0", + provider_key_id: pk.id, + }); + await seed.createApiKey({ + key_hash: createHash("sha256").update(caller).digest("hex"), + allowed_models: [`${caller}-nova`], + }); + await waitConfigPropagation(async () => { + const res = await fetch(`${app.proxyUrl}/v1/models`, { + headers: { authorization: `Bearer ${caller}` }, + }); + await res.text(); + return res.status === 200; + }); + return app; +} + +async function chatWithImage(app: SpawnedApp, caller: string, url: string) { + const res = await fetch(`${app.proxyUrl}/v1/chat/completions`, { + method: "POST", + headers: { authorization: `Bearer ${caller}`, "content-type": "application/json" }, + body: JSON.stringify({ + model: `${caller}-nova`, + messages: [{ role: "user", content: [text("hi"), image(url)] }], + }), + }); + return { status: res.status, text: await res.text() }; +} + +const NOVA_IMAGE = { image: { format: "png", source: { bytes: PNG_B64 } } }; + +describe("chat media downloads with outbound.ssrf_protection.enabled: false", () => { + let app: SpawnedApp | undefined; + let upstream: RecordingUpstream | undefined; + let media: MediaServer | undefined; + let etcdReachable = false; + const CALLER = "sk-media-ssrf-off"; + + beforeAll(async () => { + etcdReachable = await new EtcdClient().ping(); + if (!etcdReachable) return; + upstream = await startUpstream(); + media = await startMediaServer(); + app = await spawnNovaGateway(CALLER, upstream.baseUrl, { + extra: { outbound: { ssrf_protection: { enabled: false } } }, + }); + }); + + afterAll(async () => { + await app?.exit(); + await upstream?.close(); + await media?.close(); + }); + + test("a loopback address is downloaded like any other, directly and through a redirect", async (ctx) => { + if (!etcdReachable || !app || !upstream || !media) return ctx.skip(); + for (const path of ["/cat.png", "/to-private.png"]) { + const before = upstream.received.length; + const res = await chatWithImage(app, CALLER, `http://127.0.0.1:${media.port}${path}`); + expect(res.status, res.text).toBe(200); + const sent = JSON.parse(upstream.received[before].body); + expect(sent.messages[0].content).toContainEqual(NOVA_IMAGE); + } + }); +}); + +describe("chat media downloads go through the environment proxy", () => { + let upstream: RecordingUpstream | undefined; + let media: MediaServer | undefined; + let unlistedMedia: MediaServer | undefined; + let tlsMedia: Awaited>; + let proxy: Awaited> | undefined; + let guarded: SpawnedApp | undefined; + let open: SpawnedApp | undefined; + let etcdReachable = false; + const GUARDED = "sk-media-proxy-guarded"; + const OPEN = "sk-media-proxy-open"; + + beforeAll(async () => { + etcdReachable = await new EtcdClient().ping(); + if (!etcdReachable) return; + upstream = await startUpstream(); + media = await startMediaServer(); + unlistedMedia = await startMediaServer(); + tlsMedia = await startHttpsMediaServer(); + proxy = await startForwardProxy(); + // The provider upstream is on 127.0.0.1 and stays direct; every + // `localhost` URL is the proxy's. + const proxyEnv = { + HTTP_PROXY: proxy.url, + HTTPS_PROXY: proxy.url, + http_proxy: proxy.url, + https_proxy: proxy.url, + NO_PROXY: "127.0.0.1", + no_proxy: "127.0.0.1", + }; + const allowed = [`localhost:${media.port}`]; + if (tlsMedia) allowed.push(`localhost:${tlsMedia.port}`); + [guarded, open] = await Promise.all([ + spawnNovaGateway(GUARDED, upstream.baseUrl, { + extraEnv: proxyEnv, + extra: { + // The https media server's certificate is self-signed. + upstream: { tls: { verify: false } }, + outbound: { ssrf_protection: { allowed_hosts: allowed } }, + }, + }), + spawnNovaGateway(OPEN, upstream.baseUrl, { + extraEnv: proxyEnv, + extra: { outbound: { ssrf_protection: { enabled: false } } }, + }), + ]); + }); + + afterAll(async () => { + await guarded?.exit(); + await open?.exit(); + await upstream?.close(); + await media?.close(); + await unlistedMedia?.close(); + await tlsMedia?.close(); + await proxy?.close(); + }); + + const ready = (ctx: { skip: () => void }) => { + if (!etcdReachable || !guarded || !open || !upstream || !media || !unlistedMedia || !proxy) { + ctx.skip(); + return false; + } + return true; + }; + + test("an allow-listed http:// host is fetched through HTTP_PROXY", async (ctx) => { + if (!ready(ctx)) return; + const url = `http://localhost:${media!.port}/cat.png`; + const before = upstream!.received.length; + const res = await chatWithImage(guarded!, GUARDED, url); + expect(res.status, res.text).toBe(200); + expect(JSON.parse(upstream!.received[before].body).messages[0].content).toContainEqual( + NOVA_IMAGE, + ); + expect(proxy!.seen).toContain(`GET ${url}`); + }); + + test("an https:// host is tunnelled through HTTPS_PROXY and follows upstream.tls.verify", async (ctx) => { + if (!ready(ctx)) return; + if (!tlsMedia) return ctx.skip(); + const before = upstream!.received.length; + const res = await chatWithImage(guarded!, GUARDED, `https://localhost:${tlsMedia.port}/cat.png`); + expect(res.status, res.text).toBe(200); + expect(JSON.parse(upstream!.received[before].body).messages[0].content).toContainEqual( + NOVA_IMAGE, + ); + expect(proxy!.seen).toContain(`CONNECT localhost:${tlsMedia.port}`); + expect(tlsMedia.hits()).toBeGreaterThan(0); + }); + + test("with protection on, a name resolving to a private address is refused before the proxy sees it", async (ctx) => { + if (!ready(ctx)) return; + const url = `http://localhost:${unlistedMedia!.port}/cat.png`; + const res = await chatWithImage(guarded!, GUARDED, url); + expect(res.status, res.text).toBe(400); + expect(JSON.parse(res.text).error.message).toContain("outbound.ssrf_protection.allowed_hosts"); + expect(proxy!.seen.filter((l) => l.includes(`:${unlistedMedia!.port}`))).toEqual([]); + expect(unlistedMedia!.hits.get("/cat.png") ?? 0).toBe(0); + }); + + test("with protection off, the same host goes through the proxy unchecked", async (ctx) => { + if (!ready(ctx)) return; + const url = `http://localhost:${unlistedMedia!.port}/cat.png`; + const res = await chatWithImage(open!, OPEN, url); + expect(res.status, res.text).toBe(200); + expect(proxy!.seen).toContain(`GET ${url}`); + }); +}); diff --git a/tests/e2e/src/cases/claude-platform-native-messages-e2e.test.ts b/tests/e2e/src/cases/claude-platform-native-messages-e2e.test.ts index 152623ac..1b63fc6a 100644 --- a/tests/e2e/src/cases/claude-platform-native-messages-e2e.test.ts +++ b/tests/e2e/src/cases/claude-platform-native-messages-e2e.test.ts @@ -318,10 +318,8 @@ describe("Claude on Amazon Bedrock and Vertex AI on their own Messages routes", [`SLS_CRED_${CREDENTIAL_REF.toUpperCase()}_AK_SECRET`]: "mock-ak-secret", }, extra: { - upstream: { - media_fetch: { allowed_hosts: [`localhost:${upstream.port}`] }, - bedrock: { mantle_endpoint: upstream.baseUrl }, - }, + upstream: { bedrock: { mantle_endpoint: upstream.baseUrl } }, + outbound: { ssrf_protection: { allowed_hosts: [`localhost:${upstream.port}`] } }, }, }); const seed = new SeedClient(etcd, app.etcdPrefix);