diff --git a/fcvm-fleet-broadcast.tf b/fcvm-fleet-broadcast.tf new file mode 100644 index 00000000..43acca60 --- /dev/null +++ b/fcvm-fleet-broadcast.tf @@ -0,0 +1,108 @@ +# fcvm-fleet-broadcast.tf +# +# The wall's broadcast (ejc3/fcvm docs/fleet/ARCHITECTURE.md, "Broadcast mode"): past the direct-mode capacity (500 +# viewers, ejc3/fcvm fleet/renderer/README.md "Viewers per renderer"), viewers watch one composite of the whole wall that +# the coordinator encodes and pushes to an Amazon IVS low-latency channel. Measured 2026-10-27 (ejc3/fcvm#1384): 1.79 s +# glass to glass at the median, 0.21 s to the first frame. +# +# - One IVS channel in us-east-1 (IVS is not offered in us-east-2), STANDARD (IVS transcodes the 720p input into 720p, +# 480p, 360p and 160p), LOW latency, private: a playback request needs a token signed with the channel's playback key +# pair. The coordinator holds the private key and signs a token per /live connection; /live is behind Cloudflare +# Access on staging, so only Access-authenticated viewers get one. +# - The secret fcvm-fleet/broadcast (us-east-2, with the fleet's other secrets): the ingest URL with the stream key, the +# playback URL, the channel ARN and the playback private key. Only the coordinator's task role reads it. +# - The coordinator's task grows to 2 vCPU and 4 GB for the decode and the 720p encode (1.69 cores measured on a Xeon), +# and runs with --broadcast-listen, --broadcast-secret and --direct-capacity (fcvm-fleet.tf). +# +# Cost: $2.00 an hour while the channel is live (the coordinator pushes only while the fleet is awake, and the fleet +# sleeps after 10 minutes with no viewer), $0.072 per viewer-hour at 720p and $0.036 at 480p (North America, first +# 10,000 hours a month), and the larger task about $43 a month. +# +# The gate: local.fcvm_fleet_broadcast. Keep it off until admission-enforcing host and tile AMIs are pinned and +# running (ejc3/aws#722). Enabling also needs a coordinator image with --broadcast-listen (ejc3/fcvm#1385 and its stack). + +locals { + fcvm_fleet_broadcast = false + # The broadcast aggregator's private listener (layout and stats) inside the coordinator's task. + fcvm_fleet_broadcast_port = 20310 + # Seats for direct streams; the rest watch the broadcast. + fcvm_fleet_direct_capacity = 500 +} + +resource "aws_ivs_channel" "fcvm_fleet_broadcast" { + count = local.fcvm_fleet_broadcast ? 1 : 0 + provider = aws.fcvm_fleet_east1 + name = "fcvm-fleet-stage-broadcast" + latency_mode = "LOW" + type = "STANDARD" + authorized = true + tags = { Project = "fcvm-fleet", Purpose = "fcvm-fleet-broadcast" } +} + +# The channel's default stream key (IVS makes one with the channel). +data "aws_ivs_stream_key" "fcvm_fleet_broadcast" { + count = local.fcvm_fleet_broadcast ? 1 : 0 + provider = aws.fcvm_fleet_east1 + channel_arn = aws_ivs_channel.fcvm_fleet_broadcast[0].arn +} + +# The playback key pair: IVS keeps the public half; the private half signs viewers' tokens (ES384). +resource "tls_private_key" "fcvm_fleet_broadcast_playback" { + count = local.fcvm_fleet_broadcast ? 1 : 0 + algorithm = "ECDSA" + ecdsa_curve = "P384" +} + +resource "aws_ivs_playback_key_pair" "fcvm_fleet_broadcast" { + count = local.fcvm_fleet_broadcast ? 1 : 0 + provider = aws.fcvm_fleet_east1 + name = "fcvm-fleet-stage-broadcast" + public_key = tls_private_key.fcvm_fleet_broadcast_playback[0].public_key_pem + tags = { Project = "fcvm-fleet" } +} + +resource "aws_secretsmanager_secret" "fcvm_fleet_broadcast" { + count = local.fcvm_fleet_broadcast ? 1 : 0 + provider = aws.fcvm_fleet + name = "fcvm-fleet/broadcast" + description = "The wall's IVS broadcast: ingest URL with stream key, playback URL, channel ARN, playback private key (JSON). Written by fcvm-fleet-broadcast.tf." + recovery_window_in_days = 7 + tags = { Name = "fcvm-fleet/broadcast", Project = "fcvm-fleet" } +} + +resource "aws_secretsmanager_secret_version" "fcvm_fleet_broadcast" { + count = local.fcvm_fleet_broadcast ? 1 : 0 + provider = aws.fcvm_fleet + secret_id = aws_secretsmanager_secret.fcvm_fleet_broadcast[0].id + # ejc3/fcvm src/fleet/coord/broadcast.rs, BroadcastSecret. + secret_string = jsonencode({ + ingest_url = "rtmps://${aws_ivs_channel.fcvm_fleet_broadcast[0].ingest_endpoint}:443/app/${data.aws_ivs_stream_key.fcvm_fleet_broadcast[0].value}" + playback_url = aws_ivs_channel.fcvm_fleet_broadcast[0].playback_url + channel_arn = aws_ivs_channel.fcvm_fleet_broadcast[0].arn + playback_key_pem = tls_private_key.fcvm_fleet_broadcast_playback[0].private_key_pem_pkcs8 + }) +} + +# The coordinator's inline policies already fill its aggregate 10 KiB budget. This +# managed policy grants only this secret read without changing those permissions. +resource "aws_iam_policy" "fcvm_fleet_coordinator_broadcast" { + count = local.fcvm_fleet_broadcast ? 1 : 0 + provider = aws.fcvm_fleet + name = "fcvm-fleet-coordinator-broadcast-secret" + policy = jsonencode({ + Version = "2012-10-17" + Statement = [{ + Sid = "BroadcastSecret" + Effect = "Allow" + Action = "secretsmanager:GetSecretValue" + Resource = aws_secretsmanager_secret.fcvm_fleet_broadcast[0].arn + }] + }) +} + +resource "aws_iam_role_policy_attachment" "fcvm_fleet_coordinator_broadcast" { + count = local.fcvm_fleet_broadcast ? 1 : 0 + provider = aws.fcvm_fleet + role = aws_iam_role.fcvm_fleet_coordinator_task.name + policy_arn = aws_iam_policy.fcvm_fleet_coordinator_broadcast[0].arn +} diff --git a/fcvm-fleet.tf b/fcvm-fleet.tf index e5d5e509..524291e9 100644 --- a/fcvm-fleet.tf +++ b/fcvm-fleet.tf @@ -23,7 +23,7 @@ # us-east-1, so 8 is $62.59 an hour. The account's on-demand G and VT quota there (L-DB2E81BA) is 384 vCPU, 4 hosts at 96 # each, less any on-demand EC2 tiles (4 vCPU each); launches beyond the quota fail. # - local.fcvm_fleet_coordinator_enabled: the coordinator service (one Fargate task, 0.5 vCPU and 1 GB ARM64, about $15 a -# month). Its initial image is fcvm-fleet/coordinator:; +# month; 2 vCPU and 4 GB, about $58, with the broadcast, fcvm-fleet-broadcast.tf). Its initial image is fcvm-fleet/coordinator:; # main-only CI releases immutable digests through fcvm-fleet-ci.tf. variable "fcvm_fleet_host_amis" { @@ -143,6 +143,9 @@ locals { ["--dns-subzone", local.fcvm_fleet_names_zone, "--dns-subzone-ns", join(",", local.fcvm_fleet_names_ns), "--dns-listen", "0.0.0.0:${local.fcvm_fleet_dns_port}"], # The EC2 half's aggregator beside the coordinator: its layout on loopback, browsers on 443 of the task's own addresses. local.fcvm_fleet_aggregated ? ["--ec2-aggregate-listen", "127.0.0.1:${local.fcvm_fleet_aggregate_port}", "--ec2-aggregate-tls", "--ec2-aggregate-tls-dir", "/run/fcvm-fleet/tls"] : [], + # The broadcast (fcvm-fleet-broadcast.tf): its aggregator's private listener, the secret it pushes with and signs + # playback tokens from, and the seats for direct streams. + local.fcvm_fleet_broadcast ? ["--broadcast-listen", "127.0.0.1:${local.fcvm_fleet_broadcast_port}", "--broadcast-secret", aws_secretsmanager_secret.fcvm_fleet_broadcast[0].name, "--direct-capacity", tostring(local.fcvm_fleet_direct_capacity)] : [], # Every viewer word is checked by Claude Haiku 5.5 on Bedrock before it enters the scene (the task role's # CheckViewerWordsWithClaudeHaiku55). Without a checker the coordinator refuses every word. ["--text-check", "bedrock", "--text-check-model", local.bedrock_haiku_profile, "--text-check-region", local.bedrock_haiku_region], @@ -991,10 +994,11 @@ resource "aws_ecs_task_definition" "fcvm_fleet_coordinator" { family = "fcvm-fleet-coordinator" requires_compatibilities = ["FARGATE"] network_mode = "awsvpc" - cpu = "512" - memory = "1024" - execution_role_arn = aws_iam_role.fcvm_fleet_coordinator_execution.arn - task_role_arn = aws_iam_role.fcvm_fleet_coordinator_task.arn + # 2 vCPU and 4 GB with the broadcast: its tile decodes and 720p encode took 1.69 cores on a Xeon (ejc3/fcvm#1384). + cpu = local.fcvm_fleet_broadcast ? "2048" : "512" + memory = local.fcvm_fleet_broadcast ? "4096" : "1024" + execution_role_arn = aws_iam_role.fcvm_fleet_coordinator_execution.arn + task_role_arn = aws_iam_role.fcvm_fleet_coordinator_task.arn # The release function changes only the image and records its digest in the # release parameter. Track that revision without ignoring infrastructure drift. track_latest = true @@ -1074,9 +1078,10 @@ resource "aws_ecs_task_definition" "fcvm_fleet_coordinator" { } # A Fargate task gets an IPv6 address in a dual-stack subnet only with this account setting; the EC2 half's aggregator is -# named at it (agg-ec2- AAAA). Without it the aggregator has its A record only. +# named at it (agg-ec2- AAAA), and the broadcast reads every tile at its IPv6 name. Without it the aggregator has +# its A record only. resource "aws_ecs_account_setting_default" "fcvm_fleet_dual_stack" { - count = local.fcvm_fleet_aggregated ? 1 : 0 + count = local.fcvm_fleet_aggregated || local.fcvm_fleet_broadcast ? 1 : 0 provider = aws.fcvm_fleet name = "dualStackIPv6" value = "enabled" @@ -1084,7 +1089,14 @@ resource "aws_ecs_account_setting_default" "fcvm_fleet_dual_stack" { resource "aws_ecs_service" "fcvm_fleet_coordinator" { # The listener too: ECS refuses a target group that no load balancer uses yet, and the listener is what attaches it. - depends_on = [ephemeral.aws_ssm_parameter.fcvm_coordinator_release_guard, aws_lb_listener.fcvm_fleet_dns] + # First enablement must publish the broadcast credentials and read grant before + # starting a coordinator with --broadcast-secret. Empty gated collections are valid. + depends_on = [ + ephemeral.aws_ssm_parameter.fcvm_coordinator_release_guard, + aws_lb_listener.fcvm_fleet_dns, + aws_secretsmanager_secret_version.fcvm_fleet_broadcast, + aws_iam_role_policy_attachment.fcvm_fleet_coordinator_broadcast, + ] provider = aws.fcvm_fleet name = "fcvm-fleet-coordinator" cluster = aws_ecs_cluster.fcvm_fleet.id diff --git a/scripts/test-fcvm-fleet.py b/scripts/test-fcvm-fleet.py index b76d76a3..50c2290a 100644 --- a/scripts/test-fcvm-fleet.py +++ b/scripts/test-fcvm-fleet.py @@ -225,7 +225,7 @@ class SecretTests(unittest.TestCase): def test_only_the_certificate_container_is_created_here(self): names = re.findall(r'resource "aws_secretsmanager_secret" "[^"]+" \{[^}]*?\bname\s+= "([^"]+)"', TF, re.S) self.assertEqual(names, ["fcvm-fleet/tls-wildcard"]) - self.assertNotIn("aws_secretsmanager_secret_version", TF) + self.assertNotIn('resource "aws_secretsmanager_secret_version"', TF) def test_hosts_and_tiles_read_only_what_they_need(self): host = block('resource "aws_iam_role_policy" "fcvm_fleet_host"') @@ -609,7 +609,8 @@ def test_everything_it_adds_is_gated_by_it(self): self.assertIn('containerPath = "/run/fcvm-fleet"', task) self.assertIn('"/run/fcvm-fleet/tls"', cmd) dual = block('resource "aws_ecs_account_setting_default" "fcvm_fleet_dual_stack"') - self.assertIn("count = local.fcvm_fleet_aggregated ? 1 : 0", dual) + # The broadcast reads every tile at its IPv6 name too (fcvm-fleet-broadcast.tf). + self.assertIn("count = local.fcvm_fleet_aggregated || local.fcvm_fleet_broadcast ? 1 : 0", dual) def test_the_host_aggregator_port_follows_the_slots(self): self.assertIn( @@ -618,5 +619,49 @@ def test_the_host_aggregator_port_follows_the_slots(self): # The private listener is outside the admitted range. self.assertFalse(20200 <= priv <= 20208, priv) +class BroadcastTests(unittest.TestCase): + """The broadcast (ejc3/fcvm "Broadcast mode") is one switch: the coordinator's flags, its task size, its + secret and the private IVS channel all follow it.""" + + BC = (ROOT / "fcvm-fleet-broadcast.tf").read_text() + + def test_the_coordinator_gets_its_flags_and_size_from_the_switch(self): + cmd = TF.split("fcvm_fleet_coordinator_command = concat(", 1)[1].split("\n )\n", 1)[0] + self.assertIn('local.fcvm_fleet_broadcast ? ["--broadcast-listen", "127.0.0.1:', cmd) + self.assertIn('"--broadcast-secret", aws_secretsmanager_secret.fcvm_fleet_broadcast[0].name', cmd) + self.assertIn('"--direct-capacity", tostring(local.fcvm_fleet_direct_capacity)', cmd) + task = block('resource "aws_ecs_task_definition" "fcvm_fleet_coordinator"') + self.assertIn('cpu = local.fcvm_fleet_broadcast ? "2048" : "512"', task) + self.assertIn('memory = local.fcvm_fleet_broadcast ? "4096" : "1024"', task) + + def test_the_channel_is_private_and_only_the_coordinator_reads_its_secret(self): + ch = block('resource "aws_ivs_channel" "fcvm_fleet_broadcast"', self.BC) + self.assertIn("authorized = true", ch) + self.assertIn('latency_mode = "LOW"', ch) + pol = block('resource "aws_iam_policy" "fcvm_fleet_coordinator_broadcast"', self.BC) + self.assertIn('Action = "secretsmanager:GetSecretValue"', pol) + self.assertIn("Resource = aws_secretsmanager_secret.fcvm_fleet_broadcast[0].arn", pol) + self.assertEqual(pol.count("Effect"), 1) + self.assertNotIn('resource "aws_iam_role_policy"', self.BC) + attachment = block('resource "aws_iam_role_policy_attachment" "fcvm_fleet_coordinator_broadcast"', self.BC) + self.assertIn("role = aws_iam_role.fcvm_fleet_coordinator_task.name", attachment) + self.assertIn("policy_arn = aws_iam_policy.fcvm_fleet_coordinator_broadcast[0].arn", attachment) + for r in re.findall(r'resource "([a-z_]+)" "fcvm_fleet_broadcast[a-z_]*"', self.BC): + self.assertIn(r, {"aws_ivs_channel", "tls_private_key", "aws_ivs_playback_key_pair", "aws_secretsmanager_secret", + "aws_secretsmanager_secret_version", "aws_iam_policy", "aws_iam_role_policy_attachment"}, r) + # Every one of its seven resources and its one data source is counted by the switch. + self.assertEqual(len(re.findall(r"count\s+= local\.fcvm_fleet_broadcast \? 1 : 0", self.BC)), 8) + + def test_first_rollout_waits_for_credentials_and_the_read_grant(self): + service = block('resource "aws_ecs_service" "fcvm_fleet_coordinator"') + dependencies = re.search(r"depends_on\s*=\s*\[(.*?)\]", service, re.S).group(1) + self.assertEqual(set(re.findall(r"\b(?:ephemeral\.)?aws_[\w.]+", dependencies)), { + "ephemeral.aws_ssm_parameter.fcvm_coordinator_release_guard", + "aws_lb_listener.fcvm_fleet_dns", + "aws_secretsmanager_secret_version.fcvm_fleet_broadcast", + "aws_iam_role_policy_attachment.fcvm_fleet_coordinator_broadcast", + }) + + if __name__ == "__main__": unittest.main(verbosity=2, buffer=True)