fcvm-fleet: the wall's broadcast on Amazon IVS

fcvm-fleet-broadcast.tf

@@ -0,0 +1,108 @@
+# fcvm-fleet-broadcast.tf
+#
+# The wall's broadcast (ejc3/fcvm docs/fleet/ARCHITECTURE.md, "Broadcast mode"): past the direct-mode capacity (500
+# viewers, ejc3/fcvm fleet/renderer/README.md "Viewers per renderer"), viewers watch one composite of the whole wall that
+# the coordinator encodes and pushes to an Amazon IVS low-latency channel. Measured 2026-10-27 (ejc3/fcvm#1384): 1.79 s
+# glass to glass at the median, 0.21 s to the first frame.
+#
+# - One IVS channel in us-east-1 (IVS is not offered in us-east-2), STANDARD (IVS transcodes the 720p input into 720p,
+#   480p, 360p and 160p), LOW latency, private: a playback request needs a token signed with the channel's playback key
+#   pair. The coordinator holds the private key and signs a token per /live connection; /live is behind Cloudflare
+#   Access on staging, so only Access-authenticated viewers get one.
+# - The secret fcvm-fleet/broadcast (us-east-2, with the fleet's other secrets): the ingest URL with the stream key, the
+#   playback URL, the channel ARN and the playback private key. Only the coordinator's task role reads it.
+# - The coordinator's task grows to 2 vCPU and 4 GB for the decode and the 720p encode (1.69 cores measured on a Xeon),
+#   and runs with --broadcast-listen, --broadcast-secret and --direct-capacity (fcvm-fleet.tf).
+#
+# Cost: $2.00 an hour while the channel is live (the coordinator pushes only while the fleet is awake, and the fleet
+# sleeps after 10 minutes with no viewer), $0.072 per viewer-hour at 720p and $0.036 at 480p (North America, first
+# 10,000 hours a month), and the larger task about $43 a month.
+#
+# The gate: local.fcvm_fleet_broadcast. Keep it off until admission-enforcing host and tile AMIs are pinned and
+# running (ejc3/aws#722). Enabling also needs a coordinator image with --broadcast-listen (ejc3/fcvm#1385 and its stack).
+
+locals {
+  fcvm_fleet_broadcast = false
+  # The broadcast aggregator's private listener (layout and stats) inside the coordinator's task.
+  fcvm_fleet_broadcast_port = 20310
+  # Seats for direct streams; the rest watch the broadcast.
+  fcvm_fleet_direct_capacity = 500
+}
+
+resource "aws_ivs_channel" "fcvm_fleet_broadcast" {
+  count        = local.fcvm_fleet_broadcast ? 1 : 0
+  provider     = aws.fcvm_fleet_east1
+  name         = "fcvm-fleet-stage-broadcast"
+  latency_mode = "LOW"
+  type         = "STANDARD"
+  authorized   = true
+  tags         = { Project = "fcvm-fleet", Purpose = "fcvm-fleet-broadcast" }
+}
+
+# The channel's default stream key (IVS makes one with the channel).
+data "aws_ivs_stream_key" "fcvm_fleet_broadcast" {
+  count       = local.fcvm_fleet_broadcast ? 1 : 0
+  provider    = aws.fcvm_fleet_east1
+  channel_arn = aws_ivs_channel.fcvm_fleet_broadcast[0].arn
+}
+
+# The playback key pair: IVS keeps the public half; the private half signs viewers' tokens (ES384).
+resource "tls_private_key" "fcvm_fleet_broadcast_playback" {
+  count       = local.fcvm_fleet_broadcast ? 1 : 0
+  algorithm   = "ECDSA"
+  ecdsa_curve = "P384"
+}
+
+resource "aws_ivs_playback_key_pair" "fcvm_fleet_broadcast" {
+  count      = local.fcvm_fleet_broadcast ? 1 : 0
+  provider   = aws.fcvm_fleet_east1
+  name       = "fcvm-fleet-stage-broadcast"
+  public_key = tls_private_key.fcvm_fleet_broadcast_playback[0].public_key_pem
+  tags       = { Project = "fcvm-fleet" }
+}
+
+resource "aws_secretsmanager_secret" "fcvm_fleet_broadcast" {
+  count                   = local.fcvm_fleet_broadcast ? 1 : 0
+  provider                = aws.fcvm_fleet
+  name                    = "fcvm-fleet/broadcast"
+  description             = "The wall's IVS broadcast: ingest URL with stream key, playback URL, channel ARN, playback private key (JSON). Written by fcvm-fleet-broadcast.tf."
+  recovery_window_in_days = 7
+  tags                    = { Name = "fcvm-fleet/broadcast", Project = "fcvm-fleet" }
+}
+
+resource "aws_secretsmanager_secret_version" "fcvm_fleet_broadcast" {
+  count     = local.fcvm_fleet_broadcast ? 1 : 0
+  provider  = aws.fcvm_fleet
+  secret_id = aws_secretsmanager_secret.fcvm_fleet_broadcast[0].id
+  # ejc3/fcvm src/fleet/coord/broadcast.rs, BroadcastSecret.
+  secret_string = jsonencode({
+    ingest_url       = "rtmps://${aws_ivs_channel.fcvm_fleet_broadcast[0].ingest_endpoint}:443/app/${data.aws_ivs_stream_key.fcvm_fleet_broadcast[0].value}"
+    playback_url     = aws_ivs_channel.fcvm_fleet_broadcast[0].playback_url
+    channel_arn      = aws_ivs_channel.fcvm_fleet_broadcast[0].arn
+    playback_key_pem = tls_private_key.fcvm_fleet_broadcast_playback[0].private_key_pem_pkcs8
+  })
+}
+
+# The coordinator's inline policies already fill its aggregate 10 KiB budget. This
+# managed policy grants only this secret read without changing those permissions.
+resource "aws_iam_policy" "fcvm_fleet_coordinator_broadcast" {
+  count    = local.fcvm_fleet_broadcast ? 1 : 0
+  provider = aws.fcvm_fleet
+  name     = "fcvm-fleet-coordinator-broadcast-secret"
+  policy = jsonencode({
+    Version = "2012-10-17"
+    Statement = [{
+      Sid      = "BroadcastSecret"
+      Effect   = "Allow"
+      Action   = "secretsmanager:GetSecretValue"
+      Resource = aws_secretsmanager_secret.fcvm_fleet_broadcast[0].arn
+    }]
+  })
+}
+
+resource "aws_iam_role_policy_attachment" "fcvm_fleet_coordinator_broadcast" {
+  count      = local.fcvm_fleet_broadcast ? 1 : 0
+  provider   = aws.fcvm_fleet
+  role       = aws_iam_role.fcvm_fleet_coordinator_task.name
+  policy_arn = aws_iam_policy.fcvm_fleet_coordinator_broadcast[0].arn
+}

fcvm-fleet.tf

@@ -23,7 +23,7 @@
 #   us-east-1, so 8 is $62.59 an hour. The account's on-demand G and VT quota there (L-DB2E81BA) is 384 vCPU, 4 hosts at 96
 #   each, less any on-demand EC2 tiles (4 vCPU each); launches beyond the quota fail.
 # - local.fcvm_fleet_coordinator_enabled: the coordinator service (one Fargate task, 0.5 vCPU and 1 GB ARM64, about $15 a
-#   month). Its initial image is fcvm-fleet/coordinator:<local.fcvm_fleet_coordinator_tag>;
+#   month; 2 vCPU and 4 GB, about $58, with the broadcast, fcvm-fleet-broadcast.tf). Its initial image is fcvm-fleet/coordinator:<local.fcvm_fleet_coordinator_tag>;
 #   main-only CI releases immutable digests through fcvm-fleet-ci.tf.
 
 variable "fcvm_fleet_host_amis" {
@@ -143,6 +143,9 @@ locals {
     ["--dns-subzone", local.fcvm_fleet_names_zone, "--dns-subzone-ns", join(",", local.fcvm_fleet_names_ns), "--dns-listen", "0.0.0.0:${local.fcvm_fleet_dns_port}"],
     # The EC2 half's aggregator beside the coordinator: its layout on loopback, browsers on 443 of the task's own addresses.
     local.fcvm_fleet_aggregated ? ["--ec2-aggregate-listen", "127.0.0.1:${local.fcvm_fleet_aggregate_port}", "--ec2-aggregate-tls", "--ec2-aggregate-tls-dir", "/run/fcvm-fleet/tls"] : [],
+    # The broadcast (fcvm-fleet-broadcast.tf): its aggregator's private listener, the secret it pushes with and signs
+    # playback tokens from, and the seats for direct streams.
+    local.fcvm_fleet_broadcast ? ["--broadcast-listen", "127.0.0.1:${local.fcvm_fleet_broadcast_port}", "--broadcast-secret", aws_secretsmanager_secret.fcvm_fleet_broadcast[0].name, "--direct-capacity", tostring(local.fcvm_fleet_direct_capacity)] : [],
     # Every viewer word is checked by Claude Haiku 5.5 on Bedrock before it enters the scene (the task role's
     # CheckViewerWordsWithClaudeHaiku55). Without a checker the coordinator refuses every word.
     ["--text-check", "bedrock", "--text-check-model", local.bedrock_haiku_profile, "--text-check-region", local.bedrock_haiku_region],
@@ -991,10 +994,11 @@ resource "aws_ecs_task_definition" "fcvm_fleet_coordinator" {
   family                   = "fcvm-fleet-coordinator"
   requires_compatibilities = ["FARGATE"]
   network_mode             = "awsvpc"
-  cpu                      = "512"
-  memory                   = "1024"
-  execution_role_arn       = aws_iam_role.fcvm_fleet_coordinator_execution.arn
-  task_role_arn            = aws_iam_role.fcvm_fleet_coordinator_task.arn
+  # 2 vCPU and 4 GB with the broadcast: its tile decodes and 720p encode took 1.69 cores on a Xeon (ejc3/fcvm#1384).
+  cpu                = local.fcvm_fleet_broadcast ? "2048" : "512"
+  memory             = local.fcvm_fleet_broadcast ? "4096" : "1024"
+  execution_role_arn = aws_iam_role.fcvm_fleet_coordinator_execution.arn
+  task_role_arn      = aws_iam_role.fcvm_fleet_coordinator_task.arn
   # The release function changes only the image and records its digest in the
   # release parameter. Track that revision without ignoring infrastructure drift.
   track_latest = true
@@ -1074,17 +1078,25 @@ resource "aws_ecs_task_definition" "fcvm_fleet_coordinator" {
 }
 
 # A Fargate task gets an IPv6 address in a dual-stack subnet only with this account setting; the EC2 half's aggregator is
-# named at it (agg-ec2-<token> AAAA). Without it the aggregator has its A record only.
+# named at it (agg-ec2-<token> AAAA), and the broadcast reads every tile at its IPv6 name. Without it the aggregator has
+# its A record only.
 resource "aws_ecs_account_setting_default" "fcvm_fleet_dual_stack" {
-  count    = local.fcvm_fleet_aggregated ? 1 : 0
+  count    = local.fcvm_fleet_aggregated || local.fcvm_fleet_broadcast ? 1 : 0
   provider = aws.fcvm_fleet
   name     = "dualStackIPv6"
   value    = "enabled"
 }
 
 resource "aws_ecs_service" "fcvm_fleet_coordinator" {
   # The listener too: ECS refuses a target group that no load balancer uses yet, and the listener is what attaches it.
-  depends_on      = [ephemeral.aws_ssm_parameter.fcvm_coordinator_release_guard, aws_lb_listener.fcvm_fleet_dns]
+  # First enablement must publish the broadcast credentials and read grant before
+  # starting a coordinator with --broadcast-secret. Empty gated collections are valid.
+  depends_on = [
+    ephemeral.aws_ssm_parameter.fcvm_coordinator_release_guard,
+    aws_lb_listener.fcvm_fleet_dns,
+    aws_secretsmanager_secret_version.fcvm_fleet_broadcast,
+    aws_iam_role_policy_attachment.fcvm_fleet_coordinator_broadcast,
+  ]
   provider        = aws.fcvm_fleet
   name            = "fcvm-fleet-coordinator"
   cluster         = aws_ecs_cluster.fcvm_fleet.id

scripts/test-fcvm-fleet.py

@@ -225,7 +225,7 @@ class SecretTests(unittest.TestCase):
     def test_only_the_certificate_container_is_created_here(self):
         names = re.findall(r'resource "aws_secretsmanager_secret" "[^"]+" \{[^}]*?\bname\s+= "([^"]+)"', TF, re.S)
         self.assertEqual(names, ["fcvm-fleet/tls-wildcard"])
-        self.assertNotIn("aws_secretsmanager_secret_version", TF)
+        self.assertNotIn('resource "aws_secretsmanager_secret_version"', TF)
 
     def test_hosts_and_tiles_read_only_what_they_need(self):
         host = block('resource "aws_iam_role_policy" "fcvm_fleet_host"')
@@ -609,7 +609,8 @@ def test_everything_it_adds_is_gated_by_it(self):
         self.assertIn('containerPath = "/run/fcvm-fleet"', task)
         self.assertIn('"/run/fcvm-fleet/tls"', cmd)
         dual = block('resource "aws_ecs_account_setting_default" "fcvm_fleet_dual_stack"')
-        self.assertIn("count    = local.fcvm_fleet_aggregated ? 1 : 0", dual)
+        # The broadcast reads every tile at its IPv6 name too (fcvm-fleet-broadcast.tf).
+        self.assertIn("count    = local.fcvm_fleet_aggregated || local.fcvm_fleet_broadcast ? 1 : 0", dual)
 
     def test_the_host_aggregator_port_follows_the_slots(self):
         self.assertIn(
@@ -618,5 +619,49 @@ def test_the_host_aggregator_port_follows_the_slots(self):
         # The private listener is outside the admitted range.
         self.assertFalse(20200 <= priv <= 20208, priv)
 
+class BroadcastTests(unittest.TestCase):
+    """The broadcast (ejc3/fcvm "Broadcast mode") is one switch: the coordinator's flags, its task size, its
+    secret and the private IVS channel all follow it."""
+
+    BC = (ROOT / "fcvm-fleet-broadcast.tf").read_text()
+
+    def test_the_coordinator_gets_its_flags_and_size_from_the_switch(self):
+        cmd = TF.split("fcvm_fleet_coordinator_command = concat(", 1)[1].split("\n  )\n", 1)[0]
+        self.assertIn('local.fcvm_fleet_broadcast ? ["--broadcast-listen", "127.0.0.1:', cmd)
+        self.assertIn('"--broadcast-secret", aws_secretsmanager_secret.fcvm_fleet_broadcast[0].name', cmd)
+        self.assertIn('"--direct-capacity", tostring(local.fcvm_fleet_direct_capacity)', cmd)
+        task = block('resource "aws_ecs_task_definition" "fcvm_fleet_coordinator"')
+        self.assertIn('cpu                = local.fcvm_fleet_broadcast ? "2048" : "512"', task)
+        self.assertIn('memory             = local.fcvm_fleet_broadcast ? "4096" : "1024"', task)
+
+    def test_the_channel_is_private_and_only_the_coordinator_reads_its_secret(self):
+        ch = block('resource "aws_ivs_channel" "fcvm_fleet_broadcast"', self.BC)
+        self.assertIn("authorized   = true", ch)
+        self.assertIn('latency_mode = "LOW"', ch)
+        pol = block('resource "aws_iam_policy" "fcvm_fleet_coordinator_broadcast"', self.BC)
+        self.assertIn('Action   = "secretsmanager:GetSecretValue"', pol)
+        self.assertIn("Resource = aws_secretsmanager_secret.fcvm_fleet_broadcast[0].arn", pol)
+        self.assertEqual(pol.count("Effect"), 1)
+        self.assertNotIn('resource "aws_iam_role_policy"', self.BC)
+        attachment = block('resource "aws_iam_role_policy_attachment" "fcvm_fleet_coordinator_broadcast"', self.BC)
+        self.assertIn("role       = aws_iam_role.fcvm_fleet_coordinator_task.name", attachment)
+        self.assertIn("policy_arn = aws_iam_policy.fcvm_fleet_coordinator_broadcast[0].arn", attachment)
+        for r in re.findall(r'resource "([a-z_]+)" "fcvm_fleet_broadcast[a-z_]*"', self.BC):
+            self.assertIn(r, {"aws_ivs_channel", "tls_private_key", "aws_ivs_playback_key_pair", "aws_secretsmanager_secret",
+                              "aws_secretsmanager_secret_version", "aws_iam_policy", "aws_iam_role_policy_attachment"}, r)
+        # Every one of its seven resources and its one data source is counted by the switch.
+        self.assertEqual(len(re.findall(r"count\s+= local\.fcvm_fleet_broadcast \? 1 : 0", self.BC)), 8)
+
+    def test_first_rollout_waits_for_credentials_and_the_read_grant(self):
+        service = block('resource "aws_ecs_service" "fcvm_fleet_coordinator"')
+        dependencies = re.search(r"depends_on\s*=\s*\[(.*?)\]", service, re.S).group(1)
+        self.assertEqual(set(re.findall(r"\b(?:ephemeral\.)?aws_[\w.]+", dependencies)), {
+            "ephemeral.aws_ssm_parameter.fcvm_coordinator_release_guard",
+            "aws_lb_listener.fcvm_fleet_dns",
+            "aws_secretsmanager_secret_version.fcvm_fleet_broadcast",
+            "aws_iam_role_policy_attachment.fcvm_fleet_coordinator_broadcast",
+        })
+
+
 if __name__ == "__main__":
     unittest.main(verbosity=2, buffer=True)