build: replace ko with make images using podman or docker

.dockerignore

@@ -7,8 +7,10 @@ tmp/
 bin/*
 !bin/linux_amd64
 !bin/linux_amd64/ax-task-runner
+!bin/linux_amd64/ax-server
 !bin/linux_arm64
 !bin/linux_arm64/ax-task-runner
+!bin/linux_arm64/ax-server
 ax
 ax-server
 ax-task-runner

CONTRIBUTING.md

@@ -36,8 +36,7 @@ When you start on an issue or pull request, assign yourself to it, so others kno
 ### Prerequisites
 
 - **Go 1.27+**
-- **[`ko`](https://ko.build/)** (for building and deploying control plane images)
-- **Docker** or **Podman** (for building the Linux task-runner container image)
+- **Podman** or **Docker** (for building the Linux container images)
 - A Kubernetes cluster with [Agent Substrate](https://github.com/agent-substrate/substrate) installed
 
 ### Building Binaries
@@ -74,26 +73,21 @@ Or run via `go test`:
 go test -v ./...
 ```
 
-### Building the Task Runner Image
+### Building the Images
 
-Cross-compile the task runner for `linux/amd64` and package the container image:
+Cross-compile `ax-server` and `ax-task-runner` for `linux/amd64`, build their container images (with `podman` if available, otherwise `docker`), and push them to `$AX_IMAGE_REPO`:
 
 ```bash
-make build-task-runner
-```
-
-To build and push to a remote container registry:
-
-```bash
-make push-task-runner TASK_RUNNER_REPO=gcr.io/<your-project>/ax-task-runner
+export AX_IMAGE_REPO=<your-registry>   # e.g. gcr.io/my-project
+make images
 ```
 
 ### Deploying to Kubernetes
 
 Deploy Redis and server components to your cluster in the `ax-system` namespace:
 
 ```bash
-make deploy AX_IMAGE_REPO=gcr.io/<your-project>/ax-images
+make deploy
 ```
 
 ---

Makefile

@@ -15,22 +15,27 @@
 SHELL := /bin/bash
 
 # Configuration
-AX_IMAGE_REPO ?= gcr.io/ax-substrate/ate-images
-TASK_RUNNER_REPO ?= $(AX_IMAGE_REPO)/ax-task-runner
-CONTAINER_CLI ?= $(shell which podman 2>/dev/null || which docker 2>/dev/null)
+#
+# AX_IMAGE_REPO is required by `make images` and `make deploy`: it is the
+# registry the images are pushed to and pulled from, e.g.
+# AX_IMAGE_REPO=gcr.io/my-project.
+AX_IMAGE_REPO ?=
+IMAGE_TAG ?= latest
+IMAGE_ARCH ?= amd64
+AX_SERVER_IMAGE ?= $(AX_IMAGE_REPO)/ax-server:$(IMAGE_TAG)
+AX_TASK_RUNNER_IMAGE ?= $(AX_IMAGE_REPO)/ax-task-runner:$(IMAGE_TAG)
 
-.PHONY: all build build-binaries build-task-runner install push push-task-runner deploy deploy-server deploy-redis apply-example test test-e2e test-e2e-head e2e-setup e2e-run e2e-cleanup clean
+# Prefer podman, fall back to docker.
+CONTAINER_CLI ?= $(shell command -v podman 2>/dev/null || command -v docker 2>/dev/null)
 
-all: build
+.PHONY: build install images check-image-repo deploy deploy-server deploy-redis apply-example test test-e2e test-e2e-head e2e-setup e2e-run e2e-cleanup clean
 
 ## --------------------------------------
 ## Build Targets
 ## --------------------------------------
 
 # Build all local binaries (ax CLI, server)
-build: build-binaries
-
-build-binaries:
+build:
 	@echo "==> Building local binaries (ax, ax-server)..."
 	@mkdir -p bin
 	go build -trimpath -ldflags="-s -w" -o bin/ax ./cmd/ax
@@ -41,38 +46,46 @@ install:
 	@echo "==> Installing ax CLI to $$(go env GOPATH)/bin..."
 	go install -trimpath -ldflags="-s -w" ./cmd/ax
 
-# Cross-compile ax-task-runner for Linux amd64 and build container image with Python, Antigravity, and git/curl
-build-task-runner:
-	@echo "==> Cross-compiling ax-task-runner for linux/amd64..."
-	@mkdir -p bin/linux_amd64
-	GOOS=linux GOARCH=amd64 CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o bin/linux_amd64/ax-task-runner ./cmd/ax-task-runner
-	@echo "==> Building container image $(TASK_RUNNER_REPO):latest using $(CONTAINER_CLI)..."
-	$(CONTAINER_CLI) build --platform linux/amd64 -t $(TASK_RUNNER_REPO):latest -f Dockerfile.task-runner .
-
-# Push task-runner container image to registry
-push-task-runner: build-task-runner
-	@echo "==> Pushing task runner image to $(TASK_RUNNER_REPO):latest..."
-	$(CONTAINER_CLI) push $(TASK_RUNNER_REPO):latest
-	@echo "==> Current pushed digest:"
-	@gcloud container images list-tags $(TASK_RUNNER_REPO) --filter="tags=latest" --format="get(digest)"
-
-# Build and push all images
-push: push-task-runner
+check-image-repo:
+	@if [ -z "$(AX_IMAGE_REPO)" ]; then \
+		echo "error: AX_IMAGE_REPO is not set (e.g. AX_IMAGE_REPO=gcr.io/my-project)"; exit 1; \
+	fi
+
+# Cross-compile ax-server and ax-task-runner for linux/$(IMAGE_ARCH), then build
+# and push their container images.
+images: check-image-repo
+	@if [ -z "$(CONTAINER_CLI)" ]; then \
+		echo "error: neither podman nor docker found in PATH (or set CONTAINER_CLI)"; exit 1; \
+	fi
+	@echo "==> Cross-compiling ax-server and ax-task-runner for linux/$(IMAGE_ARCH)..."
+	@mkdir -p bin/linux_$(IMAGE_ARCH)
+	GOOS=linux GOARCH=$(IMAGE_ARCH) CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o bin/linux_$(IMAGE_ARCH)/ax-server ./cmd/ax-server
+	GOOS=linux GOARCH=$(IMAGE_ARCH) CGO_ENABLED=0 go build -trimpath -ldflags="-s -w" -o bin/linux_$(IMAGE_ARCH)/ax-task-runner ./cmd/ax-task-runner
+	@echo "==> Building $(AX_SERVER_IMAGE) using $(CONTAINER_CLI)..."
+	$(CONTAINER_CLI) build --platform linux/$(IMAGE_ARCH) --build-arg TARGETARCH=$(IMAGE_ARCH) -t $(AX_SERVER_IMAGE) -f Dockerfile.server .
+	@echo "==> Building $(AX_TASK_RUNNER_IMAGE) using $(CONTAINER_CLI)..."
+	$(CONTAINER_CLI) build --platform linux/$(IMAGE_ARCH) --build-arg TARGETARCH=$(IMAGE_ARCH) -t $(AX_TASK_RUNNER_IMAGE) -f Dockerfile.task-runner .
+	@echo "==> Pushing $(AX_SERVER_IMAGE)..."
+	$(CONTAINER_CLI) push $(AX_SERVER_IMAGE)
+	@echo "==> Pushing $(AX_TASK_RUNNER_IMAGE)..."
+	$(CONTAINER_CLI) push $(AX_TASK_RUNNER_IMAGE)
 
 ## --------------------------------------
 ## Deployment Targets
 ## --------------------------------------
 
-# Deploy all AX components to Kubernetes (Redis, ax-server)
+# Deploy all AX components to Kubernetes (Redis, ax-server). Run `make images` first.
 deploy: deploy-redis deploy-server
 
 deploy-redis:
 	@echo "==> Deploying Redis to ax-system namespace..."
 	kubectl apply -f deploy/redis.yaml
 
-deploy-server:
-	@echo "==> Building and deploying ax-server using ko..."
-	KO_DOCKER_REPO=$(AX_IMAGE_REPO) ko apply -f deploy/ax-server.yaml
+deploy-server: check-image-repo
+	@echo "==> Deploying ax-server ($(AX_SERVER_IMAGE))..."
+	sed -e 's|$${AX_SERVER_IMAGE}|$(AX_SERVER_IMAGE)|g' \
+		-e 's|$${AX_DEFAULT_TASK_IMAGE}|$(AX_TASK_RUNNER_IMAGE)|g' \
+		deploy/ax-server.yaml | kubectl apply -f -
 
 # Apply example task and resources
 apply-example:

README.md

@@ -64,7 +64,7 @@ AX schedules every task as a sandboxed actor on [Agent Substrate](https://github
 
 - A Kubernetes cluster with **Agent Substrate** installed (see below)
 - [Go](https://go.dev/doc/install) and `kubectl`
-- [`ko`](https://ko.build/) (`brew install ko`) and a container registry your cluster can pull from
+- [Podman](https://podman.io/) or [Docker](https://docs.docker.com/get-docker/), authenticated to a container registry your cluster can pull from
 
 To install Agent Substrate, follow the instructions in the [Substrate README](https://github.com/agent-substrate/substrate#readme). Substrate lands in the `ate-system` namespace and exposes its Control API at `api.ate-system.svc.cluster.local:443`, which is where AX expects to find it. Verify it is up before moving on:
 
@@ -80,17 +80,26 @@ go install github.com/google/ax/cmd/ax@latest
 
 This puts the `ax` binary in `$(go env GOPATH)/bin`. Make sure that directory is on your `PATH`.
 
-### 2. Deploy the control plane
+### 2. Build and push the images
+
+Point `AX_IMAGE_REPO` at your registry, then build the `ax-server` and `ax-task-runner` images and push them to it:
+
+```bash
+export AX_IMAGE_REPO=<your-registry>   # e.g. gcr.io/my-project
+make images
+```
+
+### 3. Deploy the control plane
 
 With the [prerequisites](#prerequisites) in place — most importantly a reachable Agent Substrate Control API — deploy the AX control plane:
 
 ```bash
-make deploy AX_IMAGE_REPO=<your-registry>
+make deploy
 ```
 
-This deploys Redis, then builds and deploys the control plane images with `ko`. Everything lands in the `ax-system` namespace.
+This deploys Redis and the `ax-server` image you just pushed, configured to use your `ax-task-runner` image for tasks that do not set `spec.image`. Everything lands in the `ax-system` namespace.
 
-### 3. Run your first task
+### 4. Run your first task
 
 ```bash
 ax apply -f examples/task.yaml       # Task + Workspace + Model in one file

deploy/ax-server.yaml

@@ -68,8 +68,8 @@ spec:
       serviceAccountName: ax-server
       containers:
         - name: ax-server
-          image: ko://github.com/google/ax/cmd/ax-server
-          imagePullPolicy: IfNotPresent
+          image: ${AX_SERVER_IMAGE}
+          imagePullPolicy: Always
           args:
             - "--addr=:8080"
             - "--redis-addr=ax-redis.ax-system.svc.cluster.local:6379"
@@ -82,6 +82,8 @@ spec:
           env:
             - name: ATENET_ROUTER_ADDR
               value: "atenet-router.ate-system.svc.cluster.local:80"
+            - name: AX_DEFAULT_TASK_IMAGE
+              value: "${AX_DEFAULT_TASK_IMAGE}"
             - name: AX_SNAPSHOTS_BUCKET
               value: "gs://dberkov-gke-dev3/ate-env/"
           ports:

docs/development.md

@@ -3,17 +3,15 @@
 ## Prerequisites
 
 - Go 1.27+
-- [`ko`](https://ko.build/) for building and deploying control plane images
-- Docker or Podman for the task runner image
+- Podman or Docker for building container images
 - A Kubernetes cluster and kubeconfig
 
 ## Build
 
 ```bash
 make build                 # bin/ax, bin/ax-server
 make install               # install the ax CLI into $(go env GOPATH)/bin
-make build-task-runner     # cross-compile the runner for linux/amd64 and build its image
-make push-task-runner      # ...and push it (set TASK_RUNNER_REPO)
+make images                # build and push the ax-server and ax-task-runner images to $AX_IMAGE_REPO
 ```
 
 ## Test

docs/runner.md

@@ -56,8 +56,7 @@ The `/workspace` volume is what survives suspend and resume. Agent Substrate sna
 `ax-task-runner` lives in `cmd/ax-task-runner` and is a thin wrapper over the `runner` Go package. It implements everything above and is documented from the inside in [Sandbox](sandbox.md). Its image, built from `Dockerfile.task-runner`, is Python 3.12 with `git`, `curl`, `openssh-client`, and the Antigravity agent installed, because the goal-based workspace bootstrap hands the goal to Antigravity.
 
 ```bash
-make build-task-runner     # cross-compile for linux/amd64 and build the image
-make push-task-runner      # push it; set TASK_RUNNER_REPO to choose the registry
+make images                # cross-compile for linux/amd64, build, and push to $AX_IMAGE_REPO
 ```
 
 ## Replacing the default runner

hack/setup-e2e-kind.sh

@@ -252,7 +252,7 @@ log "Building ax (host), ax-server and ax-task-runner (linux/${GOARCH})..."
 
 log "Building and pushing images to ${REGISTRY_ADDR}..."
 AX_SERVER_IMAGE="$(build_and_push "${REGISTRY_ADDR}/ax-server:e2e" \
-  --build-arg TARGETARCH="${GOARCH}" -f "${E2E_DIR}/Dockerfile.server")"
+  --build-arg TARGETARCH="${GOARCH}" -f "${REPO_ROOT}/Dockerfile.server")"
 # The E2E task-runner image extends the production image with test fixtures.
 docker build -q -t "${REGISTRY_ADDR}/ax-task-runner:base" \
   --build-arg TARGETARCH="${GOARCH}" -f "${REPO_ROOT}/Dockerfile.task-runner" "${REPO_ROOT}" > /dev/null

test/e2e/README.md

@@ -13,7 +13,6 @@ the `ax` CLI, against a configurable Substrate branch, tag, or commit.
 | `harness_test.go` | `TestMain`, cluster connection setup, and shared helpers |
 | `e2e_test.go` | The test cases |
 | `manifests/` | Namespace, WorkerPool, Redis, and ax-server manifests (rendered with `envsubst`) |
-| `Dockerfile.server` | ax-server image |
 | `Dockerfile.task-runner.e2e` | Extends `Dockerfile.task-runner` with test fixtures (a sample Git repo) |
 
 ## Quick Start