Phase 2 (part 4): move the Zephyr-only kernel code to src/kernel

AGENTS.md

@@ -31,7 +31,7 @@ moving code, follow it:
 
 * **Place code in the right tier.** OS/HAL/boot/inter-core → a kernel-only directory
   (`kernel/`, which is built into the `sof_kernel` library, and the directories not yet
-  moved there: `arch/`, `drivers/`, `platform/`, `probe/`, `logging/`, `trace/`); audio
+  moved there: `arch/`, `drivers/`, `platform/`, `trace/`); audio
   application logic → `audio/`; reusable pure code → a shared-library directory (`lib/`,
   `math/`, `module/`) kept free of privileged operations. Each top-level directory's
   `README.md` states its tier in a "Runs in:" banner — keep it accurate.

scripts/kernel_user_partition.txt

@@ -28,13 +28,9 @@ src/lib/objpool.c
 
 # Kernel sources inside the directories above.
 [not-user-sources]
-src/audio/base_fw.c
-src/audio/base_fw_intel.c
 src/audio/dai-legacy.c
 src/audio/host-legacy.c
 src/audio/intel_uaol.c
-src/audio/mic_privacy_manager/*
-src/audio/module_adapter/library/userspace_proxy.c
 
 # SOF headers that only kernel code may include.
 # Not listed on purpose: rtos/init.h, which only provides the

src/Kconfig

@@ -10,8 +10,6 @@ rsource "audio/Kconfig"
 
 rsource "trace/Kconfig"
 
-rsource "probe/Kconfig"
-
 rsource "samples/Kconfig"
 
 rsource "schedule/Kconfig"
@@ -23,10 +21,6 @@ rsource "ipc/Kconfig"
 
 rsource "math/Kconfig"
 
-rsource "library_manager/Kconfig"
-
-rsource "debug/telemetry/Kconfig"
-
 rsource "debug/debug_stream/Kconfig"
 
 rsource "debug/tester/Kconfig"

src/audio/CMakeLists.txt

@@ -16,11 +16,6 @@ add_subdirectory(buffers)
 add_subdirectory(pcm_converter)
 add_subdirectory(pipeline)
 
-if(CONFIG_COMP_BASEFW_IPC4 AND NOT CONFIG_LIBRARY)
-	add_local_sources(sof base_fw.c)
-endif()
-add_local_sources_ifdef(CONFIG_IPC4_BASE_FW_INTEL sof base_fw_intel.c)
-
 # directories and files included conditionally (alphabetical order)
 if(CONFIG_COMP_ARIA)
 	add_subdirectory(aria)
@@ -143,9 +138,6 @@ if(CONFIG_COMP_KPB AND NOT CONFIG_LIBRARY)
 		kpb.c
 	)
 endif()
-if(CONFIG_INTEL_ADSP_MIC_PRIVACY)
-	add_subdirectory(mic_privacy_manager)
-endif()
 if(CONFIG_COMP_TONE)
 	add_subdirectory(tone)
 endif()

src/audio/module_adapter/CMakeLists.txt

@@ -84,7 +84,6 @@ endif()
 	endif()
 
 	zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_PROXY
-		library/userspace_proxy.c
 		library/userspace_proxy_user.c
 	)
 

src/audio/module_adapter/README.md

@@ -1,7 +1,8 @@
 # Module Adapter Architecture
 
 > **Runs in:** Boundary / bridge — the module adapter is the container that runs audio modules
-> in **user** threads (DP threads, or the userspace proxy in `library/userspace_proxy.c`) while
+> in **user** threads (DP threads, or the userspace proxy in `library/userspace_proxy_user.c`,
+> whose kernel half is `src/kernel/audio/module_adapter/library/userspace_proxy.c`) while
 > being driven from the **kernel** pipeline. It hosts the `mod_*` allocation syscalls and sets
 > up per-module memory domains and object grants. See
 > [User/kernel split](../../include/sof/userspace/README.md).

src/debug/CMakeLists.txt

@@ -6,8 +6,5 @@ is_zephyr(zephyr)
 if(zephyr) ###  Zephyr ###
 
 add_subdirectory(debug_stream)
-add_subdirectory(telemetry)
-
-zephyr_library_sources_ifdef(CONFIG_COLD_STORE_EXECUTE_DEBUG dram.c)
 
 endif() # Zephyr

src/include/sof/userspace/README.md

@@ -74,8 +74,8 @@ its subdirectories span all tiers.
 
 | Tier | Meaning | Directories |
 |------|---------|-------------|
-| **Kernel-only** | OS integration, HAL, boot, inter-core. Never executes in a user thread. Must not be reached directly from user code — only via a syscall. | `kernel/` (`kernel/idc/`, `kernel/init/`), `arch/`, `drivers/`, `platform/`, `probe/` *, `logging/` *, `trace/` * |
-| **Boundary / bridge** | Owns the syscalls and the thread / memory-domain setup. Split internally: some translation units run kernel-side, some user-side (e.g. `handler-kernel.c` vs `handler-user.c`, `zephyr_ll.c` vs `zephyr_ll_user.c`). | `schedule/`, `ipc/`, `library_manager/` *, `debug/debug_stream/` *, and (in the Zephyr integration tree) `zephyr/syscall/` |
+| **Kernel-only** | OS integration, HAL, boot, inter-core. Never executes in a user thread. Must not be reached directly from user code — only via a syscall. | `kernel/` (see [its README](../../../kernel/README.md)), `arch/`, `drivers/`, `platform/`, `trace/` * |
+| **Boundary / bridge** | Owns the syscalls and the thread / memory-domain setup. Split internally: some translation units run kernel-side, some user-side (e.g. `handler-kernel.c` vs `handler-user.c`). | `schedule/`, `ipc/`, `debug/debug_stream/` *, and (in the Zephyr integration tree) `zephyr/syscall/` |
 | **Application** | Audio pipeline / component logic that runs in user threads under userspace configs. | `audio/` |
 | **Shared library** | Context-agnostic code linked into whichever context calls it. Must stay "user-safe": no privileged operations (no direct MMIO, no spinlocks held across syscalls, no raw kernel object access) so it is safe when pulled into a user build. | `math/`, `module/` |
 | **Mixed** | A directory whose files span more than one tier; see its own README for per-file classification. | `lib/` |
@@ -123,7 +123,7 @@ syscall by subsystem:
 | DAI | `dai_get_device` | `sof/lib/dai-zephyr.h` | `zephyr/syscall/dai.c` / `lib/dai.c` |
 | CPU | `cpu_get_id` | `sof/lib/cpu.h` | `zephyr/syscall/cpu.c` / `zephyr/lib/cpu.c` |
 | Module management | `mod_balloc_align`, `mod_alloc_ext`, `mod_free`, `mod_free_all`, `mod_data_blob_handler_new`, `mod_fast_get` | `sof/audio/module_adapter/module/generic.h` | scattered: `audio/module_adapter/module/generic.c` |
-| Library manager | `lib_manager_free_module` | `sof/lib_manager.h` | scattered: `library_manager/lib_manager.c` |
+| Library manager | `lib_manager_free_module` | `sof/lib_manager.h` | scattered: `kernel/library_manager/lib_manager.c` |
 | Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `schedule/zephyr_ll.c`, `schedule/zephyr_dp_schedule*.c` |
 | Debug | `debug_stream_slot_send_record` | `user/debug_stream_slot.h` | scattered: `debug/debug_stream/debug_stream_slot.c` |
 
@@ -152,7 +152,7 @@ marker at its declaration site.* Conversely, marked data is **user-writable**, s
 must never trust it — re-validate on every use. DP modules running under
 `SOF_USERSPACE_APPLICATION` access data through their per-module memory domain set up by the
 proxy or DP thread infrastructure, not through `APP_TASK_*` / `APP_SYSUSER_*` markers. See
-[`schedule/zephyr_ll_user.c`](../../../schedule/zephyr_ll_user.c): `zephyr_ll_user_heap()`
+[`kernel/schedule/zephyr_ll_user.c`](../../../kernel/schedule/zephyr_ll_user.c): `zephyr_ll_user_heap()`
 returns the address of the kernel-only heap object rather than a pointer stored in
 user-writable memory, and `zephyr_ll_user_heap_verify()` checks the heap pointer that user
 threads pass to the heap system calls.

src/ipc/CMakeLists.txt

@@ -19,12 +19,6 @@ if(zephyr) ###  Zephyr ###
 # dma-copy only used for dma-trace
 zephyr_library_sources_ifdef(CONFIG_TRACE dma-copy.c)
 
-if (CONFIG_SOC_SERIES_INTEL_CAVS_V25 OR CONFIG_SOC_SERIES_INTEL_ADSP_ACE)
-	zephyr_library_sources(
-		ipc-zephyr.c
-	)
-endif()
-
 else() ###  Not Zephyr ###
 
 if (CONFIG_TRACE)

src/kernel/CMakeLists.txt

@@ -10,5 +10,13 @@ if(zephyr)
 	set(ZEPHYR_CURRENT_LIBRARY sof_kernel)
 endif()
 
+add_subdirectory(audio)
+add_subdirectory(debug)
 add_subdirectory(idc)
 add_subdirectory(init)
+add_subdirectory(ipc)
+add_subdirectory(lib)
+add_subdirectory(library_manager)
+add_subdirectory(logging)
+add_subdirectory(probe)
+add_subdirectory(schedule)

src/kernel/Kconfig

@@ -2,4 +2,10 @@
 
 # Code that only runs in kernel mode, see scripts/kernel_user_partition.txt.
 
+rsource "debug/telemetry/Kconfig"
+
 rsource "idc/Kconfig"
+
+rsource "library_manager/Kconfig"
+
+rsource "probe/Kconfig"

src/kernel/README.md

@@ -0,0 +1,28 @@
+# Kernel-only code (`src/kernel`)
+
+> **Runs in:** Kernel-only — everything here runs in supervisor context and never executes in
+> a user thread. User code reaches it only through system calls. See
+> [User/kernel split](../include/sof/userspace/README.md).
+
+This directory collects the SOF code that only runs in kernel mode: OS, hardware and
+memory-domain facing code such as boot and initialization, inter-core communication and
+module loading. Kernel-only code elsewhere in `src/` moves here one subsystem at a time as part
+of the kernel / user-space partition.
+
+## Layout
+
+Each subdirectory keeps the path the code had under `src/`, so `src/<path>` moves to
+`src/kernel/<path>`. For example `src/audio/base_fw.c` becomes `src/kernel/audio/base_fw.c`.
+When only some files of a directory are kernel-only, only those files move, and the rest of
+the directory stays where it is.
+
+## Build
+
+Zephyr builds everything below this directory into the `sof_kernel` library instead of
+`modules_sof`. [`CMakeLists.txt`](CMakeLists.txt) makes `sof_kernel` the current Zephyr
+library for this directory tree, so the CMake files of moved directories do not change. Their
+Kconfig files are sourced from [`Kconfig`](Kconfig).
+
+Code here may use kernel-only headers. User-space code must not call into it directly; the
+checker in `scripts/check_kernel_includes.py` reports kernel-only includes from user-space
+sources.

src/kernel/audio/CMakeLists.txt

@@ -0,0 +1,16 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+if(CONFIG_COMP_BASEFW_IPC4 AND NOT CONFIG_LIBRARY)
+	add_local_sources(sof base_fw.c)
+endif()
+add_local_sources_ifdef(CONFIG_IPC4_BASE_FW_INTEL sof base_fw_intel.c)
+if(CONFIG_INTEL_ADSP_MIC_PRIVACY)
+	add_subdirectory(mic_privacy_manager)
+endif()
+
+is_zephyr(zephyr)
+if(zephyr)
+	zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_PROXY
+		module_adapter/library/userspace_proxy.c
+	)
+endif()

src/audio/base_fw_intel.c → src/kernel/audio/base_fw_intel.c

@@ -28,7 +28,7 @@
 #include <ipc4/base_fw.h>
 #include <ipc4/alh.h>
 #include <rimage/sof/user/manifest.h>
-#include "copier/copier_gain.h"
+#include "../../audio/copier/copier_gain.h"
 
 #if CONFIG_INTEL_ADSP_MIC_PRIVACY
 #include <sof/audio/mic_privacy_manager.h>

src/kernel/debug/CMakeLists.txt

@@ -0,0 +1,10 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+is_zephyr(zephyr)
+if(zephyr) ###  Zephyr ###
+
+add_subdirectory(telemetry)
+
+zephyr_library_sources_ifdef(CONFIG_COLD_STORE_EXECUTE_DEBUG dram.c)
+
+endif() # Zephyr

src/kernel/ipc/CMakeLists.txt

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+is_zephyr(zephyr)
+if(zephyr)
+	if (CONFIG_SOC_SERIES_INTEL_CAVS_V25 OR CONFIG_SOC_SERIES_INTEL_ADSP_ACE)
+		zephyr_library_sources(ipc-zephyr.c)
+	endif()
+endif()

src/kernel/lib/CMakeLists.txt

@@ -0,0 +1,18 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+if(CONFIG_AMS)
+  add_local_sources(sof ams.c)
+endif()
+
+if(CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL)
+  add_local_sources(sof cpu-clk-manager.c)
+endif()
+
+is_zephyr(zephyr)
+if(zephyr) ###  Zephyr ###
+
+  if(NOT CONFIG_SOF_ZEPHYR_NO_SOF_CLOCK)
+    add_local_sources(sof clk.c)
+  endif()
+
+endif()

src/kernel/schedule/CMakeLists.txt

@@ -0,0 +1,8 @@
+# SPDX-License-Identifier: BSD-3-Clause
+
+is_zephyr(zephyr)
+if(zephyr)
+	zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_LL
+		zephyr_ll_user.c
+	)
+endif()

src/lib/CMakeLists.txt

@@ -15,21 +15,9 @@ if(CONFIG_HAVE_AGENT)
   add_local_sources(sof agent.c)
 endif()
 
-if(CONFIG_AMS)
-  add_local_sources(sof ams.c)
-endif()
-
-if(CONFIG_KCPS_DYNAMIC_CLOCK_CONTROL)
-  add_local_sources(sof cpu-clk-manager.c)
-endif()
-
 is_zephyr(zephyr)
 if(zephyr) ###  Zephyr ###
 
   add_local_sources(sof ${common_files})
 
-  if(NOT CONFIG_SOF_ZEPHYR_NO_SOF_CLOCK)
-    add_local_sources(sof clk.c)
-  endif()
-
 endif()

src/lib/README.md

@@ -3,10 +3,10 @@
 > **Runs in:** Mixed — this directory contains code in more than one tier. `dma.c` and `dai.c`
 > host the kernel-side implementations (`z_impl_sof_dma_get/put`, `z_impl_dai_get_device`) of
 > syscalls whose validators live in `zephyr/syscall/`; treat them as boundary/kernel code.
-> Several other files (`clk.c`, `notifier.c`, `cpu-clk-manager.c`) take Zephyr spinlocks or
-> perform privileged platform callbacks and are similarly restricted to kernel context. Truly
-> context-agnostic helpers (`lib.c`, `objpool.c`, `ams.c`) can safely be linked into user
-> builds. See [User/kernel split](../include/sof/userspace/README.md).
+> `notifier.c` takes Zephyr spinlocks and is similarly restricted to kernel context. The clock,
+> KCPS and AMS code (`clk.c`, `cpu-clk-manager.c`, `ams.c`) is kernel-only and lives in
+> `src/kernel/lib`. Truly context-agnostic helpers (`lib.c`, `objpool.c`) can safely be linked
+> into user builds. See [User/kernel split](../include/sof/userspace/README.md).
 
 This directory collects cross-cutting helpers that don't belong to a single subsystem. When
 adding code here, decide first which tier it belongs to: a pure helper (no locks, no

src/platform/posix/include/platform/lib/clk.h

@@ -7,7 +7,7 @@
 #define CPU_LPRO_FREQ_IDX 0
 #define CPU_LOWEST_FREQ_IDX CPU_LPRO_FREQ_IDX
 
-/* This is not a platform function, it's defined in src/lib/clk.c.
+/* This is not a platform function, it's defined in src/kernel/lib/clk.c.
  * But the declaration has historically been in the platform layer, so
  * it's repeated here.
  */

src/schedule/CMakeLists.txt

@@ -54,8 +54,4 @@ zephyr_library_sources_ifdef(CONFIG_ZEPHYR_TWB_SCHEDULER
 	zephyr_twb_schedule.c
 )
 
-zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_LL
-	zephyr_ll_user.c
-)
-
 endif()

src/schedule/README.md

@@ -1,8 +1,8 @@
 # SOF Scheduling Architecture
 
 > **Runs in:** Boundary / bridge — this directory owns the LL/DP schedulers and their thread +
-> memory-domain setup. `zephyr_ll.c` and `zephyr_domain.c` are kernel-side; `zephyr_ll_user.c`
-> sets up the LL user memory domain (kernel-executed) and supports the LL user thread.
+> memory-domain setup. `zephyr_ll.c` and `zephyr_domain.c` are kernel-side; the LL user memory
+> domain and heap are set up by `src/kernel/schedule/zephyr_ll_user.c`.
 > `zephyr_dp_schedule_thread.c` is boundary code that can create and manage both kernel and
 > `K_USER` DP threads depending on configuration; `zephyr_dp_schedule_application.c` is the
 > DP application-mode path. This directory hosts the `zephyr_ll_task_sem_*` and

tools/rimage/config/lnl.toml.h

@@ -75,7 +75,7 @@
 #endif
 
 #ifdef CONFIG_PROBE
-#include <probe/probe.toml>
+#include <kernel/probe/probe.toml>
 #endif
 
 #if defined(CONFIG_COMP_MUX) || defined(LLEXT_FORCE_ALL_MODULAR)

tools/rimage/config/mtl.toml.h

@@ -71,7 +71,7 @@
 #endif
 
 #ifdef CONFIG_PROBE
-#include <probe/probe.toml>
+#include <kernel/probe/probe.toml>
 #endif
 
 #if defined(CONFIG_COMP_MUX) || defined(LLEXT_FORCE_ALL_MODULAR)

tools/rimage/config/nvl.toml.h

@@ -75,7 +75,7 @@ index = __COUNTER__
 #endif
 
 #ifdef CONFIG_PROBE
-#include <probe/probe.toml>
+#include <kernel/probe/probe.toml>
 #endif
 
 #if defined(CONFIG_COMP_MUX) || defined(LLEXT_FORCE_ALL_MODULAR)

tools/rimage/config/ptl.toml.h

@@ -75,7 +75,7 @@ index = __COUNTER__
 #endif
 
 #ifdef CONFIG_PROBE
-#include <probe/probe.toml>
+#include <kernel/probe/probe.toml>
 #endif
 
 #if defined(CONFIG_COMP_MUX) || defined(LLEXT_FORCE_ALL_MODULAR)

tools/rimage/config/tgl-h.toml.h

@@ -37,7 +37,7 @@
 #endif
 
 #ifdef CONFIG_PROBE
-#include <probe/probe.toml>
+#include <kernel/probe/probe.toml>
 #endif
 
 #if defined(CONFIG_COMP_SRC) || defined(LLEXT_FORCE_ALL_MODULAR)

tools/rimage/config/tgl.toml.h

@@ -37,7 +37,7 @@
 #endif
 
 #ifdef CONFIG_PROBE
-#include <probe/probe.toml>
+#include <kernel/probe/probe.toml>
 #endif
 
 #if defined(CONFIG_COMP_SRC) || defined(LLEXT_FORCE_ALL_MODULAR)

tools/rimage/config/wcl.toml.h

@@ -75,7 +75,7 @@ index = __COUNTER__
 #endif
 
 #ifdef CONFIG_PROBE
-#include <probe/probe.toml>
+#include <kernel/probe/probe.toml>
 #endif
 
 #if defined(CONFIG_COMP_MUX) || defined(LLEXT_FORCE_ALL_MODULAR)