From ee8217a6c75056a65f2a25effccabb05ef6bf154 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 15:04:51 +0000 Subject: [PATCH 01/19] arch: move src/arch to src/kernel/arch The architecture code (Xtensa cache and interrupt support, the host build's architecture layer and its library defconfig) is kernel-only. Move it to src/kernel/arch. Zephyr only uses it through include directories and the Xtensa cache attribute source listed in zephyr/CMakeLists.txt; update those paths. The host builds now add it from src/kernel. Update the configuration search paths in kconfig.cmake and defconfigs.cmake, the testbench, tplg_parser and app CMake, the MediaTek tools script and the docs. Its Kconfig is sourced from the same place in src/Kconfig as before, so the Kconfig definition order does not change. References to the removed XTOS build output (build__gcc/src/arch/xtensa) in old scripts and .gitignore are left as they are. No functional change. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- AGENTS.md | 2 +- CMakeLists.txt | 2 +- app/CMakeLists.txt | 2 +- scripts/cmake/defconfigs.cmake | 2 +- scripts/cmake/kconfig.cmake | 6 +++--- scripts/rebuild-testbench.sh | 2 +- src/CMakeLists.txt | 1 - src/Kconfig | 2 +- src/include/sof/userspace/README.md | 2 +- src/kernel/CMakeLists.txt | 3 +++ src/{ => kernel}/arch/CMakeLists.txt | 0 src/{ => kernel}/arch/Kconfig | 0 src/{ => kernel}/arch/README.md | 4 ++-- src/{ => kernel}/arch/host/CMakeLists.txt | 2 +- src/{ => kernel}/arch/host/configs/library_defconfig | 0 src/{ => kernel}/arch/host/include/arch/atomic.h | 0 src/{ => kernel}/arch/host/include/arch/compiler_info.h | 0 src/{ => kernel}/arch/host/include/arch/debug/backtrace.h | 0 src/{ => kernel}/arch/host/include/arch/debug/panic.h | 0 src/{ => kernel}/arch/host/include/arch/drivers/idc.h | 0 src/{ => kernel}/arch/host/include/arch/drivers/interrupt.h | 0 src/{ => kernel}/arch/host/include/arch/drivers/timer.h | 0 src/{ => kernel}/arch/host/include/arch/lib/cache.h | 0 src/{ => kernel}/arch/host/include/arch/lib/cpu.h | 0 src/{ => kernel}/arch/host/include/arch/lib/wait.h | 0 src/{ => kernel}/arch/host/include/arch/schedule/task.h | 0 src/{ => kernel}/arch/host/include/arch/sof.h | 0 src/{ => kernel}/arch/host/include/arch/string.h | 0 src/{ => kernel}/arch/host/lib/CMakeLists.txt | 0 src/{ => kernel}/arch/host/lib/ams.c | 0 src/{ => kernel}/arch/host/lib/notifier.c | 0 src/{ => kernel}/arch/xtensa/Kconfig | 0 src/{ => kernel}/arch/xtensa/drivers/cache_attr.c | 0 .../arch/xtensa/include/arch/drivers/interrupt.h | 0 src/platform/mtk/tools/build.sh | 2 +- tools/testbench/CMakeLists.txt | 2 +- tools/testbench/README.md | 6 +++--- tools/tplg_parser/CMakeLists.txt | 2 +- zephyr/CMakeLists.txt | 4 ++-- 39 files changed, 24 insertions(+), 22 deletions(-) rename src/{ => kernel}/arch/CMakeLists.txt (100%) rename src/{ => kernel}/arch/Kconfig (100%) rename src/{ => kernel}/arch/README.md (72%) rename src/{ => kernel}/arch/host/CMakeLists.txt (95%) rename src/{ => kernel}/arch/host/configs/library_defconfig (100%) rename src/{ => kernel}/arch/host/include/arch/atomic.h (100%) rename src/{ => kernel}/arch/host/include/arch/compiler_info.h (100%) rename src/{ => kernel}/arch/host/include/arch/debug/backtrace.h (100%) rename src/{ => kernel}/arch/host/include/arch/debug/panic.h (100%) rename src/{ => kernel}/arch/host/include/arch/drivers/idc.h (100%) rename src/{ => kernel}/arch/host/include/arch/drivers/interrupt.h (100%) rename src/{ => kernel}/arch/host/include/arch/drivers/timer.h (100%) rename src/{ => kernel}/arch/host/include/arch/lib/cache.h (100%) rename src/{ => kernel}/arch/host/include/arch/lib/cpu.h (100%) rename src/{ => kernel}/arch/host/include/arch/lib/wait.h (100%) rename src/{ => kernel}/arch/host/include/arch/schedule/task.h (100%) rename src/{ => kernel}/arch/host/include/arch/sof.h (100%) rename src/{ => kernel}/arch/host/include/arch/string.h (100%) rename src/{ => kernel}/arch/host/lib/CMakeLists.txt (100%) rename src/{ => kernel}/arch/host/lib/ams.c (100%) rename src/{ => kernel}/arch/host/lib/notifier.c (100%) rename src/{ => kernel}/arch/xtensa/Kconfig (100%) rename src/{ => kernel}/arch/xtensa/drivers/cache_attr.c (100%) rename src/{ => kernel}/arch/xtensa/include/arch/drivers/interrupt.h (100%) diff --git a/AGENTS.md b/AGENTS.md index 6a2f4cdfee5b..2a807612c145 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,7 @@ moving code, follow it: * **Place code in the right tier.** OS/HAL/boot/inter-core → a kernel-only directory (`kernel/`, which is built into the `sof_kernel` library, and the directories not yet - moved there: `arch/`, `platform/`, `trace/`); audio + moved there: `platform/`, `trace/`); audio application logic → `audio/`; reusable pure code → a shared-library directory (`lib/`, `math/`, `module/`) kept free of privileged operations. Each top-level directory's `README.md` states its tier in a "Runs in:" banner — keep it accurate. diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fe6aa02b614..1af14e4b4089 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,6 +144,6 @@ endif() if(BUILD_UNIT_TESTS) enable_testing() - add_subdirectory(src/arch/${ARCH}) + add_subdirectory(src/kernel/arch/${ARCH}) add_subdirectory(test) endif() diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index fe9b208fe99f..5537caecda09 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -27,6 +27,6 @@ target_sources(app PRIVATE ) zephyr_library_include_directories(app PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/../src/arch/xtensa/include + ${CMAKE_CURRENT_SOURCE_DIR}/../src/kernel/arch/xtensa/include ${CMAKE_CURRENT_SOURCE_DIR}/../src/include ) diff --git a/scripts/cmake/defconfigs.cmake b/scripts/cmake/defconfigs.cmake index 9bdbd23539a8..84d7eb0fe752 100644 --- a/scripts/cmake/defconfigs.cmake +++ b/scripts/cmake/defconfigs.cmake @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Looks for defconfig files in arch directories where kconfig.cmake looks too. -set(DEFCONFIGS_DIRECTORY "${PROJECT_SOURCE_DIR}/src/arch/${ARCH}/configs") +set(DEFCONFIGS_DIRECTORY "${PROJECT_SOURCE_DIR}/src/kernel/arch/${ARCH}/configs") file(GLOB DEFCONFIG_PATHS "${DEFCONFIGS_DIRECTORY}/*_defconfig") # Adds dependency on defconfigs directory diff --git a/scripts/cmake/kconfig.cmake b/scripts/cmake/kconfig.cmake index f885598387bf..7d3dd4481b18 100644 --- a/scripts/cmake/kconfig.cmake +++ b/scripts/cmake/kconfig.cmake @@ -13,14 +13,14 @@ if(NOT INIT_CONFIG_found) set(INIT_CONFIG "initial.config" CACHE STRING "Initial .config file") # - ".' is the top source directory. -# - "src/arch/host/configs" holds the library (testbench) configurations +# - "src/kernel/arch/host/configs" holds the library (testbench) configurations # and is also used by defconfigs.cmake. # - "test/cmocka/configs" holds the unit test configurations. # - First found wins. # - If two directories ever use the same defconfig name then a full # path must be used, e.g.: -DINIT_CONFIG=test/cmocka/configs/collision_defconfig -set(init_config_search_list "." "src/arch/host/configs" "test/cmocka/configs") +set(init_config_search_list "." "src/kernel/arch/host/configs" "test/cmocka/configs") find_file(INIT_CONFIG_found NAMES ${INIT_CONFIG} @@ -46,7 +46,7 @@ message(FATAL_ERROR "Initial configuration missing, no ${INIT_CONFIG} found. " "Provide a ${PROJECT_SOURCE_DIR}/initial.config file or specify some " "other -DINIT_CONFIG=location relative to '${PROJECT_SOURCE_DIR}/' or " - "'${PROJECT_SOURCE_DIR}/src/arch/*/configs/'" + "'${PROJECT_SOURCE_DIR}/src/kernel/arch/*/configs/'" ) endif() diff --git a/scripts/rebuild-testbench.sh b/scripts/rebuild-testbench.sh index 996d16f45a8c..578632dfa1cb 100755 --- a/scripts/rebuild-testbench.sh +++ b/scripts/rebuild-testbench.sh @@ -116,7 +116,7 @@ For temporary, interactive Kconfiguration use: $BUILD_BACKEND -C $TESTBENCH_DIR/$BUILD_DIR_NAME/sof_ep/build/ menuconfig -Permanent configuration is "src/arch/host/configs/library_defconfig". +Permanent configuration is "src/kernel/arch/host/configs/library_defconfig". For instant, incremental build: diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a4d31797de8f..d364361e9a0e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,7 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause add_subdirectory(platform) -add_subdirectory(arch) add_subdirectory(ipc) add_subdirectory(audio) add_subdirectory(lib) diff --git a/src/Kconfig b/src/Kconfig index 6dbd3c396c04..d72155332d4a 100644 --- a/src/Kconfig +++ b/src/Kconfig @@ -1,6 +1,6 @@ # SPDX-License-Identifier: BSD-3-Clause -rsource "arch/Kconfig" +rsource "kernel/arch/Kconfig" rsource "platform/Kconfig" diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index d342bbc941f7..70fdece0ed35 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -74,7 +74,7 @@ its subdirectories span all tiers. | Tier | Meaning | Directories | |------|---------|-------------| -| **Kernel-only** | OS integration, HAL, boot, inter-core. Never executes in a user thread. Must not be reached directly from user code — only via a syscall. | `kernel/` (see [its README](../../../kernel/README.md)), `arch/`, `platform/`, `trace/` * | +| **Kernel-only** | OS integration, HAL, boot, inter-core. Never executes in a user thread. Must not be reached directly from user code — only via a syscall. | `kernel/` (see [its README](../../../kernel/README.md)), `platform/`, `trace/` * | | **Boundary / bridge** | Owns the syscalls and the thread / memory-domain setup. Split internally: some translation units run kernel-side, some user-side (e.g. `handler-kernel.c` vs `handler-user.c`). | `schedule/`, `ipc/`, `debug/debug_stream/` *, and (in the Zephyr integration tree) `zephyr/syscall/` | | **Application** | Audio pipeline / component logic that runs in user threads under userspace configs. | `audio/` | | **Shared library** | Context-agnostic code linked into whichever context calls it. Must stay "user-safe": no privileged operations (no direct MMIO, no spinlocks held across syscalls, no raw kernel object access) so it is safe when pulled into a user build. | `math/`, `module/` | diff --git a/src/kernel/CMakeLists.txt b/src/kernel/CMakeLists.txt index b14881ad3b7a..8e9afe9e6293 100644 --- a/src/kernel/CMakeLists.txt +++ b/src/kernel/CMakeLists.txt @@ -21,6 +21,9 @@ if(zephyr) add_subdirectory(logging) add_subdirectory(probe) add_subdirectory(schedule) +else() + # Only built for the host + add_subdirectory(arch) endif() # Built for Zephyr and for the host diff --git a/src/arch/CMakeLists.txt b/src/kernel/arch/CMakeLists.txt similarity index 100% rename from src/arch/CMakeLists.txt rename to src/kernel/arch/CMakeLists.txt diff --git a/src/arch/Kconfig b/src/kernel/arch/Kconfig similarity index 100% rename from src/arch/Kconfig rename to src/kernel/arch/Kconfig diff --git a/src/arch/README.md b/src/kernel/arch/README.md similarity index 72% rename from src/arch/README.md rename to src/kernel/arch/README.md index df7999635033..c69391224118 100644 --- a/src/arch/README.md +++ b/src/kernel/arch/README.md @@ -1,8 +1,8 @@ -# Architecture Support (`src/arch`) +# Architecture Support (`src/kernel/arch`) > **Runs in:** Kernel-only — CPU-architecture primitives (Xtensa) execute in supervisor context > and are never linked into a user thread. See -> [User/kernel split](../include/sof/userspace/README.md). +> [User/kernel split](../../include/sof/userspace/README.md). This directory contains architecture-specific code. It sits below the RTOS and audio application and has no user-space entry points. diff --git a/src/arch/host/CMakeLists.txt b/src/kernel/arch/host/CMakeLists.txt similarity index 95% rename from src/arch/host/CMakeLists.txt rename to src/kernel/arch/host/CMakeLists.txt index 75314f00f556..bef4fdf9ec83 100644 --- a/src/arch/host/CMakeLists.txt +++ b/src/kernel/arch/host/CMakeLists.txt @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # includes -target_include_directories(sof_public_headers INTERFACE ${PROJECT_SOURCE_DIR}/src/arch/host/include) +target_include_directories(sof_public_headers INTERFACE ${PROJECT_SOURCE_DIR}/src/kernel/arch/host/include) target_include_directories(sof_public_headers INTERFACE ${PROJECT_SOURCE_DIR}/src/platform/library/include) include(CheckCCompilerFlag) diff --git a/src/arch/host/configs/library_defconfig b/src/kernel/arch/host/configs/library_defconfig similarity index 100% rename from src/arch/host/configs/library_defconfig rename to src/kernel/arch/host/configs/library_defconfig diff --git a/src/arch/host/include/arch/atomic.h b/src/kernel/arch/host/include/arch/atomic.h similarity index 100% rename from src/arch/host/include/arch/atomic.h rename to src/kernel/arch/host/include/arch/atomic.h diff --git a/src/arch/host/include/arch/compiler_info.h b/src/kernel/arch/host/include/arch/compiler_info.h similarity index 100% rename from src/arch/host/include/arch/compiler_info.h rename to src/kernel/arch/host/include/arch/compiler_info.h diff --git a/src/arch/host/include/arch/debug/backtrace.h b/src/kernel/arch/host/include/arch/debug/backtrace.h similarity index 100% rename from src/arch/host/include/arch/debug/backtrace.h rename to src/kernel/arch/host/include/arch/debug/backtrace.h diff --git a/src/arch/host/include/arch/debug/panic.h b/src/kernel/arch/host/include/arch/debug/panic.h similarity index 100% rename from src/arch/host/include/arch/debug/panic.h rename to src/kernel/arch/host/include/arch/debug/panic.h diff --git a/src/arch/host/include/arch/drivers/idc.h b/src/kernel/arch/host/include/arch/drivers/idc.h similarity index 100% rename from src/arch/host/include/arch/drivers/idc.h rename to src/kernel/arch/host/include/arch/drivers/idc.h diff --git a/src/arch/host/include/arch/drivers/interrupt.h b/src/kernel/arch/host/include/arch/drivers/interrupt.h similarity index 100% rename from src/arch/host/include/arch/drivers/interrupt.h rename to src/kernel/arch/host/include/arch/drivers/interrupt.h diff --git a/src/arch/host/include/arch/drivers/timer.h b/src/kernel/arch/host/include/arch/drivers/timer.h similarity index 100% rename from src/arch/host/include/arch/drivers/timer.h rename to src/kernel/arch/host/include/arch/drivers/timer.h diff --git a/src/arch/host/include/arch/lib/cache.h b/src/kernel/arch/host/include/arch/lib/cache.h similarity index 100% rename from src/arch/host/include/arch/lib/cache.h rename to src/kernel/arch/host/include/arch/lib/cache.h diff --git a/src/arch/host/include/arch/lib/cpu.h b/src/kernel/arch/host/include/arch/lib/cpu.h similarity index 100% rename from src/arch/host/include/arch/lib/cpu.h rename to src/kernel/arch/host/include/arch/lib/cpu.h diff --git a/src/arch/host/include/arch/lib/wait.h b/src/kernel/arch/host/include/arch/lib/wait.h similarity index 100% rename from src/arch/host/include/arch/lib/wait.h rename to src/kernel/arch/host/include/arch/lib/wait.h diff --git a/src/arch/host/include/arch/schedule/task.h b/src/kernel/arch/host/include/arch/schedule/task.h similarity index 100% rename from src/arch/host/include/arch/schedule/task.h rename to src/kernel/arch/host/include/arch/schedule/task.h diff --git a/src/arch/host/include/arch/sof.h b/src/kernel/arch/host/include/arch/sof.h similarity index 100% rename from src/arch/host/include/arch/sof.h rename to src/kernel/arch/host/include/arch/sof.h diff --git a/src/arch/host/include/arch/string.h b/src/kernel/arch/host/include/arch/string.h similarity index 100% rename from src/arch/host/include/arch/string.h rename to src/kernel/arch/host/include/arch/string.h diff --git a/src/arch/host/lib/CMakeLists.txt b/src/kernel/arch/host/lib/CMakeLists.txt similarity index 100% rename from src/arch/host/lib/CMakeLists.txt rename to src/kernel/arch/host/lib/CMakeLists.txt diff --git a/src/arch/host/lib/ams.c b/src/kernel/arch/host/lib/ams.c similarity index 100% rename from src/arch/host/lib/ams.c rename to src/kernel/arch/host/lib/ams.c diff --git a/src/arch/host/lib/notifier.c b/src/kernel/arch/host/lib/notifier.c similarity index 100% rename from src/arch/host/lib/notifier.c rename to src/kernel/arch/host/lib/notifier.c diff --git a/src/arch/xtensa/Kconfig b/src/kernel/arch/xtensa/Kconfig similarity index 100% rename from src/arch/xtensa/Kconfig rename to src/kernel/arch/xtensa/Kconfig diff --git a/src/arch/xtensa/drivers/cache_attr.c b/src/kernel/arch/xtensa/drivers/cache_attr.c similarity index 100% rename from src/arch/xtensa/drivers/cache_attr.c rename to src/kernel/arch/xtensa/drivers/cache_attr.c diff --git a/src/arch/xtensa/include/arch/drivers/interrupt.h b/src/kernel/arch/xtensa/include/arch/drivers/interrupt.h similarity index 100% rename from src/arch/xtensa/include/arch/drivers/interrupt.h rename to src/kernel/arch/xtensa/include/arch/drivers/interrupt.h diff --git a/src/platform/mtk/tools/build.sh b/src/platform/mtk/tools/build.sh index 1b17c8e38cf8..8d7b30d855b3 100755 --- a/src/platform/mtk/tools/build.sh +++ b/src/platform/mtk/tools/build.sh @@ -16,7 +16,7 @@ for p in $PLATFORMS; do SRCS="$SOF/src/platform/$p/lib/dai.c $SOF/src/platform/$p/afe-platform.c" INCS="-I$SOF/src/include -I$SOF/src/platform/posix/include -I$SOF/posix/include" - INCS="$INCS -I$SOF/src/arch/host/include -I$SOF/src/platform/$p/include/platform" + INCS="$INCS -I$SOF/src/kernel/arch/host/include -I$SOF/src/platform/$p/include/platform" INCS="$INCS -I$SOF/src/platform/$p/include" DEFS="-DRELATIVE_FILE=\"mt-dai-gen.c\" -DCONFIG_CORE_COUNT=1 -DCONFIG_IPC_MAJOR_3=1" diff --git a/tools/testbench/CMakeLists.txt b/tools/testbench/CMakeLists.txt index d73d20ed2c5b..b07dc2be0e6a 100644 --- a/tools/testbench/CMakeLists.txt +++ b/tools/testbench/CMakeLists.txt @@ -9,7 +9,7 @@ include(CheckCCompilerFlag) set(default_asoc_h "/usr/include/alsa/sound/uapi/asoc.h") -file(READ "../../src/arch/host/configs/library_defconfig" library_defconfig) +file(READ "../../src/kernel/arch/host/configs/library_defconfig" library_defconfig) if(library_defconfig MATCHES "CONFIG_IPC_MAJOR_4=y") message("-- Selected IPC4 build for testbench") diff --git a/tools/testbench/README.md b/tools/testbench/README.md index 82e1630dc771..8fb56e906db4 100644 --- a/tools/testbench/README.md +++ b/tools/testbench/README.md @@ -253,10 +253,10 @@ The testbench build is by default IPC4. The IPC version can be changed by applyi patch: ``` -diff --git a/src/arch/host/configs/library_defconfig b/src/arch/host/configs/library_defconfig +diff --git a/src/kernel/arch/host/configs/library_defconfig b/src/kernel/arch/host/configs/library_defconfig index 1b9483ffe..5c944d292 100644 ---- a/src/arch/host/configs/library_defconfig -+++ b/src/arch/host/configs/library_defconfig +--- a/src/kernel/arch/host/configs/library_defconfig ++++ b/src/kernel/arch/host/configs/library_defconfig @@ -22,8 +22,8 @@ CONFIG_COMP_VOLUME=y CONFIG_COMP_VOLUME_LINEAR_RAMP=y CONFIG_COMP_VOLUME_WINDOWS_FADE=y diff --git a/tools/tplg_parser/CMakeLists.txt b/tools/tplg_parser/CMakeLists.txt index f7a447a6e6c2..716d921d45c6 100644 --- a/tools/tplg_parser/CMakeLists.txt +++ b/tools/tplg_parser/CMakeLists.txt @@ -42,7 +42,7 @@ sof_append_relative_path_definitions(sof_tplg_parser) target_include_directories(sof_tplg_parser PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/include) target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/audio) -target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/arch/host/include) +target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/kernel/arch/host/include) target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/platform/library/include) target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/posix/include) diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 8da2dc47fdc5..986b47f6ee44 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -170,7 +170,7 @@ set(RIMAGE_CONFIG_PATH ${RIMAGE_TOP}/config CACHE PATH # default SOF includes target_include_directories(SOF INTERFACE ${RIMAGE_TOP}/src/include) target_include_directories(SOF INTERFACE ${SOF_SRC_PATH}/include) -target_include_directories(SOF INTERFACE ${SOF_SRC_PATH}/arch/${ARCH}/include) +target_include_directories(SOF INTERFACE ${SOF_SRC_PATH}/kernel/arch/${ARCH}/include) target_include_directories(SOF INTERFACE ${sof_top_dir}/third_party/include) target_include_directories(SOF INTERFACE ${sof_top_dir}/zephyr/include) @@ -577,7 +577,7 @@ endif() zephyr_library_sources( # SOF core infrastructure - runs on top of Zephyr - ${SOF_SRC_PATH}/arch/xtensa/drivers/cache_attr.c + ${SOF_SRC_PATH}/kernel/arch/xtensa/drivers/cache_attr.c # Bridge wrapper between SOF and Zephyr APIs - Will shrink over time. wrapper.c From 7e6bf4219bc7cb12edb2043649471f0bd81aa380 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 15:06:01 +0000 Subject: [PATCH 02/19] platform: move src/platform to src/kernel/platform The platform code (memory maps, clocks, interrupts, platform init and the host library and posix platform layers) is kernel-only. Move it to src/kernel/platform. Point SOF_PLATFORM_PATH in zephyr/CMakeLists.txt to the new location; the two i.MX linker snippets now use it as well. The host builds now add it from src/kernel. Update the include directories of the cmocka and ztest unit tests, the testbench and tplg_parser, the platform header patterns in the include checker's manifest, CODEOWNERS, the MediaTek tools script and the docs. Its Kconfig is sourced from the same place in src/Kconfig as before, so the Kconfig definition order does not change. No functional change. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- AGENTS.md | 2 +- CODEOWNERS | 6 +++--- scripts/kernel_user_partition.txt | 2 +- src/CMakeLists.txt | 1 - src/Kconfig | 2 +- src/audio/README.md | 2 +- src/include/sof/userspace/README.md | 2 +- src/ipc/README.md | 2 +- src/kernel/CMakeLists.txt | 1 + src/kernel/arch/host/CMakeLists.txt | 2 +- src/{ => kernel}/platform/CMakeLists.txt | 0 src/{ => kernel}/platform/Kconfig | 0 src/{ => kernel}/platform/README.md | 4 ++-- .../platform/ace30/include/platform/lib/clk.h | 0 .../platform/ace30/include/platform/lib/mailbox.h | 0 .../platform/ace30/include/platform/lib/memory.h | 0 .../platform/ace30/include/platform/platform.h | 0 .../platform/ace30/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/ace30/lib/clk.c | 0 src/{ => kernel}/platform/amd/acp_7_0/acp_7_0.x.in | 0 .../amd/acp_7_0/include/arch/xtensa/config/core-isa.h | 0 .../amd/acp_7_0/include/arch/xtensa/config/core-matmap.h | 0 .../amd/acp_7_0/include/arch/xtensa/config/defs.h | 0 .../amd/acp_7_0/include/arch/xtensa/config/specreg.h | 0 .../amd/acp_7_0/include/arch/xtensa/config/system.h | 0 .../amd/acp_7_0/include/arch/xtensa/config/tie-asm.h | 0 .../platform/amd/acp_7_0/include/arch/xtensa/config/tie.h | 0 .../platform/amd/acp_7_0/include/arch/xtensa/idma.h | 0 .../amd/acp_7_0/include/arch/xtensa/tie/xt_datacache.h | 0 .../include/arch/xtensa/tie/xt_externalregisters.h | 0 .../amd/acp_7_0/include/arch/xtensa/xtensa-types.h | 0 .../amd/acp_7_0/include/platform/drivers/interrupt.h | 0 .../amd/acp_7_0/include/platform/fw_scratch_mem.h | 0 .../platform/amd/acp_7_0/include/platform/lib/memory.h | 0 .../platform/amd/acp_7_0/include/platform/platform.h | 0 .../platform/amd/acp_7_0/include/platform/platform_misc.h | 0 src/{ => kernel}/platform/amd/acp_7_0/lib/clk.c | 0 src/{ => kernel}/platform/amd/acp_7_0/platform.c | 0 .../amd/acp_7_x/include/arch/xtensa/tie/xt_datacache.h | 0 .../amd/acp_7_x/include/platform/fw_scratch_mem.h | 0 .../platform/amd/acp_7_x/include/platform/lib/memory.h | 0 .../platform/amd/acp_7_x/include/platform/platform.h | 0 .../platform/amd/acp_7_x/include/platform/platform_misc.h | 0 src/{ => kernel}/platform/amd/acp_7_x/lib/clk.c | 0 src/{ => kernel}/platform/amd/acp_7_x/platform.c | 0 .../platform/amd/common/include/platform/acp_dma.h | 0 .../platform/amd/common/include/platform/acp_dmic_dma.h | 0 .../platform/amd/common/include/platform/acp_sp_dma.h | 0 .../platform/amd/common/include/platform/drivers/idc.h | 0 .../platform/amd/common/include/platform/ipc.h | 0 .../platform/amd/common/include/platform/lib/clk.h | 0 .../platform/amd/common/include/platform/lib/cpu.h | 0 .../platform/amd/common/include/platform/lib/dai.h | 0 .../platform/amd/common/include/platform/lib/dma.h | 0 .../platform/amd/common/include/platform/lib/mailbox.h | 0 .../platform/amd/common/include/platform/lib/pm_runtime.h | 0 .../platform/amd/common/include/platform/trace/trace.h | 0 .../amd/rembrandt/include/arch/xtensa/config/core-isa.h | 0 .../rembrandt/include/arch/xtensa/config/core-matmap.h | 0 .../amd/rembrandt/include/arch/xtensa/config/defs.h | 0 .../amd/rembrandt/include/arch/xtensa/config/specreg.h | 0 .../amd/rembrandt/include/arch/xtensa/config/system.h | 0 .../amd/rembrandt/include/arch/xtensa/config/tie-asm.h | 0 .../amd/rembrandt/include/arch/xtensa/config/tie.h | 0 .../amd/rembrandt/include/arch/xtensa/tie/xt_datacache.h | 0 .../amd/rembrandt/include/platform/chip_offset_byte.h | 0 .../amd/rembrandt/include/platform/chip_registers.h | 0 .../amd/rembrandt/include/platform/drivers/interrupt.h | 0 .../amd/rembrandt/include/platform/fw_scratch_mem.h | 0 .../platform/amd/rembrandt/include/platform/lib/memory.h | 0 .../platform/amd/rembrandt/include/platform/platform.h | 0 src/{ => kernel}/platform/amd/rembrandt/lib/clk.c | 0 src/{ => kernel}/platform/amd/rembrandt/lib/dai.c | 0 src/{ => kernel}/platform/amd/rembrandt/lib/dma.c | 0 src/{ => kernel}/platform/amd/rembrandt/platform.c | 0 src/{ => kernel}/platform/amd/rembrandt/rembrandt.x.in | 0 src/{ => kernel}/platform/imx8/imx8.x.in | 0 .../platform/imx8/include/arch/xtensa/config/core-isa.h | 0 .../imx8/include/arch/xtensa/config/core-matmap.h | 0 .../platform/imx8/include/arch/xtensa/config/defs.h | 0 .../platform/imx8/include/arch/xtensa/config/specreg.h | 0 .../platform/imx8/include/arch/xtensa/config/system.h | 0 .../platform/imx8/include/arch/xtensa/config/tie-asm.h | 0 .../platform/imx8/include/arch/xtensa/config/tie.h | 0 .../platform/imx8/include/platform/drivers/idc.h | 0 .../platform/imx8/include/platform/drivers/interrupt.h | 0 src/{ => kernel}/platform/imx8/include/platform/lib/clk.h | 0 src/{ => kernel}/platform/imx8/include/platform/lib/cpu.h | 0 src/{ => kernel}/platform/imx8/include/platform/lib/dai.h | 0 src/{ => kernel}/platform/imx8/include/platform/lib/dma.h | 0 .../platform/imx8/include/platform/lib/mailbox.h | 0 .../platform/imx8/include/platform/lib/memory.h | 0 .../platform/imx8/include/platform/platform.h | 0 .../platform/imx8/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/imx8/lib/clk.c | 0 src/{ => kernel}/platform/imx8/platform.c | 0 src/{ => kernel}/platform/imx8m/imx8m.x.in | 0 .../platform/imx8m/include/arch/xtensa/config/core-isa.h | 0 .../imx8m/include/arch/xtensa/config/core-matmap.h | 0 .../platform/imx8m/include/arch/xtensa/config/defs.h | 0 .../platform/imx8m/include/arch/xtensa/config/specreg.h | 0 .../platform/imx8m/include/arch/xtensa/config/system.h | 0 .../platform/imx8m/include/arch/xtensa/config/tie-asm.h | 0 .../platform/imx8m/include/arch/xtensa/config/tie.h | 0 .../platform/imx8m/include/platform/drivers/idc.h | 0 .../platform/imx8m/include/platform/drivers/interrupt.h | 0 .../platform/imx8m/include/platform/lib/clk.h | 0 .../platform/imx8m/include/platform/lib/cpu.h | 0 .../platform/imx8m/include/platform/lib/dai.h | 0 .../platform/imx8m/include/platform/lib/dma.h | 0 .../platform/imx8m/include/platform/lib/mailbox.h | 0 .../platform/imx8m/include/platform/lib/memory.h | 0 .../platform/imx8m/include/platform/lib/pm_runtime.h | 0 .../platform/imx8m/include/platform/platform.h | 0 .../platform/imx8m/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/imx8m/lib/clk.c | 0 src/{ => kernel}/platform/imx8m/platform.c | 0 .../platform/imx8m_cm7/include/platform/lib/clk.h | 0 .../platform/imx8m_cm7/include/platform/lib/dma.h | 0 .../platform/imx8m_cm7/include/platform/lib/mailbox.h | 0 .../platform/imx8m_cm7/include/platform/lib/memory.h | 0 .../platform/imx8m_cm7/include/platform/platform.h | 0 .../platform/imx8m_cm7/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/imx8m_cm7/lib/clk.c | 0 .../platform/imx8m_cm7/linker/data-sections.ld | 0 src/{ => kernel}/platform/imx8m_cm7/platform.c | 0 src/{ => kernel}/platform/imx8ulp/imx8ulp.x.in | 0 .../imx8ulp/include/arch/xtensa/config/core-isa.h | 0 .../imx8ulp/include/arch/xtensa/config/core-matmap.h | 0 .../platform/imx8ulp/include/arch/xtensa/config/defs.h | 0 .../platform/imx8ulp/include/arch/xtensa/config/specreg.h | 0 .../platform/imx8ulp/include/arch/xtensa/config/system.h | 0 .../platform/imx8ulp/include/arch/xtensa/config/tie-asm.h | 0 .../platform/imx8ulp/include/arch/xtensa/config/tie.h | 0 .../platform/imx8ulp/include/platform/drivers/idc.h | 0 .../platform/imx8ulp/include/platform/drivers/interrupt.h | 0 .../platform/imx8ulp/include/platform/lib/clk.h | 0 .../platform/imx8ulp/include/platform/lib/cpu.h | 0 .../platform/imx8ulp/include/platform/lib/dai.h | 0 .../platform/imx8ulp/include/platform/lib/dma.h | 0 .../platform/imx8ulp/include/platform/lib/mailbox.h | 0 .../platform/imx8ulp/include/platform/lib/memory.h | 0 .../platform/imx8ulp/include/platform/platform.h | 0 .../platform/imx8ulp/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/imx8ulp/lib/clk.c | 0 src/{ => kernel}/platform/imx8ulp/platform.c | 0 .../platform/imx93_a55/include/platform/drivers/idc.h | 0 .../platform/imx93_a55/include/platform/lib/clk.h | 0 .../platform/imx93_a55/include/platform/lib/dai.h | 0 .../platform/imx93_a55/include/platform/lib/dma.h | 0 .../platform/imx93_a55/include/platform/lib/mailbox.h | 0 .../platform/imx93_a55/include/platform/lib/memory.h | 0 .../platform/imx93_a55/include/platform/platform.h | 0 .../platform/imx93_a55/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/imx93_a55/lib/clk.c | 0 src/{ => kernel}/platform/imx93_a55/platform.c | 0 .../platform/imx95/include/platform/lib/clk.h | 0 .../platform/imx95/include/platform/lib/dma.h | 0 .../platform/imx95/include/platform/lib/mailbox.h | 0 .../platform/imx95/include/platform/lib/memory.h | 0 .../platform/imx95/include/platform/platform.h | 0 .../platform/imx95/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/imx95/lib/clk.c | 0 src/{ => kernel}/platform/imx95/linker/data-sections.ld | 0 src/{ => kernel}/platform/imx95/platform.c | 0 src/{ => kernel}/platform/intel/ace/include/ace/lib/clk.h | 0 .../platform/intel/ace/include/ace/lib/mailbox.h | 0 .../platform/intel/ace/include/ace/lib/memory.h | 0 src/{ => kernel}/platform/intel/ace/include/ace/version.h | 0 .../platform/intel/ace/include/auth/intel_auth_api.h | 0 .../intel/ace/include/auth/intel_status_logger_api.h | 0 .../intel/ace/include/auth/intel_status_logger_iface.h | 0 src/{ => kernel}/platform/intel/ace/lib/watchdog.c | 0 src/{ => kernel}/platform/intel/ace/platform.c | 0 .../platform/intel/cavs/include/cavs/drivers/mn.h | 0 .../intel/cavs/include/cavs/drivers/sideband-ipc.h | 0 .../platform/intel/cavs/include/cavs/drivers/timestamp.h | 0 .../platform/intel/cavs/include/cavs/ext_manifest.h | 0 .../platform/intel/cavs/include/cavs/lib/mailbox.h | 0 .../platform/intel/cavs/include/cavs/lib/memory.h | 0 .../platform/intel/cavs/include/cavs/lib/power_down.h | 0 .../platform/intel/cavs/include/cavs/mem_window.h | 0 .../platform/intel/cavs/include/cavs/trace/trace.h | 0 .../platform/intel/cavs/include/cavs/version.h | 0 src/{ => kernel}/platform/intel/cavs/platform.c | 0 src/{ => kernel}/platform/library/CMakeLists.txt | 0 .../platform/library/include/platform/drivers/idc.h | 0 .../platform/library/include/platform/drivers/interrupt.h | 0 .../platform/library/include/platform/drivers/pmc.h | 0 .../platform/library/include/platform/lib/clk.h | 0 .../platform/library/include/platform/lib/cpu.h | 0 .../platform/library/include/platform/lib/dai.h | 0 .../platform/library/include/platform/lib/dma.h | 0 .../platform/library/include/platform/lib/ll_schedule.h | 0 .../platform/library/include/platform/lib/mailbox.h | 0 .../platform/library/include/platform/lib/memory.h | 0 .../platform/library/include/platform/lib/pm_runtime.h | 0 .../platform/library/include/platform/platform.h | 0 src/{ => kernel}/platform/library/init.c | 0 src/{ => kernel}/platform/library/lib/CMakeLists.txt | 0 src/{ => kernel}/platform/library/lib/alloc.c | 0 src/{ => kernel}/platform/library/lib/clk.c | 0 src/{ => kernel}/platform/library/lib/dai.c | 0 src/{ => kernel}/platform/library/lib/memory.c | 0 src/{ => kernel}/platform/library/lib/pm_runtime.c | 0 src/{ => kernel}/platform/library/lib/trace.c | 0 src/{ => kernel}/platform/library/panic.c | 0 src/{ => kernel}/platform/library/platform.c | 0 src/{ => kernel}/platform/library/schedule/CMakeLists.txt | 0 src/{ => kernel}/platform/library/schedule/edf_schedule.c | 0 src/{ => kernel}/platform/library/schedule/ll_schedule.c | 0 src/{ => kernel}/platform/library/schedule/schedule.c | 0 .../platform/lunarlake/include/platform/lib/clk.h | 0 .../platform/lunarlake/include/platform/lib/mailbox.h | 0 .../platform/lunarlake/include/platform/lib/memory.h | 0 .../platform/lunarlake/include/platform/platform.h | 0 .../platform/lunarlake/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/lunarlake/lib/clk.c | 0 .../platform/meteorlake/include/platform/lib/clk.h | 0 .../platform/meteorlake/include/platform/lib/mailbox.h | 0 .../platform/meteorlake/include/platform/lib/memory.h | 0 .../platform/meteorlake/include/platform/platform.h | 0 .../platform/meteorlake/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/meteorlake/lib/clk.c | 0 src/{ => kernel}/platform/mtk/dai.c | 0 src/{ => kernel}/platform/mtk/include/platform/lib/clk.h | 0 src/{ => kernel}/platform/mtk/include/platform/lib/cpu.h | 0 src/{ => kernel}/platform/mtk/include/platform/lib/dai.h | 0 src/{ => kernel}/platform/mtk/include/platform/lib/dma.h | 0 .../platform/mtk/include/platform/lib/mailbox.h | 0 .../platform/mtk/include/platform/lib/memory.h | 0 .../platform/mtk/include/platform/lib/pm_runtime.h | 0 src/{ => kernel}/platform/mtk/include/platform/platform.h | 0 .../platform/mtk/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/mtk/platform.c | 0 src/{ => kernel}/platform/mtk/tools/README.txt | 0 src/{ => kernel}/platform/mtk/tools/build.sh | 8 ++++---- src/{ => kernel}/platform/mtk/tools/mt-dai-gen.c | 0 .../platform/novalake/include/platform/lib/clk.h | 0 .../platform/novalake/include/platform/lib/mailbox.h | 0 .../platform/novalake/include/platform/lib/memory.h | 0 .../platform/novalake/include/platform/platform.h | 0 .../platform/novalake/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/novalake/lib/clk.c | 0 src/{ => kernel}/platform/posix/dai.c | 0 src/{ => kernel}/platform/posix/dma.c | 0 src/{ => kernel}/platform/posix/fuzz.c | 0 .../platform/posix/include/platform/drivers/idc.h | 0 .../platform/posix/include/platform/drivers/interrupt.h | 0 src/{ => kernel}/platform/posix/include/platform/fuzz.h | 0 .../platform/posix/include/platform/lib/clk.h | 0 .../platform/posix/include/platform/lib/cpu.h | 0 .../platform/posix/include/platform/lib/dai.h | 0 .../platform/posix/include/platform/lib/dma.h | 0 .../platform/posix/include/platform/lib/mailbox.h | 0 .../platform/posix/include/platform/lib/memory.h | 0 .../platform/posix/include/platform/lib/pm_runtime.h | 0 .../platform/posix/include/platform/platform.h | 0 .../platform/posix/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/posix/ipc.c | 0 src/{ => kernel}/platform/posix/posix.c | 0 .../platform/qemu_xtensa/include/platform/lib/clk.h | 0 .../platform/qemu_xtensa/include/platform/lib/dai.h | 0 .../platform/qemu_xtensa/include/platform/lib/dma.h | 0 .../platform/qemu_xtensa/include/platform/lib/mailbox.h | 0 .../platform/qemu_xtensa/include/platform/lib/memory.h | 0 .../platform/qemu_xtensa/include/platform/platform.h | 0 .../platform/qemu_xtensa/include/platform/trace/trace.h | 0 src/{ => kernel}/platform/qemu_xtensa/platform.c | 0 .../platform/tigerlake/include/platform/lib/mailbox.h | 0 .../platform/tigerlake/include/platform/lib/memory.h | 0 .../platform/tigerlake/include/platform/platform.h | 0 .../platform/tigerlake/include/platform/trace/trace.h | 0 test/cmocka/CMakeLists.txt | 4 ++-- test/cmocka/src/lib/alloc/CMakeLists.txt | 4 ++-- test/cmocka/src/lib/fast-get/CMakeLists.txt | 2 +- test/ztest/unit/audio/crossover/CMakeLists.txt | 2 +- test/ztest/unit/audio/drc/CMakeLists.txt | 2 +- test/ztest/unit/dsrc/CMakeLists.txt | 2 +- test/ztest/unit/fast-get/CMakeLists.txt | 2 +- test/ztest/unit/math/advanced/functions/CMakeLists.txt | 2 +- test/ztest/unit/math/auditory/CMakeLists.txt | 2 +- test/ztest/unit/math/dct/CMakeLists.txt | 2 +- test/ztest/unit/math/fft/CMakeLists.txt | 2 +- test/ztest/unit/math/fft_multi/CMakeLists.txt | 2 +- test/ztest/unit/math/matrix/CMakeLists.txt | 2 +- test/ztest/unit/objpool/CMakeLists.txt | 2 +- tools/testbench/CMakeLists.txt | 2 +- tools/tplg_parser/CMakeLists.txt | 2 +- zephyr/CMakeLists.txt | 6 +++--- 290 files changed, 38 insertions(+), 38 deletions(-) rename src/{ => kernel}/platform/CMakeLists.txt (100%) rename src/{ => kernel}/platform/Kconfig (100%) rename src/{ => kernel}/platform/README.md (81%) rename src/{ => kernel}/platform/ace30/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/ace30/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/ace30/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/ace30/include/platform/platform.h (100%) rename src/{ => kernel}/platform/ace30/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/ace30/lib/clk.c (100%) rename src/{ => kernel}/platform/amd/acp_7_0/acp_7_0.x.in (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/arch/xtensa/config/core-isa.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/arch/xtensa/config/core-matmap.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/arch/xtensa/config/defs.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/arch/xtensa/config/specreg.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/arch/xtensa/config/system.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/arch/xtensa/config/tie-asm.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/arch/xtensa/config/tie.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/arch/xtensa/idma.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_datacache.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_externalregisters.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/arch/xtensa/xtensa-types.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/platform/drivers/interrupt.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/platform/fw_scratch_mem.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/platform/platform.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/include/platform/platform_misc.h (100%) rename src/{ => kernel}/platform/amd/acp_7_0/lib/clk.c (100%) rename src/{ => kernel}/platform/amd/acp_7_0/platform.c (100%) rename src/{ => kernel}/platform/amd/acp_7_x/include/arch/xtensa/tie/xt_datacache.h (100%) rename src/{ => kernel}/platform/amd/acp_7_x/include/platform/fw_scratch_mem.h (100%) rename src/{ => kernel}/platform/amd/acp_7_x/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/amd/acp_7_x/include/platform/platform.h (100%) rename src/{ => kernel}/platform/amd/acp_7_x/include/platform/platform_misc.h (100%) rename src/{ => kernel}/platform/amd/acp_7_x/lib/clk.c (100%) rename src/{ => kernel}/platform/amd/acp_7_x/platform.c (100%) rename src/{ => kernel}/platform/amd/common/include/platform/acp_dma.h (100%) rename src/{ => kernel}/platform/amd/common/include/platform/acp_dmic_dma.h (100%) rename src/{ => kernel}/platform/amd/common/include/platform/acp_sp_dma.h (100%) rename src/{ => kernel}/platform/amd/common/include/platform/drivers/idc.h (100%) rename src/{ => kernel}/platform/amd/common/include/platform/ipc.h (100%) rename src/{ => kernel}/platform/amd/common/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/amd/common/include/platform/lib/cpu.h (100%) rename src/{ => kernel}/platform/amd/common/include/platform/lib/dai.h (100%) rename src/{ => kernel}/platform/amd/common/include/platform/lib/dma.h (100%) rename src/{ => kernel}/platform/amd/common/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/amd/common/include/platform/lib/pm_runtime.h (100%) rename src/{ => kernel}/platform/amd/common/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/arch/xtensa/config/core-isa.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/arch/xtensa/config/core-matmap.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/arch/xtensa/config/defs.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/arch/xtensa/config/specreg.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/arch/xtensa/config/system.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/arch/xtensa/config/tie-asm.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/arch/xtensa/config/tie.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/arch/xtensa/tie/xt_datacache.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/platform/chip_offset_byte.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/platform/chip_registers.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/platform/drivers/interrupt.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/platform/fw_scratch_mem.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/include/platform/platform.h (100%) rename src/{ => kernel}/platform/amd/rembrandt/lib/clk.c (100%) rename src/{ => kernel}/platform/amd/rembrandt/lib/dai.c (100%) rename src/{ => kernel}/platform/amd/rembrandt/lib/dma.c (100%) rename src/{ => kernel}/platform/amd/rembrandt/platform.c (100%) rename src/{ => kernel}/platform/amd/rembrandt/rembrandt.x.in (100%) rename src/{ => kernel}/platform/imx8/imx8.x.in (100%) rename src/{ => kernel}/platform/imx8/include/arch/xtensa/config/core-isa.h (100%) rename src/{ => kernel}/platform/imx8/include/arch/xtensa/config/core-matmap.h (100%) rename src/{ => kernel}/platform/imx8/include/arch/xtensa/config/defs.h (100%) rename src/{ => kernel}/platform/imx8/include/arch/xtensa/config/specreg.h (100%) rename src/{ => kernel}/platform/imx8/include/arch/xtensa/config/system.h (100%) rename src/{ => kernel}/platform/imx8/include/arch/xtensa/config/tie-asm.h (100%) rename src/{ => kernel}/platform/imx8/include/arch/xtensa/config/tie.h (100%) rename src/{ => kernel}/platform/imx8/include/platform/drivers/idc.h (100%) rename src/{ => kernel}/platform/imx8/include/platform/drivers/interrupt.h (100%) rename src/{ => kernel}/platform/imx8/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/imx8/include/platform/lib/cpu.h (100%) rename src/{ => kernel}/platform/imx8/include/platform/lib/dai.h (100%) rename src/{ => kernel}/platform/imx8/include/platform/lib/dma.h (100%) rename src/{ => kernel}/platform/imx8/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/imx8/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/imx8/include/platform/platform.h (100%) rename src/{ => kernel}/platform/imx8/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/imx8/lib/clk.c (100%) rename src/{ => kernel}/platform/imx8/platform.c (100%) rename src/{ => kernel}/platform/imx8m/imx8m.x.in (100%) rename src/{ => kernel}/platform/imx8m/include/arch/xtensa/config/core-isa.h (100%) rename src/{ => kernel}/platform/imx8m/include/arch/xtensa/config/core-matmap.h (100%) rename src/{ => kernel}/platform/imx8m/include/arch/xtensa/config/defs.h (100%) rename src/{ => kernel}/platform/imx8m/include/arch/xtensa/config/specreg.h (100%) rename src/{ => kernel}/platform/imx8m/include/arch/xtensa/config/system.h (100%) rename src/{ => kernel}/platform/imx8m/include/arch/xtensa/config/tie-asm.h (100%) rename src/{ => kernel}/platform/imx8m/include/arch/xtensa/config/tie.h (100%) rename src/{ => kernel}/platform/imx8m/include/platform/drivers/idc.h (100%) rename src/{ => kernel}/platform/imx8m/include/platform/drivers/interrupt.h (100%) rename src/{ => kernel}/platform/imx8m/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/imx8m/include/platform/lib/cpu.h (100%) rename src/{ => kernel}/platform/imx8m/include/platform/lib/dai.h (100%) rename src/{ => kernel}/platform/imx8m/include/platform/lib/dma.h (100%) rename src/{ => kernel}/platform/imx8m/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/imx8m/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/imx8m/include/platform/lib/pm_runtime.h (100%) rename src/{ => kernel}/platform/imx8m/include/platform/platform.h (100%) rename src/{ => kernel}/platform/imx8m/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/imx8m/lib/clk.c (100%) rename src/{ => kernel}/platform/imx8m/platform.c (100%) rename src/{ => kernel}/platform/imx8m_cm7/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/imx8m_cm7/include/platform/lib/dma.h (100%) rename src/{ => kernel}/platform/imx8m_cm7/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/imx8m_cm7/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/imx8m_cm7/include/platform/platform.h (100%) rename src/{ => kernel}/platform/imx8m_cm7/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/imx8m_cm7/lib/clk.c (100%) rename src/{ => kernel}/platform/imx8m_cm7/linker/data-sections.ld (100%) rename src/{ => kernel}/platform/imx8m_cm7/platform.c (100%) rename src/{ => kernel}/platform/imx8ulp/imx8ulp.x.in (100%) rename src/{ => kernel}/platform/imx8ulp/include/arch/xtensa/config/core-isa.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/arch/xtensa/config/core-matmap.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/arch/xtensa/config/defs.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/arch/xtensa/config/specreg.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/arch/xtensa/config/system.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/arch/xtensa/config/tie-asm.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/arch/xtensa/config/tie.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/platform/drivers/idc.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/platform/drivers/interrupt.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/platform/lib/cpu.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/platform/lib/dai.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/platform/lib/dma.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/platform/platform.h (100%) rename src/{ => kernel}/platform/imx8ulp/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/imx8ulp/lib/clk.c (100%) rename src/{ => kernel}/platform/imx8ulp/platform.c (100%) rename src/{ => kernel}/platform/imx93_a55/include/platform/drivers/idc.h (100%) rename src/{ => kernel}/platform/imx93_a55/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/imx93_a55/include/platform/lib/dai.h (100%) rename src/{ => kernel}/platform/imx93_a55/include/platform/lib/dma.h (100%) rename src/{ => kernel}/platform/imx93_a55/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/imx93_a55/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/imx93_a55/include/platform/platform.h (100%) rename src/{ => kernel}/platform/imx93_a55/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/imx93_a55/lib/clk.c (100%) rename src/{ => kernel}/platform/imx93_a55/platform.c (100%) rename src/{ => kernel}/platform/imx95/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/imx95/include/platform/lib/dma.h (100%) rename src/{ => kernel}/platform/imx95/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/imx95/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/imx95/include/platform/platform.h (100%) rename src/{ => kernel}/platform/imx95/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/imx95/lib/clk.c (100%) rename src/{ => kernel}/platform/imx95/linker/data-sections.ld (100%) rename src/{ => kernel}/platform/imx95/platform.c (100%) rename src/{ => kernel}/platform/intel/ace/include/ace/lib/clk.h (100%) rename src/{ => kernel}/platform/intel/ace/include/ace/lib/mailbox.h (100%) rename src/{ => kernel}/platform/intel/ace/include/ace/lib/memory.h (100%) rename src/{ => kernel}/platform/intel/ace/include/ace/version.h (100%) rename src/{ => kernel}/platform/intel/ace/include/auth/intel_auth_api.h (100%) rename src/{ => kernel}/platform/intel/ace/include/auth/intel_status_logger_api.h (100%) rename src/{ => kernel}/platform/intel/ace/include/auth/intel_status_logger_iface.h (100%) rename src/{ => kernel}/platform/intel/ace/lib/watchdog.c (100%) rename src/{ => kernel}/platform/intel/ace/platform.c (100%) rename src/{ => kernel}/platform/intel/cavs/include/cavs/drivers/mn.h (100%) rename src/{ => kernel}/platform/intel/cavs/include/cavs/drivers/sideband-ipc.h (100%) rename src/{ => kernel}/platform/intel/cavs/include/cavs/drivers/timestamp.h (100%) rename src/{ => kernel}/platform/intel/cavs/include/cavs/ext_manifest.h (100%) rename src/{ => kernel}/platform/intel/cavs/include/cavs/lib/mailbox.h (100%) rename src/{ => kernel}/platform/intel/cavs/include/cavs/lib/memory.h (100%) rename src/{ => kernel}/platform/intel/cavs/include/cavs/lib/power_down.h (100%) rename src/{ => kernel}/platform/intel/cavs/include/cavs/mem_window.h (100%) rename src/{ => kernel}/platform/intel/cavs/include/cavs/trace/trace.h (100%) rename src/{ => kernel}/platform/intel/cavs/include/cavs/version.h (100%) rename src/{ => kernel}/platform/intel/cavs/platform.c (100%) rename src/{ => kernel}/platform/library/CMakeLists.txt (100%) rename src/{ => kernel}/platform/library/include/platform/drivers/idc.h (100%) rename src/{ => kernel}/platform/library/include/platform/drivers/interrupt.h (100%) rename src/{ => kernel}/platform/library/include/platform/drivers/pmc.h (100%) rename src/{ => kernel}/platform/library/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/library/include/platform/lib/cpu.h (100%) rename src/{ => kernel}/platform/library/include/platform/lib/dai.h (100%) rename src/{ => kernel}/platform/library/include/platform/lib/dma.h (100%) rename src/{ => kernel}/platform/library/include/platform/lib/ll_schedule.h (100%) rename src/{ => kernel}/platform/library/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/library/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/library/include/platform/lib/pm_runtime.h (100%) rename src/{ => kernel}/platform/library/include/platform/platform.h (100%) rename src/{ => kernel}/platform/library/init.c (100%) rename src/{ => kernel}/platform/library/lib/CMakeLists.txt (100%) rename src/{ => kernel}/platform/library/lib/alloc.c (100%) rename src/{ => kernel}/platform/library/lib/clk.c (100%) rename src/{ => kernel}/platform/library/lib/dai.c (100%) rename src/{ => kernel}/platform/library/lib/memory.c (100%) rename src/{ => kernel}/platform/library/lib/pm_runtime.c (100%) rename src/{ => kernel}/platform/library/lib/trace.c (100%) rename src/{ => kernel}/platform/library/panic.c (100%) rename src/{ => kernel}/platform/library/platform.c (100%) rename src/{ => kernel}/platform/library/schedule/CMakeLists.txt (100%) rename src/{ => kernel}/platform/library/schedule/edf_schedule.c (100%) rename src/{ => kernel}/platform/library/schedule/ll_schedule.c (100%) rename src/{ => kernel}/platform/library/schedule/schedule.c (100%) rename src/{ => kernel}/platform/lunarlake/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/lunarlake/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/lunarlake/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/lunarlake/include/platform/platform.h (100%) rename src/{ => kernel}/platform/lunarlake/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/lunarlake/lib/clk.c (100%) rename src/{ => kernel}/platform/meteorlake/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/meteorlake/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/meteorlake/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/meteorlake/include/platform/platform.h (100%) rename src/{ => kernel}/platform/meteorlake/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/meteorlake/lib/clk.c (100%) rename src/{ => kernel}/platform/mtk/dai.c (100%) rename src/{ => kernel}/platform/mtk/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/mtk/include/platform/lib/cpu.h (100%) rename src/{ => kernel}/platform/mtk/include/platform/lib/dai.h (100%) rename src/{ => kernel}/platform/mtk/include/platform/lib/dma.h (100%) rename src/{ => kernel}/platform/mtk/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/mtk/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/mtk/include/platform/lib/pm_runtime.h (100%) rename src/{ => kernel}/platform/mtk/include/platform/platform.h (100%) rename src/{ => kernel}/platform/mtk/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/mtk/platform.c (100%) rename src/{ => kernel}/platform/mtk/tools/README.txt (100%) rename src/{ => kernel}/platform/mtk/tools/build.sh (61%) rename src/{ => kernel}/platform/mtk/tools/mt-dai-gen.c (100%) rename src/{ => kernel}/platform/novalake/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/novalake/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/novalake/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/novalake/include/platform/platform.h (100%) rename src/{ => kernel}/platform/novalake/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/novalake/lib/clk.c (100%) rename src/{ => kernel}/platform/posix/dai.c (100%) rename src/{ => kernel}/platform/posix/dma.c (100%) rename src/{ => kernel}/platform/posix/fuzz.c (100%) rename src/{ => kernel}/platform/posix/include/platform/drivers/idc.h (100%) rename src/{ => kernel}/platform/posix/include/platform/drivers/interrupt.h (100%) rename src/{ => kernel}/platform/posix/include/platform/fuzz.h (100%) rename src/{ => kernel}/platform/posix/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/posix/include/platform/lib/cpu.h (100%) rename src/{ => kernel}/platform/posix/include/platform/lib/dai.h (100%) rename src/{ => kernel}/platform/posix/include/platform/lib/dma.h (100%) rename src/{ => kernel}/platform/posix/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/posix/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/posix/include/platform/lib/pm_runtime.h (100%) rename src/{ => kernel}/platform/posix/include/platform/platform.h (100%) rename src/{ => kernel}/platform/posix/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/posix/ipc.c (100%) rename src/{ => kernel}/platform/posix/posix.c (100%) rename src/{ => kernel}/platform/qemu_xtensa/include/platform/lib/clk.h (100%) rename src/{ => kernel}/platform/qemu_xtensa/include/platform/lib/dai.h (100%) rename src/{ => kernel}/platform/qemu_xtensa/include/platform/lib/dma.h (100%) rename src/{ => kernel}/platform/qemu_xtensa/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/qemu_xtensa/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/qemu_xtensa/include/platform/platform.h (100%) rename src/{ => kernel}/platform/qemu_xtensa/include/platform/trace/trace.h (100%) rename src/{ => kernel}/platform/qemu_xtensa/platform.c (100%) rename src/{ => kernel}/platform/tigerlake/include/platform/lib/mailbox.h (100%) rename src/{ => kernel}/platform/tigerlake/include/platform/lib/memory.h (100%) rename src/{ => kernel}/platform/tigerlake/include/platform/platform.h (100%) rename src/{ => kernel}/platform/tigerlake/include/platform/trace/trace.h (100%) diff --git a/AGENTS.md b/AGENTS.md index 2a807612c145..2c303eff1ec0 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -31,7 +31,7 @@ moving code, follow it: * **Place code in the right tier.** OS/HAL/boot/inter-core → a kernel-only directory (`kernel/`, which is built into the `sof_kernel` library, and the directories not yet - moved there: `platform/`, `trace/`); audio + moved there: `trace/`); audio application logic → `audio/`; reusable pure code → a shared-library directory (`lib/`, `math/`, `module/`) kept free of privileged operations. Each top-level directory's `README.md` states its tier in a "Runs in:" banner — keep it accurate. diff --git a/CODEOWNERS b/CODEOWNERS index 543435d5f3c3..e06147afb8cb 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -47,9 +47,9 @@ src/audio/module_adapter/* @ranj063 @jxstelter src/audio/google/* @thesofproject/google # platforms -src/platform/imx8/** @dbaluta -src/platform/amd/** @bhiregoudar @sunilkumardommati -src/platform/mtk/** @yaochunhung @kuanhsuncheng +src/kernel/platform/imx8/** @dbaluta +src/kernel/platform/amd/** @bhiregoudar @sunilkumardommati +src/kernel/platform/mtk/** @yaochunhung @kuanhsuncheng # drivers src/drivers/intel/dmic.c @singalsu diff --git a/scripts/kernel_user_partition.txt b/scripts/kernel_user_partition.txt index b5d6cda1c33f..83bb935b082e 100644 --- a/scripts/kernel_user_partition.txt +++ b/scripts/kernel_user_partition.txt @@ -46,7 +46,7 @@ src/include/sof/lib/cpu-clk-manager.h src/include/sof/lib/mailbox.h src/include/sof/lib/perf_cnt.h src/include/sof/lib/watchdog.h -src/platform/* +src/kernel/platform/* zephyr/include/rtos/clk.h zephyr/include/rtos/idc.h zephyr/include/rtos/interrupt.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index d364361e9a0e..f68fff8f434e 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,6 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -add_subdirectory(platform) add_subdirectory(ipc) add_subdirectory(audio) add_subdirectory(lib) diff --git a/src/Kconfig b/src/Kconfig index d72155332d4a..92486532f4dd 100644 --- a/src/Kconfig +++ b/src/Kconfig @@ -2,7 +2,7 @@ rsource "kernel/arch/Kconfig" -rsource "platform/Kconfig" +rsource "kernel/platform/Kconfig" rsource "audio/Kconfig" diff --git a/src/audio/README.md b/src/audio/README.md index 73a48ec7520e..39cc910c4dfa 100644 --- a/src/audio/README.md +++ b/src/audio/README.md @@ -15,5 +15,5 @@ modules in user context. When adding a component here, keep it inside the application tier: allocate through the module allocation syscalls (`mod_*`), mark any globals shared with the kernel using the partition -markers, and never call kernel-only subsystems (`kernel/drivers/`, `platform/`, `kernel/idc/`) +markers, and never call kernel-only subsystems (`kernel/drivers/`, `kernel/platform/`, `kernel/idc/`) directly. diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index 70fdece0ed35..06f828ba7113 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -74,7 +74,7 @@ its subdirectories span all tiers. | Tier | Meaning | Directories | |------|---------|-------------| -| **Kernel-only** | OS integration, HAL, boot, inter-core. Never executes in a user thread. Must not be reached directly from user code — only via a syscall. | `kernel/` (see [its README](../../../kernel/README.md)), `platform/`, `trace/` * | +| **Kernel-only** | OS integration, HAL, boot, inter-core. Never executes in a user thread. Must not be reached directly from user code — only via a syscall. | `kernel/` (see [its README](../../../kernel/README.md)), `trace/` * | | **Boundary / bridge** | Owns the syscalls and the thread / memory-domain setup. Split internally: some translation units run kernel-side, some user-side (e.g. `handler-kernel.c` vs `handler-user.c`). | `schedule/`, `ipc/`, `debug/debug_stream/` *, and (in the Zephyr integration tree) `zephyr/syscall/` | | **Application** | Audio pipeline / component logic that runs in user threads under userspace configs. | `audio/` | | **Shared library** | Context-agnostic code linked into whichever context calls it. Must stay "user-safe": no privileged operations (no direct MMIO, no spinlocks held across syscalls, no raw kernel object access) so it is safe when pulled into a user build. | `math/`, `module/` | diff --git a/src/ipc/README.md b/src/ipc/README.md index 844c13a84f1b..c240cc54a50e 100644 --- a/src/ipc/README.md +++ b/src/ipc/README.md @@ -45,7 +45,7 @@ graph TD ## Processing Flow -When the host writes a command to the IPC mailbox and triggers an interrupt, the hardware-specific driver (`src/platform/...`) catches the IRQ and eventually calls down into the IPC framework. +When the host writes a command to the IPC mailbox and triggers an interrupt, the hardware-specific driver (`src/kernel/platform/...`) catches the IRQ and eventually calls down into the IPC framework. Different RTOS environments (Zephyr vs. bare metal SOF native) handle the thread handoff differently. In Zephyr, this leverages the `k_work` queues heavily for `ipc_work_handler`. diff --git a/src/kernel/CMakeLists.txt b/src/kernel/CMakeLists.txt index 8e9afe9e6293..633434c4a1fe 100644 --- a/src/kernel/CMakeLists.txt +++ b/src/kernel/CMakeLists.txt @@ -24,6 +24,7 @@ if(zephyr) else() # Only built for the host add_subdirectory(arch) + add_subdirectory(platform) endif() # Built for Zephyr and for the host diff --git a/src/kernel/arch/host/CMakeLists.txt b/src/kernel/arch/host/CMakeLists.txt index bef4fdf9ec83..bc700f509094 100644 --- a/src/kernel/arch/host/CMakeLists.txt +++ b/src/kernel/arch/host/CMakeLists.txt @@ -2,7 +2,7 @@ # includes target_include_directories(sof_public_headers INTERFACE ${PROJECT_SOURCE_DIR}/src/kernel/arch/host/include) -target_include_directories(sof_public_headers INTERFACE ${PROJECT_SOURCE_DIR}/src/platform/library/include) +target_include_directories(sof_public_headers INTERFACE ${PROJECT_SOURCE_DIR}/src/kernel/platform/library/include) include(CheckCCompilerFlag) if (NOT CONFIG_SOF_ZEPHYR_STRICT_HEADERS) diff --git a/src/platform/CMakeLists.txt b/src/kernel/platform/CMakeLists.txt similarity index 100% rename from src/platform/CMakeLists.txt rename to src/kernel/platform/CMakeLists.txt diff --git a/src/platform/Kconfig b/src/kernel/platform/Kconfig similarity index 100% rename from src/platform/Kconfig rename to src/kernel/platform/Kconfig diff --git a/src/platform/README.md b/src/kernel/platform/README.md similarity index 81% rename from src/platform/README.md rename to src/kernel/platform/README.md index da20eb6569d2..b3aa7014290a 100644 --- a/src/platform/README.md +++ b/src/kernel/platform/README.md @@ -1,8 +1,8 @@ -# Platform Support (`src/platform`) +# Platform Support (`src/kernel/platform`) > **Runs in:** Kernel-only — platform bring-up, memory maps, interrupt wiring and IPC mailbox > windows are supervisor-context concerns and never run in a user thread. See -> [User/kernel split](../include/sof/userspace/README.md). +> [User/kernel split](../../include/sof/userspace/README.md). This directory contains SoC/platform-specific integration (e.g. Intel ACE, MediaTek, i.MX), including the Kconfig that selects the build target such as `PANTHERLAKE` (PTL). It configures diff --git a/src/platform/ace30/include/platform/lib/clk.h b/src/kernel/platform/ace30/include/platform/lib/clk.h similarity index 100% rename from src/platform/ace30/include/platform/lib/clk.h rename to src/kernel/platform/ace30/include/platform/lib/clk.h diff --git a/src/platform/ace30/include/platform/lib/mailbox.h b/src/kernel/platform/ace30/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/ace30/include/platform/lib/mailbox.h rename to src/kernel/platform/ace30/include/platform/lib/mailbox.h diff --git a/src/platform/ace30/include/platform/lib/memory.h b/src/kernel/platform/ace30/include/platform/lib/memory.h similarity index 100% rename from src/platform/ace30/include/platform/lib/memory.h rename to src/kernel/platform/ace30/include/platform/lib/memory.h diff --git a/src/platform/ace30/include/platform/platform.h b/src/kernel/platform/ace30/include/platform/platform.h similarity index 100% rename from src/platform/ace30/include/platform/platform.h rename to src/kernel/platform/ace30/include/platform/platform.h diff --git a/src/platform/ace30/include/platform/trace/trace.h b/src/kernel/platform/ace30/include/platform/trace/trace.h similarity index 100% rename from src/platform/ace30/include/platform/trace/trace.h rename to src/kernel/platform/ace30/include/platform/trace/trace.h diff --git a/src/platform/ace30/lib/clk.c b/src/kernel/platform/ace30/lib/clk.c similarity index 100% rename from src/platform/ace30/lib/clk.c rename to src/kernel/platform/ace30/lib/clk.c diff --git a/src/platform/amd/acp_7_0/acp_7_0.x.in b/src/kernel/platform/amd/acp_7_0/acp_7_0.x.in similarity index 100% rename from src/platform/amd/acp_7_0/acp_7_0.x.in rename to src/kernel/platform/amd/acp_7_0/acp_7_0.x.in diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/core-isa.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/core-isa.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/core-isa.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/core-isa.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/core-matmap.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/core-matmap.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/core-matmap.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/core-matmap.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/defs.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/defs.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/defs.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/defs.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/specreg.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/specreg.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/specreg.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/specreg.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/system.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/system.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/system.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/system.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/tie-asm.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/tie-asm.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/tie-asm.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/tie-asm.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/tie.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/tie.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/tie.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/tie.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/idma.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/idma.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/idma.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/idma.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_datacache.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_datacache.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_datacache.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_datacache.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_externalregisters.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_externalregisters.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_externalregisters.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_externalregisters.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/xtensa-types.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/xtensa-types.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/xtensa-types.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/xtensa-types.h diff --git a/src/platform/amd/acp_7_0/include/platform/drivers/interrupt.h b/src/kernel/platform/amd/acp_7_0/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/amd/acp_7_0/include/platform/drivers/interrupt.h rename to src/kernel/platform/amd/acp_7_0/include/platform/drivers/interrupt.h diff --git a/src/platform/amd/acp_7_0/include/platform/fw_scratch_mem.h b/src/kernel/platform/amd/acp_7_0/include/platform/fw_scratch_mem.h similarity index 100% rename from src/platform/amd/acp_7_0/include/platform/fw_scratch_mem.h rename to src/kernel/platform/amd/acp_7_0/include/platform/fw_scratch_mem.h diff --git a/src/platform/amd/acp_7_0/include/platform/lib/memory.h b/src/kernel/platform/amd/acp_7_0/include/platform/lib/memory.h similarity index 100% rename from src/platform/amd/acp_7_0/include/platform/lib/memory.h rename to src/kernel/platform/amd/acp_7_0/include/platform/lib/memory.h diff --git a/src/platform/amd/acp_7_0/include/platform/platform.h b/src/kernel/platform/amd/acp_7_0/include/platform/platform.h similarity index 100% rename from src/platform/amd/acp_7_0/include/platform/platform.h rename to src/kernel/platform/amd/acp_7_0/include/platform/platform.h diff --git a/src/platform/amd/acp_7_0/include/platform/platform_misc.h b/src/kernel/platform/amd/acp_7_0/include/platform/platform_misc.h similarity index 100% rename from src/platform/amd/acp_7_0/include/platform/platform_misc.h rename to src/kernel/platform/amd/acp_7_0/include/platform/platform_misc.h diff --git a/src/platform/amd/acp_7_0/lib/clk.c b/src/kernel/platform/amd/acp_7_0/lib/clk.c similarity index 100% rename from src/platform/amd/acp_7_0/lib/clk.c rename to src/kernel/platform/amd/acp_7_0/lib/clk.c diff --git a/src/platform/amd/acp_7_0/platform.c b/src/kernel/platform/amd/acp_7_0/platform.c similarity index 100% rename from src/platform/amd/acp_7_0/platform.c rename to src/kernel/platform/amd/acp_7_0/platform.c diff --git a/src/platform/amd/acp_7_x/include/arch/xtensa/tie/xt_datacache.h b/src/kernel/platform/amd/acp_7_x/include/arch/xtensa/tie/xt_datacache.h similarity index 100% rename from src/platform/amd/acp_7_x/include/arch/xtensa/tie/xt_datacache.h rename to src/kernel/platform/amd/acp_7_x/include/arch/xtensa/tie/xt_datacache.h diff --git a/src/platform/amd/acp_7_x/include/platform/fw_scratch_mem.h b/src/kernel/platform/amd/acp_7_x/include/platform/fw_scratch_mem.h similarity index 100% rename from src/platform/amd/acp_7_x/include/platform/fw_scratch_mem.h rename to src/kernel/platform/amd/acp_7_x/include/platform/fw_scratch_mem.h diff --git a/src/platform/amd/acp_7_x/include/platform/lib/memory.h b/src/kernel/platform/amd/acp_7_x/include/platform/lib/memory.h similarity index 100% rename from src/platform/amd/acp_7_x/include/platform/lib/memory.h rename to src/kernel/platform/amd/acp_7_x/include/platform/lib/memory.h diff --git a/src/platform/amd/acp_7_x/include/platform/platform.h b/src/kernel/platform/amd/acp_7_x/include/platform/platform.h similarity index 100% rename from src/platform/amd/acp_7_x/include/platform/platform.h rename to src/kernel/platform/amd/acp_7_x/include/platform/platform.h diff --git a/src/platform/amd/acp_7_x/include/platform/platform_misc.h b/src/kernel/platform/amd/acp_7_x/include/platform/platform_misc.h similarity index 100% rename from src/platform/amd/acp_7_x/include/platform/platform_misc.h rename to src/kernel/platform/amd/acp_7_x/include/platform/platform_misc.h diff --git a/src/platform/amd/acp_7_x/lib/clk.c b/src/kernel/platform/amd/acp_7_x/lib/clk.c similarity index 100% rename from src/platform/amd/acp_7_x/lib/clk.c rename to src/kernel/platform/amd/acp_7_x/lib/clk.c diff --git a/src/platform/amd/acp_7_x/platform.c b/src/kernel/platform/amd/acp_7_x/platform.c similarity index 100% rename from src/platform/amd/acp_7_x/platform.c rename to src/kernel/platform/amd/acp_7_x/platform.c diff --git a/src/platform/amd/common/include/platform/acp_dma.h b/src/kernel/platform/amd/common/include/platform/acp_dma.h similarity index 100% rename from src/platform/amd/common/include/platform/acp_dma.h rename to src/kernel/platform/amd/common/include/platform/acp_dma.h diff --git a/src/platform/amd/common/include/platform/acp_dmic_dma.h b/src/kernel/platform/amd/common/include/platform/acp_dmic_dma.h similarity index 100% rename from src/platform/amd/common/include/platform/acp_dmic_dma.h rename to src/kernel/platform/amd/common/include/platform/acp_dmic_dma.h diff --git a/src/platform/amd/common/include/platform/acp_sp_dma.h b/src/kernel/platform/amd/common/include/platform/acp_sp_dma.h similarity index 100% rename from src/platform/amd/common/include/platform/acp_sp_dma.h rename to src/kernel/platform/amd/common/include/platform/acp_sp_dma.h diff --git a/src/platform/amd/common/include/platform/drivers/idc.h b/src/kernel/platform/amd/common/include/platform/drivers/idc.h similarity index 100% rename from src/platform/amd/common/include/platform/drivers/idc.h rename to src/kernel/platform/amd/common/include/platform/drivers/idc.h diff --git a/src/platform/amd/common/include/platform/ipc.h b/src/kernel/platform/amd/common/include/platform/ipc.h similarity index 100% rename from src/platform/amd/common/include/platform/ipc.h rename to src/kernel/platform/amd/common/include/platform/ipc.h diff --git a/src/platform/amd/common/include/platform/lib/clk.h b/src/kernel/platform/amd/common/include/platform/lib/clk.h similarity index 100% rename from src/platform/amd/common/include/platform/lib/clk.h rename to src/kernel/platform/amd/common/include/platform/lib/clk.h diff --git a/src/platform/amd/common/include/platform/lib/cpu.h b/src/kernel/platform/amd/common/include/platform/lib/cpu.h similarity index 100% rename from src/platform/amd/common/include/platform/lib/cpu.h rename to src/kernel/platform/amd/common/include/platform/lib/cpu.h diff --git a/src/platform/amd/common/include/platform/lib/dai.h b/src/kernel/platform/amd/common/include/platform/lib/dai.h similarity index 100% rename from src/platform/amd/common/include/platform/lib/dai.h rename to src/kernel/platform/amd/common/include/platform/lib/dai.h diff --git a/src/platform/amd/common/include/platform/lib/dma.h b/src/kernel/platform/amd/common/include/platform/lib/dma.h similarity index 100% rename from src/platform/amd/common/include/platform/lib/dma.h rename to src/kernel/platform/amd/common/include/platform/lib/dma.h diff --git a/src/platform/amd/common/include/platform/lib/mailbox.h b/src/kernel/platform/amd/common/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/amd/common/include/platform/lib/mailbox.h rename to src/kernel/platform/amd/common/include/platform/lib/mailbox.h diff --git a/src/platform/amd/common/include/platform/lib/pm_runtime.h b/src/kernel/platform/amd/common/include/platform/lib/pm_runtime.h similarity index 100% rename from src/platform/amd/common/include/platform/lib/pm_runtime.h rename to src/kernel/platform/amd/common/include/platform/lib/pm_runtime.h diff --git a/src/platform/amd/common/include/platform/trace/trace.h b/src/kernel/platform/amd/common/include/platform/trace/trace.h similarity index 100% rename from src/platform/amd/common/include/platform/trace/trace.h rename to src/kernel/platform/amd/common/include/platform/trace/trace.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/core-isa.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/core-isa.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/core-isa.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/core-isa.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/core-matmap.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/core-matmap.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/core-matmap.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/core-matmap.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/defs.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/defs.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/defs.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/defs.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/specreg.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/specreg.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/specreg.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/specreg.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/system.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/system.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/system.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/system.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/tie-asm.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/tie-asm.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/tie-asm.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/tie-asm.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/tie.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/tie.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/tie.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/tie.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/tie/xt_datacache.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/tie/xt_datacache.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/tie/xt_datacache.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/tie/xt_datacache.h diff --git a/src/platform/amd/rembrandt/include/platform/chip_offset_byte.h b/src/kernel/platform/amd/rembrandt/include/platform/chip_offset_byte.h similarity index 100% rename from src/platform/amd/rembrandt/include/platform/chip_offset_byte.h rename to src/kernel/platform/amd/rembrandt/include/platform/chip_offset_byte.h diff --git a/src/platform/amd/rembrandt/include/platform/chip_registers.h b/src/kernel/platform/amd/rembrandt/include/platform/chip_registers.h similarity index 100% rename from src/platform/amd/rembrandt/include/platform/chip_registers.h rename to src/kernel/platform/amd/rembrandt/include/platform/chip_registers.h diff --git a/src/platform/amd/rembrandt/include/platform/drivers/interrupt.h b/src/kernel/platform/amd/rembrandt/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/amd/rembrandt/include/platform/drivers/interrupt.h rename to src/kernel/platform/amd/rembrandt/include/platform/drivers/interrupt.h diff --git a/src/platform/amd/rembrandt/include/platform/fw_scratch_mem.h b/src/kernel/platform/amd/rembrandt/include/platform/fw_scratch_mem.h similarity index 100% rename from src/platform/amd/rembrandt/include/platform/fw_scratch_mem.h rename to src/kernel/platform/amd/rembrandt/include/platform/fw_scratch_mem.h diff --git a/src/platform/amd/rembrandt/include/platform/lib/memory.h b/src/kernel/platform/amd/rembrandt/include/platform/lib/memory.h similarity index 100% rename from src/platform/amd/rembrandt/include/platform/lib/memory.h rename to src/kernel/platform/amd/rembrandt/include/platform/lib/memory.h diff --git a/src/platform/amd/rembrandt/include/platform/platform.h b/src/kernel/platform/amd/rembrandt/include/platform/platform.h similarity index 100% rename from src/platform/amd/rembrandt/include/platform/platform.h rename to src/kernel/platform/amd/rembrandt/include/platform/platform.h diff --git a/src/platform/amd/rembrandt/lib/clk.c b/src/kernel/platform/amd/rembrandt/lib/clk.c similarity index 100% rename from src/platform/amd/rembrandt/lib/clk.c rename to src/kernel/platform/amd/rembrandt/lib/clk.c diff --git a/src/platform/amd/rembrandt/lib/dai.c b/src/kernel/platform/amd/rembrandt/lib/dai.c similarity index 100% rename from src/platform/amd/rembrandt/lib/dai.c rename to src/kernel/platform/amd/rembrandt/lib/dai.c diff --git a/src/platform/amd/rembrandt/lib/dma.c b/src/kernel/platform/amd/rembrandt/lib/dma.c similarity index 100% rename from src/platform/amd/rembrandt/lib/dma.c rename to src/kernel/platform/amd/rembrandt/lib/dma.c diff --git a/src/platform/amd/rembrandt/platform.c b/src/kernel/platform/amd/rembrandt/platform.c similarity index 100% rename from src/platform/amd/rembrandt/platform.c rename to src/kernel/platform/amd/rembrandt/platform.c diff --git a/src/platform/amd/rembrandt/rembrandt.x.in b/src/kernel/platform/amd/rembrandt/rembrandt.x.in similarity index 100% rename from src/platform/amd/rembrandt/rembrandt.x.in rename to src/kernel/platform/amd/rembrandt/rembrandt.x.in diff --git a/src/platform/imx8/imx8.x.in b/src/kernel/platform/imx8/imx8.x.in similarity index 100% rename from src/platform/imx8/imx8.x.in rename to src/kernel/platform/imx8/imx8.x.in diff --git a/src/platform/imx8/include/arch/xtensa/config/core-isa.h b/src/kernel/platform/imx8/include/arch/xtensa/config/core-isa.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/core-isa.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/core-isa.h diff --git a/src/platform/imx8/include/arch/xtensa/config/core-matmap.h b/src/kernel/platform/imx8/include/arch/xtensa/config/core-matmap.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/core-matmap.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/core-matmap.h diff --git a/src/platform/imx8/include/arch/xtensa/config/defs.h b/src/kernel/platform/imx8/include/arch/xtensa/config/defs.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/defs.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/defs.h diff --git a/src/platform/imx8/include/arch/xtensa/config/specreg.h b/src/kernel/platform/imx8/include/arch/xtensa/config/specreg.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/specreg.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/specreg.h diff --git a/src/platform/imx8/include/arch/xtensa/config/system.h b/src/kernel/platform/imx8/include/arch/xtensa/config/system.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/system.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/system.h diff --git a/src/platform/imx8/include/arch/xtensa/config/tie-asm.h b/src/kernel/platform/imx8/include/arch/xtensa/config/tie-asm.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/tie-asm.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/tie-asm.h diff --git a/src/platform/imx8/include/arch/xtensa/config/tie.h b/src/kernel/platform/imx8/include/arch/xtensa/config/tie.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/tie.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/tie.h diff --git a/src/platform/imx8/include/platform/drivers/idc.h b/src/kernel/platform/imx8/include/platform/drivers/idc.h similarity index 100% rename from src/platform/imx8/include/platform/drivers/idc.h rename to src/kernel/platform/imx8/include/platform/drivers/idc.h diff --git a/src/platform/imx8/include/platform/drivers/interrupt.h b/src/kernel/platform/imx8/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/imx8/include/platform/drivers/interrupt.h rename to src/kernel/platform/imx8/include/platform/drivers/interrupt.h diff --git a/src/platform/imx8/include/platform/lib/clk.h b/src/kernel/platform/imx8/include/platform/lib/clk.h similarity index 100% rename from src/platform/imx8/include/platform/lib/clk.h rename to src/kernel/platform/imx8/include/platform/lib/clk.h diff --git a/src/platform/imx8/include/platform/lib/cpu.h b/src/kernel/platform/imx8/include/platform/lib/cpu.h similarity index 100% rename from src/platform/imx8/include/platform/lib/cpu.h rename to src/kernel/platform/imx8/include/platform/lib/cpu.h diff --git a/src/platform/imx8/include/platform/lib/dai.h b/src/kernel/platform/imx8/include/platform/lib/dai.h similarity index 100% rename from src/platform/imx8/include/platform/lib/dai.h rename to src/kernel/platform/imx8/include/platform/lib/dai.h diff --git a/src/platform/imx8/include/platform/lib/dma.h b/src/kernel/platform/imx8/include/platform/lib/dma.h similarity index 100% rename from src/platform/imx8/include/platform/lib/dma.h rename to src/kernel/platform/imx8/include/platform/lib/dma.h diff --git a/src/platform/imx8/include/platform/lib/mailbox.h b/src/kernel/platform/imx8/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/imx8/include/platform/lib/mailbox.h rename to src/kernel/platform/imx8/include/platform/lib/mailbox.h diff --git a/src/platform/imx8/include/platform/lib/memory.h b/src/kernel/platform/imx8/include/platform/lib/memory.h similarity index 100% rename from src/platform/imx8/include/platform/lib/memory.h rename to src/kernel/platform/imx8/include/platform/lib/memory.h diff --git a/src/platform/imx8/include/platform/platform.h b/src/kernel/platform/imx8/include/platform/platform.h similarity index 100% rename from src/platform/imx8/include/platform/platform.h rename to src/kernel/platform/imx8/include/platform/platform.h diff --git a/src/platform/imx8/include/platform/trace/trace.h b/src/kernel/platform/imx8/include/platform/trace/trace.h similarity index 100% rename from src/platform/imx8/include/platform/trace/trace.h rename to src/kernel/platform/imx8/include/platform/trace/trace.h diff --git a/src/platform/imx8/lib/clk.c b/src/kernel/platform/imx8/lib/clk.c similarity index 100% rename from src/platform/imx8/lib/clk.c rename to src/kernel/platform/imx8/lib/clk.c diff --git a/src/platform/imx8/platform.c b/src/kernel/platform/imx8/platform.c similarity index 100% rename from src/platform/imx8/platform.c rename to src/kernel/platform/imx8/platform.c diff --git a/src/platform/imx8m/imx8m.x.in b/src/kernel/platform/imx8m/imx8m.x.in similarity index 100% rename from src/platform/imx8m/imx8m.x.in rename to src/kernel/platform/imx8m/imx8m.x.in diff --git a/src/platform/imx8m/include/arch/xtensa/config/core-isa.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/core-isa.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/core-isa.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/core-isa.h diff --git a/src/platform/imx8m/include/arch/xtensa/config/core-matmap.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/core-matmap.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/core-matmap.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/core-matmap.h diff --git a/src/platform/imx8m/include/arch/xtensa/config/defs.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/defs.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/defs.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/defs.h diff --git a/src/platform/imx8m/include/arch/xtensa/config/specreg.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/specreg.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/specreg.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/specreg.h diff --git a/src/platform/imx8m/include/arch/xtensa/config/system.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/system.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/system.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/system.h diff --git a/src/platform/imx8m/include/arch/xtensa/config/tie-asm.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/tie-asm.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/tie-asm.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/tie-asm.h diff --git a/src/platform/imx8m/include/arch/xtensa/config/tie.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/tie.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/tie.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/tie.h diff --git a/src/platform/imx8m/include/platform/drivers/idc.h b/src/kernel/platform/imx8m/include/platform/drivers/idc.h similarity index 100% rename from src/platform/imx8m/include/platform/drivers/idc.h rename to src/kernel/platform/imx8m/include/platform/drivers/idc.h diff --git a/src/platform/imx8m/include/platform/drivers/interrupt.h b/src/kernel/platform/imx8m/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/imx8m/include/platform/drivers/interrupt.h rename to src/kernel/platform/imx8m/include/platform/drivers/interrupt.h diff --git a/src/platform/imx8m/include/platform/lib/clk.h b/src/kernel/platform/imx8m/include/platform/lib/clk.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/clk.h rename to src/kernel/platform/imx8m/include/platform/lib/clk.h diff --git a/src/platform/imx8m/include/platform/lib/cpu.h b/src/kernel/platform/imx8m/include/platform/lib/cpu.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/cpu.h rename to src/kernel/platform/imx8m/include/platform/lib/cpu.h diff --git a/src/platform/imx8m/include/platform/lib/dai.h b/src/kernel/platform/imx8m/include/platform/lib/dai.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/dai.h rename to src/kernel/platform/imx8m/include/platform/lib/dai.h diff --git a/src/platform/imx8m/include/platform/lib/dma.h b/src/kernel/platform/imx8m/include/platform/lib/dma.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/dma.h rename to src/kernel/platform/imx8m/include/platform/lib/dma.h diff --git a/src/platform/imx8m/include/platform/lib/mailbox.h b/src/kernel/platform/imx8m/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/mailbox.h rename to src/kernel/platform/imx8m/include/platform/lib/mailbox.h diff --git a/src/platform/imx8m/include/platform/lib/memory.h b/src/kernel/platform/imx8m/include/platform/lib/memory.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/memory.h rename to src/kernel/platform/imx8m/include/platform/lib/memory.h diff --git a/src/platform/imx8m/include/platform/lib/pm_runtime.h b/src/kernel/platform/imx8m/include/platform/lib/pm_runtime.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/pm_runtime.h rename to src/kernel/platform/imx8m/include/platform/lib/pm_runtime.h diff --git a/src/platform/imx8m/include/platform/platform.h b/src/kernel/platform/imx8m/include/platform/platform.h similarity index 100% rename from src/platform/imx8m/include/platform/platform.h rename to src/kernel/platform/imx8m/include/platform/platform.h diff --git a/src/platform/imx8m/include/platform/trace/trace.h b/src/kernel/platform/imx8m/include/platform/trace/trace.h similarity index 100% rename from src/platform/imx8m/include/platform/trace/trace.h rename to src/kernel/platform/imx8m/include/platform/trace/trace.h diff --git a/src/platform/imx8m/lib/clk.c b/src/kernel/platform/imx8m/lib/clk.c similarity index 100% rename from src/platform/imx8m/lib/clk.c rename to src/kernel/platform/imx8m/lib/clk.c diff --git a/src/platform/imx8m/platform.c b/src/kernel/platform/imx8m/platform.c similarity index 100% rename from src/platform/imx8m/platform.c rename to src/kernel/platform/imx8m/platform.c diff --git a/src/platform/imx8m_cm7/include/platform/lib/clk.h b/src/kernel/platform/imx8m_cm7/include/platform/lib/clk.h similarity index 100% rename from src/platform/imx8m_cm7/include/platform/lib/clk.h rename to src/kernel/platform/imx8m_cm7/include/platform/lib/clk.h diff --git a/src/platform/imx8m_cm7/include/platform/lib/dma.h b/src/kernel/platform/imx8m_cm7/include/platform/lib/dma.h similarity index 100% rename from src/platform/imx8m_cm7/include/platform/lib/dma.h rename to src/kernel/platform/imx8m_cm7/include/platform/lib/dma.h diff --git a/src/platform/imx8m_cm7/include/platform/lib/mailbox.h b/src/kernel/platform/imx8m_cm7/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/imx8m_cm7/include/platform/lib/mailbox.h rename to src/kernel/platform/imx8m_cm7/include/platform/lib/mailbox.h diff --git a/src/platform/imx8m_cm7/include/platform/lib/memory.h b/src/kernel/platform/imx8m_cm7/include/platform/lib/memory.h similarity index 100% rename from src/platform/imx8m_cm7/include/platform/lib/memory.h rename to src/kernel/platform/imx8m_cm7/include/platform/lib/memory.h diff --git a/src/platform/imx8m_cm7/include/platform/platform.h b/src/kernel/platform/imx8m_cm7/include/platform/platform.h similarity index 100% rename from src/platform/imx8m_cm7/include/platform/platform.h rename to src/kernel/platform/imx8m_cm7/include/platform/platform.h diff --git a/src/platform/imx8m_cm7/include/platform/trace/trace.h b/src/kernel/platform/imx8m_cm7/include/platform/trace/trace.h similarity index 100% rename from src/platform/imx8m_cm7/include/platform/trace/trace.h rename to src/kernel/platform/imx8m_cm7/include/platform/trace/trace.h diff --git a/src/platform/imx8m_cm7/lib/clk.c b/src/kernel/platform/imx8m_cm7/lib/clk.c similarity index 100% rename from src/platform/imx8m_cm7/lib/clk.c rename to src/kernel/platform/imx8m_cm7/lib/clk.c diff --git a/src/platform/imx8m_cm7/linker/data-sections.ld b/src/kernel/platform/imx8m_cm7/linker/data-sections.ld similarity index 100% rename from src/platform/imx8m_cm7/linker/data-sections.ld rename to src/kernel/platform/imx8m_cm7/linker/data-sections.ld diff --git a/src/platform/imx8m_cm7/platform.c b/src/kernel/platform/imx8m_cm7/platform.c similarity index 100% rename from src/platform/imx8m_cm7/platform.c rename to src/kernel/platform/imx8m_cm7/platform.c diff --git a/src/platform/imx8ulp/imx8ulp.x.in b/src/kernel/platform/imx8ulp/imx8ulp.x.in similarity index 100% rename from src/platform/imx8ulp/imx8ulp.x.in rename to src/kernel/platform/imx8ulp/imx8ulp.x.in diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/core-isa.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/core-isa.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/core-isa.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/core-isa.h diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/core-matmap.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/core-matmap.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/core-matmap.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/core-matmap.h diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/defs.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/defs.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/defs.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/defs.h diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/specreg.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/specreg.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/specreg.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/specreg.h diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/system.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/system.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/system.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/system.h diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/tie-asm.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/tie-asm.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/tie-asm.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/tie-asm.h diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/tie.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/tie.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/tie.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/tie.h diff --git a/src/platform/imx8ulp/include/platform/drivers/idc.h b/src/kernel/platform/imx8ulp/include/platform/drivers/idc.h similarity index 100% rename from src/platform/imx8ulp/include/platform/drivers/idc.h rename to src/kernel/platform/imx8ulp/include/platform/drivers/idc.h diff --git a/src/platform/imx8ulp/include/platform/drivers/interrupt.h b/src/kernel/platform/imx8ulp/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/imx8ulp/include/platform/drivers/interrupt.h rename to src/kernel/platform/imx8ulp/include/platform/drivers/interrupt.h diff --git a/src/platform/imx8ulp/include/platform/lib/clk.h b/src/kernel/platform/imx8ulp/include/platform/lib/clk.h similarity index 100% rename from src/platform/imx8ulp/include/platform/lib/clk.h rename to src/kernel/platform/imx8ulp/include/platform/lib/clk.h diff --git a/src/platform/imx8ulp/include/platform/lib/cpu.h b/src/kernel/platform/imx8ulp/include/platform/lib/cpu.h similarity index 100% rename from src/platform/imx8ulp/include/platform/lib/cpu.h rename to src/kernel/platform/imx8ulp/include/platform/lib/cpu.h diff --git a/src/platform/imx8ulp/include/platform/lib/dai.h b/src/kernel/platform/imx8ulp/include/platform/lib/dai.h similarity index 100% rename from src/platform/imx8ulp/include/platform/lib/dai.h rename to src/kernel/platform/imx8ulp/include/platform/lib/dai.h diff --git a/src/platform/imx8ulp/include/platform/lib/dma.h b/src/kernel/platform/imx8ulp/include/platform/lib/dma.h similarity index 100% rename from src/platform/imx8ulp/include/platform/lib/dma.h rename to src/kernel/platform/imx8ulp/include/platform/lib/dma.h diff --git a/src/platform/imx8ulp/include/platform/lib/mailbox.h b/src/kernel/platform/imx8ulp/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/imx8ulp/include/platform/lib/mailbox.h rename to src/kernel/platform/imx8ulp/include/platform/lib/mailbox.h diff --git a/src/platform/imx8ulp/include/platform/lib/memory.h b/src/kernel/platform/imx8ulp/include/platform/lib/memory.h similarity index 100% rename from src/platform/imx8ulp/include/platform/lib/memory.h rename to src/kernel/platform/imx8ulp/include/platform/lib/memory.h diff --git a/src/platform/imx8ulp/include/platform/platform.h b/src/kernel/platform/imx8ulp/include/platform/platform.h similarity index 100% rename from src/platform/imx8ulp/include/platform/platform.h rename to src/kernel/platform/imx8ulp/include/platform/platform.h diff --git a/src/platform/imx8ulp/include/platform/trace/trace.h b/src/kernel/platform/imx8ulp/include/platform/trace/trace.h similarity index 100% rename from src/platform/imx8ulp/include/platform/trace/trace.h rename to src/kernel/platform/imx8ulp/include/platform/trace/trace.h diff --git a/src/platform/imx8ulp/lib/clk.c b/src/kernel/platform/imx8ulp/lib/clk.c similarity index 100% rename from src/platform/imx8ulp/lib/clk.c rename to src/kernel/platform/imx8ulp/lib/clk.c diff --git a/src/platform/imx8ulp/platform.c b/src/kernel/platform/imx8ulp/platform.c similarity index 100% rename from src/platform/imx8ulp/platform.c rename to src/kernel/platform/imx8ulp/platform.c diff --git a/src/platform/imx93_a55/include/platform/drivers/idc.h b/src/kernel/platform/imx93_a55/include/platform/drivers/idc.h similarity index 100% rename from src/platform/imx93_a55/include/platform/drivers/idc.h rename to src/kernel/platform/imx93_a55/include/platform/drivers/idc.h diff --git a/src/platform/imx93_a55/include/platform/lib/clk.h b/src/kernel/platform/imx93_a55/include/platform/lib/clk.h similarity index 100% rename from src/platform/imx93_a55/include/platform/lib/clk.h rename to src/kernel/platform/imx93_a55/include/platform/lib/clk.h diff --git a/src/platform/imx93_a55/include/platform/lib/dai.h b/src/kernel/platform/imx93_a55/include/platform/lib/dai.h similarity index 100% rename from src/platform/imx93_a55/include/platform/lib/dai.h rename to src/kernel/platform/imx93_a55/include/platform/lib/dai.h diff --git a/src/platform/imx93_a55/include/platform/lib/dma.h b/src/kernel/platform/imx93_a55/include/platform/lib/dma.h similarity index 100% rename from src/platform/imx93_a55/include/platform/lib/dma.h rename to src/kernel/platform/imx93_a55/include/platform/lib/dma.h diff --git a/src/platform/imx93_a55/include/platform/lib/mailbox.h b/src/kernel/platform/imx93_a55/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/imx93_a55/include/platform/lib/mailbox.h rename to src/kernel/platform/imx93_a55/include/platform/lib/mailbox.h diff --git a/src/platform/imx93_a55/include/platform/lib/memory.h b/src/kernel/platform/imx93_a55/include/platform/lib/memory.h similarity index 100% rename from src/platform/imx93_a55/include/platform/lib/memory.h rename to src/kernel/platform/imx93_a55/include/platform/lib/memory.h diff --git a/src/platform/imx93_a55/include/platform/platform.h b/src/kernel/platform/imx93_a55/include/platform/platform.h similarity index 100% rename from src/platform/imx93_a55/include/platform/platform.h rename to src/kernel/platform/imx93_a55/include/platform/platform.h diff --git a/src/platform/imx93_a55/include/platform/trace/trace.h b/src/kernel/platform/imx93_a55/include/platform/trace/trace.h similarity index 100% rename from src/platform/imx93_a55/include/platform/trace/trace.h rename to src/kernel/platform/imx93_a55/include/platform/trace/trace.h diff --git a/src/platform/imx93_a55/lib/clk.c b/src/kernel/platform/imx93_a55/lib/clk.c similarity index 100% rename from src/platform/imx93_a55/lib/clk.c rename to src/kernel/platform/imx93_a55/lib/clk.c diff --git a/src/platform/imx93_a55/platform.c b/src/kernel/platform/imx93_a55/platform.c similarity index 100% rename from src/platform/imx93_a55/platform.c rename to src/kernel/platform/imx93_a55/platform.c diff --git a/src/platform/imx95/include/platform/lib/clk.h b/src/kernel/platform/imx95/include/platform/lib/clk.h similarity index 100% rename from src/platform/imx95/include/platform/lib/clk.h rename to src/kernel/platform/imx95/include/platform/lib/clk.h diff --git a/src/platform/imx95/include/platform/lib/dma.h b/src/kernel/platform/imx95/include/platform/lib/dma.h similarity index 100% rename from src/platform/imx95/include/platform/lib/dma.h rename to src/kernel/platform/imx95/include/platform/lib/dma.h diff --git a/src/platform/imx95/include/platform/lib/mailbox.h b/src/kernel/platform/imx95/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/imx95/include/platform/lib/mailbox.h rename to src/kernel/platform/imx95/include/platform/lib/mailbox.h diff --git a/src/platform/imx95/include/platform/lib/memory.h b/src/kernel/platform/imx95/include/platform/lib/memory.h similarity index 100% rename from src/platform/imx95/include/platform/lib/memory.h rename to src/kernel/platform/imx95/include/platform/lib/memory.h diff --git a/src/platform/imx95/include/platform/platform.h b/src/kernel/platform/imx95/include/platform/platform.h similarity index 100% rename from src/platform/imx95/include/platform/platform.h rename to src/kernel/platform/imx95/include/platform/platform.h diff --git a/src/platform/imx95/include/platform/trace/trace.h b/src/kernel/platform/imx95/include/platform/trace/trace.h similarity index 100% rename from src/platform/imx95/include/platform/trace/trace.h rename to src/kernel/platform/imx95/include/platform/trace/trace.h diff --git a/src/platform/imx95/lib/clk.c b/src/kernel/platform/imx95/lib/clk.c similarity index 100% rename from src/platform/imx95/lib/clk.c rename to src/kernel/platform/imx95/lib/clk.c diff --git a/src/platform/imx95/linker/data-sections.ld b/src/kernel/platform/imx95/linker/data-sections.ld similarity index 100% rename from src/platform/imx95/linker/data-sections.ld rename to src/kernel/platform/imx95/linker/data-sections.ld diff --git a/src/platform/imx95/platform.c b/src/kernel/platform/imx95/platform.c similarity index 100% rename from src/platform/imx95/platform.c rename to src/kernel/platform/imx95/platform.c diff --git a/src/platform/intel/ace/include/ace/lib/clk.h b/src/kernel/platform/intel/ace/include/ace/lib/clk.h similarity index 100% rename from src/platform/intel/ace/include/ace/lib/clk.h rename to src/kernel/platform/intel/ace/include/ace/lib/clk.h diff --git a/src/platform/intel/ace/include/ace/lib/mailbox.h b/src/kernel/platform/intel/ace/include/ace/lib/mailbox.h similarity index 100% rename from src/platform/intel/ace/include/ace/lib/mailbox.h rename to src/kernel/platform/intel/ace/include/ace/lib/mailbox.h diff --git a/src/platform/intel/ace/include/ace/lib/memory.h b/src/kernel/platform/intel/ace/include/ace/lib/memory.h similarity index 100% rename from src/platform/intel/ace/include/ace/lib/memory.h rename to src/kernel/platform/intel/ace/include/ace/lib/memory.h diff --git a/src/platform/intel/ace/include/ace/version.h b/src/kernel/platform/intel/ace/include/ace/version.h similarity index 100% rename from src/platform/intel/ace/include/ace/version.h rename to src/kernel/platform/intel/ace/include/ace/version.h diff --git a/src/platform/intel/ace/include/auth/intel_auth_api.h b/src/kernel/platform/intel/ace/include/auth/intel_auth_api.h similarity index 100% rename from src/platform/intel/ace/include/auth/intel_auth_api.h rename to src/kernel/platform/intel/ace/include/auth/intel_auth_api.h diff --git a/src/platform/intel/ace/include/auth/intel_status_logger_api.h b/src/kernel/platform/intel/ace/include/auth/intel_status_logger_api.h similarity index 100% rename from src/platform/intel/ace/include/auth/intel_status_logger_api.h rename to src/kernel/platform/intel/ace/include/auth/intel_status_logger_api.h diff --git a/src/platform/intel/ace/include/auth/intel_status_logger_iface.h b/src/kernel/platform/intel/ace/include/auth/intel_status_logger_iface.h similarity index 100% rename from src/platform/intel/ace/include/auth/intel_status_logger_iface.h rename to src/kernel/platform/intel/ace/include/auth/intel_status_logger_iface.h diff --git a/src/platform/intel/ace/lib/watchdog.c b/src/kernel/platform/intel/ace/lib/watchdog.c similarity index 100% rename from src/platform/intel/ace/lib/watchdog.c rename to src/kernel/platform/intel/ace/lib/watchdog.c diff --git a/src/platform/intel/ace/platform.c b/src/kernel/platform/intel/ace/platform.c similarity index 100% rename from src/platform/intel/ace/platform.c rename to src/kernel/platform/intel/ace/platform.c diff --git a/src/platform/intel/cavs/include/cavs/drivers/mn.h b/src/kernel/platform/intel/cavs/include/cavs/drivers/mn.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/drivers/mn.h rename to src/kernel/platform/intel/cavs/include/cavs/drivers/mn.h diff --git a/src/platform/intel/cavs/include/cavs/drivers/sideband-ipc.h b/src/kernel/platform/intel/cavs/include/cavs/drivers/sideband-ipc.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/drivers/sideband-ipc.h rename to src/kernel/platform/intel/cavs/include/cavs/drivers/sideband-ipc.h diff --git a/src/platform/intel/cavs/include/cavs/drivers/timestamp.h b/src/kernel/platform/intel/cavs/include/cavs/drivers/timestamp.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/drivers/timestamp.h rename to src/kernel/platform/intel/cavs/include/cavs/drivers/timestamp.h diff --git a/src/platform/intel/cavs/include/cavs/ext_manifest.h b/src/kernel/platform/intel/cavs/include/cavs/ext_manifest.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/ext_manifest.h rename to src/kernel/platform/intel/cavs/include/cavs/ext_manifest.h diff --git a/src/platform/intel/cavs/include/cavs/lib/mailbox.h b/src/kernel/platform/intel/cavs/include/cavs/lib/mailbox.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/lib/mailbox.h rename to src/kernel/platform/intel/cavs/include/cavs/lib/mailbox.h diff --git a/src/platform/intel/cavs/include/cavs/lib/memory.h b/src/kernel/platform/intel/cavs/include/cavs/lib/memory.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/lib/memory.h rename to src/kernel/platform/intel/cavs/include/cavs/lib/memory.h diff --git a/src/platform/intel/cavs/include/cavs/lib/power_down.h b/src/kernel/platform/intel/cavs/include/cavs/lib/power_down.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/lib/power_down.h rename to src/kernel/platform/intel/cavs/include/cavs/lib/power_down.h diff --git a/src/platform/intel/cavs/include/cavs/mem_window.h b/src/kernel/platform/intel/cavs/include/cavs/mem_window.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/mem_window.h rename to src/kernel/platform/intel/cavs/include/cavs/mem_window.h diff --git a/src/platform/intel/cavs/include/cavs/trace/trace.h b/src/kernel/platform/intel/cavs/include/cavs/trace/trace.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/trace/trace.h rename to src/kernel/platform/intel/cavs/include/cavs/trace/trace.h diff --git a/src/platform/intel/cavs/include/cavs/version.h b/src/kernel/platform/intel/cavs/include/cavs/version.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/version.h rename to src/kernel/platform/intel/cavs/include/cavs/version.h diff --git a/src/platform/intel/cavs/platform.c b/src/kernel/platform/intel/cavs/platform.c similarity index 100% rename from src/platform/intel/cavs/platform.c rename to src/kernel/platform/intel/cavs/platform.c diff --git a/src/platform/library/CMakeLists.txt b/src/kernel/platform/library/CMakeLists.txt similarity index 100% rename from src/platform/library/CMakeLists.txt rename to src/kernel/platform/library/CMakeLists.txt diff --git a/src/platform/library/include/platform/drivers/idc.h b/src/kernel/platform/library/include/platform/drivers/idc.h similarity index 100% rename from src/platform/library/include/platform/drivers/idc.h rename to src/kernel/platform/library/include/platform/drivers/idc.h diff --git a/src/platform/library/include/platform/drivers/interrupt.h b/src/kernel/platform/library/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/library/include/platform/drivers/interrupt.h rename to src/kernel/platform/library/include/platform/drivers/interrupt.h diff --git a/src/platform/library/include/platform/drivers/pmc.h b/src/kernel/platform/library/include/platform/drivers/pmc.h similarity index 100% rename from src/platform/library/include/platform/drivers/pmc.h rename to src/kernel/platform/library/include/platform/drivers/pmc.h diff --git a/src/platform/library/include/platform/lib/clk.h b/src/kernel/platform/library/include/platform/lib/clk.h similarity index 100% rename from src/platform/library/include/platform/lib/clk.h rename to src/kernel/platform/library/include/platform/lib/clk.h diff --git a/src/platform/library/include/platform/lib/cpu.h b/src/kernel/platform/library/include/platform/lib/cpu.h similarity index 100% rename from src/platform/library/include/platform/lib/cpu.h rename to src/kernel/platform/library/include/platform/lib/cpu.h diff --git a/src/platform/library/include/platform/lib/dai.h b/src/kernel/platform/library/include/platform/lib/dai.h similarity index 100% rename from src/platform/library/include/platform/lib/dai.h rename to src/kernel/platform/library/include/platform/lib/dai.h diff --git a/src/platform/library/include/platform/lib/dma.h b/src/kernel/platform/library/include/platform/lib/dma.h similarity index 100% rename from src/platform/library/include/platform/lib/dma.h rename to src/kernel/platform/library/include/platform/lib/dma.h diff --git a/src/platform/library/include/platform/lib/ll_schedule.h b/src/kernel/platform/library/include/platform/lib/ll_schedule.h similarity index 100% rename from src/platform/library/include/platform/lib/ll_schedule.h rename to src/kernel/platform/library/include/platform/lib/ll_schedule.h diff --git a/src/platform/library/include/platform/lib/mailbox.h b/src/kernel/platform/library/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/library/include/platform/lib/mailbox.h rename to src/kernel/platform/library/include/platform/lib/mailbox.h diff --git a/src/platform/library/include/platform/lib/memory.h b/src/kernel/platform/library/include/platform/lib/memory.h similarity index 100% rename from src/platform/library/include/platform/lib/memory.h rename to src/kernel/platform/library/include/platform/lib/memory.h diff --git a/src/platform/library/include/platform/lib/pm_runtime.h b/src/kernel/platform/library/include/platform/lib/pm_runtime.h similarity index 100% rename from src/platform/library/include/platform/lib/pm_runtime.h rename to src/kernel/platform/library/include/platform/lib/pm_runtime.h diff --git a/src/platform/library/include/platform/platform.h b/src/kernel/platform/library/include/platform/platform.h similarity index 100% rename from src/platform/library/include/platform/platform.h rename to src/kernel/platform/library/include/platform/platform.h diff --git a/src/platform/library/init.c b/src/kernel/platform/library/init.c similarity index 100% rename from src/platform/library/init.c rename to src/kernel/platform/library/init.c diff --git a/src/platform/library/lib/CMakeLists.txt b/src/kernel/platform/library/lib/CMakeLists.txt similarity index 100% rename from src/platform/library/lib/CMakeLists.txt rename to src/kernel/platform/library/lib/CMakeLists.txt diff --git a/src/platform/library/lib/alloc.c b/src/kernel/platform/library/lib/alloc.c similarity index 100% rename from src/platform/library/lib/alloc.c rename to src/kernel/platform/library/lib/alloc.c diff --git a/src/platform/library/lib/clk.c b/src/kernel/platform/library/lib/clk.c similarity index 100% rename from src/platform/library/lib/clk.c rename to src/kernel/platform/library/lib/clk.c diff --git a/src/platform/library/lib/dai.c b/src/kernel/platform/library/lib/dai.c similarity index 100% rename from src/platform/library/lib/dai.c rename to src/kernel/platform/library/lib/dai.c diff --git a/src/platform/library/lib/memory.c b/src/kernel/platform/library/lib/memory.c similarity index 100% rename from src/platform/library/lib/memory.c rename to src/kernel/platform/library/lib/memory.c diff --git a/src/platform/library/lib/pm_runtime.c b/src/kernel/platform/library/lib/pm_runtime.c similarity index 100% rename from src/platform/library/lib/pm_runtime.c rename to src/kernel/platform/library/lib/pm_runtime.c diff --git a/src/platform/library/lib/trace.c b/src/kernel/platform/library/lib/trace.c similarity index 100% rename from src/platform/library/lib/trace.c rename to src/kernel/platform/library/lib/trace.c diff --git a/src/platform/library/panic.c b/src/kernel/platform/library/panic.c similarity index 100% rename from src/platform/library/panic.c rename to src/kernel/platform/library/panic.c diff --git a/src/platform/library/platform.c b/src/kernel/platform/library/platform.c similarity index 100% rename from src/platform/library/platform.c rename to src/kernel/platform/library/platform.c diff --git a/src/platform/library/schedule/CMakeLists.txt b/src/kernel/platform/library/schedule/CMakeLists.txt similarity index 100% rename from src/platform/library/schedule/CMakeLists.txt rename to src/kernel/platform/library/schedule/CMakeLists.txt diff --git a/src/platform/library/schedule/edf_schedule.c b/src/kernel/platform/library/schedule/edf_schedule.c similarity index 100% rename from src/platform/library/schedule/edf_schedule.c rename to src/kernel/platform/library/schedule/edf_schedule.c diff --git a/src/platform/library/schedule/ll_schedule.c b/src/kernel/platform/library/schedule/ll_schedule.c similarity index 100% rename from src/platform/library/schedule/ll_schedule.c rename to src/kernel/platform/library/schedule/ll_schedule.c diff --git a/src/platform/library/schedule/schedule.c b/src/kernel/platform/library/schedule/schedule.c similarity index 100% rename from src/platform/library/schedule/schedule.c rename to src/kernel/platform/library/schedule/schedule.c diff --git a/src/platform/lunarlake/include/platform/lib/clk.h b/src/kernel/platform/lunarlake/include/platform/lib/clk.h similarity index 100% rename from src/platform/lunarlake/include/platform/lib/clk.h rename to src/kernel/platform/lunarlake/include/platform/lib/clk.h diff --git a/src/platform/lunarlake/include/platform/lib/mailbox.h b/src/kernel/platform/lunarlake/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/lunarlake/include/platform/lib/mailbox.h rename to src/kernel/platform/lunarlake/include/platform/lib/mailbox.h diff --git a/src/platform/lunarlake/include/platform/lib/memory.h b/src/kernel/platform/lunarlake/include/platform/lib/memory.h similarity index 100% rename from src/platform/lunarlake/include/platform/lib/memory.h rename to src/kernel/platform/lunarlake/include/platform/lib/memory.h diff --git a/src/platform/lunarlake/include/platform/platform.h b/src/kernel/platform/lunarlake/include/platform/platform.h similarity index 100% rename from src/platform/lunarlake/include/platform/platform.h rename to src/kernel/platform/lunarlake/include/platform/platform.h diff --git a/src/platform/lunarlake/include/platform/trace/trace.h b/src/kernel/platform/lunarlake/include/platform/trace/trace.h similarity index 100% rename from src/platform/lunarlake/include/platform/trace/trace.h rename to src/kernel/platform/lunarlake/include/platform/trace/trace.h diff --git a/src/platform/lunarlake/lib/clk.c b/src/kernel/platform/lunarlake/lib/clk.c similarity index 100% rename from src/platform/lunarlake/lib/clk.c rename to src/kernel/platform/lunarlake/lib/clk.c diff --git a/src/platform/meteorlake/include/platform/lib/clk.h b/src/kernel/platform/meteorlake/include/platform/lib/clk.h similarity index 100% rename from src/platform/meteorlake/include/platform/lib/clk.h rename to src/kernel/platform/meteorlake/include/platform/lib/clk.h diff --git a/src/platform/meteorlake/include/platform/lib/mailbox.h b/src/kernel/platform/meteorlake/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/meteorlake/include/platform/lib/mailbox.h rename to src/kernel/platform/meteorlake/include/platform/lib/mailbox.h diff --git a/src/platform/meteorlake/include/platform/lib/memory.h b/src/kernel/platform/meteorlake/include/platform/lib/memory.h similarity index 100% rename from src/platform/meteorlake/include/platform/lib/memory.h rename to src/kernel/platform/meteorlake/include/platform/lib/memory.h diff --git a/src/platform/meteorlake/include/platform/platform.h b/src/kernel/platform/meteorlake/include/platform/platform.h similarity index 100% rename from src/platform/meteorlake/include/platform/platform.h rename to src/kernel/platform/meteorlake/include/platform/platform.h diff --git a/src/platform/meteorlake/include/platform/trace/trace.h b/src/kernel/platform/meteorlake/include/platform/trace/trace.h similarity index 100% rename from src/platform/meteorlake/include/platform/trace/trace.h rename to src/kernel/platform/meteorlake/include/platform/trace/trace.h diff --git a/src/platform/meteorlake/lib/clk.c b/src/kernel/platform/meteorlake/lib/clk.c similarity index 100% rename from src/platform/meteorlake/lib/clk.c rename to src/kernel/platform/meteorlake/lib/clk.c diff --git a/src/platform/mtk/dai.c b/src/kernel/platform/mtk/dai.c similarity index 100% rename from src/platform/mtk/dai.c rename to src/kernel/platform/mtk/dai.c diff --git a/src/platform/mtk/include/platform/lib/clk.h b/src/kernel/platform/mtk/include/platform/lib/clk.h similarity index 100% rename from src/platform/mtk/include/platform/lib/clk.h rename to src/kernel/platform/mtk/include/platform/lib/clk.h diff --git a/src/platform/mtk/include/platform/lib/cpu.h b/src/kernel/platform/mtk/include/platform/lib/cpu.h similarity index 100% rename from src/platform/mtk/include/platform/lib/cpu.h rename to src/kernel/platform/mtk/include/platform/lib/cpu.h diff --git a/src/platform/mtk/include/platform/lib/dai.h b/src/kernel/platform/mtk/include/platform/lib/dai.h similarity index 100% rename from src/platform/mtk/include/platform/lib/dai.h rename to src/kernel/platform/mtk/include/platform/lib/dai.h diff --git a/src/platform/mtk/include/platform/lib/dma.h b/src/kernel/platform/mtk/include/platform/lib/dma.h similarity index 100% rename from src/platform/mtk/include/platform/lib/dma.h rename to src/kernel/platform/mtk/include/platform/lib/dma.h diff --git a/src/platform/mtk/include/platform/lib/mailbox.h b/src/kernel/platform/mtk/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/mtk/include/platform/lib/mailbox.h rename to src/kernel/platform/mtk/include/platform/lib/mailbox.h diff --git a/src/platform/mtk/include/platform/lib/memory.h b/src/kernel/platform/mtk/include/platform/lib/memory.h similarity index 100% rename from src/platform/mtk/include/platform/lib/memory.h rename to src/kernel/platform/mtk/include/platform/lib/memory.h diff --git a/src/platform/mtk/include/platform/lib/pm_runtime.h b/src/kernel/platform/mtk/include/platform/lib/pm_runtime.h similarity index 100% rename from src/platform/mtk/include/platform/lib/pm_runtime.h rename to src/kernel/platform/mtk/include/platform/lib/pm_runtime.h diff --git a/src/platform/mtk/include/platform/platform.h b/src/kernel/platform/mtk/include/platform/platform.h similarity index 100% rename from src/platform/mtk/include/platform/platform.h rename to src/kernel/platform/mtk/include/platform/platform.h diff --git a/src/platform/mtk/include/platform/trace/trace.h b/src/kernel/platform/mtk/include/platform/trace/trace.h similarity index 100% rename from src/platform/mtk/include/platform/trace/trace.h rename to src/kernel/platform/mtk/include/platform/trace/trace.h diff --git a/src/platform/mtk/platform.c b/src/kernel/platform/mtk/platform.c similarity index 100% rename from src/platform/mtk/platform.c rename to src/kernel/platform/mtk/platform.c diff --git a/src/platform/mtk/tools/README.txt b/src/kernel/platform/mtk/tools/README.txt similarity index 100% rename from src/platform/mtk/tools/README.txt rename to src/kernel/platform/mtk/tools/README.txt diff --git a/src/platform/mtk/tools/build.sh b/src/kernel/platform/mtk/tools/build.sh similarity index 61% rename from src/platform/mtk/tools/build.sh rename to src/kernel/platform/mtk/tools/build.sh index 8d7b30d855b3..e38d16b7051b 100755 --- a/src/platform/mtk/tools/build.sh +++ b/src/kernel/platform/mtk/tools/build.sh @@ -13,11 +13,11 @@ SOF=`cd ../../../..; /bin/pwd` for p in $PLATFORMS; do - SRCS="$SOF/src/platform/$p/lib/dai.c $SOF/src/platform/$p/afe-platform.c" + SRCS="$SOF/src/kernel/platform/$p/lib/dai.c $SOF/src/kernel/platform/$p/afe-platform.c" - INCS="-I$SOF/src/include -I$SOF/src/platform/posix/include -I$SOF/posix/include" - INCS="$INCS -I$SOF/src/kernel/arch/host/include -I$SOF/src/platform/$p/include/platform" - INCS="$INCS -I$SOF/src/platform/$p/include" + INCS="-I$SOF/src/include -I$SOF/src/kernel/platform/posix/include -I$SOF/posix/include" + INCS="$INCS -I$SOF/src/kernel/arch/host/include -I$SOF/src/kernel/platform/$p/include/platform" + INCS="$INCS -I$SOF/src/kernel/platform/$p/include" DEFS="-DRELATIVE_FILE=\"mt-dai-gen.c\" -DCONFIG_CORE_COUNT=1 -DCONFIG_IPC_MAJOR_3=1" diff --git a/src/platform/mtk/tools/mt-dai-gen.c b/src/kernel/platform/mtk/tools/mt-dai-gen.c similarity index 100% rename from src/platform/mtk/tools/mt-dai-gen.c rename to src/kernel/platform/mtk/tools/mt-dai-gen.c diff --git a/src/platform/novalake/include/platform/lib/clk.h b/src/kernel/platform/novalake/include/platform/lib/clk.h similarity index 100% rename from src/platform/novalake/include/platform/lib/clk.h rename to src/kernel/platform/novalake/include/platform/lib/clk.h diff --git a/src/platform/novalake/include/platform/lib/mailbox.h b/src/kernel/platform/novalake/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/novalake/include/platform/lib/mailbox.h rename to src/kernel/platform/novalake/include/platform/lib/mailbox.h diff --git a/src/platform/novalake/include/platform/lib/memory.h b/src/kernel/platform/novalake/include/platform/lib/memory.h similarity index 100% rename from src/platform/novalake/include/platform/lib/memory.h rename to src/kernel/platform/novalake/include/platform/lib/memory.h diff --git a/src/platform/novalake/include/platform/platform.h b/src/kernel/platform/novalake/include/platform/platform.h similarity index 100% rename from src/platform/novalake/include/platform/platform.h rename to src/kernel/platform/novalake/include/platform/platform.h diff --git a/src/platform/novalake/include/platform/trace/trace.h b/src/kernel/platform/novalake/include/platform/trace/trace.h similarity index 100% rename from src/platform/novalake/include/platform/trace/trace.h rename to src/kernel/platform/novalake/include/platform/trace/trace.h diff --git a/src/platform/novalake/lib/clk.c b/src/kernel/platform/novalake/lib/clk.c similarity index 100% rename from src/platform/novalake/lib/clk.c rename to src/kernel/platform/novalake/lib/clk.c diff --git a/src/platform/posix/dai.c b/src/kernel/platform/posix/dai.c similarity index 100% rename from src/platform/posix/dai.c rename to src/kernel/platform/posix/dai.c diff --git a/src/platform/posix/dma.c b/src/kernel/platform/posix/dma.c similarity index 100% rename from src/platform/posix/dma.c rename to src/kernel/platform/posix/dma.c diff --git a/src/platform/posix/fuzz.c b/src/kernel/platform/posix/fuzz.c similarity index 100% rename from src/platform/posix/fuzz.c rename to src/kernel/platform/posix/fuzz.c diff --git a/src/platform/posix/include/platform/drivers/idc.h b/src/kernel/platform/posix/include/platform/drivers/idc.h similarity index 100% rename from src/platform/posix/include/platform/drivers/idc.h rename to src/kernel/platform/posix/include/platform/drivers/idc.h diff --git a/src/platform/posix/include/platform/drivers/interrupt.h b/src/kernel/platform/posix/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/posix/include/platform/drivers/interrupt.h rename to src/kernel/platform/posix/include/platform/drivers/interrupt.h diff --git a/src/platform/posix/include/platform/fuzz.h b/src/kernel/platform/posix/include/platform/fuzz.h similarity index 100% rename from src/platform/posix/include/platform/fuzz.h rename to src/kernel/platform/posix/include/platform/fuzz.h diff --git a/src/platform/posix/include/platform/lib/clk.h b/src/kernel/platform/posix/include/platform/lib/clk.h similarity index 100% rename from src/platform/posix/include/platform/lib/clk.h rename to src/kernel/platform/posix/include/platform/lib/clk.h diff --git a/src/platform/posix/include/platform/lib/cpu.h b/src/kernel/platform/posix/include/platform/lib/cpu.h similarity index 100% rename from src/platform/posix/include/platform/lib/cpu.h rename to src/kernel/platform/posix/include/platform/lib/cpu.h diff --git a/src/platform/posix/include/platform/lib/dai.h b/src/kernel/platform/posix/include/platform/lib/dai.h similarity index 100% rename from src/platform/posix/include/platform/lib/dai.h rename to src/kernel/platform/posix/include/platform/lib/dai.h diff --git a/src/platform/posix/include/platform/lib/dma.h b/src/kernel/platform/posix/include/platform/lib/dma.h similarity index 100% rename from src/platform/posix/include/platform/lib/dma.h rename to src/kernel/platform/posix/include/platform/lib/dma.h diff --git a/src/platform/posix/include/platform/lib/mailbox.h b/src/kernel/platform/posix/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/posix/include/platform/lib/mailbox.h rename to src/kernel/platform/posix/include/platform/lib/mailbox.h diff --git a/src/platform/posix/include/platform/lib/memory.h b/src/kernel/platform/posix/include/platform/lib/memory.h similarity index 100% rename from src/platform/posix/include/platform/lib/memory.h rename to src/kernel/platform/posix/include/platform/lib/memory.h diff --git a/src/platform/posix/include/platform/lib/pm_runtime.h b/src/kernel/platform/posix/include/platform/lib/pm_runtime.h similarity index 100% rename from src/platform/posix/include/platform/lib/pm_runtime.h rename to src/kernel/platform/posix/include/platform/lib/pm_runtime.h diff --git a/src/platform/posix/include/platform/platform.h b/src/kernel/platform/posix/include/platform/platform.h similarity index 100% rename from src/platform/posix/include/platform/platform.h rename to src/kernel/platform/posix/include/platform/platform.h diff --git a/src/platform/posix/include/platform/trace/trace.h b/src/kernel/platform/posix/include/platform/trace/trace.h similarity index 100% rename from src/platform/posix/include/platform/trace/trace.h rename to src/kernel/platform/posix/include/platform/trace/trace.h diff --git a/src/platform/posix/ipc.c b/src/kernel/platform/posix/ipc.c similarity index 100% rename from src/platform/posix/ipc.c rename to src/kernel/platform/posix/ipc.c diff --git a/src/platform/posix/posix.c b/src/kernel/platform/posix/posix.c similarity index 100% rename from src/platform/posix/posix.c rename to src/kernel/platform/posix/posix.c diff --git a/src/platform/qemu_xtensa/include/platform/lib/clk.h b/src/kernel/platform/qemu_xtensa/include/platform/lib/clk.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/lib/clk.h rename to src/kernel/platform/qemu_xtensa/include/platform/lib/clk.h diff --git a/src/platform/qemu_xtensa/include/platform/lib/dai.h b/src/kernel/platform/qemu_xtensa/include/platform/lib/dai.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/lib/dai.h rename to src/kernel/platform/qemu_xtensa/include/platform/lib/dai.h diff --git a/src/platform/qemu_xtensa/include/platform/lib/dma.h b/src/kernel/platform/qemu_xtensa/include/platform/lib/dma.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/lib/dma.h rename to src/kernel/platform/qemu_xtensa/include/platform/lib/dma.h diff --git a/src/platform/qemu_xtensa/include/platform/lib/mailbox.h b/src/kernel/platform/qemu_xtensa/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/lib/mailbox.h rename to src/kernel/platform/qemu_xtensa/include/platform/lib/mailbox.h diff --git a/src/platform/qemu_xtensa/include/platform/lib/memory.h b/src/kernel/platform/qemu_xtensa/include/platform/lib/memory.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/lib/memory.h rename to src/kernel/platform/qemu_xtensa/include/platform/lib/memory.h diff --git a/src/platform/qemu_xtensa/include/platform/platform.h b/src/kernel/platform/qemu_xtensa/include/platform/platform.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/platform.h rename to src/kernel/platform/qemu_xtensa/include/platform/platform.h diff --git a/src/platform/qemu_xtensa/include/platform/trace/trace.h b/src/kernel/platform/qemu_xtensa/include/platform/trace/trace.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/trace/trace.h rename to src/kernel/platform/qemu_xtensa/include/platform/trace/trace.h diff --git a/src/platform/qemu_xtensa/platform.c b/src/kernel/platform/qemu_xtensa/platform.c similarity index 100% rename from src/platform/qemu_xtensa/platform.c rename to src/kernel/platform/qemu_xtensa/platform.c diff --git a/src/platform/tigerlake/include/platform/lib/mailbox.h b/src/kernel/platform/tigerlake/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/tigerlake/include/platform/lib/mailbox.h rename to src/kernel/platform/tigerlake/include/platform/lib/mailbox.h diff --git a/src/platform/tigerlake/include/platform/lib/memory.h b/src/kernel/platform/tigerlake/include/platform/lib/memory.h similarity index 100% rename from src/platform/tigerlake/include/platform/lib/memory.h rename to src/kernel/platform/tigerlake/include/platform/lib/memory.h diff --git a/src/platform/tigerlake/include/platform/platform.h b/src/kernel/platform/tigerlake/include/platform/platform.h similarity index 100% rename from src/platform/tigerlake/include/platform/platform.h rename to src/kernel/platform/tigerlake/include/platform/platform.h diff --git a/src/platform/tigerlake/include/platform/trace/trace.h b/src/kernel/platform/tigerlake/include/platform/trace/trace.h similarity index 100% rename from src/platform/tigerlake/include/platform/trace/trace.h rename to src/kernel/platform/tigerlake/include/platform/trace/trace.h diff --git a/test/cmocka/CMakeLists.txt b/test/cmocka/CMakeLists.txt index 288282d11457..98a64af7d584 100644 --- a/test/cmocka/CMakeLists.txt +++ b/test/cmocka/CMakeLists.txt @@ -42,9 +42,9 @@ else() endif() if(CONFIG_CAVS) - target_include_directories(sof_options INTERFACE ${PROJECT_SOURCE_DIR}/src/platform/intel/cavs/include) + target_include_directories(sof_options INTERFACE ${PROJECT_SOURCE_DIR}/src/kernel/platform/intel/cavs/include) elseif(CONFIG_ACE) - target_include_directories(sof_options INTERFACE ${PROJECT_SOURCE_DIR}/src/platform/intel/ace/include) + target_include_directories(sof_options INTERFACE ${PROJECT_SOURCE_DIR}/src/kernel/platform/intel/ace/include) endif() # linker script, just for log entries diff --git a/test/cmocka/src/lib/alloc/CMakeLists.txt b/test/cmocka/src/lib/alloc/CMakeLists.txt index 2ec7c8f606db..10bef8205b4b 100644 --- a/test/cmocka/src/lib/alloc/CMakeLists.txt +++ b/test/cmocka/src/lib/alloc/CMakeLists.txt @@ -3,7 +3,7 @@ cmocka_test(alloc alloc.c ${PROJECT_SOURCE_DIR}/src/lib/alloc.c - ${PROJECT_SOURCE_DIR}/src/platform/library/lib/memory.c + ${PROJECT_SOURCE_DIR}/src/kernel/platform/library/lib/memory.c ) -target_include_directories(sof_options INTERFACE ${PROJECT_SOURCE_DIR}/src/platform/intel/cavs/include) +target_include_directories(sof_options INTERFACE ${PROJECT_SOURCE_DIR}/src/kernel/platform/intel/cavs/include) diff --git a/test/cmocka/src/lib/fast-get/CMakeLists.txt b/test/cmocka/src/lib/fast-get/CMakeLists.txt index 8449320f4e6b..cf542366bcce 100644 --- a/test/cmocka/src/lib/fast-get/CMakeLists.txt +++ b/test/cmocka/src/lib/fast-get/CMakeLists.txt @@ -5,7 +5,7 @@ cmocka_test(fast-get-tests ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/zephyr/lib/fast-get.c ${PROJECT_SOURCE_DIR}/src/lib/alloc.c - ${PROJECT_SOURCE_DIR}/src/platform/library/lib/memory.c + ${PROJECT_SOURCE_DIR}/src/kernel/platform/library/lib/memory.c ) target_link_libraries(fast-get-tests PRIVATE "-Wl,--wrap=rzalloc,--wrap=rmalloc,--wrap=rfree") diff --git a/test/ztest/unit/audio/crossover/CMakeLists.txt b/test/ztest/unit/audio/crossover/CMakeLists.txt index 39fe53fccf8c..cc599bc4e512 100644 --- a/test/ztest/unit/audio/crossover/CMakeLists.txt +++ b/test/ztest/unit/audio/crossover/CMakeLists.txt @@ -12,7 +12,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/src/audio/crossover ${PROJECT_BINARY_DIR}/include/generated ) diff --git a/test/ztest/unit/audio/drc/CMakeLists.txt b/test/ztest/unit/audio/drc/CMakeLists.txt index 4c83db706257..2faef72b825c 100644 --- a/test/ztest/unit/audio/drc/CMakeLists.txt +++ b/test/ztest/unit/audio/drc/CMakeLists.txt @@ -17,7 +17,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/src/audio ${SOF_ROOT}/test/cmocka/include ${PROJECT_BINARY_DIR}/include/generated diff --git a/test/ztest/unit/dsrc/CMakeLists.txt b/test/ztest/unit/dsrc/CMakeLists.txt index a198ace9f231..db31f8a536d8 100644 --- a/test/ztest/unit/dsrc/CMakeLists.txt +++ b/test/ztest/unit/dsrc/CMakeLists.txt @@ -11,7 +11,7 @@ include(${SOF_ROOT}/scripts/cmake/misc.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ) # Define SOF-specific configurations for unit testing diff --git a/test/ztest/unit/fast-get/CMakeLists.txt b/test/ztest/unit/fast-get/CMakeLists.txt index 3fe6a87680f4..b58587e15794 100644 --- a/test/ztest/unit/fast-get/CMakeLists.txt +++ b/test/ztest/unit/fast-get/CMakeLists.txt @@ -15,7 +15,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${PROJECT_BINARY_DIR}/include/generated # For uuid-registry.h ) diff --git a/test/ztest/unit/math/advanced/functions/CMakeLists.txt b/test/ztest/unit/math/advanced/functions/CMakeLists.txt index 615fc86a7db6..0afee53408b7 100644 --- a/test/ztest/unit/math/advanced/functions/CMakeLists.txt +++ b/test/ztest/unit/math/advanced/functions/CMakeLists.txt @@ -15,7 +15,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/test/cmocka/include ${PROJECT_BINARY_DIR}/include/generated # For uuid-registry.h ) diff --git a/test/ztest/unit/math/auditory/CMakeLists.txt b/test/ztest/unit/math/auditory/CMakeLists.txt index 6126b05d3e3d..d09c724d80af 100644 --- a/test/ztest/unit/math/auditory/CMakeLists.txt +++ b/test/ztest/unit/math/auditory/CMakeLists.txt @@ -16,7 +16,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/test/ztest/unit/common # Reference test vectors are #included in place from the CMocka tree # rather than copied, so both frameworks share identical data. diff --git a/test/ztest/unit/math/dct/CMakeLists.txt b/test/ztest/unit/math/dct/CMakeLists.txt index 1bdf36a69247..3bddbf06e6ea 100644 --- a/test/ztest/unit/math/dct/CMakeLists.txt +++ b/test/ztest/unit/math/dct/CMakeLists.txt @@ -16,7 +16,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/test/ztest/unit/common # Reference test vectors are #included in place from the CMocka tree # rather than copied, so both frameworks share identical data. diff --git a/test/ztest/unit/math/fft/CMakeLists.txt b/test/ztest/unit/math/fft/CMakeLists.txt index 6c6432bc3efe..920f41bdb642 100644 --- a/test/ztest/unit/math/fft/CMakeLists.txt +++ b/test/ztest/unit/math/fft/CMakeLists.txt @@ -16,7 +16,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/src/audio ${SOF_ROOT}/test/ztest/unit/common ${SOF_ROOT}/test/cmocka/include diff --git a/test/ztest/unit/math/fft_multi/CMakeLists.txt b/test/ztest/unit/math/fft_multi/CMakeLists.txt index b08ee40c0884..3287559f3050 100644 --- a/test/ztest/unit/math/fft_multi/CMakeLists.txt +++ b/test/ztest/unit/math/fft_multi/CMakeLists.txt @@ -16,7 +16,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/test/ztest/unit/common ${SOF_ROOT}/test/cmocka/include # Reference test vectors are #included in place from the CMocka tree diff --git a/test/ztest/unit/math/matrix/CMakeLists.txt b/test/ztest/unit/math/matrix/CMakeLists.txt index 6399f3c8ce1f..1dbfea88a08a 100644 --- a/test/ztest/unit/math/matrix/CMakeLists.txt +++ b/test/ztest/unit/math/matrix/CMakeLists.txt @@ -16,7 +16,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/test/ztest/unit/common # Reference test vectors are #included in place from the CMocka tree # rather than copied, so both frameworks share identical data. diff --git a/test/ztest/unit/objpool/CMakeLists.txt b/test/ztest/unit/objpool/CMakeLists.txt index 2ce499607463..f19b8a3e926c 100644 --- a/test/ztest/unit/objpool/CMakeLists.txt +++ b/test/ztest/unit/objpool/CMakeLists.txt @@ -11,7 +11,7 @@ include(${SOF_ROOT}/scripts/cmake/misc.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ) # Define SOF-specific configurations for unit testing diff --git a/tools/testbench/CMakeLists.txt b/tools/testbench/CMakeLists.txt index b07dc2be0e6a..ec949fc6ed14 100644 --- a/tools/testbench/CMakeLists.txt +++ b/tools/testbench/CMakeLists.txt @@ -39,7 +39,7 @@ set(sof_binary_directory "${PROJECT_BINARY_DIR}/sof_ep/build") set(config_h ${sof_binary_directory}/library_autoconfig.h) -target_include_directories(${testbench} PRIVATE "${sof_source_directory}/src/platform/library/include") +target_include_directories(${testbench} PRIVATE "${sof_source_directory}/src/kernel/platform/library/include") target_include_directories(${testbench} PRIVATE "${sof_source_directory}/src/audio") # Configuration time, make copy diff --git a/tools/tplg_parser/CMakeLists.txt b/tools/tplg_parser/CMakeLists.txt index 716d921d45c6..353808a3ef15 100644 --- a/tools/tplg_parser/CMakeLists.txt +++ b/tools/tplg_parser/CMakeLists.txt @@ -43,7 +43,7 @@ target_include_directories(sof_tplg_parser PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/i target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/include) target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/audio) target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/kernel/arch/host/include) -target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/platform/library/include) +target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/kernel/platform/library/include) target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/posix/include) # Configuration time, make copy diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 986b47f6ee44..a72da6c50b42 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -156,7 +156,7 @@ zephyr_interface_library_named(SOF) cmake_path(SET sof_top_dir NORMALIZE "${CMAKE_CURRENT_SOURCE_DIR}/..") set(SOF_SRC_PATH "${sof_top_dir}/src") -set(SOF_PLATFORM_PATH "${SOF_SRC_PATH}/platform") +set(SOF_PLATFORM_PATH "${SOF_SRC_PATH}/kernel/platform") set(SOF_DRIVERS_PATH "${SOF_SRC_PATH}/kernel/drivers") set(SOF_TRACE_PATH "${SOF_SRC_PATH}/trace") @@ -364,7 +364,7 @@ if (CONFIG_SOC_MIMX8ML8_M7) ) # SOF-specific linker script additions - zephyr_linker_sources(DATA_SECTIONS ${sof_top_dir}/src/platform/imx8m_cm7/linker/data-sections.ld) + zephyr_linker_sources(DATA_SECTIONS ${sof_top_dir}/src/kernel/platform/imx8m_cm7/linker/data-sections.ld) set(PLATFORM "imx8m_cm7") @@ -428,7 +428,7 @@ if (CONFIG_SOC_MIMX9596_M7) ) # SOF-specific linker script additions - zephyr_linker_sources(DATA_SECTIONS ${sof_top_dir}/src/platform/imx95/linker/data-sections.ld) + zephyr_linker_sources(DATA_SECTIONS ${sof_top_dir}/src/kernel/platform/imx95/linker/data-sections.ld) set(PLATFORM "imx95") From d377fa711651d71db87ae8234d6df8722120850c Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 15:08:32 +0000 Subject: [PATCH 03/19] zephyr: build all of src/kernel into sof_kernel zephyr/CMakeLists.txt lists the platform, driver and Xtensa cache attribute sources itself, mixed with the Zephyr integration sources in zephyr/lib, so they are built into modules_sof although they now live under src/kernel. After all sources have been added, move every source under src/kernel from modules_sof to sof_kernel, so that sof_kernel holds exactly the code in src/kernel however it was added. Both libraries are created in the same directory and link the same SOF interface library, so the compile flags do not change. No functional change. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- zephyr/CMakeLists.txt | 19 +++++++++++++++++++ 1 file changed, 19 insertions(+) diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index a72da6c50b42..419acf5bb92e 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -666,6 +666,25 @@ include(../scripts/cmake/version.cmake) include(../scripts/cmake/uuid-registry.cmake) +# Everything under src/kernel belongs in sof_kernel, also the platform, driver +# and architecture sources that this file lists itself rather than adding +# them from src/kernel/CMakeLists.txt. Move them out of modules_sof. +get_target_property(modules_sof_sources modules_sof SOURCES) +cmake_path(SET sof_kernel_src_dir NORMALIZE "${SOF_SRC_PATH}/kernel/") +set(sof_kernel_listed_sources) +foreach(src ${modules_sof_sources}) + cmake_path(ABSOLUTE_PATH src NORMALIZE OUTPUT_VARIABLE abs_src) + string(FIND "${abs_src}" "${sof_kernel_src_dir}" pos) + if(pos EQUAL 0) + list(APPEND sof_kernel_listed_sources ${src}) + endif() +endforeach() +if(sof_kernel_listed_sources) + list(REMOVE_ITEM modules_sof_sources ${sof_kernel_listed_sources}) + set_property(TARGET modules_sof PROPERTY SOURCES ${modules_sof_sources}) + target_sources(sof_kernel PRIVATE ${sof_kernel_listed_sources}) +endif() + # Create Trace realtive file paths sof_append_relative_path_definitions(modules_sof) sof_append_relative_path_definitions(sof_kernel) From 6416a3d06e416a9ffbf43ed7810b361e993a4d6f Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 19:24:21 +0000 Subject: [PATCH 04/19] logging: move the SOF probe log backend options to src/kernel/logging The Kconfig options of the SOF probe log backend (CONFIG_LOG_BACKEND_SOF_PROBE*) are defined in src/trace/Kconfig, which is otherwise only about the SOF DMA trace. Move them to a Kconfig file next to the backend, in src/kernel/logging, sourced from src/kernel/Kconfig. The option values are unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/kernel/Kconfig | 2 ++ src/kernel/logging/Kconfig | 40 ++++++++++++++++++++++++++++++++++++++ src/trace/Kconfig | 39 ------------------------------------- 3 files changed, 42 insertions(+), 39 deletions(-) create mode 100644 src/kernel/logging/Kconfig diff --git a/src/kernel/Kconfig b/src/kernel/Kconfig index f92df62d911a..537ad2eda22c 100644 --- a/src/kernel/Kconfig +++ b/src/kernel/Kconfig @@ -10,4 +10,6 @@ rsource "idc/Kconfig" rsource "library_manager/Kconfig" +rsource "logging/Kconfig" + rsource "probe/Kconfig" diff --git a/src/kernel/logging/Kconfig b/src/kernel/logging/Kconfig new file mode 100644 index 000000000000..80900349ec9e --- /dev/null +++ b/src/kernel/logging/Kconfig @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: BSD-3-Clause + +config LOG_BACKEND_SOF_PROBE + bool "Logging backend with SOF probes" + depends on LOG + select LOG_OUTPUT + help + Enable backend for log output via SOF probe interface. + Probe interface allows to transmit logs and PCM data, muxed over + a shared DMA channel. + Logging is enabled by setting up a probe point with + probe purpose value of PROBE_PURPOSE_LOGGING. + +config LOG_BACKEND_SOF_PROBE_OUTPUT_AUTO_ENABLE + bool "Automatically enable probes logging when probe dma is started" + depends on LOG_BACKEND_SOF_PROBE + help + Automatically enable logging as soon as probes DMA + starts. This option simplifies probes logging back + end usage as there is no need to enable logs after + every boot. The probes DMA is still needed before the + log output can be enabled. + +config LOG_BACKEND_SOF_PROBE_OUTPUT_DICTIONARY + bool "Dictionary" + select LOG_DICTIONARY_SUPPORT + help + Set output format of the SOF probe logging backend to + Zephyr log dictionary. + The resulting log file can be parsed with tools in + zephyr/scripts/logging/dictionary + A dictionary for the binary is created during build. + +config LOG_BACKEND_SOF_PROBE_OUTPUT + int "Set logging output format" + default 1 if LOG_MIPI_SYST_ENABLE + default 2 if LOG_BACKEND_SOF_PROBE_OUTPUT_DICTIONARY + default 0 # Text encoding is default + help + Set the logging format for SOF probe output. diff --git a/src/trace/Kconfig b/src/trace/Kconfig index 92ccb461ed18..6ed586707234 100644 --- a/src/trace/Kconfig +++ b/src/trace/Kconfig @@ -97,43 +97,4 @@ config TRACE_BURST_COUNT Amount of messages that will pass through the filter even if sent in rapid succession. Allowed message burst size before filter suppresses the message. -config LOG_BACKEND_SOF_PROBE - bool "Logging backend with SOF probes" - depends on LOG - select LOG_OUTPUT - help - Enable backend for log output via SOF probe interface. - Probe interface allows to transmit logs and PCM data, muxed over - a shared DMA channel. - Logging is enabled by setting up a probe point with - probe purpose value of PROBE_PURPOSE_LOGGING. - -config LOG_BACKEND_SOF_PROBE_OUTPUT_AUTO_ENABLE - bool "Automatically enable probes logging when probe dma is started" - depends on LOG_BACKEND_SOF_PROBE - help - Automatically enable logging as soon as probes DMA - starts. This option simplifies probes logging back - end usage as there is no need to enable logs after - every boot. The probes DMA is still needed before the - log output can be enabled. - -config LOG_BACKEND_SOF_PROBE_OUTPUT_DICTIONARY - bool "Dictionary" - select LOG_DICTIONARY_SUPPORT - help - Set output format of the SOF probe logging backend to - Zephyr log dictionary. - The resulting log file can be parsed with tools in - zephyr/scripts/logging/dictionary - A dictionary for the binary is created during build. - -config LOG_BACKEND_SOF_PROBE_OUTPUT - int "Set logging output format" - default 1 if LOG_MIPI_SYST_ENABLE - default 2 if LOG_BACKEND_SOF_PROBE_OUTPUT_DICTIONARY - default 0 # Text encoding is default - help - Set the logging format for SOF probe output. - endmenu From 91d93eddfe7b10173b4733a879ea09f1f9267b87 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 19:25:52 +0000 Subject: [PATCH 05/19] trace: print host build traces without CONFIG_TRACE The host builds (testbench, host library) print their traces to stderr through the CONFIG_LIBRARY part of trace-soflogger.h, which is only included when CONFIG_TRACE is set. CONFIG_TRACE also builds the SOF DMA trace, which the host builds do not need. Move the host printing to a header of its own, trace-host.h, and use it for CONFIG_LIBRARY builds. It maps tr_err(), tr_warn(), tr_info(), tr_dbg() and the component trace macros to the same levels and output as before, which is what the host library got with CONFIG_TRACEV and CONFIG_TRACEE set. Disable CONFIG_TRACE for the host library, build the host trace support unconditionally, and drop CONFIG_TRACEV from library_defconfig. The host library code is unchanged; only the trace contexts move from the .trace_ctx section, which only the firmware dictionary logger uses, to normal data. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/trace/trace-host.h | 75 +++++++++++++++++++ src/include/sof/trace/trace.h | 6 ++ .../arch/host/configs/library_defconfig | 1 - .../platform/library/lib/CMakeLists.txt | 5 +- src/kernel/platform/library/lib/trace.c | 1 - src/trace/Kconfig | 2 +- test/cmocka/src/common_mocks.c | 2 +- 7 files changed, 84 insertions(+), 8 deletions(-) create mode 100644 src/include/sof/trace/trace-host.h diff --git a/src/include/sof/trace/trace-host.h b/src/include/sof/trace/trace-host.h new file mode 100644 index 000000000000..bb3298ccf45d --- /dev/null +++ b/src/include/sof/trace/trace-host.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2016,2024 Intel Corporation. + */ + +/** + * \file include/sof/trace/trace-host.h + * \brief Trace output of the host builds (testbench, host library). + * \ingroup trace_logging_api + * + * Messages are printed to stderr when their level is at or below + * host_trace_level, which the testbench sets with its -d option. + */ + +#ifndef __SOF_TRACE_TRACE_HOST_H__ +#define __SOF_TRACE_TRACE_HOST_H__ + +#include +#include +#include +#include /* LOG_LEVEL_... */ + +/* trace level used on host configurations */ +extern int host_trace_level; + +char *get_trace_class(uint32_t trace_class); + +#define _log_message(level, ctx, id_1, id_2, format, ...) \ +do { \ + (void)ctx; \ + (void)id_1; \ + (void)id_2; \ + struct timeval tv; \ + char slevel[5][4] = {"dis", "err", "wrn", "inf", "dbg"}; \ + char *msg = "(%s:%d) " format; \ + if (level <= host_trace_level) { \ + gettimeofday(&tv, NULL); \ + fprintf(stderr, "trace: [%ld.%6.6ld] <%s> ", tv.tv_sec, tv.tv_usec, \ + slevel[level]); \ + fprintf(stderr, msg, strrchr(__FILE__, '/') + 1, \ + __LINE__, ##__VA_ARGS__); \ + fprintf(stderr, "\n"); \ + } \ +} while (0) + +/* tracing from device (component, pipeline, dai, ...) */ + +#define trace_dev_err(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ + _log_message(LOG_LEVEL_CRITICAL, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) + +#define trace_dev_warn(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ + _log_message(LOG_LEVEL_WARNING, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) + +#define trace_dev_info(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ + _log_message(LOG_LEVEL_INFO, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) + +#define trace_dev_dbg(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ + _log_message(LOG_LEVEL_VERBOSE, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) + +/* tracing from infrastructure part */ + +#define tr_err(ctx, fmt, ...) \ + _log_message(LOG_LEVEL_CRITICAL, ctx, _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) +#define tr_warn(ctx, fmt, ...) \ + _log_message(LOG_LEVEL_WARNING, ctx, _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) +#define tr_info(ctx, fmt, ...) \ + _log_message(LOG_LEVEL_INFO, ctx, _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) +#define tr_dbg(ctx, fmt, ...) \ + _log_message(LOG_LEVEL_VERBOSE, ctx, _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) + +#endif /* __SOF_TRACE_TRACE_HOST_H__ */ diff --git a/src/include/sof/trace/trace.h b/src/include/sof/trace/trace.h index 0639ac4adbd9..007abe866731 100644 --- a/src/include/sof/trace/trace.h +++ b/src/include/sof/trace/trace.h @@ -45,6 +45,10 @@ #define tr_info(ctx, fmt, ...) LOG_INF(fmt, ##__VA_ARGS__) #define tr_dbg(ctx, fmt, ...) LOG_DBG(fmt, ##__VA_ARGS__) +#elif CONFIG_LIBRARY && !defined(__ZEPHYR__) + +#include "trace-host.h" + #elif CONFIG_TRACE #include "trace-soflogger.h" @@ -94,6 +98,7 @@ static inline int trace_filter_update(const struct trace_filter *filter) #define trace_error_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) +#if !CONFIG_LIBRARY || defined(__ZEPHYR__) #define trace_dev_err(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ trace_event_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ get_id_m(dev), get_subid_m(dev), \ @@ -110,6 +115,7 @@ static inline int trace_filter_update(const struct trace_filter *filter) trace_event_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ get_id_m(dev), get_subid_m(dev), \ fmt, ##__VA_ARGS__) +#endif /* !CONFIG_LIBRARY || __ZEPHYR__ */ static inline void mtrace_printf(int log_level, const char *format_str, ...) { diff --git a/src/kernel/arch/host/configs/library_defconfig b/src/kernel/arch/host/configs/library_defconfig index 3a72dc85c79a..a7328c55a916 100644 --- a/src/kernel/arch/host/configs/library_defconfig +++ b/src/kernel/arch/host/configs/library_defconfig @@ -32,5 +32,4 @@ CONFIG_IPC_MAJOR_3=n CONFIG_IPC_MAJOR_4=y CONFIG_LIBRARY=y CONFIG_MATH_IIR_DF2T=y -CONFIG_TRACEV=y CONFIG_XT_RUN=y diff --git a/src/kernel/platform/library/lib/CMakeLists.txt b/src/kernel/platform/library/lib/CMakeLists.txt index 05965cd482ee..b8d0224bc4aa 100644 --- a/src/kernel/platform/library/lib/CMakeLists.txt +++ b/src/kernel/platform/library/lib/CMakeLists.txt @@ -6,8 +6,5 @@ add_local_sources(sof dai.c pm_runtime.c memory.c + trace.c ) - -if (CONFIG_TRACE) - add_local_sources(sof trace.c) -endif() diff --git a/src/kernel/platform/library/lib/trace.c b/src/kernel/platform/library/lib/trace.c index 7ca847afabec..c949a81f5f41 100644 --- a/src/kernel/platform/library/lib/trace.c +++ b/src/kernel/platform/library/lib/trace.c @@ -9,7 +9,6 @@ #include #include -#include /* enable trace by default in testbench */ int host_trace_level = LOG_LEVEL_ERROR; diff --git a/src/trace/Kconfig b/src/trace/Kconfig index 6ed586707234..c60f2598ca79 100644 --- a/src/trace/Kconfig +++ b/src/trace/Kconfig @@ -6,7 +6,7 @@ menu "Trace" config TRACE bool "Trace" - default n if ZEPHYR_SOF_MODULE + default n if ZEPHYR_SOF_MODULE || LIBRARY default y help Enable SOF DMA based traces compatible with the sof-logger tool. diff --git a/test/cmocka/src/common_mocks.c b/test/cmocka/src/common_mocks.c index 633108a0730e..0e47a23a4174 100644 --- a/test/cmocka/src/common_mocks.c +++ b/test/cmocka/src/common_mocks.c @@ -181,7 +181,7 @@ void WEAK __panic(uint32_t p, const char *filename, uint32_t linenum) exit(EXIT_FAILURE); } -#if CONFIG_TRACE +#if CONFIG_TRACE && !CONFIG_LIBRARY void WEAK trace_log_filtered(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list args) From 4104527dedd69a0fe49c80ebfc9700d29cb82be3 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 19:45:05 +0000 Subject: [PATCH 06/19] trace: remove the SOF DMA trace The SOF DMA trace (CONFIG_TRACE) sends dictionary-encoded traces for the sof-logger tool over a host DMA channel. Zephyr builds use Zephyr logging instead and cannot enable it at all: zephyr/include/sof/trace/ preproc.h stops the build if CONFIG_TRACE is set. Since the previous patch the host builds do not use it either, so nothing builds it any more. Remove it: - src/trace with its Kconfig (CONFIG_TRACE, CONFIG_TRACEV, CONFIG_TRACEE, CONFIG_TRACEM and the trace filtering options), trace-soflogger.h and dma-trace.h; - the IPC DMA copy helper (src/ipc/dma-copy.c) and CONFIG_TRACE_CHANNEL, used only by the DMA trace; - the code under CONFIG_TRACE in the IPC3 handler (the DMA trace and trace filter commands; the stub that accepts trace commands without a DMA trace stays), the i.MX and AMD platform init, the system init, the panic handler and the cmocka mocks, and the DMA trace calls in the IPC3 PM gate handler; - the trace position helpers ipc_build_trace_posn() and ipc_trigger_trace_xfer(), the trace and DMA trace fields of struct sof, and dump_hex() with the two CONFIG_TRACEV blocks in the Waves module that used it; - the CONFIG_TRACE=n settings in the board and application configs, and the CONFIG_TRACE_CHANNEL settings in the cmocka defconfigs. trace.h keeps the Zephyr logging, host and no-op trace macros. The IPC trace structures in include/ipc/trace.h are host ABI and stay. The dma_copy_*() declarations and DMA_TRACE_CHANNEL are still referenced by the legacy (non-native driver) probe and AMD DMA code and are left for the removal of that code. No firmware build compiled the DMA trace itself. The firmware does change in two ways: the unused ipc_build_trace_posn() and ipc_trigger_trace_xfer() are no longer built, and struct sof is two pointers smaller, which changes the field offsets in the code that uses it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- app/boards/acp_7_0_adsp.conf | 1 - app/boards/acp_7_x_adsp.conf | 1 - app/boards/imx8mp_evk_mimx8ml8_adsp.conf | 1 - app/boards/intel_adsp_ace40_nvl.conf | 1 - app/boards/intel_adsp_ace40_nvls.conf | 1 - app/prj.conf | 1 - doc/sof.doxygen.in | 1 - posix/include/rtos/sof.h | 6 - scripts/kernel_user_partition.txt | 1 - src/CMakeLists.txt | 3 - src/Kconfig | 2 - src/audio/Kconfig | 8 - src/audio/module_adapter/module/waves/waves.c | 39 -- src/debug/panic.c | 4 - src/include/sof/debug/debug.h | 29 - src/include/sof/ipc/common.h | 6 - src/include/sof/ipc/msg.h | 12 - src/include/sof/trace/dma-trace.h | 172 ----- src/include/sof/trace/trace-soflogger.h | 476 ------------- src/include/sof/trace/trace.h | 75 +- src/ipc/CMakeLists.txt | 8 - src/ipc/dma-copy.c | 159 ----- src/ipc/ipc4/helper.c | 11 - src/kernel/init/init.c | 5 - src/kernel/ipc/ipc3/handler.c | 164 ----- src/kernel/ipc/ipc3/helper.c | 12 - src/kernel/platform/amd/acp_7_0/platform.c | 1 - src/kernel/platform/amd/acp_7_x/platform.c | 1 - src/kernel/platform/amd/rembrandt/platform.c | 12 - src/kernel/platform/imx8m/platform.c | 7 - src/kernel/platform/imx8ulp/platform.c | 9 - .../tigerlake/include/platform/lib/memory.h | 4 - src/trace/CMakeLists.txt | 5 - src/trace/Kconfig | 100 --- src/trace/dma-trace.c | 645 ------------------ src/trace/trace.c | 579 ---------------- test/cmocka/configs/acp_7_0_defconfig | 1 - test/cmocka/configs/acp_7_x_defconfig | 1 - test/cmocka/configs/rembrandt_defconfig | 1 - test/cmocka/src/common_mocks.c | 37 - zephyr/CMakeLists.txt | 2 - zephyr/include/rtos/sof.h | 6 - zephyr/include/sof/trace/preproc.h | 5 - zephyr/include/sof/trace/trace.h | 3 +- 44 files changed, 13 insertions(+), 2605 deletions(-) delete mode 100644 src/include/sof/trace/dma-trace.h delete mode 100644 src/include/sof/trace/trace-soflogger.h delete mode 100644 src/ipc/dma-copy.c delete mode 100644 src/trace/CMakeLists.txt delete mode 100644 src/trace/Kconfig delete mode 100644 src/trace/dma-trace.c delete mode 100644 src/trace/trace.c diff --git a/app/boards/acp_7_0_adsp.conf b/app/boards/acp_7_0_adsp.conf index 5eb2ff9f7b95..96ffd80506c2 100644 --- a/app/boards/acp_7_0_adsp.conf +++ b/app/boards/acp_7_0_adsp.conf @@ -18,7 +18,6 @@ CONFIG_DMA_AMD_ACP_TDM=y CONFIG_DAI_AMD_TDM=y CONFIG_AMS=n CONFIG_WRAP_ACTUAL_POSITION=y -CONFIG_TRACE=n CONFIG_COMP_VOLUME=y CONFIG_COMP_SRC=n CONFIG_COMP_FIR=n diff --git a/app/boards/acp_7_x_adsp.conf b/app/boards/acp_7_x_adsp.conf index 44847f4c77de..36af59b502f5 100644 --- a/app/boards/acp_7_x_adsp.conf +++ b/app/boards/acp_7_x_adsp.conf @@ -19,7 +19,6 @@ CONFIG_DAI_AMD_SDW=y CONFIG_DAI_AMD_TDM=y CONFIG_AMS=n CONFIG_WRAP_ACTUAL_POSITION=y -CONFIG_TRACE=n CONFIG_COMP_VOLUME=y CONFIG_COMP_SRC=n CONFIG_COMP_FIR=n diff --git a/app/boards/imx8mp_evk_mimx8ml8_adsp.conf b/app/boards/imx8mp_evk_mimx8ml8_adsp.conf index da9e56d759a4..aee1c653d332 100644 --- a/app/boards/imx8mp_evk_mimx8ml8_adsp.conf +++ b/app/boards/imx8mp_evk_mimx8ml8_adsp.conf @@ -5,7 +5,6 @@ CONFIG_KPB_FORCE_COPY_TYPE_NORMAL=n CONFIG_DMA=y CONFIG_DMA_NXP_SDMA=y CONFIG_DAI_NXP_MICFIL=y -CONFIG_TRACE=n CONFIG_SHARED_INTERRUPTS=y CONFIG_ZEPHYR_NATIVE_DRIVERS=y CONFIG_SAI_HAS_MCLK_CONFIG_OPTION=y diff --git a/app/boards/intel_adsp_ace40_nvl.conf b/app/boards/intel_adsp_ace40_nvl.conf index 4576e5a45210..bc62ec76e877 100644 --- a/app/boards/intel_adsp_ace40_nvl.conf +++ b/app/boards/intel_adsp_ace40_nvl.conf @@ -31,7 +31,6 @@ CONFIG_LIBRARY_BUILD_LIB=y CONFIG_LIBRARY_DEFAULT_MODULAR=y # SOF / logging -CONFIG_TRACE=n CONFIG_SOF_LOG_LEVEL_INF=y # Zephyr / OS features diff --git a/app/boards/intel_adsp_ace40_nvls.conf b/app/boards/intel_adsp_ace40_nvls.conf index 67cdb0e2cb86..0013a290bc4a 100644 --- a/app/boards/intel_adsp_ace40_nvls.conf +++ b/app/boards/intel_adsp_ace40_nvls.conf @@ -31,7 +31,6 @@ CONFIG_LIBRARY_BUILD_LIB=y CONFIG_LIBRARY_DEFAULT_MODULAR=y # SOF / logging -CONFIG_TRACE=n CONFIG_SOF_LOG_LEVEL_INF=y # Zephyr / OS features diff --git a/app/prj.conf b/app/prj.conf index 5c22e19d6e0e..b989ac6d1ac4 100644 --- a/app/prj.conf +++ b/app/prj.conf @@ -15,7 +15,6 @@ CONFIG_HAVE_AGENT=n CONFIG_MINIMAL_LIBC=y CONFIG_ZEPHYR_LOG=y -CONFIG_TRACE=n CONFIG_LOG=y CONFIG_LOG_PRINTK=y # Log processing is offloaded to a low-priority thread. diff --git a/doc/sof.doxygen.in b/doc/sof.doxygen.in index 02750d0f324f..c4d6e196c18a 100644 --- a/doc/sof.doxygen.in +++ b/doc/sof.doxygen.in @@ -45,7 +45,6 @@ PREDEFINED = "__attribute__(x)= " \ "CONFIG_FORMAT_S16LE=1" \ "CONFIG_FORMAT_S32LE=1" \ "CONFIG_FORMAT_S24LE=1" \ - "CONFIG_TRACE=1" \ "CONFIG_IPC4=1" \ "CONFIG_IPC3=1" \ "CONFIG_IPC_MAJOR_4=1" \ diff --git a/posix/include/rtos/sof.h b/posix/include/rtos/sof.h index d6aa0fce66db..3487e0d11d32 100644 --- a/posix/include/rtos/sof.h +++ b/posix/include/rtos/sof.h @@ -17,7 +17,6 @@ struct clock_info; struct comp_driver_list; struct dai_info; struct dma_info; -struct dma_trace_data; struct ipc; struct ll_schedule_domain; struct mm; @@ -27,7 +26,6 @@ struct notify_data; struct pm_runtime_data; struct sa; struct timer; -struct trace; struct pipeline_posn; struct probe_pdata; @@ -49,11 +47,7 @@ struct sof { /* system agent */ struct sa *sa; - /* DMA for Trace*/ - struct dma_trace_data *dmat; - /* generic trace structure */ - struct trace *trace; /* platform clock information */ struct clock_info *clocks; diff --git a/scripts/kernel_user_partition.txt b/scripts/kernel_user_partition.txt index 83bb935b082e..39d5e1191366 100644 --- a/scripts/kernel_user_partition.txt +++ b/scripts/kernel_user_partition.txt @@ -40,7 +40,6 @@ src/include/sof/drivers/* src/include/sof/platform.h src/include/sof/llext_manager.h src/include/sof/ipc/driver.h -src/include/sof/trace/dma-trace.h src/include/sof/lib/agent.h src/include/sof/lib/cpu-clk-manager.h src/include/sof/lib/mailbox.h diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index f68fff8f434e..aa5c682264c4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -12,6 +12,3 @@ endif() add_subdirectory(kernel) -if (CONFIG_TRACE) - add_subdirectory(trace) -endif() diff --git a/src/Kconfig b/src/Kconfig index 92486532f4dd..d008574a9684 100644 --- a/src/Kconfig +++ b/src/Kconfig @@ -6,8 +6,6 @@ rsource "kernel/platform/Kconfig" rsource "audio/Kconfig" -rsource "trace/Kconfig" - rsource "samples/Kconfig" rsource "schedule/Kconfig" diff --git a/src/audio/Kconfig b/src/audio/Kconfig index dbd12f1626b2..fecbf0d25afa 100644 --- a/src/audio/Kconfig +++ b/src/audio/Kconfig @@ -422,14 +422,6 @@ config PCM_REMAPPING_CONVERTERS Enable conversion functions that perform both format conversion and channel remapping simultaneously. -config TRACE_CHANNEL - int "TRACE DMA Channel configuration" - default 0 - help - This option is used for custom configuration of trace dma channel other - than zero dma channel.Define macro DMA_TRACE_CHANNEL in platform to - configure the required channel number. - config WRAP_ACTUAL_POSITION bool "Buffer wrapping" help diff --git a/src/audio/module_adapter/module/waves/waves.c b/src/audio/module_adapter/module/waves/waves.c index 208bb7e010ff..b77dcdcee88d 100644 --- a/src/audio/module_adapter/module/waves/waves.c +++ b/src/audio/module_adapter/module/waves/waves.c @@ -432,31 +432,6 @@ static int waves_effect_revision(struct processing_module *mod) return -EINVAL; } -#if CONFIG_TRACEV - if (revision_len) { - const uint32_t *ptr = (uint32_t *)revision; - uint32_t len = revision_len / sizeof(uint32_t); - uint32_t idx = 0; - - /* get requests from codec_adapter are not supported - * printing strings is not supported - * so dumping revision string to trace log as ascii values - * if simply write a for loop here then depending on trace filtering settings - * some parts of revision might not be printed - this is highly unwanted - */ - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - } -#endif - comp_info(dev, "done"); return 0; } @@ -528,20 +503,6 @@ static int waves_effect_message(struct processing_module *mod, void *data, uint3 } } -#if CONFIG_TRACEV - /* at time of writing codec adapter does not support getting something from codec - * so response is stored to internal structure and dumped into trace messages - */ - if (response_size) { - uint32_t idx; - uint32_t len = response_size / sizeof(uint32_t); - const uint32_t *ptr = (uint32_t *)waves_codec->response; - - for (idx = 0; idx < len; ) - dump_hex(ptr, idx, len); - } -#endif - return 0; } diff --git a/src/debug/panic.c b/src/debug/panic.c index 0c56886e386a..7f163905ee83 100644 --- a/src/debug/panic.c +++ b/src/debug/panic.c @@ -87,10 +87,6 @@ void panic_dump(uint32_t p, struct sof_ipc_panic_info *panic_info, dump_panicinfo(ext_offset, panic_info); ext_offset += sizeof(struct sof_ipc_panic_info); -#if CONFIG_TRACE - trace_flush_dma_to_mbox(); -#endif - /* Dump stack frames and override panic code 'p' if ext_offset is * off stack_limit. Find stack_ptr. */ diff --git a/src/include/sof/debug/debug.h b/src/include/sof/debug/debug.h index fa3fa990b2b4..3e656e43dc87 100644 --- a/src/include/sof/debug/debug.h +++ b/src/include/sof/debug/debug.h @@ -143,34 +143,5 @@ ((n & 0x00FF0000) >> 8) | \ ((n & 0xFF000000) >> 24); }) -#if CONFIG_TRACE -/* dump up to 4 32-bit words into trace starting in ptr, idx is incremented inside - * unconditionally swaps endianness of data - * todo: use cpu_to_endianness so trace data will be independent of platform - */ -#define dump_hex(ptr, idx, len) \ - do { \ - __typeof__(idx) __i = (idx); \ - __typeof__(ptr) __p = (ptr) + __i; \ - __typeof__(idx) __l = (len) - __i; \ - __typeof__(idx) __n = __l > 4 ? 4 : __l; \ - if (__n == 4) { \ - comp_info(dev, "%08x%08x%08x%08x", bswap32(__p[0]), bswap32(__p[1]), \ - bswap32(__p[2]), bswap32(__p[3])); \ - } else if (__n == 3) { \ - comp_info(dev, "%08x%08x%08x", bswap32(__p[0]), bswap32(__p[1]), \ - bswap32(__p[2])); \ - } else if (__n == 2) { \ - comp_info(dev, "%08x%08x", bswap32(__p[0]), bswap32(__p[1])); \ - } else if (__n == 1) { \ - comp_info(dev, "%08x", bswap32(__p[0])); \ - } \ - idx += __n; \ - } while (0) -#else - -#define dump_hex() do {} while (0) - -#endif #endif /* __SOF_DEBUG_DEBUG_H__ */ diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 696cb553661d..d959a55d3b9d 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -213,12 +213,6 @@ int ipc_process_host_buffer(struct ipc *ipc, struct dma_sg_elem_array *elem_array, uint32_t *ring_size); -/** - * \brief Send DMA trace host buffer position to host. - * @return 0 on success. - */ -int ipc_dma_trace_send_position(void); - /** * \brief send a IPC buffer status notify message */ diff --git a/src/include/sof/ipc/msg.h b/src/include/sof/ipc/msg.h index 943faf6df813..528026783924 100644 --- a/src/include/sof/ipc/msg.h +++ b/src/include/sof/ipc/msg.h @@ -138,16 +138,4 @@ void ipc_build_stream_posn(struct sof_ipc_stream_posn *posn, uint32_t type, void ipc_build_comp_event(struct sof_ipc_comp_event *event, uint32_t type, uint32_t id); -/** - * \brief Check if trace buffer is ready for transmission. - * @param[in,out] avail Data available in trace buffer - */ -bool ipc_trigger_trace_xfer(uint32_t avail); - -/** - * \brief Build trace position IPC message. - * @param[in,out] posn Trace position message - */ -void ipc_build_trace_posn(struct sof_ipc_dma_trace_posn *posn); - #endif diff --git a/src/include/sof/trace/dma-trace.h b/src/include/sof/trace/dma-trace.h deleted file mode 100644 index 018576cd7ca2..000000000000 --- a/src/include/sof/trace/dma-trace.h +++ /dev/null @@ -1,172 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * - * Copyright(c) 2016 Intel Corporation. All rights reserved. - * - * Author: Yan Wang - */ - -/** - * \file include/sof/trace/dma-trace.h - * \brief Direct Memory Access (DMA) trace buffer management and host streaming. - * \ingroup trace_logging_api - */ - -#ifndef __SOF_TRACE_DMA_TRACE_H__ -#define __SOF_TRACE_DMA_TRACE_H__ - -#include -#include -#include -#include -#include -#include - -struct ipc_msg; -struct sof; - -/** - * \brief Internal ring buffer descriptor for DSP trace packets. - * \ingroup trace_logging_api - */ -struct dma_trace_buf { - void *w_ptr; /**< Buffer write pointer where new trace packets are appended */ - void *r_ptr; /**< Buffer read pointer tracking data transferred by DMA */ - void *addr; /**< Base address of the circular trace buffer */ - void *end_addr; /**< Upper boundary address of the trace buffer */ - uint32_t size; /**< Total size of the buffer in bytes */ - uint32_t avail; /**< Number of readable bytes currently available */ -}; - -/** - * \brief Main DMA trace context and state machine descriptor. - * \ingroup trace_logging_api - */ -struct dma_trace_data { - struct dma_sg_config config; /**< Scatter-gather configuration for host DMA */ - struct dma_trace_buf dmatb; /**< Internal circular buffer tracking read/write pointers */ - struct dma_copy dc; /**< DMA copy engine instance */ - struct sof_ipc_dma_trace_posn posn; /**< Position report payload sent to host */ - struct ipc_msg *msg; /**< IPC message used to report buffer position */ - uint32_t host_size; /**< Total size of host-allocated trace buffer */ - struct task dmat_work; /**< Periodic scheduling task driving DMA transfers */ - uint32_t enabled; /**< 1 if DMA trace streaming is active, 0 otherwise */ - uint32_t copy_in_progress; /**< 1 if a DMA copy operation is currently underway */ - uint32_t stream_tag; /**< Assigned host DMA stream tag */ - uint32_t active_stream_tag; /**< Current active stream tag */ - uint32_t dma_copy_align; /**< Minimal byte alignment required for DMA transfer chunks */ - uint32_t dropped_entries; /**< Cumulative counter of dropped entries due to buffer overflow */ - struct k_spinlock lock; /**< Spinlock synchronizing buffer writes and DMA triggers */ - uint64_t time_delta; /**< Offset between host wallclock and DSP hardware timer */ -}; - -/** - * \brief Early initialization of the DMA trace subsystem during bootstrap. - * \ingroup trace_logging_api - * @param[in,out] sof Global SOF system context. - * @return 0 on success, or negative error code on failure. - */ -int dma_trace_init_early(struct sof *sof); - -/** - * \brief Complete initialization of DMA trace once host channel is negotiated. - * \ingroup trace_logging_api - * @param[in,out] d DMA trace data context. - * @return 0 on success, or negative error code on failure. - */ -int dma_trace_init_complete(struct dma_trace_data *d); - -/** - * \brief Configure the host-side destination scatter-gather buffer for DMA trace. - * \ingroup trace_logging_api - * @param[in,out] d DMA trace data context. - * @param[in] elem_array Array of scatter-gather buffer page elements. - * @param[in] host_size Total size in bytes of the host buffer. - * @return 0 on success, or negative error code on failure. - */ -int dma_trace_host_buffer(struct dma_trace_data *d, - struct dma_sg_elem_array *elem_array, - uint32_t host_size); - -/** - * \brief Enable DMA trace streaming to the host. - * \ingroup trace_logging_api - * @param[in,out] d DMA trace data context. - * @return 0 on success, or negative error code on failure. - */ -int dma_trace_enable(struct dma_trace_data *d); - -/** - * \brief Disable DMA trace streaming and cancel pending periodic tasks. - * \ingroup trace_logging_api - * @param[in,out] d DMA trace data context. - */ -void dma_trace_disable(struct dma_trace_data *d); - -/** - * \brief Flush residual trace data from local DSP buffer to destination memory or mailbox. - * \ingroup trace_logging_api - * @param[out] destination Pointer to destination memory buffer, or NULL for default mailbox. - */ -void dma_trace_flush(void *destination); - -/** - * \brief Globally unpause and resume trace event logging. - * \ingroup trace_logging_api - */ -void dma_trace_on(void); - -/** - * \brief Globally pause trace event logging. - * \ingroup trace_logging_api - */ -void dma_trace_off(void); - -/** - * \brief Append a binary trace event to the circular buffer. - * \ingroup trace_logging_api - * @param[in] e Pointer to binary event packet. - * @param[in] size Packet size in bytes. - */ -void dtrace_event(const char *e, uint32_t size); - -/** - * \brief Append a binary trace event to the buffer from atomic or interrupt context. - * \ingroup trace_logging_api - * @param[in] e Pointer to binary event packet. - * @param[in] length Size of packet in bytes. - */ -void dtrace_event_atomic(const char *e, uint32_t length); - -/** - * \brief Check whether DMA trace subsystem has been initialized. - * \ingroup trace_logging_api - * @param[in] d DMA trace context. - * @return True if initialized with valid buffer memory, false otherwise. - */ -static inline bool dma_trace_initialized(const struct dma_trace_data *d) -{ - return d && d->dmatb.addr; -} - -/** - * \brief Retrieve global DMA trace data context from system runtime. - * \ingroup trace_logging_api - * @return Pointer to struct dma_trace_data. - */ -static inline struct dma_trace_data *dma_trace_data_get(void) -{ - return sof_get()->dmat; -} - -/** - * \brief Calculate contiguous bytes remaining until the end of the circular buffer. - * \ingroup trace_logging_api - * @param[in] buffer Pointer to circular trace buffer descriptor. - * @return Contiguous byte margin before write wrap-around. - */ -static inline uint32_t dtrace_calc_buf_margin(struct dma_trace_buf *buffer) -{ - return (char *)buffer->end_addr - (char *)buffer->w_ptr; -} - -#endif /* __SOF_TRACE_DMA_TRACE_H__ */ diff --git a/src/include/sof/trace/trace-soflogger.h b/src/include/sof/trace/trace-soflogger.h deleted file mode 100644 index b8b46a0f4dab..000000000000 --- a/src/include/sof/trace/trace-soflogger.h +++ /dev/null @@ -1,476 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * - * Copyright(c) 2016,2024 Intel Corporation. - */ - -/** - * \file include/sof/trace/trace-soflogger.h - * \brief Dictionary-based trace logging engine, log filtering, and mailbox trace fallback. - * \ingroup trace_logging_api - */ - -#ifndef __SOF_TRACE_TRACE_SOFLOGGER_H__ -#define __SOF_TRACE_TRACE_SOFLOGGER_H__ - -#if !CONFIG_LIBRARY -#include -#endif - -#if CONFIG_LIBRARY -#include -#endif - -struct sof; -struct trace; -struct tr_ctx; - -/** - * \brief Runtime trace filter criteria for components and pipelines. - * \ingroup trace_logging_api - */ -struct trace_filter { - uint32_t uuid_id; /**< Component UUID type identifier, or 0 for wildcard */ - int32_t comp_id; /**< Component ID, or -1 for wildcard */ - int32_t pipe_id; /**< Pipeline ID, or -1 for wildcard */ - int32_t log_level; /**< Target log level threshold (LOG_LEVEL_*) */ -}; - -/** The start of this linker output MUST match the 'ldc_entry_header' - * struct defined in the logger program running in user space. - */ -#define _DECLARE_LOG_ENTRY(lvl, format, comp_class, n_params) \ - __section(".static_log." #lvl) \ - static const struct { \ - uint32_t level; \ - uint32_t component_class; \ - uint32_t params_num; \ - uint32_t line_idx; \ - uint32_t file_name_len; \ - uint32_t text_len; \ - const char file_name[sizeof(RELATIVE_FILE)]; \ - const char text[sizeof(format)]; \ - } log_entry = { \ - lvl, \ - comp_class, \ - n_params, \ - __LINE__, \ - sizeof(RELATIVE_FILE), \ - sizeof(format), \ - RELATIVE_FILE, \ - format \ - } - -#define _TRACE_EVENT_MAX_ARGUMENT_COUNT 4 - -/** - * \brief Retrieve global trace subsystem context from SOF runtime. - * \ingroup trace_logging_api - * @return Pointer to struct trace. - */ -static inline struct trace *trace_get(void) -{ - return sof_get()->trace; -} - -#include -#include /* LOG_LEVEL_... */ - -/* - * trace_event macro definition - * - * trace_event() macro is used for logging events that occur at runtime. - * It comes in 2 main flavours, atomic and non-atomic. Depending of definitions - * above, it might also propagate log messages to mbox if desired. - * - * First argument is always class of event being logged, as defined in - * user/trace.h - TRACE_CLASS_* (deprecated - do not use). - * Second argument is string literal in printf format, followed by up to 4 - * parameters (uint32_t), that are used to expand into string fromat when - * parsing log data. - * - * All compile-time accessible data (verbosity, class, source file name, line - * index and string literal) are linked into .static_log_entries section - * of binary and then extracted by smex, so they do not contribute to loadable - * image size. This way more elaborate log messages are possible and encouraged, - * for better debugging experience, without worrying about runtime performance. - */ - -/* Map the different trace_xxxx_with_ids(... ) levels to the - * _trace_event_with_ids(level_xxxx, ...) macro shared across log - * levels. - */ -#define trace_event_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_event_with_ids(LOG_LEVEL_INFO, class, ctx, id_1, id_2, \ - format, ##__VA_ARGS__) - -#define trace_event_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_event_atomic_with_ids(LOG_LEVEL_INFO, class, ctx, id_1, id_2, \ - format, ##__VA_ARGS__) - -#define trace_warn_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_event_with_ids(LOG_LEVEL_WARNING, class, ctx, id_1, id_2, \ - format, ##__VA_ARGS__) - -#define trace_warn_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_event_atomic_with_ids(LOG_LEVEL_WARNING, class, \ - ctx, id_1, id_2, \ - format, ##__VA_ARGS__) - -/** - * \brief Flush buffered trace data from DMA buffer to Mailbox Window. - * \ingroup trace_logging_api - */ -void trace_flush_dma_to_mbox(void); - -/** - * \brief Globally resume trace event logging. - * \ingroup trace_logging_api - */ -void trace_on(void); - -/** - * \brief Globally pause trace event logging. - * \ingroup trace_logging_api - */ -void trace_off(void); - -/** - * \brief Initialize the trace subsystem and allocate global trace buffers. - * \ingroup trace_logging_api - * @param[in,out] sof Global SOF runtime context. - */ -void trace_init(struct sof *sof); - -/** - * \brief Function pointer type for formatted trace logger backends. - * \ingroup trace_logging_api - * @param[in] send_atomic True if logging from atomic or interrupt context. - * @param[in] log_entry Pointer to static log entry descriptor. - * @param[in] ctx Pointer to caller trace context. - * @param[in] lvl Severity log level (LOG_LEVEL_*). - * @param[in] id_1 Primary component/pipe identifier. - * @param[in] id_2 Secondary component/subid identifier. - * @param[in] arg_count Number of variable arguments. - * @param[in] args Variable arguments list. - */ -typedef void (*log_func_t)(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list args); - -/** - * \brief Emit a filtered dictionary trace message. - * \ingroup trace_logging_api - * @param[in] send_atomic True if logging from atomic context. - * @param[in] log_entry Pointer to static log entry. - * @param[in] ctx Trace context. - * @param[in] lvl Log level. - * @param[in] id_1 Primary identifier. - * @param[in] id_2 Secondary identifier. - * @param[in] arg_count Argument count. - * @param[in] args Argument list. - */ -void trace_log_filtered(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list args); - -/** - * \brief Emit an unfiltered dictionary trace message directly to buffers. - * \ingroup trace_logging_api - * @param[in] send_atomic True if logging from atomic context. - * @param[in] log_entry Pointer to static log entry. - * @param[in] ctx Trace context. - * @param[in] lvl Log level. - * @param[in] id_1 Primary identifier. - * @param[in] id_2 Secondary identifier. - * @param[in] arg_count Argument count. - * @param[in] args Argument list. - */ -void trace_log_unfiltered(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list args); - -/** - * \brief Populate an IPC trace filter element array. - * \ingroup trace_logging_api - * @param[in,out] elem Pointer to current output filter element. - * @param[in] end Boundary pointer of element array. - * @param[in] filter Source trace filter configuration. - * @return Pointer to next available filter element slot. - */ -struct sof_ipc_trace_filter_elem *trace_filter_fill(struct sof_ipc_trace_filter_elem *elem, - struct sof_ipc_trace_filter_elem *end, - struct trace_filter *filter); - -/** - * \brief Update runtime trace filtering configuration from IPC payload. - * \ingroup trace_logging_api - * @param[in] elem Trace filter descriptor to apply. - * @return 0 on success, or negative error code on failure. - */ -int trace_filter_update(const struct trace_filter *elem); - -#define _trace_event_with_ids(lvl, class, ctx, id_1, id_2, format, ...) \ - _log_message(trace_log_filtered, false, lvl, class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -#define _trace_event_atomic_with_ids(lvl, class, ctx, id_1, id_2, format, ...) \ - _log_message(trace_log_filtered, true, lvl, class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -/** - * \brief Appends one SOF dictionary entry and log statement to shared memory mtrace. - * \ingroup trace_logging_api - * @param atomic_context Take the trace->lock if false. - * @param log_entry_pointer Dictionary index produced by _DECLARE_LOG_ENTRY macro. - * @param n_args Number of variable arguments. - */ -void mtrace_dict_entry(bool atomic_context, uint32_t log_entry_pointer, int n_args, ...); - -/** - * \brief Posts a fully prepared log header and log entry packet to mtrace buffer. - * \ingroup trace_logging_api - * @param complete_packet Pointer to prepared log packet buffer. - * @param length Packet size in bytes. - */ -void mtrace_event(const char *complete_packet, uint32_t length); - -#ifdef CONFIG_TRACEM /* Send everything to shared memory too */ -# ifdef __ZEPHYR__ -/* We don't use Zephyr's dictionary yet so there's not enough space for - * DEBUG messages - */ -# define MTRACE_DUPLICATION_LEVEL LOG_LEVEL_INFO -# else -# define MTRACE_DUPLICATION_LEVEL LOG_LEVEL_DEBUG -# endif -#else /* copy only ERRORS */ -# define MTRACE_DUPLICATION_LEVEL LOG_LEVEL_ERROR -#endif /* CONFIG_TRACEM */ - -/** - * \brief Internal dictionary logger formatting and dispatch helper. - * \ingroup trace_logging_api - * @param sofdict_logf Backend log emission callback function. - * @param atomic True if executed in atomic context. - * @param log_entry Pointer to static log entry. - * @param ctx Trace context. - * @param lvl Log severity level. - * @param id_1 Primary identifier. - * @param id_2 Secondary identifier. - * @param arg_count Number of format arguments. - */ -void _log_sofdict(log_func_t sofdict_logf, bool atomic, const void *log_entry, - const struct tr_ctx *ctx, const uint32_t lvl, - uint32_t id_1, uint32_t id_2, int arg_count, ...); - -/* _log_message() */ - -#ifdef CONFIG_LIBRARY - -#include - -/* trace level used on host configurations */ -extern int host_trace_level; - -char *get_trace_class(uint32_t trace_class); -#define _log_message(ignored_log_func, atomic, level, comp_class, ctx, id_1, id_2, format, ...) \ -do { \ - (void)ctx; \ - (void)id_1; \ - (void)id_2; \ - struct timeval tv; \ - char slevel[5][4] = {"dis", "err", "wrn", "inf", "dbg"}; \ - char *msg = "(%s:%d) " format; \ - if (level <= host_trace_level) { \ - gettimeofday(&tv, NULL); \ - fprintf(stderr, "trace: [%ld.%6.6ld] <%s> ", tv.tv_sec, tv.tv_usec, \ - slevel[level]); \ - fprintf(stderr, msg, strrchr(__FILE__, '/') + 1, \ - __LINE__, ##__VA_ARGS__); \ - fprintf(stderr, "\n"); \ - } \ -} while (0) - -#define trace_point(x) do {} while (0) - -#else /* CONFIG_LIBRARY */ - -#define trace_point(x) platform_trace_point(x) - -#define BASE_LOG_ASSERT_FAIL_MSG \ -unsupported_amount_of_params_in_trace_event\ -_thrown_from_macro_BASE_LOG_in_trace_h - -#define CT_ASSERT(COND, MESSAGE) \ - ((void)sizeof(char[1 - 2 * !(COND)])) - -#define trace_check_size_uint32(a) \ - CT_ASSERT(sizeof(a) <= sizeof(uint32_t), "error: trace argument is bigger than a uint32_t"); - -#define STATIC_ASSERT_ARG_SIZE(...) \ - META_MAP(1, trace_check_size_uint32, __VA_ARGS__) - -/** _log_message is where the memory-saving dictionary magic described - * above happens: the "format" string argument is moved to a special - * linker section and replaced by a &log_entry pointer to it. This must - * be a macro for the source location to be meaningful. - */ -#define _log_message(log_func, atomic, lvl, comp_class, ctx, id_1, id_2, format, ...) \ -do { \ - _DECLARE_LOG_ENTRY(lvl, format, comp_class, \ - META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__)); \ - STATIC_ASSERT_ARG_SIZE(__VA_ARGS__); \ - STATIC_ASSERT(_TRACE_EVENT_MAX_ARGUMENT_COUNT >= \ - META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__), \ - BASE_LOG_ASSERT_FAIL_MSG \ - ); \ - _log_sofdict(log_func, atomic, &log_entry, ctx, lvl, id_1, id_2, \ - META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__), ##__VA_ARGS__); \ - _log_nodict(atomic, META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__), \ - lvl, format, ##__VA_ARGS__); \ -} while (0) - -#ifdef __ZEPHYR__ -/* Just like XTOS, only the most urgent messages go to limited - * shared memory. - */ -#define _log_nodict(atomic, arg_count, lvl, format, ...) \ -do { \ - if ((lvl) <= MTRACE_DUPLICATION_LEVEL) \ - printk("%llu " format "\n", k_cycle_get_64(), \ - ##__VA_ARGS__); \ -} while (0) -#else -#define _log_nodict(atomic, n_args, lvl, format, ...) -#endif - -#endif /* CONFIG_LIBRARY */ - -#if CONFIG_TRACEV -/* Enable tr_dbg() statements by defining tracev_...() */ -#define tracev_event_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_event_with_ids(LOG_LEVEL_VERBOSE, class, \ - ctx, id_1, id_2, \ - format, ##__VA_ARGS__) - -#define tracev_event_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_event_atomic_with_ids(LOG_LEVEL_VERBOSE, class, \ - ctx, id_1, id_2, \ - format, ##__VA_ARGS__) - -#else /* CONFIG_TRACEV */ -#define tracev_event_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) -#define tracev_event_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -#endif /* CONFIG_TRACEV */ - -/* The _error_ level has 2, 1 or 0 backends depending on Kconfig */ -#if CONFIG_TRACEE -/* LOG_LEVEL_CRITICAL messages are duplicated to the mail box */ -#define _trace_error_with_ids(class, ctx, id_1, id_2, format, ...) \ - _log_message(trace_log_filtered, true, LOG_LEVEL_CRITICAL, class, ctx, id_1, \ - id_2, format, ##__VA_ARGS__) -#define trace_error_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_error_with_ids(class, ctx, id_1, id_2, format, ##__VA_ARGS__) -#define trace_error_atomic_with_ids(...) trace_error_with_ids(__VA_ARGS__) - -#elif CONFIG_TRACE -/* Goes to trace_log_filtered() too but with a downgraded, LOG_INFO level */ -#define trace_error_with_ids(...) trace_event_with_ids(__VA_ARGS__) -#define trace_error_atomic_with_ids(...) \ - trace_event_atomic_with_ids(__VA_ARGS__) - -#endif /* CONFIG_TRACEE, CONFIG_TRACE */ - -/* tracing from device (component, pipeline, dai, ...) */ - -/** \brief Trace from a device on err level. - * - * @param get_ctx_m Macro that can retrieve trace context from dev - * @param get_id_m Macro that can retrieve device's id0 from the dev - * @param get_subid_m Macro that can retrieve device's id1 from the dev - * @param dev Device - * @param fmt Format followed by parameters - * @param ... Parameters - */ -#define trace_dev_err(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_error_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) - -/** \brief Trace from a device on warning level. */ -#define trace_dev_warn(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_warn_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) - -/** \brief Trace from a device on info level. */ -#define trace_dev_info(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_event_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) - -/** \brief Trace from a device on dbg level. */ -#define trace_dev_dbg(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - tracev_event_with_ids(_TRACE_INV_CLASS, \ - get_ctx_m(dev), get_id_m(dev), \ - get_subid_m(dev), fmt, ##__VA_ARGS__) - -/* tracing from infrastructure part */ - -#define tr_err_atomic(ctx, fmt, ...) \ - trace_error_atomic_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, \ - fmt, ##__VA_ARGS__) - -#define tr_warn_atomic(ctx, fmt, ...) \ - trace_warn_atomic_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, \ - fmt, ##__VA_ARGS__) - -#define tr_info_atomic(ctx, fmt, ...) \ - trace_event_atomic_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, \ - fmt, ##__VA_ARGS__) - -#define tr_dbg_atomic(ctx, fmt, ...) \ - tracev_event_atomic_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, \ - fmt, ##__VA_ARGS__) - -#define tr_err(ctx, fmt, ...) \ - trace_error_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) - -#define tr_warn(ctx, fmt, ...) \ - trace_warn_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) - -#define tr_info(ctx, fmt, ...) \ - trace_event_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) - -/* tracev_ output depends on CONFIG_TRACEV=y */ -#define tr_dbg(ctx, fmt, ...) \ - tracev_event_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) - -/** Direct, low-level access to mbox / shared memory logging when DMA - * tracing is either not initialized yet or disabled or found broken for - * any reason. - * To keep it simpler than and with minimal dependencies on - * the huge number of lines above, this does not check arguments at compile - * time. - * There is neither log level filtering, throttling or any other - * advanced feature. - */ -#define mtrace_printf(log_level, format_str, ...) \ - do { \ - STATIC_ASSERT(META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__) \ - <= _TRACE_EVENT_MAX_ARGUMENT_COUNT, \ - too_many_mtrace_printf_arguments); \ - _DECLARE_LOG_ENTRY(log_level, format_str, _TRACE_INV_CLASS, \ - META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__)); \ - mtrace_dict_entry(true, (uint32_t)&log_entry, \ - META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__), \ - ##__VA_ARGS__); \ - } while (0) - -#endif diff --git a/src/include/sof/trace/trace.h b/src/include/sof/trace/trace.h index 007abe866731..188796d24606 100644 --- a/src/include/sof/trace/trace.h +++ b/src/include/sof/trace/trace.h @@ -49,10 +49,6 @@ #include "trace-host.h" -#elif CONFIG_TRACE - -#include "trace-soflogger.h" - #else #define tr_err(ctx, fmt, ...) \ @@ -64,64 +60,23 @@ #define tr_dbg(ctx, fmt, ...) \ trace_unused(_TRACE_INV_CLASS, ctx, _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) -#endif - -#ifndef CONFIG_TRACE - -struct trace_filter; -struct sof; - -#define trace_event_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) -#define trace_event_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -#define trace_warn_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) -#define trace_warn_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -#define tracev_event_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -#define trace_point(x) do {} while (0) - -static inline void trace_flush_dma_to_mbox(void) { } -static inline void trace_on(void) { } -static inline void trace_off(void) { } -static inline void trace_init(struct sof *sof) { } -static inline int trace_filter_update(const struct trace_filter *filter) - { return 0; } - -#define trace_error_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) -#define trace_error_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -#if !CONFIG_LIBRARY || defined(__ZEPHYR__) #define trace_dev_err(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_event_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) + trace_unused(_TRACE_INV_CLASS, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) #define trace_dev_warn(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_event_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) + trace_unused(_TRACE_INV_CLASS, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) #define trace_dev_info(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_event_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) + trace_unused(_TRACE_INV_CLASS, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) #define trace_dev_dbg(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_event_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) -#endif /* !CONFIG_LIBRARY || __ZEPHYR__ */ + trace_unused(_TRACE_INV_CLASS, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) -static inline void mtrace_printf(int log_level, const char *format_str, ...) -{ -}; +#endif -#endif /* !CONFIG_TRACE */ +/* Boot progress markers; no longer reported anywhere */ +#define trace_point(x) do {} while (0) /** Default value when there is no specific pipeline, dev, dai, etc. */ #define _TRACE_INV_ID -1 @@ -140,12 +95,6 @@ struct tr_ctx { uint32_t level; /**< Default log level */ }; -#if defined(UNIT_TEST) || !defined(CONFIG_TRACE) -#define TRACE_CONTEXT_SECTION -#else -#define TRACE_CONTEXT_SECTION __section(".trace_ctx") -#endif - /** * Declares trace context. * @param ctx_name (Symbol) name. @@ -153,7 +102,7 @@ struct tr_ctx { * @param default_log_level Default log level. */ #define DECLARE_TR_CTX(ctx_name, uuid, default_log_level) \ - struct tr_ctx ctx_name TRACE_CONTEXT_SECTION = { \ + struct tr_ctx ctx_name = { \ .uuid_p = uuid, \ .level = default_log_level, \ } diff --git a/src/ipc/CMakeLists.txt b/src/ipc/CMakeLists.txt index f12ac465acd0..3608369cda7d 100644 --- a/src/ipc/CMakeLists.txt +++ b/src/ipc/CMakeLists.txt @@ -13,14 +13,6 @@ add_local_sources(sof is_zephyr(zephyr) if(zephyr) ### Zephyr ### -# dma-copy only used for dma-trace -zephyr_library_sources_ifdef(CONFIG_TRACE dma-copy.c) -else() ### Not Zephyr ### - -if (CONFIG_TRACE) - add_local_sources(sof - dma-copy.c) -endif() endif() # Zephyr diff --git a/src/ipc/dma-copy.c b/src/ipc/dma-copy.c deleted file mode 100644 index b7f55a90292e..000000000000 --- a/src/ipc/dma-copy.c +++ /dev/null @@ -1,159 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause -// -// Copyright(c) 2016 Intel Corporation. All rights reserved. -// -// Author: Liam Girdwood -// Keyon Jie - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -/* tracing */ -LOG_MODULE_REGISTER(dma_copy, CONFIG_SOF_LOG_LEVEL); - -SOF_DEFINE_REG_UUID(dma_copy); - -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(dmacpy_tr, SOF_UUID(dma_copy_uuid), LOG_LEVEL_INFO); - -static struct dma_sg_elem *sg_get_elem_at(struct dma_sg_config *host_sg, - int32_t *offset) -{ - struct dma_sg_elem *host_sg_elem; - int i; - int32_t _offset = *offset; - - /* find host element with host_offset */ - for (i = 0; i < host_sg->elem_array.count; i++) { - - host_sg_elem = host_sg->elem_array.elems + i; - - /* is offset in this elem ? */ - if (_offset >= 0 && _offset < host_sg_elem->size) { - *offset = _offset; - return host_sg_elem; - } - - _offset -= host_sg_elem->size; - } - - /* host offset in beyond end of SG buffer */ - tr_err(&dmacpy_tr, "host offset in beyond end of SG buffer"); - return NULL; -} - -/* Copy DSP memory to host memory. - * Copies DSP memory to host in a single PAGE_SIZE or smaller block. Does not - * waits/sleeps and can be used in IRQ context. - */ - -static int -dma_copy_to_host_flags(struct dma_copy *dc, struct dma_sg_config *host_sg, - int32_t host_offset, void *local_ptr, int32_t size, - uint32_t flags) -{ - struct dma_sg_config config; - struct dma_sg_elem *host_sg_elem; - struct dma_sg_elem local_sg_elem; - int32_t err; - int32_t offset = host_offset; - - if (size <= 0) - return 0; - - /* find host element with host_offset */ - host_sg_elem = sg_get_elem_at(host_sg, &offset); - if (!host_sg_elem) - return -EINVAL; - - /* set up DMA configuration */ - config.direction = DMA_DIR_LMEM_TO_HMEM; - config.src_width = sizeof(uint32_t); - config.dest_width = sizeof(uint32_t); - config.cyclic = 0; - config.irq_disabled = false; - dma_sg_init(&config.elem_array); - - /* configure local DMA elem */ - local_sg_elem.dest = host_sg_elem->dest + offset; - local_sg_elem.src = (uintptr_t)local_ptr; - if (size >= HOST_PAGE_SIZE - offset) - local_sg_elem.size = HOST_PAGE_SIZE - offset; - else - local_sg_elem.size = size; - - config.elem_array.elems = &local_sg_elem; - config.elem_array.count = 1; - - /* start the DMA */ - err = dma_set_config_legacy(dc->chan, &config); - if (err < 0) - return err; - - err = dma_copy_legacy(dc->chan, local_sg_elem.size, - flags); - if (err < 0) - return err; - - /* bytes copied */ - return local_sg_elem.size; -} - -int dma_copy_to_host(struct dma_copy *dc, struct dma_sg_config *host_sg, - int32_t host_offset, void *local_ptr, int32_t size) -{ - return dma_copy_to_host_flags(dc, host_sg, host_offset, local_ptr, size, - DMA_COPY_ONE_SHOT | DMA_COPY_BLOCKING); -} - -int dma_copy_to_host_nowait(struct dma_copy *dc, struct dma_sg_config *host_sg, - int32_t host_offset, void *local_ptr, int32_t size) -{ - return dma_copy_to_host_flags(dc, host_sg, host_offset, local_ptr, size, - DMA_COPY_ONE_SHOT); -} - -int dma_copy_new(struct dma_copy *dc) -{ - uint32_t dir, cap, dev; - - /* request HDA DMA in the dir LMEM->HMEM with shared access */ - dir = DMA_DIR_LMEM_TO_HMEM; - dev = DMA_DEV_HOST; - cap = 0; - dc->dmac = dma_get(dir, cap, dev, DMA_ACCESS_SHARED); - if (!dc->dmac) { - tr_err(&dmacpy_tr, "dc->dmac = NULL"); - return -ENODEV; - } - - /* get DMA channel from DMAC0 */ - dc->chan = dma_channel_get_legacy(dc->dmac, CONFIG_TRACE_CHANNEL); - if (!dc->chan) { - tr_err(&dmacpy_tr, "dc->chan is NULL"); - return -ENODEV; - } - - return 0; -} - -int dma_copy_set_stream_tag(struct dma_copy *dc, uint32_t stream_tag) -{ - /* get DMA channel from DMAC */ - dc->chan = dma_channel_get_legacy(dc->dmac, stream_tag - 1); - if (!dc->chan) { - tr_err(&dmacpy_tr, "dc->chan is NULL"); - return -EINVAL; - } - - return 0; -} - diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index d5a0e095b3f4..18690cb9f28c 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -76,17 +76,6 @@ void ipc_build_comp_event(struct sof_ipc_comp_event *event, uint32_t type, { } -bool ipc_trigger_trace_xfer(uint32_t avail) -{ - return avail >= DMA_TRACE_LOCAL_SIZE / 2; -} - -void ipc_build_trace_posn(struct sof_ipc_dma_trace_posn *posn) -{ - posn->rhdr.hdr.cmd = SOF_IPC4_NOTIF_HEADER(SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS); - posn->rhdr.hdr.size = 0; -} - #if CONFIG_LIBRARY static inline unsigned char *ipc4_get_comp_new_data(void) { diff --git a/src/kernel/init/init.c b/src/kernel/init/init.c index a9acfed5e9ac..20f0f1b2fe1f 100644 --- a/src/kernel/init/init.c +++ b/src/kernel/init/init.c @@ -221,11 +221,6 @@ __cold static int primary_core_init(int argc, char *argv[], struct sof *sof) sys_clock_hw_cycles_per_sec()); #endif -#if CONFIG_TRACE - trace_point(TRACE_BOOT_SYS_TRACES); - trace_init(sof); -#endif - print_version_banner(); trace_point(TRACE_BOOT_SYS_NOTIFIER); diff --git a/src/kernel/ipc/ipc3/handler.c b/src/kernel/ipc/ipc3/handler.c index 38694e10984d..58dc12eee1ec 100644 --- a/src/kernel/ipc/ipc3/handler.c +++ b/src/kernel/ipc/ipc3/handler.c @@ -35,9 +35,6 @@ #include #include #include -#if CONFIG_TRACE -#include -#endif #include #include #include @@ -759,10 +756,6 @@ static int ipc_pm_gate(uint32_t header) tr_info(&ipc_tr, "ipc: pm gate flags 0x%x", pm_gate.flags); - /* pause dma trace firstly if needed */ - if (pm_gate.flags & SOF_PM_NO_TRACE) - trace_off(); - if (pm_gate.flags & SOF_PM_PPG) { pm_runtime_disable(PM_RUNTIME_DSP, PLATFORM_PRIMARY_CORE_ID); } else { @@ -775,10 +768,6 @@ static int ipc_pm_gate(uint32_t header) pm_runtime_enable(PM_RUNTIME_DSP, PLATFORM_PRIMARY_CORE_ID); } - /* resume dma trace if needed */ - if (!(pm_gate.flags & SOF_PM_NO_TRACE)) - trace_on(); - return 0; } @@ -806,158 +795,6 @@ static int ipc_glb_pm_message(uint32_t header) } } -#if CONFIG_TRACE -/* - * Debug IPC Operations. - */ -static void ipc_dma_trace_free(uint32_t header) -{ - struct dma_trace_data *dmat = dma_trace_data_get(); - - dma_trace_disable(dmat); -} - -static int ipc_dma_trace_config(uint32_t header) -{ -#if CONFIG_HOST_PTABLE - struct dma_sg_elem_array elem_array; - uint32_t ring_size; -#endif - struct dma_trace_data *dmat = dma_trace_data_get(); - struct ipc *ipc = ipc_get(); - struct sof_ipc_dma_trace_params_ext params; - int err; - - if (!dmat) { - mtrace_printf(LOG_LEVEL_ERROR, - "failed: dmat not initialized"); - return -ENOMEM; - } - - /* copy message with ABI safe method */ - IPC_COPY_CMD(params, ipc->comp_data); - - if (iCS(header) == SOF_IPC_TRACE_DMA_PARAMS_EXT) - /* As version 5.12 Linux sends the monotonic - * ktime_get(). Search for - * "SOF_IPC_TRACE_DMA_PARAMS_EXT" in your particular - * kernel version. - */ - dmat->time_delta = k_ns_to_cyc_near64(params.timestamp_ns) - sof_cycle_get_64(); - else - dmat->time_delta = 0; - -#if CONFIG_HOST_PTABLE - err = ipc_process_host_buffer(ipc, ¶ms.buffer, - SOF_IPC_STREAM_CAPTURE, - &elem_array, - &ring_size); - if (err < 0) - goto processing_error; - - err = dma_trace_host_buffer(dmat, &elem_array, ring_size); - if (err < 0) { - ipc_cmd_err(&ipc_tr, "ipc: trace failed to set host buffers %d", - err); - goto error; - } -#else - /* stream tag of capture stream for DMA trace */ - dmat->stream_tag = params.stream_tag; - - /* host buffer size for DMA trace */ - dmat->host_size = params.buffer.size; -#endif /* CONFIG_HOST_PTABLE */ - - err = dma_trace_enable(dmat); - if (err < 0) { - ipc_cmd_err(&ipc_tr, "ipc: failed to enable trace %d", err); - goto error; - } - - return 0; - -error: -#if CONFIG_HOST_PTABLE - dma_sg_free(NULL, &elem_array); - -processing_error: -#endif - - return err; -} - -static int ipc_trace_filter_update(uint32_t header) -{ - struct sof_ipc_trace_filter_elem *next_elem; - struct sof_ipc_trace_filter_elem *elem; - struct sof_ipc_trace_filter_elem *end; - struct sof_ipc_trace_filter *packet; - struct trace_filter filter; - struct ipc *ipc = ipc_get(); - int ret = 0; - int cnt; - - packet = ipc->comp_data; - - /* validation, packet->hdr.size has already been compared with SOF_IPC_MSG_MAX_SIZE */ - if (sizeof(*packet) + sizeof(*elem) * packet->elem_cnt != packet->hdr.size) { - ipc_cmd_err(&ipc_tr, "trace_filter_update failed, elem_cnt %d is inconsistent with hdr.size %d", - packet->elem_cnt, packet->hdr.size); - return -EINVAL; - } - - tr_info(&ipc_tr, "ipc: trace_filter_update received, size %d elems", - packet->elem_cnt); - - elem = packet->elems; - end = &packet->elems[packet->elem_cnt]; - - /* read each filter set and update selected components trace settings */ - while (elem != end) { - next_elem = trace_filter_fill(elem, end, &filter); - if (!next_elem) - return -EINVAL; - - cnt = trace_filter_update(&filter); - if (cnt < 0) { - ipc_cmd_err(&ipc_tr, "trace_filter_update failed for UUID key 0x%X, comp %d.%d and log level %d", - filter.uuid_id, filter.pipe_id, filter.comp_id, - filter.log_level); - ret = cnt; - } else { - tr_info(&ipc_tr, "trace_filter_update for UUID key 0x%X, comp %d.%d affected %d components", - filter.uuid_id, filter.pipe_id, filter.comp_id, - cnt); - } - - elem = next_elem; - } - - return ret; -} - -static int ipc_glb_trace_message(uint32_t header) -{ - uint32_t cmd = iCS(header); - - tr_info(&ipc_tr, "ipc: debug cmd 0x%x", cmd); - - switch (cmd) { - case SOF_IPC_TRACE_DMA_PARAMS: - case SOF_IPC_TRACE_DMA_PARAMS_EXT: - return ipc_dma_trace_config(header); - case SOF_IPC_TRACE_DMA_FREE: - ipc_dma_trace_free(header); - return 0; - case SOF_IPC_TRACE_FILTER_UPDATE: - return ipc_trace_filter_update(header); - default: - ipc_cmd_err(&ipc_tr, "ipc: unknown debug cmd 0x%x", cmd); - return -EINVAL; - } -} -#else /* CONFIG_TRACE */ static int ipc_glb_trace_message(uint32_t header) { /* Return success, as the protocol provides no way to inform @@ -966,7 +803,6 @@ static int ipc_glb_trace_message(uint32_t header) */ return 0; } -#endif /* CONFIG_TRACE */ static int ipc_glb_gdb_debug(uint32_t header) { diff --git a/src/kernel/ipc/ipc3/helper.c b/src/kernel/ipc/ipc3/helper.c index 2fb7b52e1f4a..76303be2c267 100644 --- a/src/kernel/ipc/ipc3/helper.c +++ b/src/kernel/ipc/ipc3/helper.c @@ -61,18 +61,6 @@ void ipc_build_comp_event(struct sof_ipc_comp_event *event, uint32_t type, event->src_comp_id = id; } -bool ipc_trigger_trace_xfer(uint32_t avail) -{ - return true; -} - -void ipc_build_trace_posn(struct sof_ipc_dma_trace_posn *posn) -{ - posn->rhdr.hdr.cmd = SOF_IPC_GLB_TRACE_MSG | - SOF_IPC_TRACE_DMA_POSITION; - posn->rhdr.hdr.size = sizeof(*posn); -} - static const struct comp_driver *get_drv(struct sof_ipc_comp *comp) { struct comp_driver_list *drivers = comp_drivers_get(); diff --git a/src/kernel/platform/amd/acp_7_0/platform.c b/src/kernel/platform/amd/acp_7_0/platform.c index 72162bfd68f4..6d40612c22d7 100644 --- a/src/kernel/platform/amd/acp_7_0/platform.c +++ b/src/kernel/platform/amd/acp_7_0/platform.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include diff --git a/src/kernel/platform/amd/acp_7_x/platform.c b/src/kernel/platform/amd/acp_7_x/platform.c index 37d6d3d45ed9..740f2eb687aa 100644 --- a/src/kernel/platform/amd/acp_7_x/platform.c +++ b/src/kernel/platform/amd/acp_7_x/platform.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include diff --git a/src/kernel/platform/amd/rembrandt/platform.c b/src/kernel/platform/amd/rembrandt/platform.c index 1c26e8e8fcfa..b915c4963438 100644 --- a/src/kernel/platform/amd/rembrandt/platform.c +++ b/src/kernel/platform/amd/rembrandt/platform.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -158,17 +157,6 @@ int platform_init(struct sof *sof) ret = dai_init(sof); if (ret < 0) return -ENODEV; -#if CONFIG_TRACE - /* Initialize DMA for Trace*/ - trace_point(TRACE_BOOT_PLATFORM_DMA_TRACE); - sof->dmat->config.elem_array.elems = rzalloc(SOF_MEM_FLAG_KERNEL, - sizeof(struct dma_sg_elem) * 1); - sof->dmat->config.elem_array.count = 1; - sof->dmat->config.elem_array.elems->dest = 0x03800000; - sof->dmat->config.elem_array.elems->size = 65536; - sof->dmat->config.scatter = 0; - dma_trace_init_complete(sof->dmat); -#endif /* show heap status */ heap_trace_all(1); return 0; diff --git a/src/kernel/platform/imx8m/platform.c b/src/kernel/platform/imx8m/platform.c index 46e60a7e8608..8bc719dc52ec 100644 --- a/src/kernel/platform/imx8m/platform.c +++ b/src/kernel/platform/imx8m/platform.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -172,12 +171,6 @@ int platform_init(struct sof *sof) if (ret < 0) return -ENODEV; -#if CONFIG_TRACE - /* Initialize DMA for Trace*/ - trace_point(TRACE_BOOT_PLATFORM_DMA_TRACE); - dma_trace_init_complete(sof->dmat); -#endif - /* show heap status */ heap_trace_all(1); diff --git a/src/kernel/platform/imx8ulp/platform.c b/src/kernel/platform/imx8ulp/platform.c index 88e6dfb8d56d..96706dd9854a 100644 --- a/src/kernel/platform/imx8ulp/platform.c +++ b/src/kernel/platform/imx8ulp/platform.c @@ -25,9 +25,6 @@ #include #include #include -#if CONFIG_TRACE -#include -#endif #include #include #include @@ -171,12 +168,6 @@ int platform_init(struct sof *sof) if (ret < 0) return -ENODEV; -#if CONFIG_TRACE - /* Initialize DMA for Trace*/ - trace_point(TRACE_BOOT_PLATFORM_DMA_TRACE); - dma_trace_init_complete(sof->dmat); -#endif - /* show heap status */ heap_trace_all(1); diff --git a/src/kernel/platform/tigerlake/include/platform/lib/memory.h b/src/kernel/platform/tigerlake/include/platform/lib/memory.h index 63802920ac95..bf82e2483b8f 100644 --- a/src/kernel/platform/tigerlake/include/platform/lib/memory.h +++ b/src/kernel/platform/tigerlake/include/platform/lib/memory.h @@ -208,11 +208,7 @@ /* window 3 */ #define SRAM_TRACE_BASE (SRAM_STREAM_BASE + SRAM_STREAM_SIZE) -#if CONFIG_TRACE -#define SRAM_TRACE_SIZE 0x2000 -#else #define SRAM_TRACE_SIZE 0x0 -#endif #define HP_SRAM_WIN0_BASE SRAM_SW_REG_BASE #define HP_SRAM_WIN0_SIZE (SRAM_SW_REG_SIZE + SRAM_OUTBOX_SIZE) diff --git a/src/trace/CMakeLists.txt b/src/trace/CMakeLists.txt deleted file mode 100644 index 7fab6cf5eaaa..000000000000 --- a/src/trace/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause - -if(CONFIG_TRACE) - add_local_sources(sof dma-trace.c trace.c) -endif() diff --git a/src/trace/Kconfig b/src/trace/Kconfig deleted file mode 100644 index c60f2598ca79..000000000000 --- a/src/trace/Kconfig +++ /dev/null @@ -1,100 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause - -# Trace configs - -menu "Trace" - -config TRACE - bool "Trace" - default n if ZEPHYR_SOF_MODULE || LIBRARY - default y - help - Enable SOF DMA based traces compatible with the sof-logger tool. - With Zephyr, RTOS side events are not seen in the SOF trace, so native - Zephyr logging (CONFIG_ZEPHYR_LOG) is recommended instead. - -config TRACEV - bool "Trace verbose" - depends on TRACE - default n - help - Enabling verbose traces. tr_dbg() statements depend on this at - compile-time and on run-time filtering below too. - -config TRACEE - bool "Trace error" - depends on TRACE - default y - help - Sending error traces by mailbox additionally. - -config TRACEM - bool "Trace mailbox" - depends on TRACE - default n - help - Sending all traces by mailbox additionally. - -config TRACE_FILTERING - bool "Trace filtering" - depends on TRACE - default y - help - Filtering of trace messages based on their verbosity level and/or frequency. - -config TRACE_FILTERING_VERBOSITY - bool "Filter by verbosity" - depends on TRACE_FILTERING - default y - help - Filtering by log verbosity level, where maximum verbosity allowed is specified for each - context and may be adjusted in runtime. Most basic feature found in every logger. - -config TRACE_FILTERING_ADAPTIVE - bool "Adaptive rate limiting" - depends on TRACE_FILTERING - default y - help - Adaptive filtering of repeating trace messages, tracking up to - CONFIG_TRACE_RECENT_ENTRIES_COUNT and suppressing messages repeated in less than - CONFIG_TRACE_RECENT_TIME_THRESHOLD microseconds. This filtering and suppression are - turned off as soon as the user makes any runtime configuration change using the - sof-logger -F flag. Warning: log message parameters (formatted with a conversion - specifier %_) are ignored when comparing repeated messages and they are dropped by - this filtering. - -config TRACE_RECENT_ENTRIES_COUNT - int "How many recent log messages are stored" - depends on TRACE_FILTERING_ADAPTIVE - default 5 - help - The most recent log messages are stored to match the currently processed - message. The size of that window affects effectiveness and performance. More recent - entries allow to better filter repetitive messages out, but also slightly decrease - performance due to the increased number of comparisons necessary. - -config TRACE_RECENT_TIME_THRESHOLD - int "Period of time considered recent (microseconds)" - depends on TRACE_FILTERING_ADAPTIVE - default 1500 - range 1 TRACE_RECENT_MAX_TIME - help - Period of time during which entries are tracked and will be suppressed if reported again. - Entries expire after this time. - -config TRACE_RECENT_MAX_TIME - int "Maximum period of time that message can be suppressed (microseconds)" - depends on TRACE_FILTERING_ADAPTIVE - default 5000000 - help - Maximum amount of time message can be suppressed for, due to repeated suppression. - -config TRACE_BURST_COUNT - int "Allowed amount of rapidly repeated messages, that will not be suppressed by the filter" - depends on TRACE_FILTERING_ADAPTIVE - default 4 - help - Amount of messages that will pass through the filter even if sent in rapid succession. - Allowed message burst size before filter suppresses the message. - -endmenu diff --git a/src/trace/dma-trace.c b/src/trace/dma-trace.c deleted file mode 100644 index 328bb2767f65..000000000000 --- a/src/trace/dma-trace.c +++ /dev/null @@ -1,645 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause -// -// Copyright(c) 2016 Intel Corporation. All rights reserved. -// -// Author: Yan Wang - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __ZEPHYR__ -#include -#endif - -#include -#include -#include - -LOG_MODULE_REGISTER(dma_trace, CONFIG_SOF_LOG_LEVEL); - -SOF_DEFINE_REG_UUID(dma_trace); - -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(dt_tr, SOF_UUID(dma_trace_uuid), LOG_LEVEL_INFO); - -SOF_DEFINE_REG_UUID(dma_trace_task); - -static int dma_trace_get_avail_data(struct dma_trace_data *d, - struct dma_trace_buf *buffer, - int avail); - -/** Periodically runs and starts the DMA even when the buffer is not - * full. - */ -static enum task_state trace_work(void *data) -{ - struct dma_trace_data *d = data; - struct dma_trace_buf *buffer = &d->dmatb; - struct dma_sg_config *config = &d->config; - k_spinlock_key_t key; - uint32_t avail = buffer->avail; - int32_t size; - uint32_t overflow; - - /* The host DMA channel is not available */ - if (!d->dc.chan) - return SOF_TASK_STATE_RESCHEDULE; - - if (!ipc_trigger_trace_xfer(avail)) - return SOF_TASK_STATE_RESCHEDULE; - - /* make sure we don't write more than buffer */ - if (avail > DMA_TRACE_LOCAL_SIZE) { - overflow = avail - DMA_TRACE_LOCAL_SIZE; - avail = DMA_TRACE_LOCAL_SIZE; - } else { - overflow = 0; - } - - /* dma gateway supports wrap mode copy, but GPDMA doesn't - * support, so do it differently based on HW features - */ - size = dma_trace_get_avail_data(d, buffer, avail); - - /* any data to copy ? */ - if (size == 0) { - return SOF_TASK_STATE_RESCHEDULE; - } - - d->posn.overflow = overflow; - - /* DMA trace copying is working */ - d->copy_in_progress = 1; - - /* copy this section to host */ - size = dma_copy_to_host(&d->dc, config, d->posn.host_offset, - buffer->r_ptr, size); - if (size < 0) { - tr_err(&dt_tr, "dma_copy_to_host() failed"); - goto out; - } - - /* update host pointer and check for wrap */ - d->posn.host_offset += size; - if (d->posn.host_offset >= d->host_size) - d->posn.host_offset -= d->host_size; - - /* update local pointer and check for wrap */ - buffer->r_ptr = (char *)buffer->r_ptr + size; - if (buffer->r_ptr >= buffer->end_addr) - buffer->r_ptr = (char *)buffer->r_ptr - DMA_TRACE_LOCAL_SIZE; - - ipc_msg_send(d->msg, &d->posn, false); - -out: - key = k_spin_lock(&d->lock); - - /* disregard any old messages and don't resend them if we overflow */ - if (size > 0) { - if (d->posn.overflow) - buffer->avail = DMA_TRACE_LOCAL_SIZE - size; - else - buffer->avail -= size; - } - - /* DMA trace copying is done, allow reschedule */ - d->copy_in_progress = 0; - - k_spin_unlock(&d->lock, key); - - /* reschedule the trace copying work */ - return SOF_TASK_STATE_RESCHEDULE; -} - -/** Do this early so we can log at initialization time even before the - * DMA runs. The rest happens later in dma_trace_init_complete() and - * dma_trace_enable() - */ -int dma_trace_init_early(struct sof *sof) -{ - int ret; - - /* If this assert is wrong then traces have been corrupting - * random parts of memory. Some functions run before _and_ after - * DMA trace initialization and we don't want to ask them to - * never trace. So dma_trace_initialized() must be either - * clearly false/NULL or clearly true, we can't tolerate random - * uninitialized values in sof->dmat etc. - */ - assert(!dma_trace_initialized(sof->dmat)); - - sof->dmat = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*sof->dmat)); - if (!sof->dmat) { - mtrace_printf(LOG_LEVEL_ERROR, - "alloc failed"); - return -ENOMEM; - } - - dma_sg_init(&sof->dmat->config.elem_array); - k_spinlock_init(&sof->dmat->lock); - - ipc_build_trace_posn(&sof->dmat->posn); - sof->dmat->msg = ipc_msg_init(NULL, sof->dmat->posn.rhdr.hdr.cmd, - sof->dmat->posn.rhdr.hdr.size); - if (!sof->dmat->msg) { - ret = -ENOMEM; - goto err; - } - - return 0; - -err: - mtrace_printf(LOG_LEVEL_ERROR, - "failed: %d", ret); - - rfree(sof->dmat); - sof->dmat = NULL; - - return ret; -} - -/** Run after dma_trace_init_early() and before dma_trace_enable() */ -int dma_trace_init_complete(struct dma_trace_data *d) -{ - int ret = 0; - - tr_info(&dt_tr, "entry"); - - if (!d) { - mtrace_printf(LOG_LEVEL_ERROR, - "failed, no dma_trace_data"); - return -ENOMEM; - } - - /* init DMA copy context */ - ret = dma_copy_new(&d->dc); - if (ret < 0) { - mtrace_printf(LOG_LEVEL_ERROR, - "dma_copy_new() failed: %d", ret); - goto out; - } -#if CONFIG_ZEPHYR_NATIVE_DRIVERS - ret = dma_get_attribute(d->dc.dmac->z_dev, DMA_ATTR_COPY_ALIGNMENT, - &d->dma_copy_align); -#else - ret = dma_get_attribute_legacy(d->dc.dmac, DMA_ATTR_COPY_ALIGNMENT, - &d->dma_copy_align); -#endif - if (ret < 0) { - mtrace_printf(LOG_LEVEL_ERROR, - "dma_get_attribute() failed: %d", ret); - - goto out; - } - - schedule_task_init_ll(&d->dmat_work, SOF_UUID(dma_trace_task_uuid), - SOF_SCHEDULE_LL_TIMER, - SOF_TASK_PRI_MED, trace_work, d, 0, 0); - -out: - - return ret; -} - -#if (CONFIG_HOST_PTABLE) -int dma_trace_host_buffer(struct dma_trace_data *d, - struct dma_sg_elem_array *elem_array, - uint32_t host_size) -{ - d->host_size = host_size; - d->config.elem_array = *elem_array; - - return 0; -} -#endif - -static void dma_trace_buffer_free(struct dma_trace_data *d) -{ - struct dma_trace_buf *buffer = &d->dmatb; - k_spinlock_key_t key; - - key = k_spin_lock(&d->lock); - - rfree(buffer->addr); - memset(buffer, 0, sizeof(*buffer)); - - k_spin_unlock(&d->lock, key); -} - -static int dma_trace_buffer_init(struct dma_trace_data *d) -{ - struct dma_trace_buf *buffer = &d->dmatb; - void *buf; - k_spinlock_key_t key; - uint32_t addr_align; - int err; - - /* - * Keep the existing dtrace buffer to avoid memory leak, unlikely to - * happen if host correctly using the dma_trace_disable(). - * - * The buffer can not be freed up here as it is likely in use. - * The (re-)initialization will happen in dma_trace_start() when it is - * safe to do (the DMA is stopped) - */ - if (dma_trace_initialized(d)) - return 0; - - if (!d || !d->dc.dmac) { - mtrace_printf(LOG_LEVEL_ERROR, - "failed, no DMAC! d=%p", d); - return -ENODEV; - } -#if CONFIG_ZEPHYR_NATIVE_DRIVERS - err = dma_get_attribute(d->dc.dmac->z_dev, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, - &addr_align); -#else - err = dma_get_attribute_legacy(d->dc.dmac, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, - &addr_align); -#endif - if (err < 0) - return err; - - /* For DMA to work properly the buffer must be correctly aligned */ - buf = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, - DMA_TRACE_LOCAL_SIZE, addr_align); - if (!buf) { - mtrace_printf(LOG_LEVEL_ERROR, "alloc failed"); - return -ENOMEM; - } - - bzero(buf, DMA_TRACE_LOCAL_SIZE); - dcache_writeback_region((__sparse_force void __sparse_cache *)buf, DMA_TRACE_LOCAL_SIZE); - - /* initialise the DMA buffer, whole sequence in section */ - key = k_spin_lock(&d->lock); - - buffer->addr = buf; - buffer->size = DMA_TRACE_LOCAL_SIZE; - buffer->w_ptr = buffer->addr; - buffer->r_ptr = buffer->addr; - buffer->end_addr = (char *)buffer->addr + buffer->size; - buffer->avail = 0; - - k_spin_unlock(&d->lock, key); - -#ifdef __ZEPHYR__ -#define ZEPHYR_VER_OPT " zephyr:" STRINGIFY(BUILD_VERSION) -#else -#define ZEPHYR_VER_OPT -#endif - - /* STRINGIFY(SOF_SRC_HASH) is part of the format string so it - * goes to the .ldc file and does not go to the firmware - * binary. It will be different from SOF_SRC_HASH in case of - * mismatch. - */ -#define SOF_BANNER_COMMON \ - "FW ABI 0x%x DBG ABI 0x%x tags SOF:" SOF_GIT_TAG ZEPHYR_VER_OPT \ - " src hash 0x%08x (ldc hash " STRINGIFY(SOF_SRC_HASH) ")" - - /* It should be the very first sent log for easy identification. */ - mtrace_printf(LOG_LEVEL_INFO, - "SHM: " SOF_BANNER_COMMON, - SOF_ABI_VERSION, SOF_ABI_DBG_VERSION, SOF_SRC_HASH); - - /* Use a different, DMA: prefix to ease identification of log files */ - tr_info(&dt_tr, - "DMA: " SOF_BANNER_COMMON, - SOF_ABI_VERSION, SOF_ABI_DBG_VERSION, SOF_SRC_HASH); - - return 0; -} - -static int dma_trace_get_avail_data(struct dma_trace_data *d, - struct dma_trace_buf *buffer, - int avail) -{ - uint32_t hsize; - uint32_t lsize; - int32_t size; - - /* copy to host in sections if we wrap */ - lsize = avail; - hsize = avail; - - if (avail == 0) - return 0; - - /* host buffer wrap ? */ - if (d->posn.host_offset + avail > d->host_size) - hsize = d->host_size - d->posn.host_offset; - - /* local buffer wrap ? */ - if ((char *)buffer->r_ptr + avail > (char *)buffer->end_addr) - lsize = (char *)buffer->end_addr - (char *)buffer->r_ptr; - - /* get smallest size */ - if (hsize < lsize) - size = hsize; - else - size = lsize; - - return size; -} - -/** Invoked remotely by SOF_IPC_TRACE_DMA_PARAMS* Depends on - * dma_trace_init_complete() - */ -int dma_trace_enable(struct dma_trace_data *d) -{ - int err; - - /* Allocate and initialize the dma trace buffer if needed */ - err = dma_trace_buffer_init(d); - if (err < 0) - return err; - - /* validate DMA context */ - if (!d->dc.dmac || !d->dc.chan) { - tr_err_atomic(&dt_tr, "not valid"); - err = -ENODEV; - goto out; - } - - d->enabled = 1; - schedule_task(&d->dmat_work, DMA_TRACE_PERIOD, DMA_TRACE_PERIOD); - -out: - if (err < 0) - dma_trace_buffer_free(d); - - return err; -} - -void dma_trace_disable(struct dma_trace_data *d) -{ - /* cancel trace work */ - schedule_task_cancel(&d->dmat_work); - - if (d->dc.chan) { - dma_stop_legacy(d->dc.chan); - dma_channel_put_legacy(d->dc.chan); - d->dc.chan = NULL; - } - -#if (CONFIG_HOST_PTABLE) - /* Free up the host SG if it is set */ - if (d->host_size) { - dma_sg_free(NULL, &d->config.elem_array); - d->host_size = 0; - } -#endif -} - -/** Sends all pending DMA messages to mailbox (for emergencies) */ -void dma_trace_flush(void *t) -{ - struct dma_trace_data *trace_data = dma_trace_data_get(); - struct dma_trace_buf *buffer = NULL; - uint32_t avail; - int32_t size; - int32_t wrap_count; - int ret; - - if (!dma_trace_initialized(trace_data)) - return; - - buffer = &trace_data->dmatb; - avail = buffer->avail; - - /* number of bytes to flush */ - if (avail > DMA_FLUSH_TRACE_SIZE) { - size = DMA_FLUSH_TRACE_SIZE; - } else { - /* check for buffer wrap */ - if (buffer->w_ptr > buffer->r_ptr) - size = (char *)buffer->w_ptr - (char *)buffer->r_ptr; - else - size = (char *)buffer->end_addr - - (char *)buffer->r_ptr + - (char *)buffer->w_ptr - - (char *)buffer->addr; - } - - size = MIN(size, MAILBOX_TRACE_SIZE); - - /* invalidate trace data */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)t, size); - - /* check for buffer wrap */ - if ((char *)buffer->w_ptr - size < (char *)buffer->addr) { - wrap_count = (char *)buffer->w_ptr - (char *)buffer->addr; - ret = memcpy_s(t, size - wrap_count, - (char *)buffer->end_addr - - (size - wrap_count), size - wrap_count); - assert(!ret); - ret = memcpy_s((char *)t + (size - wrap_count), wrap_count, - buffer->addr, wrap_count); - assert(!ret); - } else { - ret = memcpy_s(t, size, (char *)buffer->w_ptr - size, size); - assert(!ret); - } - - /* writeback trace data */ - dcache_writeback_region((__sparse_force void __sparse_cache *)t, size); - -} - -void dma_trace_on(void) -{ - if (!dma_trace_initialized(sof_get()->dmat)) - return; - - struct dma_trace_data *trace_data = dma_trace_data_get(); - - if (trace_data->enabled) { - return; - } - - trace_data->enabled = 1; - schedule_task(&trace_data->dmat_work, DMA_TRACE_PERIOD, - DMA_TRACE_PERIOD); - -} - -void dma_trace_off(void) -{ - struct dma_trace_data *trace_data = dma_trace_data_get(); - - if (!trace_data->enabled) { - return; - } - - schedule_task_cancel(&trace_data->dmat_work); - trace_data->enabled = 0; - -} - -static int dtrace_calc_buf_overflow(struct dma_trace_buf *buffer, - uint32_t length) -{ - uint32_t margin; - uint32_t overflow_margin; - uint32_t overflow = 0; - - margin = dtrace_calc_buf_margin(buffer); - - /* overflow calculating */ - if (buffer->w_ptr < buffer->r_ptr) - overflow_margin = (char *)buffer->r_ptr - - (char *)buffer->w_ptr - 1; - else - overflow_margin = margin + (char *)buffer->r_ptr - - (char *)buffer->addr - 1; - - if (overflow_margin < length) - overflow = length - overflow_margin; - - return overflow; -} - -/** Ring buffer implementation, drops on overflow. */ -static void dtrace_add_event(const char *e, uint32_t length) -{ - struct dma_trace_data *trace_data = dma_trace_data_get(); - struct dma_trace_buf *buffer = &trace_data->dmatb; - uint32_t margin; - uint32_t overflow; - int ret; - - margin = dtrace_calc_buf_margin(buffer); - overflow = dtrace_calc_buf_overflow(buffer, length); - - /* tracing dropped entries */ - if (trace_data->dropped_entries) { - if (!overflow) { - /* - * if any dropped entries have appeared and there - * is not any overflow, their amount will be logged - */ - uint32_t tmp_dropped_entries = - trace_data->dropped_entries; - trace_data->dropped_entries = 0; - mtrace_printf(LOG_LEVEL_WARNING, - "number of dropped logs = %u", - tmp_dropped_entries); - } - } - - /* checking overflow */ - if (!overflow) { - /* check for buffer wrap */ - if (margin > length) { - /* no wrap */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)buffer->w_ptr, - length); - ret = memcpy_s(buffer->w_ptr, length, e, length); - assert(!ret); - dcache_writeback_region((__sparse_force void __sparse_cache *)buffer->w_ptr, - length); - buffer->w_ptr = (char *)buffer->w_ptr + length; - } else { - /* data is bigger than remaining margin so we wrap */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)buffer->w_ptr, - margin); - ret = memcpy_s(buffer->w_ptr, margin, e, margin); - assert(!ret); - dcache_writeback_region((__sparse_force void __sparse_cache *)buffer->w_ptr, - margin); - buffer->w_ptr = buffer->addr; - - dcache_invalidate_region((__sparse_force void __sparse_cache *)buffer->w_ptr, - length - margin); - ret = memcpy_s(buffer->w_ptr, length - margin, - e + margin, length - margin); - assert(!ret); - dcache_writeback_region((__sparse_force void __sparse_cache *)buffer->w_ptr, - length - margin); - buffer->w_ptr = (char *)buffer->w_ptr + length - margin; - } - - buffer->avail += length; - trace_data->posn.messages++; - } else { - /* if there is not enough memory for new log, we drop it */ - trace_data->dropped_entries++; - } - -} - -/** Main dma-trace entry point */ -void dtrace_event(const char *e, uint32_t length) -{ - struct dma_trace_data *trace_data = dma_trace_data_get(); - struct dma_trace_buf *buffer = NULL; - k_spinlock_key_t key; - - if (!dma_trace_initialized(trace_data) || - length > DMA_TRACE_LOCAL_SIZE / 8 || length == 0) { - return; - } - - buffer = &trace_data->dmatb; - - key = k_spin_lock(&trace_data->lock); - dtrace_add_event(e, length); - - /* if DMA trace copying is working or secondary core - * don't check if local buffer is half full - */ - if (trace_data->copy_in_progress || - cpu_get_id() != PLATFORM_PRIMARY_CORE_ID) { - k_spin_unlock(&trace_data->lock, key); - return; - } - - k_spin_unlock(&trace_data->lock, key); - - /* schedule copy now if buffer > 50% full */ - if (trace_data->enabled && - buffer->avail >= (DMA_TRACE_LOCAL_SIZE / 2)) { - reschedule_task(&trace_data->dmat_work, - DMA_TRACE_RESCHEDULE_TIME); - /* reschedule should not be interrupted - * just like we are in copy progress - */ - trace_data->copy_in_progress = 1; - } - -} - -void dtrace_event_atomic(const char *e, uint32_t length) -{ - struct dma_trace_data *trace_data = dma_trace_data_get(); - - if (!dma_trace_initialized(trace_data) || - length > DMA_TRACE_LOCAL_SIZE / 8 || length == 0) { - return; - } - - dtrace_add_event(e, length); -} diff --git a/src/trace/trace.c b/src/trace/trace.c deleted file mode 100644 index bc17c848a902..000000000000 --- a/src/trace/trace.c +++ /dev/null @@ -1,579 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause -// -// Copyright(c) 2016 Intel Corporation. All rights reserved. -// -// Author: Liam Girdwood -// Artur Kloniecki -// Karol Trzcinski - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* every trace calls uses IPC context in this file */ -LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); - -extern struct tr_ctx dt_tr; - -#if CONFIG_TRACE_FILTERING_ADAPTIVE -struct recent_log_entry { - uint32_t entry_id; - uint64_t message_ts; - uint64_t first_suppression_ts; - uint32_t trigger_count; -}; - -struct recent_trace_context { - struct recent_log_entry recent_entries[CONFIG_TRACE_RECENT_ENTRIES_COUNT]; -}; -#endif /* CONFIG_TRACE_FILTERING_ADAPTIVE */ - -/** MAILBOX_TRACE_BASE ring buffer */ -struct trace { - uintptr_t pos ; /**< offset of the next byte to write */ - uint32_t enable; -#if CONFIG_TRACE_FILTERING_ADAPTIVE - bool user_filter_override; /* whether filtering was overridden by user or not */ -#endif /* CONFIG_TRACE_FILTERING_ADAPTIVE */ - struct k_spinlock lock; /* locking mechanism */ - -#if CONFIG_TRACE_FILTERING_ADAPTIVE - struct recent_trace_context trace_core_context[CONFIG_CORE_COUNT]; -#endif -}; - -/* calculates total message size, both header and payload in bytes */ -#define MESSAGE_SIZE(args_num) \ - (sizeof(struct log_entry_header) + args_num * sizeof(uint32_t)) - -/* calculated total message size in dwords */ -#define MESSAGE_SIZE_DWORDS(args_num) \ - (MESSAGE_SIZE(args_num) / sizeof(uint32_t)) - -/* calculates offset to param_idx of payload */ -#define PAYLOAD_OFFSET(param_idx) \ - (MESSAGE_SIZE_DWORDS(0) + param_idx) - -#define TRACE_ID_MASK ((1 << TRACE_ID_LENGTH) - 1) - -static void put_header(void *dst, const struct sof_uuid_entry *uid, - uint32_t id_1, uint32_t id_2, - uint32_t entry, uint64_t timestamp) -{ - struct dma_trace_data *trace_data = dma_trace_data_get(); - /* Support very early tracing */ - uint64_t delta = dma_trace_initialized(trace_data) ? trace_data->time_delta : 0; - struct log_entry_header header; - int ret; - - header.uid = (uintptr_t)uid; - header.id_0 = id_1 & TRACE_ID_MASK; - header.id_1 = id_2 & TRACE_ID_MASK; - header.core_id = cpu_get_id(); - header.timestamp = timestamp + delta; - header.log_entry_address = entry; - - ret = memcpy_s(dst, sizeof(header), &header, sizeof(header)); - assert(!ret); -} - -#ifndef __ZEPHYR__ -/** Ring buffer for the mailbox trace */ -void mtrace_event(const char *data, uint32_t length) -{ - struct trace *trace = trace_get(); - char *t = (char *)MAILBOX_TRACE_BASE; - const int32_t available = MAILBOX_TRACE_SIZE - trace->pos; - - if (available < length) { /* wrap */ - memset(t + trace->pos, 0xff, available); - dcache_writeback_region((__sparse_force char __sparse_cache *)t + trace->pos, - available); - trace->pos = 0; - } - - memcpy_s(t + trace->pos, MAILBOX_TRACE_SIZE - trace->pos, - data, length); - dcache_writeback_region((__sparse_force char __sparse_cache *)t + trace->pos, length); - trace->pos += length; -} -#endif /* __ZEPHYR__ */ - -#if CONFIG_TRACE_FILTERING_VERBOSITY -/** - * \brief Runtime trace filtering based on verbosity level - * \param lvl log level (LOG_LEVEL_ ERROR, INFO, DEBUG ...) - * \param uuid uuid address - * \return false when trace is filtered out, otherwise true - */ -static inline bool trace_filter_verbosity(uint32_t lvl, const struct tr_ctx *ctx) -{ - STATIC_ASSERT(LOG_LEVEL_CRITICAL < LOG_LEVEL_VERBOSE, - LOG_LEVEL_CRITICAL_MUST_HAVE_LOWEST_VALUE); - return lvl <= ctx->level; -} -#endif /* CONFIG_TRACE_FILTERING_VERBOSITY */ - -#if CONFIG_TRACE_FILTERING_ADAPTIVE -/** Report how many times an entry was suppressed and clear it. */ -static void emit_suppressed_entry(struct recent_log_entry *entry) -{ - _log_message(trace_log_unfiltered, false, LOG_LEVEL_INFO, _TRACE_INV_CLASS, &dt_tr, - _TRACE_INV_ID, _TRACE_INV_ID, "Suppressed %u similar messages: %pQ", - entry->trigger_count - CONFIG_TRACE_BURST_COUNT, - (void *)entry->entry_id); - - memset(entry, 0, sizeof(*entry)); -} - -/** Flush entries that have not been seen again in the last - * CONFIG_TRACE_RECENT_TIME_THRESHOLD microseconds before current_ts. - */ -static void emit_recent_entries(uint64_t current_ts) -{ - struct trace *trace = trace_get(); - struct recent_log_entry *recent_entries = - trace->trace_core_context[cpu_get_id()].recent_entries; - int i; - - /* Check if any tracked entries were dormant long enough to unsuppress them */ - for (i = 0; i < CONFIG_TRACE_RECENT_ENTRIES_COUNT; i++) { - if (recent_entries[i].entry_id) { - if (current_ts - recent_entries[i].message_ts > - CONFIG_TRACE_RECENT_TIME_THRESHOLD) { - if (recent_entries[i].trigger_count > CONFIG_TRACE_BURST_COUNT) - emit_suppressed_entry(&recent_entries[i]); - else - memset(&recent_entries[i], 0, sizeof(recent_entries[i])); - } - } - } - -} - -/** - * \brief Runtime trace flood suppression - * \param log_level log verbosity level - * \param entry ID of log entry - * \param message_ts message timestamp - * \return true when the message must be printed by the caller because it was not filtered - */ -static bool trace_filter_flood(uint32_t log_level, uint32_t entry, uint64_t message_ts) -{ - struct trace *trace = trace_get(); - struct recent_log_entry *recent_entries = - trace->trace_core_context[cpu_get_id()].recent_entries; - struct recent_log_entry *oldest_entry = recent_entries; - int i; - - /* don't attempt to suppress debug messages using this method, it would be uneffective */ - if (log_level >= LOG_LEVEL_DEBUG) - return true; - - /* check if same log entry was sent recently */ - for (i = 0; i < CONFIG_TRACE_RECENT_ENTRIES_COUNT; i++) { - /* Identify the oldest (or empty) entry to evict if we have no match */ - if (oldest_entry->first_suppression_ts > recent_entries[i].first_suppression_ts) - oldest_entry = &recent_entries[i]; - - if (recent_entries[i].entry_id == entry) { - /* We have a match but include this message in this burst only if the - * burst: - - 1. hasn't lasted for too long; - - 2. hasn't been quiet for too long. - */ - if (message_ts - recent_entries[i].first_suppression_ts < - CONFIG_TRACE_RECENT_MAX_TIME && - message_ts - recent_entries[i].message_ts < - CONFIG_TRACE_RECENT_TIME_THRESHOLD) { - recent_entries[i].trigger_count++; - /* Refresh last seen time */ - recent_entries[i].message_ts = message_ts; - - /* Allow the start of a burst to be printed normally */ - return recent_entries[i].trigger_count <= CONFIG_TRACE_BURST_COUNT; - } - - /* Emit and clear this burst */ - if (recent_entries[i].trigger_count > CONFIG_TRACE_BURST_COUNT) - emit_suppressed_entry(&recent_entries[i]); - else - memset(&recent_entries[i], 0, sizeof(recent_entries[i])); - - return true; - } - } - - /* Make room for tracking new entry, by emitting the oldest one in the filter */ - if (oldest_entry->entry_id && oldest_entry->trigger_count > CONFIG_TRACE_BURST_COUNT) - emit_suppressed_entry(oldest_entry); - - /* Start a new burst */ - oldest_entry->entry_id = entry; - oldest_entry->message_ts = message_ts; - oldest_entry->first_suppression_ts = message_ts; - oldest_entry->trigger_count = 1; - - return true; -} -#endif /* CONFIG_TRACE_FILTERING_ADAPTIVE */ - -/** Implementation shared and invoked by both adaptive filtering and - * not. Serializes events into trace messages and passes them to - * dtrace_event() - */ -static void dma_trace_log(bool send_atomic, uint32_t log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list vargs) -{ - uint32_t data[MESSAGE_SIZE_DWORDS(_TRACE_EVENT_MAX_ARGUMENT_COUNT)]; - const int message_size = MESSAGE_SIZE(arg_count); - int i; - - /* fill log content. arg_count is in the dictionary. */ - put_header(data, ctx->uuid_p, id_1, id_2, log_entry, sof_cycle_get_64_safe()); - - for (i = 0; i < arg_count; ++i) - data[PAYLOAD_OFFSET(i)] = va_arg(vargs, uint32_t); - - /* send event by */ - if (send_atomic) - dtrace_event_atomic((const char *)data, message_size); - else - dtrace_event((const char *)data, message_size); - -} - -void trace_log_unfiltered(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list vl) -{ - struct trace *trace = trace_get(); - - if (!trace->enable) { - return; - } - - dma_trace_log(send_atomic, (uint32_t)log_entry, ctx, lvl, id_1, id_2, arg_count, vl); -} - -void trace_log_filtered(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list vl) -{ - struct trace *trace = trace_get(); - - if (!trace->enable) { - return; - } - -#if CONFIG_TRACE_FILTERING_VERBOSITY - if (!trace_filter_verbosity(lvl, ctx)) - return; -#endif /* CONFIG_TRACE_FILTERING_VERBOSITY */ - -#if CONFIG_TRACE_FILTERING_ADAPTIVE - if (!trace->user_filter_override) { - const uint64_t current_ts = sof_cycle_get_64_safe(); - - emit_recent_entries(current_ts); - - if (!trace_filter_flood(lvl, (uint32_t)log_entry, current_ts)) - return; - } -#endif /* CONFIG_TRACE_FILTERING_ADAPTIVE */ - - dma_trace_log(send_atomic, (uint32_t)log_entry, ctx, lvl, id_1, id_2, arg_count, vl); -} - -struct sof_ipc_trace_filter_elem *trace_filter_fill(struct sof_ipc_trace_filter_elem *elem, - struct sof_ipc_trace_filter_elem *end, - struct trace_filter *filter) -{ - filter->log_level = -1; - filter->uuid_id = 0; - filter->comp_id = -1; - filter->pipe_id = -1; - - while (elem <= end) { - switch (elem->key & SOF_IPC_TRACE_FILTER_ELEM_TYPE_MASK) { - case SOF_IPC_TRACE_FILTER_ELEM_SET_LEVEL: - filter->log_level = elem->value; - break; - case SOF_IPC_TRACE_FILTER_ELEM_BY_UUID: - filter->uuid_id = elem->value; - break; - case SOF_IPC_TRACE_FILTER_ELEM_BY_COMP: - filter->comp_id = elem->value; - break; - case SOF_IPC_TRACE_FILTER_ELEM_BY_PIPE: - filter->pipe_id = elem->value; - break; - default: - tr_err(&ipc_tr, "Invalid SOF_IPC_TRACE_FILTER_ELEM 0x%x", - elem->key); - return NULL; - } - - /* each filter set must be terminated with FIN flag and have new log level */ - if (elem->key & SOF_IPC_TRACE_FILTER_ELEM_FIN) { - if (filter->log_level < 0) { - tr_err(&ipc_tr, "Each trace filter set must specify new log level"); - return NULL; - } else { - return elem + 1; - } - } - - ++elem; - } - - tr_err(&ipc_tr, "Trace filter elements set is not properly terminated"); - return NULL; -} - -/* update global components, which tr_ctx is stored inside special section */ -static int trace_filter_update_global(int32_t log_level, uint32_t uuid_id) -{ - int cnt = 0; -#if !defined(CONFIG_LIBRARY) - extern void *_trace_ctx_start; - extern void *_trace_ctx_end; - struct tr_ctx *ptr = (struct tr_ctx *)&_trace_ctx_start; - struct tr_ctx *end = (struct tr_ctx *)&_trace_ctx_end; - - /* iterate over global `tr_ctx` entries located in their own section */ - /* cppcheck-suppress comparePointers */ - while (ptr < end) { - /* - * when looking for specific uuid element, - * then find, update and stop searching - */ - if ((uintptr_t)ptr->uuid_p == uuid_id) { - ptr->level = log_level; - return 1; - } - /* otherwise each element should be updated */ - if (!ptr->uuid_p) { - ptr->level = log_level; - ++cnt; - } - ++ptr; - } -#endif - - return cnt; -} - -/* return trace context from any ipc component type */ -static struct tr_ctx *trace_filter_ipc_comp_context(struct ipc_comp_dev *icd) -{ - switch (icd->type) { - case COMP_TYPE_COMPONENT: - return &icd->cd->tctx; - case COMP_TYPE_BUFFER: - return &icd->cb->tctx; - case COMP_TYPE_PIPELINE: - return &icd->pipeline->tctx; - /* each COMP_TYPE must be specified */ - default: - tr_err(&ipc_tr, "Unknown trace context for ipc component type 0x%X", - icd->type); - return NULL; - } -} - -/* update ipc components, wchich tr_ctx may be read from ipc_comp_dev structure */ -static int trace_filter_update_instances(int32_t log_level, uint32_t uuid_id, - int32_t pipe_id, int32_t comp_id) -{ - struct ipc *ipc = ipc_get(); - struct ipc_comp_dev *icd; - struct list_item *clist; - struct tr_ctx *ctx; - bool correct_comp; - int cnt = 0; - - /* compare each ipc component with filter settings and update log level after pass */ - list_for_item(clist, &ipc->comp_list) { - icd = container_of(clist, struct ipc_comp_dev, list); - ctx = trace_filter_ipc_comp_context(icd); - if (!ctx) - return -EINVAL; - correct_comp = comp_id == -1 || icd->id == comp_id; /* todo: icd->topo_id */ - correct_comp &= uuid_id == 0 || (uintptr_t)ctx->uuid_p == uuid_id; - correct_comp &= pipe_id == -1 || ipc_comp_pipe_id(icd) == pipe_id; - if (correct_comp) { - ctx->level = log_level; - ++cnt; - } - - } - return cnt; -} - -int trace_filter_update(const struct trace_filter *filter) -{ - int ret = 0; -#if CONFIG_TRACE_FILTERING_ADAPTIVE - struct trace *trace = trace_get(); - - if (!trace->user_filter_override) { - trace->user_filter_override = true; - tr_info(&ipc_tr, "Adaptive filtering disabled by user"); - } -#endif /* CONFIG_TRACE_FILTERING_ADAPTIVE */ - - /* validate log level, LOG_LEVEL_CRITICAL has low value, LOG_LEVEL_VERBOSE high */ - if (filter->log_level < LOG_LEVEL_CRITICAL || - filter->log_level > LOG_LEVEL_VERBOSE) - return -EINVAL; - - /* update `*`, `name*` or global `name` */ - if (filter->pipe_id == -1 && filter->comp_id == -1) - ret = trace_filter_update_global(filter->log_level, filter->uuid_id); - - /* update `*`, `name*`, `nameX.*` or `nameX.Y`, `name` may be '*' */ - ret += trace_filter_update_instances(filter->log_level, filter->uuid_id, - filter->pipe_id, filter->comp_id); - return ret > 0 ? ret : -EINVAL; -} - -/** Sends all pending DMA messages to mailbox (for emergencies) */ -void trace_flush_dma_to_mbox(void) -{ - struct trace *trace = trace_get(); - volatile uint64_t *t; - k_spinlock_key_t key; - - key = k_spin_lock(&trace->lock); - - /* get mailbox position */ - t = (volatile uint64_t *)(MAILBOX_TRACE_BASE + trace->pos); - - /* flush dma trace messages */ - dma_trace_flush((void *)t); - - k_spin_unlock(&trace->lock, key); -} - -void trace_on(void) -{ - struct trace *trace = trace_get(); - k_spinlock_key_t key; - - key = k_spin_lock(&trace->lock); - - trace->enable = 1; - dma_trace_on(); - - k_spin_unlock(&trace->lock, key); -} - -void trace_off(void) -{ - struct trace *trace = trace_get(); - k_spinlock_key_t key; - - key = k_spin_lock(&trace->lock); - - trace->enable = 0; - dma_trace_off(); - - k_spin_unlock(&trace->lock, key); -} - -void trace_init(struct sof *sof) -{ - sof->trace = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*sof->trace)); - if (!sof->trace) { - mtrace_printf(LOG_LEVEL_ERROR, "allocation failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - - sof->trace->enable = 1; - sof->trace->pos = 0; -#if CONFIG_TRACE_FILTERING_ADAPTIVE - sof->trace->user_filter_override = false; -#endif /* CONFIG_TRACE_FILTERING_ADAPTIVE */ - k_spinlock_init(&sof->trace->lock); - -#ifndef __ZEPHYR__ - /* Zephyr owns and has already initialized this buffer (and - * likely has already logged to it by the time we get here). - * Don't touch. - */ - bzero((void *)MAILBOX_TRACE_BASE, MAILBOX_TRACE_SIZE); - dcache_writeback_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_TRACE_BASE, - MAILBOX_TRACE_SIZE); -#endif - - dma_trace_init_early(sof); -} - -static void mtrace_dict_entry_vl(bool atomic_context, uint32_t dict_entry_address, - int n_args, va_list ap) -{ - int i; - char packet[MESSAGE_SIZE(_TRACE_EVENT_MAX_ARGUMENT_COUNT)]; - uint32_t *args = (uint32_t *)&packet[MESSAGE_SIZE(0)]; - const uint64_t tstamp = sof_cycle_get_64_safe(); - - put_header(packet, dt_tr.uuid_p, _TRACE_INV_ID, _TRACE_INV_ID, - dict_entry_address, tstamp); - - for (i = 0; i < MIN(n_args, _TRACE_EVENT_MAX_ARGUMENT_COUNT); i++) - args[i] = va_arg(ap, uint32_t); - - if (atomic_context) { - mtrace_event(packet, MESSAGE_SIZE(n_args)); - } else { - struct trace * const trace = trace_get(); - k_spinlock_key_t key; - - key = k_spin_lock(&trace->lock); - mtrace_event(packet, MESSAGE_SIZE(n_args)); - k_spin_unlock(&trace->lock, key); - } -} - -void mtrace_dict_entry(bool atomic_context, uint32_t dict_entry_address, int n_args, ...) -{ - va_list ap; - - va_start(ap, n_args); - mtrace_dict_entry_vl(atomic_context, dict_entry_address, n_args, ap); - va_end(ap); -} - -void _log_sofdict(log_func_t sofdict_logf, bool atomic, const void *log_entry, - const struct tr_ctx *ctx, const uint32_t lvl, - uint32_t id_1, uint32_t id_2, int arg_count, ...) -{ - va_list ap; - -#ifndef __ZEPHYR__ /* for Zephyr see _log_nodict() in trace.h */ - if (lvl <= MTRACE_DUPLICATION_LEVEL) { - va_start(ap, arg_count); - mtrace_dict_entry_vl(atomic, (uint32_t)log_entry, arg_count, ap); - va_end(ap); - } -#endif - - va_start(ap, arg_count); - sofdict_logf(atomic, log_entry, ctx, lvl, id_1, id_2, arg_count, ap); - va_end(ap); -} diff --git a/test/cmocka/configs/acp_7_0_defconfig b/test/cmocka/configs/acp_7_0_defconfig index c5e3d4952294..0f4a75aadf9c 100644 --- a/test/cmocka/configs/acp_7_0_defconfig +++ b/test/cmocka/configs/acp_7_0_defconfig @@ -1,6 +1,5 @@ CONFIG_ACP_7_0=y CONFIG_RIMAGE_SIGNING_SCHEMA="acp_7_0" -CONFIG_TRACE_CHANNEL=7 CONFIG_WRAP_ACTUAL_POSITION=y CONFIG_AMD_BT=n CONFIG_AMD_SP=n diff --git a/test/cmocka/configs/acp_7_x_defconfig b/test/cmocka/configs/acp_7_x_defconfig index d978611fadde..8f321d541d62 100644 --- a/test/cmocka/configs/acp_7_x_defconfig +++ b/test/cmocka/configs/acp_7_x_defconfig @@ -1,6 +1,5 @@ CONFIG_ACP_7_X=y CONFIG_RIMAGE_SIGNING_SCHEMA="acp_7_3" -CONFIG_TRACE_CHANNEL=7 CONFIG_WRAP_ACTUAL_POSITION=y CONFIG_CORE_COUNT=1 CONFIG_FORMAT_CONVERT_HIFI3=n diff --git a/test/cmocka/configs/rembrandt_defconfig b/test/cmocka/configs/rembrandt_defconfig index 6d5a00745050..a7a66e29f918 100644 --- a/test/cmocka/configs/rembrandt_defconfig +++ b/test/cmocka/configs/rembrandt_defconfig @@ -1,6 +1,5 @@ CONFIG_REMBRANDT=y CONFIG_RIMAGE_SIGNING_SCHEMA="rmb" -CONFIG_TRACE_CHANNEL=7 CONFIG_WRAP_ACTUAL_POSITION=y CONFIG_AMD_BT=n CONFIG_AMD_SP=n diff --git a/test/cmocka/src/common_mocks.c b/test/cmocka/src/common_mocks.c index 0e47a23a4174..53a5bc667f8e 100644 --- a/test/cmocka/src/common_mocks.c +++ b/test/cmocka/src/common_mocks.c @@ -181,31 +181,6 @@ void WEAK __panic(uint32_t p, const char *filename, uint32_t linenum) exit(EXIT_FAILURE); } -#if CONFIG_TRACE && !CONFIG_LIBRARY -void WEAK trace_log_filtered(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, - va_list args) -{ - (void) send_atomic; - (void) log_entry; - (void) ctx; - (void) lvl; - (void) id_1; - (void) id_2; - (void) arg_count; - (void) args; -} - -void WEAK _log_sofdict(log_func_t sofdict_logf, bool atomic, const void *log_entry, - const struct tr_ctx *ctx, const uint32_t lvl, - uint32_t id_1, uint32_t id_2, int arg_count, ...) -{ -} - -void WEAK trace_flush_dma_to_mbox(void) -{ -} -#endif #if CONFIG_LIBRARY volatile void *task_context_get(void); @@ -425,18 +400,6 @@ int WEAK arch_cpu_is_core_enabled(int id) int WEAK test_bench_trace = 1; int WEAK debug; -/* ... but not always in unit tests */ -#if CONFIG_TRACE -/* look up subsystem class name from table */ -char * WEAK get_trace_class(uint32_t trace_class) -{ - (void)trace_class; - /* todo: trace class is deprecated, - * uuid should be used only - */ - return "unknown"; -} -#endif uint8_t * WEAK get_library_mailbox(void) { diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 419acf5bb92e..b6e3ab4d7792 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -158,7 +158,6 @@ set(SOF_SRC_PATH "${sof_top_dir}/src") set(SOF_PLATFORM_PATH "${SOF_SRC_PATH}/kernel/platform") set(SOF_DRIVERS_PATH "${SOF_SRC_PATH}/kernel/drivers") -set(SOF_TRACE_PATH "${SOF_SRC_PATH}/trace") set(RIMAGE_TOP ${sof_top_dir}/tools/rimage) @@ -242,7 +241,6 @@ add_subdirectory(../src/kernel/ kernel_unused_install/) add_subdirectory(../src/lib/ lib_unused_install/) add_subdirectory(../src/samples/ samples_unused_install/) add_subdirectory(../src/schedule/ schedule_unused_install/) -add_subdirectory(../src/trace/ trace_unused_install/) add_subdirectory(test/) # Old way below: all .c files added by this giant CMake file. diff --git a/zephyr/include/rtos/sof.h b/zephyr/include/rtos/sof.h index 05726a55fd5f..72b05f7bc8ed 100644 --- a/zephyr/include/rtos/sof.h +++ b/zephyr/include/rtos/sof.h @@ -25,7 +25,6 @@ struct clock_info; struct comp_driver_list; struct dai_info; struct dma_info; -struct dma_trace_data; struct ipc; struct ll_schedule_domain; struct mm; @@ -36,7 +35,6 @@ struct ams_shared_context; struct notify_data; struct pm_runtime_data; struct sa; -struct trace; struct pipeline_posn; struct probe_pdata; @@ -60,11 +58,7 @@ struct sof { /* system agent */ struct sa *sa; - /* DMA for Trace*/ - struct dma_trace_data *dmat; - /* generic trace structure */ - struct trace *trace; /* platform clock information */ struct clock_info *clocks; diff --git a/zephyr/include/sof/trace/preproc.h b/zephyr/include/sof/trace/preproc.h index 83776e6fea4e..d0333c7751c7 100644 --- a/zephyr/include/sof/trace/preproc.h +++ b/zephyr/include/sof/trace/preproc.h @@ -6,11 +6,6 @@ #ifndef __SOF_TRACE_PREPROC_H__ #define __SOF_TRACE_PREPROC_H__ -#ifdef CONFIG_TRACE -#error "SOF dma-trace (CONFIG_TRACE) not supported in Zephyr builds. \ - Please use CONFIG_ZEPHYR_LOG instead." -#endif - #ifndef CONFIG_ZEPHYR_LOG /* diff --git a/zephyr/include/sof/trace/trace.h b/zephyr/include/sof/trace/trace.h index 3032a7147783..2a71baaf5c70 100644 --- a/zephyr/include/sof/trace/trace.h +++ b/zephyr/include/sof/trace/trace.h @@ -31,8 +31,7 @@ uint64_t platform_timer_get(struct timer *timer); /* * Override SOF mtrace_printf() macro for now to support Zephyr's shared - * memory logger. Note the DMA trace can be copied to the shared memory - * too thanks to CONFIG_TRACEM. + * memory logger. */ #undef mtrace_printf From 99a62024ff30e36d5bf14f5f1ec1a0201a1dd8c7 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 19:45:52 +0000 Subject: [PATCH 07/19] docs: kernel-only code is now all in src/kernel With src/trace removed, src/kernel holds all directories of kernel-only code. Update AGENTS.md and the user-space README, which listed the directories not yet moved there, and the init README, which still described the DMA trace initialization. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- AGENTS.md | 11 +++++------ src/include/sof/userspace/README.md | 8 ++++---- src/kernel/init/README.md | 2 +- 3 files changed, 10 insertions(+), 11 deletions(-) diff --git a/AGENTS.md b/AGENTS.md index 2c303eff1ec0..3eb2efa4321b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,12 +29,11 @@ memory-partition markers and Kconfig hierarchy are documented in [`src/include/sof/userspace/README.md`](src/include/sof/userspace/README.md). When adding or moving code, follow it: -* **Place code in the right tier.** OS/HAL/boot/inter-core → a kernel-only directory - (`kernel/`, which is built into the `sof_kernel` library, and the directories not yet - moved there: `trace/`); audio - application logic → `audio/`; reusable pure code → a shared-library directory (`lib/`, - `math/`, `module/`) kept free of privileged operations. Each top-level directory's - `README.md` states its tier in a "Runs in:" banner — keep it accurate. +* **Place code in the right tier.** OS/HAL/boot/inter-core → the kernel-only directory + (`kernel/`, which is built into the `sof_kernel` library); audio application logic → + `audio/`; reusable pure code → a shared-library directory (`lib/`, `math/`, `module/`) kept + free of privileged operations. Each top-level directory's `README.md` states its tier in a + "Runs in:" banner — keep it accurate. * **Cross the boundary only via a syscall.** Add a `__syscall` declaration, put the `z_vrfy_` validator in the boundary tier (preferably `zephyr/syscall/.c`), validate every user-supplied argument, and add the call to diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index 06f828ba7113..143f2ec8b2ee 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -74,7 +74,7 @@ its subdirectories span all tiers. | Tier | Meaning | Directories | |------|---------|-------------| -| **Kernel-only** | OS integration, HAL, boot, inter-core. Never executes in a user thread. Must not be reached directly from user code — only via a syscall. | `kernel/` (see [its README](../../../kernel/README.md)), `trace/` * | +| **Kernel-only** | OS integration, HAL, boot, inter-core. Never executes in a user thread. Must not be reached directly from user code — only via a syscall. | `kernel/` (see [its README](../../../kernel/README.md)) | | **Boundary / bridge** | Owns the syscalls and the thread / memory-domain setup. Split internally: some translation units run kernel-side, some user-side (e.g. `handler-kernel.c` vs `handler-user.c`). | `schedule/`, `ipc/`, `debug/debug_stream/` *, and (in the Zephyr integration tree) `zephyr/syscall/` | | **Application** | Audio pipeline / component logic that runs in user threads under userspace configs. | `audio/` | | **Shared library** | Context-agnostic code linked into whichever context calls it. Must stay "user-safe": no privileged operations (no direct MMIO, no spinlocks held across syscalls, no raw kernel object access) so it is safe when pulled into a user build. | `math/`, `module/` | @@ -82,9 +82,9 @@ its subdirectories span all tiers. Directories marked * do not yet have a "Runs in:" README banner. -Kernel-only code is being moved under `src/kernel/`, which Zephyr builds into a library of its -own, `sof_kernel`, instead of `modules_sof`. The other kernel-only directories in the table -follow one at a time. +Kernel-only code lives under `src/kernel/`, which Zephyr builds into a library of its own, +`sof_kernel`, instead of `modules_sof`. Source files that mix kernel and user code are split, +and their kernel halves moved there, separately. Classification is grep-verifiable — see [Verifying the taxonomy](#verifying-the-taxonomy). diff --git a/src/kernel/init/README.md b/src/kernel/init/README.md index b20d490550f2..f1e811f847c6 100644 --- a/src/kernel/init/README.md +++ b/src/kernel/init/README.md @@ -47,7 +47,7 @@ sequenceDiagram ``` 1. **Context Initialization**: Sets up the global `struct sof` context. -2. **Logging and Tracing**: Initializes Zephyr's logging timestamps and SOF's DMA trace infrastructure (`trace_init()`), printing the firmware version and ABI banner. +2. **Logging**: Initializes Zephyr's logging timestamps and prints the firmware version and ABI banner. 3. **System Subsystems**: - Initializes the system notifier (`init_system_notify()`) for inter-component messaging. - Sets up runtime power management (`pm_runtime_init()`). From d2af1242900c69c7cba7b1dbb90b6e8f5c9663c3 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 21:51:23 +0000 Subject: [PATCH 08/19] ipc: split the kernel part of ipc-common.c into src/kernel/ipc ipc-common.c mixes code that only runs in kernel mode with code that also runs in the user-space IPC thread. Move the kernel part to src/kernel/ipc/ipc-common-kernel.c: - the IPC context and ipc_get() of user-space LL builds; - the IPC task, command completion and IDC forwarding to other cores; - the outgoing message queue and its work queue, and the kernel half of the ipc_msg_send() and ipc_msg_list_remove() system calls; - the setup of the user-space IPC threads, their kernel-only object copies and command forwarding to them; - ipc_init(). ipc-common.c keeps message allocation, the component lookups and, for user-space LL builds, the user-space IPC thread function and the default command dispatch. The user thread entry point and its event bits are now shared through sof/ipc/common.h. The unit tests that build ipc-common.c also build the kernel part. The code is unchanged. Only the Xtensa assembler's alignment padding and its choice between narrow and wide encodings of the same instructions differ in a few functions, as do the line numbers the moved functions pass to the assertion handler in builds with assertions (qemu_xtensa). Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/ipc/common.h | 13 + src/ipc/ipc-common.c | 760 +--------------- src/kernel/ipc/CMakeLists.txt | 2 +- src/kernel/ipc/ipc-common-kernel.c | 810 ++++++++++++++++++ test/cmocka/src/audio/buffer/CMakeLists.txt | 4 + .../cmocka/src/audio/component/CMakeLists.txt | 1 + test/cmocka/src/audio/dcblock/CMakeLists.txt | 1 + test/cmocka/src/audio/eq_fir/CMakeLists.txt | 1 + test/cmocka/src/audio/eq_iir/CMakeLists.txt | 1 + test/cmocka/src/audio/mixer/CMakeLists.txt | 1 + .../src/audio/pcm_converter/CMakeLists.txt | 1 + test/cmocka/src/audio/pipeline/CMakeLists.txt | 3 + test/cmocka/src/audio/selector/CMakeLists.txt | 1 + test/cmocka/src/audio/volume/CMakeLists.txt | 1 + test/cmocka/src/math/fft/CMakeLists.txt | 1 + test/ztest/unit/audio/drc/CMakeLists.txt | 1 + test/ztest/unit/math/fft/CMakeLists.txt | 1 + 17 files changed, 845 insertions(+), 758 deletions(-) create mode 100644 src/kernel/ipc/ipc-common-kernel.c diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index d959a55d3b9d..88bd598a9f08 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -327,6 +327,19 @@ void ipc_complete_cmd(struct ipc *ipc); extern bool ipc_enter_gdb; #ifdef CONFIG_SOF_USERSPACE_LL +/* Events that wake the user-space IPC thread */ +#define IPC_USER_EVENT_CMD BIT(0) +#define IPC_USER_EVENT_STOP BIT(1) + +/** + * @brief User-space IPC thread entry point. + * + * @param p1 struct ipc_user shared with the kernel + * @param p2 Core ID + * @param p3 Unused + */ +void ipc_user_thread_fn(void *p1, void *p2, void *p3); + /** * @brief Forward an IPC command to the user-space thread and wait for it. * diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index bbc4a9b8a7d3..6c8126f51a7a 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -48,23 +48,9 @@ #include -LOG_MODULE_REGISTER(ipc, CONFIG_SOF_LOG_LEVEL); +/* The IPC code that runs in kernel mode is in src/kernel/ipc/ipc-common-kernel.c */ -SOF_DEFINE_REG_UUID(ipc); - -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(ipc_tr, SOF_UUID(ipc_uuid), LOG_LEVEL_INFO); - -#ifdef CONFIG_SOF_USERSPACE_LL -K_APPMEM_PARTITION_DEFINE(ipc_context_part); - -K_APP_BMEM(ipc_context_part) static struct ipc ipc_context; - -struct ipc *ipc_get(void) -{ - return &ipc_context; -} -#endif +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); struct ipc_msg *ipc_msg_w_ext_init(struct k_heap *heap, uint32_t header, uint32_t extension, uint32_t size) @@ -97,46 +83,6 @@ struct ipc_msg *ipc_msg_w_ext_init(struct k_heap *heap, uint32_t header, return msg; } -int ipc_process_on_core(uint32_t core, bool blocking) -{ - struct ipc *ipc = ipc_get(); - struct idc_msg msg = { .header = IDC_MSG_IPC, .core = core, }; - int ret; - - /* check if requested core is enabled */ - if (!cpu_is_core_enabled(core)) { - tr_err(&ipc_tr, "core #%d is disabled", core); - return -EACCES; - } - -#if CONFIG_IPC_MAJOR_3 - /* The other core will write there its response */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - ((struct sof_ipc_cmd_hdr *)ipc->comp_data)->size); -#endif - - /* - * If the primary core is waiting for secondary cores to complete, it - * will also reply to the host - */ - if (!blocking) { - k_spinlock_key_t key; - - ipc->core = core; - key = k_spin_lock(&ipc->lock); - ipc->task_mask |= IPC_TASK_SECONDARY_CORE; - k_spin_unlock(&ipc->lock, key); - } - - /* send IDC message */ - ret = idc_send_msg(&msg, blocking ? IDC_BLOCKING : IDC_NON_BLOCKING); - if (ret < 0) - return ret; - - /* reply written by other core */ - return 1; -} - /* * Components, buffers and pipelines are stored in the same lists, hence * type and ID have to be used for the identification. @@ -198,301 +144,7 @@ struct ipc_comp_dev *ipc_get_ppl_comp(struct ipc *ipc, uint32_t pipeline_id, int return next_ppl_icd; } -void ipc_send_queued_msg(void) -{ - struct ipc *ipc = ipc_get(); - struct ipc_msg *msg; - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - - if (ipc->pm_prepare_D3) - goto out; - - /* any messages to send ? */ - if (list_is_empty(&ipc->msg_list)) - goto out; - - msg = list_first_item(&ipc->msg_list, struct ipc_msg, - list); - - if (ipc_platform_send_msg(msg) == 0) { - /* Remove the message from the list if it has been successfully sent. */ - list_item_del(&msg->list); - /* Invoke a callback to notify that the message has been sent. */ - if (msg->callback) - msg->callback(msg); -#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS - /* Increment performance counters */ - io_perf_monitor_update_data(ipc->io_perf_out_msg_count, 1); -#endif - } -out: - k_spin_unlock(&ipc->lock, key); -} - -#ifdef __ZEPHYR__ -static K_THREAD_STACK_DEFINE(ipc_send_wq_stack, CONFIG_STACK_SIZE_IPC_TX); - -/* - * Kernel-only: these must not live in struct ipc, which user threads can - * write with CONFIG_SOF_USERSPACE_LL. The work queue embeds the k_thread of - * a supervisor thread and the work item holds the handler it calls. - */ -static struct k_work_q ipc_send_wq; -static struct k_work_delayable ipc_send_work; - -struct k_work_q *ipc_send_wq_get(void) -{ - return &ipc_send_wq; -} -#endif - -static void schedule_ipc_worker(void) -{ - /* - * note: in XTOS builds, this is handled in - * task_main_primary_core() - */ -#ifdef __ZEPHYR__ - k_work_schedule_for_queue(&ipc_send_wq, &ipc_send_work, K_USEC(IPC_PERIOD_USEC)); -#endif -} - -__cold void ipc_msg_send_direct(struct ipc_msg *msg, void *data) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - int ret; - - assert_can_be_cold(); - - key = k_spin_lock(&ipc->lock); - - /* copy mailbox data to message if not already copied */ - if (data && msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE && - msg->tx_data != data) { - ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); - assert(!ret); - } - - ipc_platform_send_msg_direct(msg); - - k_spin_unlock(&ipc->lock, key); -} - -void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - int ret; - - key = k_spin_lock(&ipc->lock); - - /* copy mailbox data to message if not already copied */ - if (data && (msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE) && - msg->tx_data != data) { - ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); - assert(!ret); - } - - /* - * note: This function can be executed in LL or EDF context, from any core. - * In Zephyr builds, there is IPC queue that is always handled by the primary core, - * whereas submitting to the queue is allowed from any core. Therefore disable option - * of sending IPC immediately by any context/core to secure IPC registers/mailbox - * access. - */ -#ifndef __ZEPHYR__ - /* try to send critical notifications right away */ - if (high_priority) { - ret = ipc_platform_send_msg(msg); - if (!ret) { - k_spin_unlock(&ipc->lock, key); - return; - } - } -#endif - /* add to queue unless already there */ - if (list_is_empty(&msg->list)) { - if (high_priority) - list_item_prepend(&msg->list, &ipc->msg_list); - else - list_item_append(&msg->list, &ipc->msg_list); - } - - schedule_ipc_worker(); - - k_spin_unlock(&ipc->lock, key); -} -EXPORT_SYMBOL(ipc_msg_send); - -#ifdef CONFIG_USERSPACE -void z_vrfy_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) -{ - K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); - - K_OOPS(K_SYSCALL_VERIFY(msg->tx_size <= SOF_IPC_MSG_MAX_SIZE)); - - K_OOPS(K_SYSCALL_VERIFY(list_is_empty(&msg->list))); - - if (msg->tx_size > 0) { - K_OOPS(K_SYSCALL_VERIFY(msg->tx_data != NULL)); - K_OOPS(K_SYSCALL_MEMORY_WRITE(msg->tx_data, msg->tx_size)); - } - - /* If data != NULL and tx_size > 0, verify the data buffer */ - if (data && msg->tx_size > 0) - K_OOPS(K_SYSCALL_MEMORY_READ(data, msg->tx_size)); - - z_impl_ipc_msg_send(msg, data, high_priority); -} -#include -#endif /* CONFIG_USERSPACE */ - -void z_impl_ipc_msg_list_remove(struct ipc_msg *msg) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - list_item_del(&msg->list); - k_spin_unlock(&ipc->lock, key); -} - -#ifdef CONFIG_USERSPACE -void z_vrfy_ipc_msg_list_remove(struct ipc_msg *msg) -{ - struct ipc *ipc = ipc_get(); - struct list_item *mlist, *_mlist; - bool found = false; - - K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); - - /* - * special case: empty list was passed. we can't trust where - * list->prev points to, so do not pass to - * z_impl_ipc_msg_list_remove(), but handle here - */ - if (list_is_empty(&msg->list)) - return; - - list_for_item_safe(mlist, _mlist, &ipc->msg_list) { - if (mlist == &msg->list) { - found = true; - break; - } - } - K_OOPS(K_SYSCALL_VERIFY(found)); - z_impl_ipc_msg_list_remove(msg); -} -#include -#endif /* CONFIG_USERSPACE */ - -#ifdef __ZEPHYR__ -static void ipc_work_handler(struct k_work *work) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - - ipc_send_queued_msg(); - - key = k_spin_lock(&ipc->lock); - - if (!list_is_empty(&ipc->msg_list) && !ipc->pm_prepare_D3) - schedule_ipc_worker(); - - k_spin_unlock(&ipc->lock, key); -} -#endif - -void ipc_schedule_process(struct ipc *ipc) -{ -#if CONFIG_TWB_IPC_TASK - schedule_task(ipc->ipc_task, 0, IPC_PERIOD_USEC); -#else - schedule_task(&ipc->ipc_task, 0, IPC_PERIOD_USEC); -#endif -} - #ifdef CONFIG_SOF_USERSPACE_LL -/* Generic user-space IPC handling thread infrastructure. Protocol-specific - * command interpretation lives in ipc_user_thread_dispatch(), implemented by - * the active IPC major (see src/ipc/ipc4/). - */ - -#define IPC_USER_EVENT_CMD BIT(0) -#define IPC_USER_EVENT_STOP BIT(1) - -SOF_DEFINE_REG_UUID(sec_core_init); - -static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, - CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); - -/* - * Kernel-only copies of the IPC user thread state. struct ipc_user and the - * ipc->ipc_user_pdata pointer are both writable by user threads, so kernel - * code must use these instead of reading kernel object pointers from there. - */ -static struct ipc_user *ipc_user_kptr; -static struct k_sem *ipc_user_ksem; -static struct k_event *ipc_user_kevent[CONFIG_CORE_COUNT]; -static struct k_thread *ipc_user_kthread[CONFIG_CORE_COUNT]; -static bool ipc_user_kinit_needed[CONFIG_CORE_COUNT]; - -/** - * @brief Forward an IPC command to the user-space thread. - * - * Called from kernel context (IPC EDF task) to forward the message words - * to the user-space thread for processing. Sets IPC_TASK_IN_THREAD in - * task_mask so the host is not signaled until the user thread completes. - * Blocks until the user thread finishes processing and returns the result. - * - * @param primary Primary message word - * @param extension Extension message word - * @return Result from user thread processing - */ -int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core) -{ - struct ipc *ipc = ipc_get(); - struct ipc_user *pdata = ipc_user_kptr; - k_spinlock_key_t key; - int ret; - - if (core >= CONFIG_CORE_COUNT) - return -EINVAL; - - LOG_DBG("IPC: forward cmd %08x", primary); - - /* Copy message words — original buffer may be reused */ - pdata->ipc_msg_pri = primary; - pdata->ipc_msg_ext = extension; - pdata->ipc = ipc; - - /* Prevent host completion until user thread finishes */ - key = k_spin_lock(&ipc->lock); - ipc->task_mask |= IPC_TASK_IN_THREAD; - k_spin_unlock(&ipc->lock, key); - - /* Wake the user thread */ - k_event_set(ipc_user_kevent[core], IPC_USER_EVENT_CMD); - - /* Wait for user thread to complete */ - ret = k_sem_take(ipc_user_ksem, K_MSEC(100)); - if (ret) { - LOG_ERR("IPC user: sem error %d\n", ret); - /* fall through to complete the cmd */ - } - - /* Clear the task mask bit and check for completion */ - key = k_spin_lock(&ipc->lock); - ipc->task_mask &= ~IPC_TASK_IN_THREAD; - ipc_complete_cmd(ipc); - k_spin_unlock(&ipc->lock, key); - - return !ret ? pdata->result : ret; -} - /** * @brief Protocol-specific dispatch of a forwarded IPC command. * @@ -509,7 +161,7 @@ __weak int ipc_user_thread_dispatch(struct ipc_user *ipc_user) * User-space IPC thread entry point. p1 points to the ipc_user context * shared with the kernel launcher. */ -static void ipc_user_thread_fn(void *p1, void *p2, void *p3) +void ipc_user_thread_fn(void *p1, void *p2, void *p3) { struct ipc_user *ipc_user = p1; unsigned int core = POINTER_TO_UINT(p2); @@ -540,410 +192,4 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) break; } } - -__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) -{ - char thread_name[] = "ll_user0"; - struct k_thread *thread; - struct k_event *event; - int ret; - - assert_can_be_cold(); - - /* - * Build the objects in locals and publish them to struct ipc_user only - * at the end. When a secondary core is brought up, user threads on - * other cores are already running and can overwrite struct ipc_user, - * so its fields must not be read back here. - */ - event = k_object_alloc(K_OBJ_EVENT); - if (!event) { - LOG_ERR("user IPC event alloc failed"); - return -ENOMEM; - } - k_event_init(event); - - thread = k_object_alloc(K_OBJ_THREAD); - if (!thread) { - LOG_ERR("user IPC thread alloc failed"); - ret = -ENOMEM; - goto e_event; - } - - k_thread_create(thread, ipc_user_stack[core], - CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, - ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, - -1, K_USER, K_FOREVER); - - k_thread_cpu_pin(thread, core); - thread_name[sizeof(thread_name) - 2] = '0' + core; - k_thread_name_set(thread, thread_name); - - /* - * Each userspace IPC thread must be able to wait on its private event - * and signal completion on the primary core semaphore - */ - k_thread_access_grant(thread, ipc_user_ksem, event); - user_grant_dai_access_all(thread); - user_grant_dma_access_all(thread); - k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); - user_ll_grant_access(thread, core); - pipeline_posn_grant_access(thread); - - ipc_user_kevent[core] = event; - ipc_user_kthread[core] = thread; - - /* for ipc_user_thread_fn() and ipc_thread_user() in user context */ - ipc_user->event[core] = event; - ipc_user->thread[core] = thread; - - return 0; - -e_event: - k_object_free(event); - - return ret; -} - -__cold int ipc_user_init_secondary(unsigned int core) -{ - struct ipc_user *ipc_user = ipc_user_kptr; - struct k_thread *audio_thread; - int ret; - - if (core >= CONFIG_CORE_COUNT) - return -EINVAL; - - ret = ipc_user_init_thread(ipc_user, core); - - if (ret < 0) - return ret; - - assert_can_be_cold(); - - k_thread_start(ipc_user_kthread[core]); - - struct task *task = zephyr_ll_task_alloc(); - - if (!task) { - LOG_ERR("user LL task allocation failed"); - k_panic(); - } - - schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, core, 0); - - audio_thread = scheduler_init_context(task); - if (!audio_thread) { - LOG_ERR("user LL thread init failed"); - k_panic(); - } - - k_thread_access_grant(ipc_user_kthread[core], audio_thread); - ipc_user->audio_thread[core] = audio_thread; - - return 0; -} - -struct k_thread *ipc_thread_user(unsigned int core) -{ - if (core >= CONFIG_CORE_COUNT) - return NULL; - - /* user threads can not read the kernel-only copy */ - if (k_is_user_context()) - return ipc_get()->ipc_user_pdata->thread[core]; - - return ipc_user_kthread[core]; -} - -struct ipc_user *ipc_user_get(void) -{ - return ipc_user_kptr; -} - -void ipc_user_wait_core_init(unsigned int core) -{ - if (core >= CONFIG_CORE_COUNT) - return; - - if (ipc_user_kinit_needed[core]) { - /* wait for IPC thread (ipc_user_thread_fn()) */ - k_sem_take(ipc_user_ksem, K_FOREVER); - ipc_user_kinit_needed[core] = false; - } -} - -/* Primary core only */ -__cold static void ipc_user_init(void) -{ - struct ipc *ipc = ipc_get(); - struct k_thread *audio_thread; - struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ipc_user), 0); - int ret, core; - - if (!ipc_user) { - LOG_ERR("user IPC pdata alloc failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - - /* sof_heap_alloc() does not zero, start with clean state (e.g. ppl_phase) */ - memset(ipc_user, 0, sizeof(*ipc_user)); - - assert_can_be_cold(); - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { - if (core != PLATFORM_PRIMARY_CORE_ID) - ipc_user_kinit_needed[core] = true; - } - - ipc_user_ksem = k_object_alloc(K_OBJ_SEM); - if (!ipc_user_ksem) { - LOG_ERR("user IPC sem alloc failed"); - k_panic(); - } - ipc_user_kptr = ipc_user; - ipc_user->sem = ipc_user_ksem; - - ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &ipc_context_part); - if (ret < 0) - LOG_WRN("ipc context partition add failed: %d", ret); - - /* - * Grant user-space read access to the .coldrodata section in IMR. The - * prepare path walks component code that may reference __cold_rodata - * data. - * - * .cold text is deliberately NOT added as a partition: Zephyr already - * maps it executable in the shared ring. - */ -#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM - extern char __coldrodata_start[]; - extern char _imr_end[]; - - if (&_imr_end[0] > &__coldrodata_start[0]) { - struct k_mem_partition cold_part = { - .start = (uintptr_t)__coldrodata_start, - .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, - CONFIG_MMU_PAGE_SIZE), - .attr = K_MEM_PARTITION_P_RO_U_RO, - }; - - ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); - if (ret < 0) - LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", - cold_part.size, cold_part.start, ret); - } -#endif /* CONFIG_COLD_STORE_EXECUTE_DRAM */ - - k_sem_init(ipc_user_ksem, 0, 1); - - ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); - if (ret < 0) { - LOG_ERR("user IPC thread initialization failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - - ret = user_access_to_mailbox(zephyr_ll_mem_domain(), - ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); - if (ret < 0) { - LOG_ERR("ipc user: mailbox access grant failed: %d", ret); - sof_panic(SOF_IPC_PANIC_IPC); - } - - /* Store references in ipc struct so kernel handler can forward commands */ - ipc->ipc_user_pdata = ipc_user; - - struct task *task = zephyr_ll_task_alloc(); - - if (!task) { - LOG_ERR("task allocation failed"); - k_panic(); - } - - schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); - audio_thread = scheduler_init_context(task); - if (!audio_thread) { - LOG_ERR("user LL thread init failed"); - k_panic(); - } - - /* Grant ipc_user thread permission on the audio thread object. - * Needed so user-space dai_common_new() can call - * k_thread_access_grant(audio_thread, dai_mutex) from user context. - */ - k_thread_access_grant(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID], audio_thread); - ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; - - k_thread_start(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); - - /* Wait for user thread startup — consumes the initial k_sem_give from thread */ - k_sem_take(ipc_user_ksem, K_FOREVER); -} -#else /* CONFIG_SOF_USERSPACE_LL */ -static void ipc_user_init(void) -{ -} #endif /* CONFIG_SOF_USERSPACE_LL */ - -__cold int ipc_init(struct sof *sof) -{ - struct k_heap *heap; - struct ipc *ipc; - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "entry"); - -#ifdef CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); - - ipc = ipc_get(); - memset(ipc, 0, sizeof(*ipc)); - ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ipc->ll_alloc), 0); - if (!ipc->ll_alloc) { - tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc"); - sof_panic(SOF_IPC_PANIC_IPC); - } - ipc->ll_alloc->heap = heap; - ipc->ll_alloc->vreg = NULL; -#else - heap = NULL; - - /* init ipc data */ - ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*ipc)); - if (!ipc) { - tr_err(&ipc_tr, "Unable to allocate IPC data"); - return -ENOMEM; - } - sof->ipc = ipc; -#endif /* CONFIG_SOF_USERSPACE_LL */ - - ipc->comp_data = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - SOF_IPC_MSG_MAX_SIZE, 0); - if (!ipc->comp_data) { - tr_err(&ipc_tr, "Unable to allocate IPC component data"); - sof_panic(SOF_IPC_PANIC_IPC); - } - memset(ipc->comp_data, 0, SOF_IPC_MSG_MAX_SIZE); - - k_spinlock_init(&ipc->lock); - list_init(&ipc->msg_list); - list_init(&ipc->comp_list); - -#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS - struct io_perf_data_item init_data = {IO_PERF_IPC_ID, - cpu_get_id(), - IO_PERF_INPUT_DIRECTION, - IO_PERF_POWERED_UP_ENABLED, - IO_PERF_D0IX_POWER_MODE, - 0, 0, 0 }; - io_perf_monitor_init_data(&ipc->io_perf_in_msg_count, &init_data); - init_data.direction = IO_PERF_OUTPUT_DIRECTION; - io_perf_monitor_init_data(&ipc->io_perf_out_msg_count, &init_data); -#endif - -#if CONFIG_SOF_BOOT_TEST_STANDALONE - LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); - return 0; -#endif - -#ifdef __ZEPHYR__ - k_tid_t thread; - - k_work_queue_init(&ipc_send_wq); - k_work_queue_start(&ipc_send_wq, ipc_send_wq_stack, - K_THREAD_STACK_SIZEOF(ipc_send_wq_stack), 1, NULL); - - thread = k_work_queue_thread_get(&ipc_send_wq); - - k_thread_suspend(thread); - -#ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_pin(thread, PLATFORM_PRIMARY_CORE_ID); -#endif - k_thread_name_set(thread, "ipc_send_wq"); - - k_thread_resume(thread); - - k_work_init_delayable(&ipc_send_work, ipc_work_handler); -#endif /* __ZEPHYR__ */ - - ipc_user_init(); - - return platform_ipc_init(ipc); -} - -/* Locking: call with ipc->lock held and interrupts disabled */ -void ipc_complete_cmd(struct ipc *ipc) -{ - /* - * We have up to three contexts, attempting to complete IPC processing: - * the original IPC EDF task, the IDC EDF task on a secondary core, or - * an LL pipeline thread, running either on the primary or one of - * secondary cores. All these three contexts execute asynchronously. It - * is important to only signal the host that the IPC processing has - * completed after *all* tasks have completed. Therefore only the last - * context should do that. We accomplish this by setting IPC_TASK_* bits - * in ipc->task_mask for each used IPC context and by clearing them when - * each of those contexts completes. Only when the mask is 0 we can - * signal the host. - */ - if (ipc->task_mask) - return; - - ipc_platform_complete_cmd(ipc); -} - -bool ipc_enter_gdb; - -__attribute__((weak)) void ipc_platform_wait_ack(struct ipc *ipc) -{ - k_msleep(1); -} - -static void ipc_complete_task(void *data) -{ - struct ipc *ipc = data; - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - ipc->task_mask &= ~IPC_TASK_INLINE; - ipc_complete_cmd(ipc); - k_spin_unlock(&ipc->lock, key); -#if CONFIG_GDBSTUB - if (ipc_enter_gdb) { - ipc_enter_gdb = false; - ipc_platform_wait_ack(ipc); - gdb_init(); - } -#endif -} - -static enum task_state ipc_do_cmd(void *data) -{ - struct ipc *ipc = data; - -#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS - /* Increment performance counters */ - io_perf_monitor_update_data(ipc->io_perf_in_msg_count, 1); -#endif - - /* - * 32-bit writes are atomic and at the moment no IPC processing is - * taking place, so, no need for a lock. - */ - ipc->task_mask = IPC_TASK_INLINE; - - return ipc_platform_do_cmd(ipc); -} - -struct task_ops ipc_task_ops = { - .run = ipc_do_cmd, - .complete = ipc_complete_task, - .get_deadline = ipc_task_deadline, -}; diff --git a/src/kernel/ipc/CMakeLists.txt b/src/kernel/ipc/CMakeLists.txt index 0f9d63c0aee6..a3e7c61638c6 100644 --- a/src/kernel/ipc/CMakeLists.txt +++ b/src/kernel/ipc/CMakeLists.txt @@ -7,7 +7,7 @@ elseif (CONFIG_IPC_MAJOR_4) endif() # Files common to Zephyr and the host library -add_local_sources(sof notification_pool.c) +add_local_sources(sof ipc-common-kernel.c notification_pool.c) is_zephyr(zephyr) if(zephyr) diff --git a/src/kernel/ipc/ipc-common-kernel.c b/src/kernel/ipc/ipc-common-kernel.c new file mode 100644 index 000000000000..618ef83cc4a2 --- /dev/null +++ b/src/kernel/ipc/ipc-common-kernel.c @@ -0,0 +1,810 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2016 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Keyon Jie + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __ZEPHYR__ +#include +#include +#endif + +#ifdef CONFIG_SOF_USERSPACE_LL +#include +#endif + +#include + +/* Kernel part of the common IPC code; the part that also runs in user mode is in + * src/ipc/ipc-common.c + */ + +LOG_MODULE_REGISTER(ipc, CONFIG_SOF_LOG_LEVEL); + +SOF_DEFINE_REG_UUID(ipc); + +/* unused with Zephyr, generates no output */ +DECLARE_TR_CTX(ipc_tr, SOF_UUID(ipc_uuid), LOG_LEVEL_INFO); + +#ifdef CONFIG_SOF_USERSPACE_LL +K_APPMEM_PARTITION_DEFINE(ipc_context_part); + +K_APP_BMEM(ipc_context_part) static struct ipc ipc_context; + +struct ipc *ipc_get(void) +{ + return &ipc_context; +} +#endif + +int ipc_process_on_core(uint32_t core, bool blocking) +{ + struct ipc *ipc = ipc_get(); + struct idc_msg msg = { .header = IDC_MSG_IPC, .core = core, }; + int ret; + + /* check if requested core is enabled */ + if (!cpu_is_core_enabled(core)) { + tr_err(&ipc_tr, "core #%d is disabled", core); + return -EACCES; + } + +#if CONFIG_IPC_MAJOR_3 + /* The other core will write there its response */ + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + ((struct sof_ipc_cmd_hdr *)ipc->comp_data)->size); +#endif + + /* + * If the primary core is waiting for secondary cores to complete, it + * will also reply to the host + */ + if (!blocking) { + k_spinlock_key_t key; + + ipc->core = core; + key = k_spin_lock(&ipc->lock); + ipc->task_mask |= IPC_TASK_SECONDARY_CORE; + k_spin_unlock(&ipc->lock, key); + } + + /* send IDC message */ + ret = idc_send_msg(&msg, blocking ? IDC_BLOCKING : IDC_NON_BLOCKING); + if (ret < 0) + return ret; + + /* reply written by other core */ + return 1; +} + +void ipc_send_queued_msg(void) +{ + struct ipc *ipc = ipc_get(); + struct ipc_msg *msg; + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + + if (ipc->pm_prepare_D3) + goto out; + + /* any messages to send ? */ + if (list_is_empty(&ipc->msg_list)) + goto out; + + msg = list_first_item(&ipc->msg_list, struct ipc_msg, + list); + + if (ipc_platform_send_msg(msg) == 0) { + /* Remove the message from the list if it has been successfully sent. */ + list_item_del(&msg->list); + /* Invoke a callback to notify that the message has been sent. */ + if (msg->callback) + msg->callback(msg); +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + /* Increment performance counters */ + io_perf_monitor_update_data(ipc->io_perf_out_msg_count, 1); +#endif + } +out: + k_spin_unlock(&ipc->lock, key); +} + +#ifdef __ZEPHYR__ +static K_THREAD_STACK_DEFINE(ipc_send_wq_stack, CONFIG_STACK_SIZE_IPC_TX); + +/* + * Kernel-only: these must not live in struct ipc, which user threads can + * write with CONFIG_SOF_USERSPACE_LL. The work queue embeds the k_thread of + * a supervisor thread and the work item holds the handler it calls. + */ +static struct k_work_q ipc_send_wq; +static struct k_work_delayable ipc_send_work; + +struct k_work_q *ipc_send_wq_get(void) +{ + return &ipc_send_wq; +} +#endif + +static void schedule_ipc_worker(void) +{ + /* + * note: in XTOS builds, this is handled in + * task_main_primary_core() + */ +#ifdef __ZEPHYR__ + k_work_schedule_for_queue(&ipc_send_wq, &ipc_send_work, K_USEC(IPC_PERIOD_USEC)); +#endif +} + +__cold void ipc_msg_send_direct(struct ipc_msg *msg, void *data) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + int ret; + + assert_can_be_cold(); + + key = k_spin_lock(&ipc->lock); + + /* copy mailbox data to message if not already copied */ + if (data && msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE && + msg->tx_data != data) { + ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); + assert(!ret); + } + + ipc_platform_send_msg_direct(msg); + + k_spin_unlock(&ipc->lock, key); +} + +void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + int ret; + + key = k_spin_lock(&ipc->lock); + + /* copy mailbox data to message if not already copied */ + if (data && (msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE) && + msg->tx_data != data) { + ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); + assert(!ret); + } + + /* + * note: This function can be executed in LL or EDF context, from any core. + * In Zephyr builds, there is IPC queue that is always handled by the primary core, + * whereas submitting to the queue is allowed from any core. Therefore disable option + * of sending IPC immediately by any context/core to secure IPC registers/mailbox + * access. + */ +#ifndef __ZEPHYR__ + /* try to send critical notifications right away */ + if (high_priority) { + ret = ipc_platform_send_msg(msg); + if (!ret) { + k_spin_unlock(&ipc->lock, key); + return; + } + } +#endif + /* add to queue unless already there */ + if (list_is_empty(&msg->list)) { + if (high_priority) + list_item_prepend(&msg->list, &ipc->msg_list); + else + list_item_append(&msg->list, &ipc->msg_list); + } + + schedule_ipc_worker(); + + k_spin_unlock(&ipc->lock, key); +} +EXPORT_SYMBOL(ipc_msg_send); + +#ifdef CONFIG_USERSPACE +void z_vrfy_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + + K_OOPS(K_SYSCALL_VERIFY(msg->tx_size <= SOF_IPC_MSG_MAX_SIZE)); + + K_OOPS(K_SYSCALL_VERIFY(list_is_empty(&msg->list))); + + if (msg->tx_size > 0) { + K_OOPS(K_SYSCALL_VERIFY(msg->tx_data != NULL)); + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg->tx_data, msg->tx_size)); + } + + /* If data != NULL and tx_size > 0, verify the data buffer */ + if (data && msg->tx_size > 0) + K_OOPS(K_SYSCALL_MEMORY_READ(data, msg->tx_size)); + + z_impl_ipc_msg_send(msg, data, high_priority); +} +#include +#endif /* CONFIG_USERSPACE */ + +void z_impl_ipc_msg_list_remove(struct ipc_msg *msg) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + list_item_del(&msg->list); + k_spin_unlock(&ipc->lock, key); +} + +#ifdef CONFIG_USERSPACE +void z_vrfy_ipc_msg_list_remove(struct ipc_msg *msg) +{ + struct ipc *ipc = ipc_get(); + struct list_item *mlist, *_mlist; + bool found = false; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + + /* + * special case: empty list was passed. we can't trust where + * list->prev points to, so do not pass to + * z_impl_ipc_msg_list_remove(), but handle here + */ + if (list_is_empty(&msg->list)) + return; + + list_for_item_safe(mlist, _mlist, &ipc->msg_list) { + if (mlist == &msg->list) { + found = true; + break; + } + } + K_OOPS(K_SYSCALL_VERIFY(found)); + z_impl_ipc_msg_list_remove(msg); +} +#include +#endif /* CONFIG_USERSPACE */ + +#ifdef __ZEPHYR__ +static void ipc_work_handler(struct k_work *work) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + + ipc_send_queued_msg(); + + key = k_spin_lock(&ipc->lock); + + if (!list_is_empty(&ipc->msg_list) && !ipc->pm_prepare_D3) + schedule_ipc_worker(); + + k_spin_unlock(&ipc->lock, key); +} +#endif + +void ipc_schedule_process(struct ipc *ipc) +{ +#if CONFIG_TWB_IPC_TASK + schedule_task(ipc->ipc_task, 0, IPC_PERIOD_USEC); +#else + schedule_task(&ipc->ipc_task, 0, IPC_PERIOD_USEC); +#endif +} + +#ifdef CONFIG_SOF_USERSPACE_LL +/* Generic user-space IPC handling thread infrastructure. Protocol-specific + * command interpretation lives in ipc_user_thread_dispatch(), implemented by + * the active IPC major (see src/ipc/ipc4/). + */ + +SOF_DEFINE_REG_UUID(sec_core_init); + +static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); + +/* + * Kernel-only copies of the IPC user thread state. struct ipc_user and the + * ipc->ipc_user_pdata pointer are both writable by user threads, so kernel + * code must use these instead of reading kernel object pointers from there. + */ +static struct ipc_user *ipc_user_kptr; +static struct k_sem *ipc_user_ksem; +static struct k_event *ipc_user_kevent[CONFIG_CORE_COUNT]; +static struct k_thread *ipc_user_kthread[CONFIG_CORE_COUNT]; +static bool ipc_user_kinit_needed[CONFIG_CORE_COUNT]; + +/** + * @brief Forward an IPC command to the user-space thread. + * + * Called from kernel context (IPC EDF task) to forward the message words + * to the user-space thread for processing. Sets IPC_TASK_IN_THREAD in + * task_mask so the host is not signaled until the user thread completes. + * Blocks until the user thread finishes processing and returns the result. + * + * @param primary Primary message word + * @param extension Extension message word + * @return Result from user thread processing + */ +int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *pdata = ipc_user_kptr; + k_spinlock_key_t key; + int ret; + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + LOG_DBG("IPC: forward cmd %08x", primary); + + /* Copy message words — original buffer may be reused */ + pdata->ipc_msg_pri = primary; + pdata->ipc_msg_ext = extension; + pdata->ipc = ipc; + + /* Prevent host completion until user thread finishes */ + key = k_spin_lock(&ipc->lock); + ipc->task_mask |= IPC_TASK_IN_THREAD; + k_spin_unlock(&ipc->lock, key); + + /* Wake the user thread */ + k_event_set(ipc_user_kevent[core], IPC_USER_EVENT_CMD); + + /* Wait for user thread to complete */ + ret = k_sem_take(ipc_user_ksem, K_MSEC(100)); + if (ret) { + LOG_ERR("IPC user: sem error %d\n", ret); + /* fall through to complete the cmd */ + } + + /* Clear the task mask bit and check for completion */ + key = k_spin_lock(&ipc->lock); + ipc->task_mask &= ~IPC_TASK_IN_THREAD; + ipc_complete_cmd(ipc); + k_spin_unlock(&ipc->lock, key); + + return !ret ? pdata->result : ret; +} + +__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) +{ + char thread_name[] = "ll_user0"; + struct k_thread *thread; + struct k_event *event; + int ret; + + assert_can_be_cold(); + + /* + * Build the objects in locals and publish them to struct ipc_user only + * at the end. When a secondary core is brought up, user threads on + * other cores are already running and can overwrite struct ipc_user, + * so its fields must not be read back here. + */ + event = k_object_alloc(K_OBJ_EVENT); + if (!event) { + LOG_ERR("user IPC event alloc failed"); + return -ENOMEM; + } + k_event_init(event); + + thread = k_object_alloc(K_OBJ_THREAD); + if (!thread) { + LOG_ERR("user IPC thread alloc failed"); + ret = -ENOMEM; + goto e_event; + } + + k_thread_create(thread, ipc_user_stack[core], + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, + ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, + -1, K_USER, K_FOREVER); + + k_thread_cpu_pin(thread, core); + thread_name[sizeof(thread_name) - 2] = '0' + core; + k_thread_name_set(thread, thread_name); + + /* + * Each userspace IPC thread must be able to wait on its private event + * and signal completion on the primary core semaphore + */ + k_thread_access_grant(thread, ipc_user_ksem, event); + user_grant_dai_access_all(thread); + user_grant_dma_access_all(thread); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); + user_ll_grant_access(thread, core); + pipeline_posn_grant_access(thread); + + ipc_user_kevent[core] = event; + ipc_user_kthread[core] = thread; + + /* for ipc_user_thread_fn() and ipc_thread_user() in user context */ + ipc_user->event[core] = event; + ipc_user->thread[core] = thread; + + return 0; + +e_event: + k_object_free(event); + + return ret; +} + +__cold int ipc_user_init_secondary(unsigned int core) +{ + struct ipc_user *ipc_user = ipc_user_kptr; + struct k_thread *audio_thread; + int ret; + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + ret = ipc_user_init_thread(ipc_user, core); + + if (ret < 0) + return ret; + + assert_can_be_cold(); + + k_thread_start(ipc_user_kthread[core]); + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("user LL task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, core, 0); + + audio_thread = scheduler_init_context(task); + if (!audio_thread) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + k_thread_access_grant(ipc_user_kthread[core], audio_thread); + ipc_user->audio_thread[core] = audio_thread; + + return 0; +} + +struct k_thread *ipc_thread_user(unsigned int core) +{ + if (core >= CONFIG_CORE_COUNT) + return NULL; + + /* user threads can not read the kernel-only copy */ + if (k_is_user_context()) + return ipc_get()->ipc_user_pdata->thread[core]; + + return ipc_user_kthread[core]; +} + +struct ipc_user *ipc_user_get(void) +{ + return ipc_user_kptr; +} + +void ipc_user_wait_core_init(unsigned int core) +{ + if (core >= CONFIG_CORE_COUNT) + return; + + if (ipc_user_kinit_needed[core]) { + /* wait for IPC thread (ipc_user_thread_fn()) */ + k_sem_take(ipc_user_ksem, K_FOREVER); + ipc_user_kinit_needed[core] = false; + } +} + +/* Primary core only */ +__cold static void ipc_user_init(void) +{ + struct ipc *ipc = ipc_get(); + struct k_thread *audio_thread; + struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*ipc_user), 0); + int ret, core; + + if (!ipc_user) { + LOG_ERR("user IPC pdata alloc failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + + /* sof_heap_alloc() does not zero, start with clean state (e.g. ppl_phase) */ + memset(ipc_user, 0, sizeof(*ipc_user)); + + assert_can_be_cold(); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + if (core != PLATFORM_PRIMARY_CORE_ID) + ipc_user_kinit_needed[core] = true; + } + + ipc_user_ksem = k_object_alloc(K_OBJ_SEM); + if (!ipc_user_ksem) { + LOG_ERR("user IPC sem alloc failed"); + k_panic(); + } + ipc_user_kptr = ipc_user; + ipc_user->sem = ipc_user_ksem; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &ipc_context_part); + if (ret < 0) + LOG_WRN("ipc context partition add failed: %d", ret); + + /* + * Grant user-space read access to the .coldrodata section in IMR. The + * prepare path walks component code that may reference __cold_rodata + * data. + * + * .cold text is deliberately NOT added as a partition: Zephyr already + * maps it executable in the shared ring. + */ +#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM + extern char __coldrodata_start[]; + extern char _imr_end[]; + + if (&_imr_end[0] > &__coldrodata_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__coldrodata_start, + .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); + if (ret < 0) + LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", + cold_part.size, cold_part.start, ret); + } +#endif /* CONFIG_COLD_STORE_EXECUTE_DRAM */ + + k_sem_init(ipc_user_ksem, 0, 1); + + ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); + if (ret < 0) { + LOG_ERR("user IPC thread initialization failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + + ret = user_access_to_mailbox(zephyr_ll_mem_domain(), + ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); + if (ret < 0) { + LOG_ERR("ipc user: mailbox access grant failed: %d", ret); + sof_panic(SOF_IPC_PANIC_IPC); + } + + /* Store references in ipc struct so kernel handler can forward commands */ + ipc->ipc_user_pdata = ipc_user; + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); + audio_thread = scheduler_init_context(task); + if (!audio_thread) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + /* Grant ipc_user thread permission on the audio thread object. + * Needed so user-space dai_common_new() can call + * k_thread_access_grant(audio_thread, dai_mutex) from user context. + */ + k_thread_access_grant(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID], audio_thread); + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; + + k_thread_start(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); + + /* Wait for user thread startup — consumes the initial k_sem_give from thread */ + k_sem_take(ipc_user_ksem, K_FOREVER); +} +#else /* CONFIG_SOF_USERSPACE_LL */ +static void ipc_user_init(void) +{ +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +__cold int ipc_init(struct sof *sof) +{ + struct k_heap *heap; + struct ipc *ipc; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "entry"); + +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); + + ipc = ipc_get(); + memset(ipc, 0, sizeof(*ipc)); + ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*ipc->ll_alloc), 0); + if (!ipc->ll_alloc) { + tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc"); + sof_panic(SOF_IPC_PANIC_IPC); + } + ipc->ll_alloc->heap = heap; + ipc->ll_alloc->vreg = NULL; +#else + heap = NULL; + + /* init ipc data */ + ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*ipc)); + if (!ipc) { + tr_err(&ipc_tr, "Unable to allocate IPC data"); + return -ENOMEM; + } + sof->ipc = ipc; +#endif /* CONFIG_SOF_USERSPACE_LL */ + + ipc->comp_data = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + SOF_IPC_MSG_MAX_SIZE, 0); + if (!ipc->comp_data) { + tr_err(&ipc_tr, "Unable to allocate IPC component data"); + sof_panic(SOF_IPC_PANIC_IPC); + } + memset(ipc->comp_data, 0, SOF_IPC_MSG_MAX_SIZE); + + k_spinlock_init(&ipc->lock); + list_init(&ipc->msg_list); + list_init(&ipc->comp_list); + +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + struct io_perf_data_item init_data = {IO_PERF_IPC_ID, + cpu_get_id(), + IO_PERF_INPUT_DIRECTION, + IO_PERF_POWERED_UP_ENABLED, + IO_PERF_D0IX_POWER_MODE, + 0, 0, 0 }; + io_perf_monitor_init_data(&ipc->io_perf_in_msg_count, &init_data); + init_data.direction = IO_PERF_OUTPUT_DIRECTION; + io_perf_monitor_init_data(&ipc->io_perf_out_msg_count, &init_data); +#endif + +#if CONFIG_SOF_BOOT_TEST_STANDALONE + LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); + return 0; +#endif + +#ifdef __ZEPHYR__ + k_tid_t thread; + + k_work_queue_init(&ipc_send_wq); + k_work_queue_start(&ipc_send_wq, ipc_send_wq_stack, + K_THREAD_STACK_SIZEOF(ipc_send_wq_stack), 1, NULL); + + thread = k_work_queue_thread_get(&ipc_send_wq); + + k_thread_suspend(thread); + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, PLATFORM_PRIMARY_CORE_ID); +#endif + k_thread_name_set(thread, "ipc_send_wq"); + + k_thread_resume(thread); + + k_work_init_delayable(&ipc_send_work, ipc_work_handler); +#endif /* __ZEPHYR__ */ + + ipc_user_init(); + + return platform_ipc_init(ipc); +} + +/* Locking: call with ipc->lock held and interrupts disabled */ +void ipc_complete_cmd(struct ipc *ipc) +{ + /* + * We have up to three contexts, attempting to complete IPC processing: + * the original IPC EDF task, the IDC EDF task on a secondary core, or + * an LL pipeline thread, running either on the primary or one of + * secondary cores. All these three contexts execute asynchronously. It + * is important to only signal the host that the IPC processing has + * completed after *all* tasks have completed. Therefore only the last + * context should do that. We accomplish this by setting IPC_TASK_* bits + * in ipc->task_mask for each used IPC context and by clearing them when + * each of those contexts completes. Only when the mask is 0 we can + * signal the host. + */ + if (ipc->task_mask) + return; + + ipc_platform_complete_cmd(ipc); +} + +bool ipc_enter_gdb; + +__attribute__((weak)) void ipc_platform_wait_ack(struct ipc *ipc) +{ + k_msleep(1); +} + +static void ipc_complete_task(void *data) +{ + struct ipc *ipc = data; + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + ipc->task_mask &= ~IPC_TASK_INLINE; + ipc_complete_cmd(ipc); + k_spin_unlock(&ipc->lock, key); +#if CONFIG_GDBSTUB + if (ipc_enter_gdb) { + ipc_enter_gdb = false; + ipc_platform_wait_ack(ipc); + gdb_init(); + } +#endif +} + +static enum task_state ipc_do_cmd(void *data) +{ + struct ipc *ipc = data; + +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + /* Increment performance counters */ + io_perf_monitor_update_data(ipc->io_perf_in_msg_count, 1); +#endif + + /* + * 32-bit writes are atomic and at the moment no IPC processing is + * taking place, so, no need for a lock. + */ + ipc->task_mask = IPC_TASK_INLINE; + + return ipc_platform_do_cmd(ipc); +} + +struct task_ops ipc_task_ops = { + .run = ipc_do_cmd, + .complete = ipc_complete_task, + .get_deadline = ipc_task_deadline, +}; diff --git a/test/cmocka/src/audio/buffer/CMakeLists.txt b/test/cmocka/src/audio/buffer/CMakeLists.txt index cb60f07aa22e..805aaa2da784 100644 --- a/test/cmocka/src/audio/buffer/CMakeLists.txt +++ b/test/cmocka/src/audio/buffer/CMakeLists.txt @@ -14,6 +14,7 @@ cmocka_test(buffer_copy ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c @@ -38,6 +39,7 @@ cmocka_test(buffer_new ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c @@ -62,6 +64,7 @@ cmocka_test(buffer_wrap ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c @@ -86,6 +89,7 @@ cmocka_test(buffer_write ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c diff --git a/test/cmocka/src/audio/component/CMakeLists.txt b/test/cmocka/src/audio/component/CMakeLists.txt index b82756073b25..50389cfe4535 100644 --- a/test/cmocka/src/audio/component/CMakeLists.txt +++ b/test/cmocka/src/audio/component/CMakeLists.txt @@ -8,6 +8,7 @@ cmocka_test(comp_set_state ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c diff --git a/test/cmocka/src/audio/dcblock/CMakeLists.txt b/test/cmocka/src/audio/dcblock/CMakeLists.txt index 9c7f58e8feed..0e5754b022d5 100644 --- a/test/cmocka/src/audio/dcblock/CMakeLists.txt +++ b/test/cmocka/src/audio/dcblock/CMakeLists.txt @@ -41,6 +41,7 @@ add_library(audio_for_dcblock STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/audio/eq_fir/CMakeLists.txt b/test/cmocka/src/audio/eq_fir/CMakeLists.txt index 0bb81cc57b0a..485d1996dd8d 100644 --- a/test/cmocka/src/audio/eq_fir/CMakeLists.txt +++ b/test/cmocka/src/audio/eq_fir/CMakeLists.txt @@ -37,6 +37,7 @@ add_library(audio_for_eq_fir STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/audio/eq_iir/CMakeLists.txt b/test/cmocka/src/audio/eq_iir/CMakeLists.txt index 5087e77b4b83..53a550b6b6cf 100644 --- a/test/cmocka/src/audio/eq_iir/CMakeLists.txt +++ b/test/cmocka/src/audio/eq_iir/CMakeLists.txt @@ -40,6 +40,7 @@ add_library(audio_for_eq_iir STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/audio/mixer/CMakeLists.txt b/test/cmocka/src/audio/mixer/CMakeLists.txt index a6c1cdb1a78f..b0b086816e3b 100644 --- a/test/cmocka/src/audio/mixer/CMakeLists.txt +++ b/test/cmocka/src/audio/mixer/CMakeLists.txt @@ -9,6 +9,7 @@ cmocka_test(mixer ${PROJECT_SOURCE_DIR}/src/audio/mixer/mixer_hifi3.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/src/audio/module_adapter/module_adapter.c diff --git a/test/cmocka/src/audio/pcm_converter/CMakeLists.txt b/test/cmocka/src/audio/pcm_converter/CMakeLists.txt index 4dca80136e68..ee72311e004b 100644 --- a/test/cmocka/src/audio/pcm_converter/CMakeLists.txt +++ b/test/cmocka/src/audio/pcm_converter/CMakeLists.txt @@ -19,6 +19,7 @@ if(CONFIG_FORMAT_FLOAT) ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c diff --git a/test/cmocka/src/audio/pipeline/CMakeLists.txt b/test/cmocka/src/audio/pipeline/CMakeLists.txt index 628398576a43..27d27757d6ae 100644 --- a/test/cmocka/src/audio/pipeline/CMakeLists.txt +++ b/test/cmocka/src/audio/pipeline/CMakeLists.txt @@ -6,6 +6,7 @@ cmocka_test(pipeline_new ${PROJECT_SOURCE_DIR}/src/audio/component.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c @@ -31,6 +32,7 @@ cmocka_test(pipeline_connect_upstream ${PROJECT_SOURCE_DIR}/src/audio/component.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c @@ -56,6 +58,7 @@ cmocka_test(pipeline_free ${PROJECT_SOURCE_DIR}/src/audio/component.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c diff --git a/test/cmocka/src/audio/selector/CMakeLists.txt b/test/cmocka/src/audio/selector/CMakeLists.txt index 8ea8d969ae7c..eb8f742cc5f5 100644 --- a/test/cmocka/src/audio/selector/CMakeLists.txt +++ b/test/cmocka/src/audio/selector/CMakeLists.txt @@ -26,6 +26,7 @@ add_library(audio_for_selector STATIC ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c diff --git a/test/cmocka/src/audio/volume/CMakeLists.txt b/test/cmocka/src/audio/volume/CMakeLists.txt index d5e1a3ecb757..7a6dc7b2c854 100644 --- a/test/cmocka/src/audio/volume/CMakeLists.txt +++ b/test/cmocka/src/audio/volume/CMakeLists.txt @@ -33,6 +33,7 @@ add_library(audio_for_volume STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/math/fft/CMakeLists.txt b/test/cmocka/src/math/fft/CMakeLists.txt index f8a0510f7027..79234b7b12f9 100644 --- a/test/cmocka/src/math/fft/CMakeLists.txt +++ b/test/cmocka/src/math/fft/CMakeLists.txt @@ -18,6 +18,7 @@ cmocka_test(fft ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c diff --git a/test/ztest/unit/audio/drc/CMakeLists.txt b/test/ztest/unit/audio/drc/CMakeLists.txt index 2faef72b825c..39691eefa371 100644 --- a/test/ztest/unit/audio/drc/CMakeLists.txt +++ b/test/ztest/unit/audio/drc/CMakeLists.txt @@ -61,6 +61,7 @@ target_sources(app PRIVATE ${SOF_ROOT}/src/module/audio/sink_api.c ${SOF_ROOT}/src/kernel/ipc/ipc3/helper.c ${SOF_ROOT}/src/ipc/ipc-common.c + ${SOF_ROOT}/src/kernel/ipc/ipc-common-kernel.c ${SOF_ROOT}/src/ipc/ipc-helper.c ${SOF_ROOT}/src/lib/objpool.c ${SOF_ROOT}/src/audio/pipeline/pipeline-graph.c diff --git a/test/ztest/unit/math/fft/CMakeLists.txt b/test/ztest/unit/math/fft/CMakeLists.txt index 920f41bdb642..0c7a8f13783f 100644 --- a/test/ztest/unit/math/fft/CMakeLists.txt +++ b/test/ztest/unit/math/fft/CMakeLists.txt @@ -65,6 +65,7 @@ target_sources(app PRIVATE ${SOF_ROOT}/src/module/audio/sink_api.c ${SOF_ROOT}/src/kernel/ipc/ipc3/helper.c ${SOF_ROOT}/src/ipc/ipc-common.c + ${SOF_ROOT}/src/kernel/ipc/ipc-common-kernel.c ${SOF_ROOT}/src/ipc/ipc-helper.c ${SOF_ROOT}/src/audio/pipeline/pipeline-graph.c ${SOF_ROOT}/src/audio/pipeline/pipeline-params.c From 74bf7bdf9bd5609e5dcbbe4ac47201658d0746bd Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 22:22:35 +0000 Subject: [PATCH 09/19] ipc4: split the kernel part of handler-user.c into src/kernel/ipc/ipc4 handler-user.c mixes the IPC4 global and module message routers, which run in the kernel IPC thread, with the handlers that also run in the user-space IPC thread. Move the kernel part to src/kernel/ipc/ipc4/handler-user-kernel.c: - the global and module message routers and the reply message they fill in; - the forwarding to the user-space IPC thread: the target core lookup, the multicore SET_PIPELINE_STATE sequencing and the MOD_INIT_INSTANCE driver lookup and module loading; - the CHAIN_DMA, IPC gateway and GDB commands; - the module init that runs in the kernel (DP modules and builds without user-space LL) and the base firmware LARGE_CONFIG get and set, which use IDC and the IMR manifest; - the thin wrappers that the routers of builds without user-space LL call directly. handler-user.c keeps the pipeline state handling, the module config and large config processing and the user-space IPC thread dispatch. The pipeline ID lookup and the vendor config and kcontrol helpers, which both parts use, are no longer static and are declared in ipc4/handler.h. handler-user.c no longer needs the IPC gateway copier header. The code is unchanged, but the compiler can no longer inline or specialise the shared helpers in the callers on the other side of the split. Their callers now pass the full argument list instead of calling GCC's constant-propagated clones, ipc4_set_pipeline_state() only partially inlines ipc4_pipeline_id_get() in builds without user-space LL, and the register allocation of the routers differs. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/ipc4/handler.h | 70 ++ src/ipc/ipc4/README.md | 7 +- src/ipc/ipc4/handler-user.c | 844 +-------------------- src/kernel/ipc/ipc4/CMakeLists.txt | 1 + src/kernel/ipc/ipc4/handler-user-kernel.c | 877 ++++++++++++++++++++++ 5 files changed, 969 insertions(+), 830 deletions(-) create mode 100644 src/kernel/ipc/ipc4/handler-user-kernel.c diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h index 3335c108fa29..5fc9f749e7fb 100644 --- a/src/include/ipc4/handler.h +++ b/src/include/ipc4/handler.h @@ -12,7 +12,10 @@ #ifndef __SOF_IPC4_HANDLER_H__ #define __SOF_IPC4_HANDLER_H__ +struct comp_dev; +struct comp_driver; struct ipc4_message_request; +struct ipc4_pipeline_set_state; /** * \brief Processes an IPC4 userspace module message. @@ -69,6 +72,73 @@ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_ */ int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4); +/** + * \brief Get the first pipeline ID of a SET_PIPELINE_STATE message. + * \ingroup ipc4_envelope_api + * @param[in] ipc4 IPC4 message request structure. + * @param[in] state Copy of the SET_PIPELINE_STATE message header. + * @param[out] ppl_id Receives the pipeline ID array of a multi-pipeline + * message, NULL otherwise (may be NULL). + * @param[out] ppl_count Receives the number of pipelines (may be NULL). + * @return The first pipeline ID, or a negative error code. + */ +int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, + struct ipc4_pipeline_set_state *state, + const uint32_t **ppl_id, unsigned int *ppl_count); + +/** + * \brief Copy the kcontrol payload header of a LARGE_CONFIG_GET from the hostbox. + * \ingroup ipc4_envelope_api + * @param[in] dev Target component, NULL for the base firmware. + * @param[in] param_id Large config parameter ID. + * @param[out] data_out Reply data buffer. + * @param[in] data_size Size of the payload in the hostbox. + */ +void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t param_id, + char *data_out, uint32_t data_size); + +/** + * \brief Process a vendor config LARGE_CONFIG_GET. + * \ingroup ipc4_envelope_api + * @param[in] dev Target component, NULL for the base firmware. + * @param[in] drv Component driver. + * @param[in] init_block True for the first block. + * @param[in] final_block True for the last block. + * @param[in,out] data_off_size Payload size in, reply data size out. + * @param[out] data_out Reply data buffer. + * @param[in] data_in Request payload. + * @return IPC4 status code (IPC4_SUCCESS on success). + */ +int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, + const struct comp_driver *drv, + bool init_block, + bool final_block, + uint32_t *data_off_size, + char *data_out, + const char *data_in); + +/** + * \brief Process a vendor config LARGE_CONFIG_SET. + * \ingroup ipc4_envelope_api + * @param[in] dev Target component, NULL for the base firmware. + * @param[in] drv Component driver. + * @param[in] module_id Module ID, for error messages. + * @param[in] instance_id Instance ID, for error messages. + * @param[in] init_block True for the first block. + * @param[in] final_block True for the last block. + * @param[in] data_off_size Payload size. + * @param[in] data Request payload. + * @return IPC4 status code (IPC4_SUCCESS on success). + */ +int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, + const struct comp_driver *drv, + uint32_t module_id, + uint32_t instance_id, + bool init_block, + bool final_block, + uint32_t data_off_size, + const char *data); + /** * \brief Complete an IPC compound message transaction and signal waiting threads. * \ingroup ipc4_envelope_api diff --git a/src/ipc/ipc4/README.md b/src/ipc/ipc4/README.md index 237a7c1ecb36..70ee4d606af2 100644 --- a/src/ipc/ipc4/README.md +++ b/src/ipc/ipc4/README.md @@ -2,9 +2,10 @@ > **Runs in:** Boundary / bridge — `handler-user.c` and `notification-user.c` run in the user > thread that services modules. Their kernel counterparts, `handler-kernel.c` (the kernel -> dispatcher) and `notification.c`, are in `src/kernel/ipc/ipc4`, together with the -> `z_vrfy_`/`z_impl_` pairs for the compound-message, reply and notification syscalls. See -> [User/kernel split](../../include/sof/userspace/README.md). +> dispatcher), `handler-user-kernel.c` (the global and module message routers, which forward +> commands to the user thread) and `notification.c`, are in `src/kernel/ipc/ipc4`, together +> with the `z_vrfy_`/`z_impl_` pairs for the compound-message, reply and notification +> syscalls. See [User/kernel split](../../include/sof/userspace/README.md). This directory holds the handlers and topology parsing logic for Inter-Processor Communication Version 4. IPC4 introduces a significantly denser, compound-command structure heavily based around the concept of "pipelines" and dynamic "modules" rather than static DSP stream roles. diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 0af33fe7fd1c..58df68018242 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -44,8 +44,6 @@ #include #include -#include "../audio/copier/ipcgtw_copier.h" - /* Command format errors during fuzzing are reported for virtually all * commands, and the resulting flood of logging becomes a severe * performance penalty (i.e. we get a lot less fuzzing done per CPU @@ -57,11 +55,11 @@ #define ipc_cmd_err(...) tr_err(__VA_ARGS__) #endif -LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); +/* The IPC4 routers, which run in kernel mode, are in + * src/kernel/ipc/ipc4/handler-user-kernel.c + */ -/* Userspace message context, copied in/out by kernel IPC thread. */ -/* fw sends a fw ipc message to send the status of the last host ipc message */ -static struct ipc_msg *msg_reply; +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); #if CONFIG_LIBRARY @@ -90,54 +88,6 @@ static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data( /* * Global IPC Operations. */ -#ifdef CONFIG_SOF_USERSPACE_LL -/* - * Determine the target core for an IPC4 module message. - * Falls back to current core when no component is bound yet. - */ -static unsigned int ipc4_user_target_core_module(struct ipc4_message_request *ipc4) -{ - /* - * Also works for struct ipc4_module_large_config, struct ipc4_module_bind_unbind, - * struct ipc4_module_delete_instance - */ - struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; - uint32_t module_id = config->primary.r.module_id; - - if (module_id) { - uint32_t instance_id = config->primary.r.instance_id; - struct comp_dev *dev = ipc4_get_comp_dev(IPC4_COMP_ID(module_id, instance_id)); - - if (dev) - return dev->ipc_config.core; - } - - return cpu_get_id(); -} -#else /* CONFIG_SOF_USERSPACE_LL */ -__cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) -{ - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); -} - -__cold static int ipc4_delete_pipeline(struct ipc4_message_request *ipc4) -{ - struct ipc4_pipeline_delete *pipe; - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - pipe = (struct ipc4_pipeline_delete *)ipc4; - tr_dbg(&ipc_tr, "ipc4 delete pipeline %x:", (uint32_t)pipe->primary.r.instance_id); - - return ipc_pipeline_free(ipc, pipe->primary.r.instance_id); -} -#endif /* CONFIG_SOF_USERSPACE_LL */ - static int ipc4_pcm_params(struct ipc_comp_dev *pcm_dev) { int err, reset_err; @@ -450,9 +400,9 @@ __cold const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data_wrapper return ipc4_get_pipeline_data(); } -static int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, - struct ipc4_pipeline_set_state *state, - const uint32_t **ppl_id, unsigned int *ppl_count) +int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, + struct ipc4_pipeline_set_state *state, + const uint32_t **ppl_id, unsigned int *ppl_count) { if (!state->extension.r.multi_ppl) { if (ppl_count) @@ -648,364 +598,8 @@ static int ipc4_set_pipeline_state_phase(struct ipc4_message_request *ipc4, uint return IPC4_SUCCESS; } - -/* - * Handle a multi-pipeline SET_PIPELINE_STATE, where the pipelines are on - * different cores. The user-space IPC thread can't use IDC to reach the - * other cores, so the kernel runs the prepare phase on all pipelines and - * then the trigger phase on all pipelines, forwarding each step to the - * user-space IPC thread of the core, owning the pipeline. - */ -__cold static int ipc4_user_set_pipeline_state_multicore(struct ipc4_message_request *ipc4, - const uint32_t *ppl_id, - unsigned int ppl_count) -{ - static const uint32_t phases[] = { - IDC_PPL_STATE_PHASE_PREPARE, - IDC_PPL_STATE_PHASE_TRIGGER, - }; - struct ipc *ipc = ipc_get(); - struct ipc_user *pdata = ipc->ipc_user_pdata; - struct ipc4_pipeline_set_state state; - struct ipc_comp_dev *ppl_icd; - unsigned int i, j; - int ret; - - assert_can_be_cold(); - - for (j = 0; j < ARRAY_SIZE(phases); j++) { - for (i = 0; i < ppl_count; i++) { - ppl_icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, ppl_id[i], - IPC_COMP_ALL); - if (!ppl_icd) { - ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); - return IPC4_INVALID_RESOURCE_ID; - } - - /* Forward as a single-pipeline request */ - state.primary.dat = ipc4->primary.dat; - state.extension.dat = ipc4->extension.dat; - state.primary.r.ppl_id = ppl_id[i]; - state.extension.r.multi_ppl = 0; - - pdata->ppl_phase = phases[j]; - ret = ipc_user_forward_cmd(state.primary.dat, state.extension.dat, - ppl_icd->core); - pdata->ppl_phase = 0; - if (ret != 0) - return ret; - } - } - - return IPC4_SUCCESS; -} #endif /* CONFIG_SOF_USERSPACE_LL */ -__cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) -{ - assert_can_be_cold(); - -#if CONFIG_COMP_CHAIN_DMA - struct ipc_comp_dev *cdma_comp; - struct ipc *ipc = ipc_get(); - const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4; - int comp_id; - int ret; - - comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0); - cdma_comp = ipc_get_comp_by_id(ipc, comp_id); - - if (!cdma_comp) { - /* - * Nothing to do when the chainDMA is not allocated and asked to - * be freed - */ - if (!cdma->primary.r.allocate && !cdma->primary.r.enable) - return IPC4_SUCCESS; - - ret = ipc4_chain_manager_create(cdma); - if (ret < 0) - return IPC4_FAILURE; - - cdma_comp = ipc_get_comp_by_id(ipc, comp_id); - if (!cdma_comp) { - return IPC4_FAILURE; - } - - ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); - if (ret < 0) { - comp_free(cdma_comp->cd); - list_item_del(&cdma_comp->list); - rfree(cdma_comp); - return IPC4_FAILURE; - } - - return IPC4_SUCCESS; - } - - ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); - if (ret < 0) - return IPC4_INVALID_CHAIN_STATE_TRANSITION; - - return IPC4_SUCCESS; -#else /* CONFIG_COMP_CHAIN_DMA */ - return IPC4_UNAVAILABLE; -#endif /* CONFIG_COMP_CHAIN_DMA */ -} - -__cold static int ipc4_process_ipcgtw_cmd(struct ipc4_message_request *ipc4) -{ - assert_can_be_cold(); - -#if CONFIG_IPC4_GATEWAY - struct ipc *ipc = ipc_get(); - uint32_t reply_size = 0; - int err; - - err = copier_ipcgtw_process((const struct ipc4_ipcgtw_cmd *)ipc4, ipc->comp_data, - &reply_size); - /* reply size is returned in header extension dword */ - msg_reply->extension = reply_size; - - if (reply_size > 0) { - msg_reply->tx_data = ipc->comp_data; - msg_reply->tx_size = reply_size; - } - - return err < 0 ? IPC4_FAILURE : IPC4_SUCCESS; -#else /* CONFIG_IPC4_GATEWAY */ - ipc_cmd_err(&ipc_tr, "CONFIG_IPC4_GATEWAY is disabled"); - return IPC4_UNAVAILABLE; -#endif /* CONFIG_IPC4_GATEWAY */ -} - -static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) -{ -#if CONFIG_GDBSTUB - ipc_enter_gdb = true; - return IPC4_SUCCESS; -#else - return IPC4_UNAVAILABLE; -#endif -} - -int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, - struct ipc_msg *reply) -{ -#ifdef CONFIG_SOF_USERSPACE_LL - struct ipc *ipc = ipc_get(); -#endif - uint32_t type; - int ret; - - type = ipc4->primary.r.type; - msg_reply = reply; - - switch (type) { - case SOF_IPC4_GLB_BOOT_CONFIG: - case SOF_IPC4_GLB_ROM_CONTROL: - case SOF_IPC4_GLB_PERF_MEASUREMENTS_CMD: - case SOF_IPC4_GLB_LOAD_MULTIPLE_MODULES: - case SOF_IPC4_GLB_UNLOAD_MULTIPLE_MODULES: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - case SOF_IPC4_GLB_CHAIN_DMA: - ret = ipc4_process_chain_dma(ipc4); - break; - - /* pipeline settings */ - case SOF_IPC4_GLB_CREATE_PIPELINE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_pipeline_create *create = - (const struct ipc4_pipeline_create *)ipc4; - - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - create->extension.r.core_id); - } -#else - ret = ipc4_new_pipeline(ipc4); -#endif - break; - case SOF_IPC4_GLB_DELETE_PIPELINE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_pipeline_delete *del = (const struct ipc4_pipeline_delete *)ipc4; - struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, - del->primary.r.instance_id, - IPC_COMP_ALL); - - if (!ppl) { - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); - } -#else - ret = ipc4_delete_pipeline(ipc4); -#endif - break; - case SOF_IPC4_GLB_SET_PIPELINE_STATE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - struct ipc4_pipeline_set_state state = { - .primary.dat = ipc4->primary.dat, - .extension.dat = ipc4->extension.dat, - }; - const uint32_t *ppl_id; - unsigned int ppl_count; - int id = ipc4_pipeline_id_get(ipc4, &state, &ppl_id, &ppl_count); - if (id < 0) { - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - - struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, id, - IPC_COMP_ALL); - - if (!ppl) { - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - - /* - * Validate all pipelines before any state change and check, - * if they span multiple cores, needing per-core processing - */ - bool multicore = false; - - ret = IPC4_SUCCESS; - for (unsigned int i = 1; i < ppl_count; i++) { - struct ipc_comp_dev *icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, - ppl_id[i], IPC_COMP_ALL); - - if (!icd) { - ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - - if (icd->core != ppl->core) - multicore = true; - } - - if (ret != IPC4_SUCCESS) - break; - - if (multicore) - ret = ipc4_user_set_pipeline_state_multicore(ipc4, ppl_id, ppl_count); - else - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ppl->core); - } -#else /* CONFIG_SOF_USERSPACE_LL */ - ret = ipc4_set_pipeline_state(ipc4); -#endif /* CONFIG_SOF_USERSPACE_LL */ - break; - - case SOF_IPC4_GLB_GET_PIPELINE_STATE: - case SOF_IPC4_GLB_GET_PIPELINE_CONTEXT_SIZE: - case SOF_IPC4_GLB_SAVE_PIPELINE: - case SOF_IPC4_GLB_RESTORE_PIPELINE: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - case SOF_IPC4_GLB_INTERNAL_MESSAGE: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - /* Notification (FW to SW driver) */ - case SOF_IPC4_GLB_NOTIFICATION: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - case SOF_IPC4_GLB_IPCGATEWAY_CMD: - ret = ipc4_process_ipcgtw_cmd(ipc4); - break; - - case SOF_IPC4_GLB_ENTER_GDB: - ret = ipc_glb_gdb_debug(ipc4); - break; - - default: - ipc_cmd_err(&ipc_tr, "unsupported ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - } - - return ret; -} - -/* - * Ipc4 Module message <------> ipc3 module message - * init module <-------> create component - * bind modules <-------> connect components - * module set_large_config <-------> component cmd - * delete module <-------> free component - */ - -__cold static int ipc4_init_module_instance(struct ipc4_message_request *ipc4) -{ - const struct ipc4_module_init_instance *module_init = - (const struct ipc4_module_init_instance *)ipc4; - struct comp_dev *dev; - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)module_init->primary.r.module_id, - (uint32_t)module_init->primary.r.instance_id); - - /* Pass IPC to target core */ - if (!cpu_is_me(module_init->extension.r.core_id)) - return ipc4_process_on_core(module_init->extension.r.core_id, false); - - dev = comp_new_ipc4(module_init); - if (!dev) { - ipc_cmd_err(&ipc_tr, "error: failed to init module %x : %x", - (uint32_t)module_init->primary.r.module_id, - (uint32_t)module_init->primary.r.instance_id); - return IPC4_MOD_NOT_INITIALIZED; - } - - return 0; -} - -#ifndef CONFIG_SOF_USERSPACE_LL -__cold static int ipc4_bind_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x with %x : %x", - (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, - (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); - - return ipc4_comp_connect(ipc, bu); -} - -__cold static int ipc4_unbind_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x with %x : %x", - (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, - (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); - - return ipc4_comp_disconnect(ipc, bu); -} -#endif /* !CONFIG_SOF_USERSPACE_LL */ - /** * @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context. * @@ -1077,23 +671,8 @@ __cold int ipc4_process_module_config(struct ipc4_message_request *ipc4, return ret; } -#ifndef CONFIG_SOF_USERSPACE_LL -static int ipc4_set_get_config_module_instance(struct ipc4_message_request *ipc4, bool set) -{ - uint32_t reply_ext; - int ret; - - ret = ipc4_process_module_config(ipc4, set, &reply_ext); - - if (!set) - msg_reply->extension = reply_ext; - - return ret; -} -#endif /* !CONFIG_SOF_USERSPACE_LL */ - -__cold static void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t param_id, - char *data_out, uint32_t data_size) +__cold void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t param_id, + char *data_out, uint32_t data_size) { const char *hostbox; @@ -1122,13 +701,13 @@ __cold static void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t p } } -__cold static int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, - const struct comp_driver *drv, - bool init_block, - bool final_block, - uint32_t *data_off_size, - char *data_out, - const char *data_in) +__cold int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, + const struct comp_driver *drv, + bool init_block, + bool final_block, + uint32_t *data_off_size, + char *data_out, + const char *data_in) { const struct sof_tl * const input_tl = (struct sof_tl *)data_in; int ret; @@ -1324,119 +903,7 @@ __cold static int ipc4_process_large_config_get(struct ipc4_message_request *ipc } #endif -__cold static int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_large_config_reply reply; - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - char *data = ipc_get()->comp_data; - const struct comp_driver *drv; - struct comp_dev *dev = NULL; - uint32_t data_offset; - int ret; - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); - - /* get component dev for non-basefw since there is no - * component dev for basefw - */ - if (config->primary.r.module_id) { - uint32_t comp_id; - - comp_id = IPC4_COMP_ID(config->primary.r.module_id, - config->primary.r.instance_id); - dev = ipc4_get_comp_dev(comp_id); - if (!dev) - return IPC4_INVALID_RESOURCE_ID; - - drv = dev->drv; - - /* Pass IPC to target core */ - if (!cpu_is_me(dev->ipc_config.core)) - return ipc4_process_on_core(dev->ipc_config.core, false); - } else { - /* BaseFW module has only 0th instance */ - if (config->primary.r.instance_id) - return IPC4_INVALID_RESOURCE_ID; - - drv = ipc4_get_comp_drv(config->primary.r.module_id); - } - - if (!drv) - return IPC4_INVALID_RESOURCE_ID; - - if (!drv->ops.get_large_config) - return IPC4_INVALID_REQUEST; - - data_offset = config->extension.r.data_off_size; - - /* check for vendor param first */ - if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { - /* data_off_size is a 20-bit host-controlled field, so it can - * claim far more than the hostbox can physically hold. - */ - if (data_offset > MAILBOX_HOSTBOX_SIZE) { - ipc_cmd_err(&ipc_tr, "data_off_size %u exceeds mailbox bound", - data_offset); - return IPC4_INVALID_CONFIG_DATA_STRUCT; - } - /* For now only vendor_config case uses payload from hostbox */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - config->extension.r.data_off_size); - ret = ipc4_get_vendor_config_module_instance(dev, drv, - config->extension.r.init_block, - config->extension.r.final_block, - &data_offset, data, - (const char *)MAILBOX_HOSTBOX_BASE); - } else { -#if CONFIG_LIBRARY - data += sizeof(reply); -#endif - ipc4_prepare_for_kcontrol_get(dev, config->extension.r.large_param_id, - data, data_offset); - - ret = drv->ops.get_large_config(dev, config->extension.r.large_param_id, - config->extension.r.init_block, - config->extension.r.final_block, - &data_offset, data); - } - - /* set up ipc4 error code for reply data */ - if (ret < 0) - ret = IPC4_INVALID_RESOURCE_ID; - - /* Copy host config and overwrite */ - reply.extension.dat = config->extension.dat; - reply.extension.r.data_off_size = data_offset; - - /* The last block, no more data */ - if (!config->extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) - reply.extension.r.final_block = 1; - - /* Indicate last block if error occurs */ - if (ret) - reply.extension.r.final_block = 1; - - /* no need to allocate memory for reply msg */ - if (ret) - return ret; - - /* A module must not report more reply data than the DSP mailbox can hold. */ - if (data_offset > MAILBOX_DSPBOX_SIZE) { - ipc_cmd_err(&ipc_tr, "reply size %u exceeds mailbox bound", data_offset); - return IPC4_INVALID_CONFIG_DATA_STRUCT; - } - - msg_reply->extension = reply.extension.dat; - msg_reply->tx_size = data_offset; - msg_reply->tx_data = data; - return ret; -} - -__cold static int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, +__cold int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, const struct comp_driver *drv, uint32_t module_id, uint32_t instance_id, @@ -1586,283 +1053,6 @@ __cold int ipc4_process_large_config_set(struct ipc4_message_request *ipc4) return ret; } -__cold static int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4) -{ - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - struct comp_dev *dev = NULL; - const struct comp_driver *drv; - int ret; - - assert_can_be_cold(); - - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - config->extension.r.data_off_size); - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); - - if (config->primary.r.module_id) { - uint32_t comp_id; - - comp_id = IPC4_COMP_ID(config->primary.r.module_id, config->primary.r.instance_id); - dev = ipc4_get_comp_dev(comp_id); - if (!dev) - return IPC4_INVALID_RESOURCE_ID; - - drv = dev->drv; - - /* Pass IPC to target core */ - if (!cpu_is_me(dev->ipc_config.core)) - return ipc4_process_on_core(dev->ipc_config.core, false); - } else { - /* BaseFW module has only 0th instance */ - if (config->primary.r.instance_id) - return IPC4_INVALID_RESOURCE_ID; - - drv = ipc4_get_comp_drv(config->primary.r.module_id); - } - - if (!drv) - return IPC4_INVALID_RESOURCE_ID; - - if (!drv->ops.set_large_config) - return IPC4_INVALID_REQUEST; - - /* check for vendor param first */ - if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { - ret = ipc4_set_vendor_config_module_instance(dev, drv, - (uint32_t)config->primary.r.module_id, - (uint32_t)config->primary.r.instance_id, - config->extension.r.init_block, - config->extension.r.final_block, - config->extension.r.data_off_size, - (const char *)MAILBOX_HOSTBOX_BASE); - } else { -#if CONFIG_LIBRARY - struct ipc *ipc = ipc_get(); - const char *data = (const char *)ipc->comp_data + sizeof(*config); -#else - const char *data = (const char *)MAILBOX_HOSTBOX_BASE; -#endif - ret = drv->ops.set_large_config(dev, config->extension.r.large_param_id, - config->extension.r.init_block, config->extension.r.final_block, - config->extension.r.data_off_size, data); - if (ret < 0) { - ipc_cmd_err(&ipc_tr, "failed to set large_config_module_instance %x : %x", - (uint32_t)config->primary.r.module_id, - (uint32_t)config->primary.r.instance_id); - ret = IPC4_INVALID_RESOURCE_ID; - } - } - - return ret; -} - -#ifndef CONFIG_SOF_USERSPACE_LL -__cold static int ipc4_delete_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_delete_instance module; - struct ipc *ipc = ipc_get(); - uint32_t comp_id; - - assert_can_be_cold(); - - int ret = memcpy_s(&module, sizeof(module), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - - tr_dbg(&ipc_tr, "%x : %x", (uint32_t)module.primary.r.module_id, - (uint32_t)module.primary.r.instance_id); - - comp_id = IPC4_COMP_ID(module.primary.r.module_id, module.primary.r.instance_id); - ret = ipc_comp_free(ipc, comp_id); - if (ret < 0) { - ipc_cmd_err(&ipc_tr, "failed to delete module instance %x : %x", - (uint32_t)module.primary.r.module_id, - (uint32_t)module.primary.r.instance_id); - ret = IPC4_INVALID_RESOURCE_ID; - } - - return ret; -} -#endif /* !CONFIG_SOF_USERSPACE_LL */ - -__cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, - struct ipc_msg *reply) -{ - const struct ipc4_module_init_instance *mi; - uint32_t type; - int ret; - - assert_can_be_cold(); - - type = ipc4->primary.r.type; - msg_reply = reply; - - switch (type) { - case SOF_IPC4_MOD_INIT_INSTANCE: - mi = (const struct ipc4_module_init_instance *)ipc4; - - if (cpu_is_me(mi->extension.r.core_id) && !mi->extension.r.proc_domain && - IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { -#ifdef CONFIG_SOF_USERSPACE_LL - BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= - sizeof(((struct ipc_user *)0)->init_drv_data), - "ipc_user.init_drv_data too small for driver copy"); - - /* User-space init: kernel does driver lookup only (requires - * access to IMR manifest and driver list in kernel memory). - * Component creation (drv->ops.create) runs in user thread - * so untrusted module code does not execute in kernel context. - */ - uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, - mi->primary.r.instance_id); - const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); - - if (!drv || !drv->ops.create) { - ret = IPC4_MOD_NOT_INITIALIZED; - break; - } - - struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); - - if (ctx && drv->type == SOF_COMP_MODULE_ADAPTER) { - int err = ipc4_user_module_load(drv, mi); - - if (err < 0) { - ret = IPC4_MOD_NOT_INITIALIZED; - break; - } - } - - ipc_user_get()->init_drv = drv; - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - mi->extension.r.core_id); -#endif /* CONFIG_SOF_USERSPACE_LL */ - } else { - /* - * DP module creation starts running in kernel mode and - * switches to userspace later via a scheduler_dp_thread_ipc() - * call in module_init(). - */ - ret = ipc4_init_module_instance(ipc4); - } - break; - case SOF_IPC4_MOD_CONFIG_GET: -#ifdef CONFIG_SOF_USERSPACE_LL - /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); - if (!ret) { - struct ipc_user *pdata = ipc_user_get(); - - msg_reply->extension = pdata->reply_ext; - } -#else - ret = ipc4_set_get_config_module_instance(ipc4, false); -#endif - break; - case SOF_IPC4_MOD_CONFIG_SET: -#ifdef CONFIG_SOF_USERSPACE_LL - /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_set_get_config_module_instance(ipc4, true); -#endif - break; - case SOF_IPC4_MOD_LARGE_CONFIG_GET: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - - if (config->primary.r.module_id) { - /* Module case: forward to user thread */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); - if (!ret) { - struct ipc_user *pdata = ipc_user_get(); - - msg_reply->extension = pdata->reply_ext; - msg_reply->tx_size = pdata->reply_tx_size; - msg_reply->tx_data = pdata->reply_tx_data; - } - } else { - /* Base firmware (module_id==0): keep in kernel — - * ipc4_get_comp_drv() accesses IMR manifest which - * has no user-space partition. - */ - ret = ipc4_get_large_config_module_instance(ipc4); - } - } -#else /* CONFIG_SOF_USERSPACE_LL */ - ret = ipc4_get_large_config_module_instance(ipc4); -#endif /* CONFIG_SOF_USERSPACE_LL */ - break; - case SOF_IPC4_MOD_LARGE_CONFIG_SET: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - - if (config->primary.r.module_id) { - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); - } else { - /* Base firmware: keep in kernel (IMR access) */ - ret = ipc4_set_large_config_module_instance(ipc4); - } - } -#else - ret = ipc4_set_large_config_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_BIND: -#ifdef CONFIG_SOF_USERSPACE_LL - /* - * bind and unbind can connect LL with DP. In that case it isn't - * immediately clear whether the handler should run in the DP - * thread context or in the LL IPC thread context. The LL IPC - * thread has access to DP modules, so we have to perform - * binding in the LL IPC thread context - */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_bind_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_UNBIND: -#ifdef CONFIG_SOF_USERSPACE_LL - /* DP / LL: see comment above */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_unbind_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_DELETE_INSTANCE: -#ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_delete_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_ENTER_MODULE_RESTORE: - case SOF_IPC4_MOD_EXIT_MODULE_RESTORE: - ret = IPC4_UNAVAILABLE; - break; - default: - ret = IPC4_UNKNOWN_MESSAGE_TYPE; - break; - } - - return ret; -} - #ifdef CONFIG_SOF_USERSPACE_LL /* * IPC4 implementation of the user-thread dispatch hook. Runs in the diff --git a/src/kernel/ipc/ipc4/CMakeLists.txt b/src/kernel/ipc/ipc4/CMakeLists.txt index 53a37d553675..94c63418370c 100644 --- a/src/kernel/ipc/ipc4/CMakeLists.txt +++ b/src/kernel/ipc/ipc4/CMakeLists.txt @@ -3,6 +3,7 @@ # Files common to Zephyr and the host library add_local_sources(sof handler-kernel.c + handler-user-kernel.c logging.c notification.c ) diff --git a/src/kernel/ipc/ipc4/handler-user-kernel.c b/src/kernel/ipc/ipc4/handler-user-kernel.c new file mode 100644 index 000000000000..ebf9f406dfe2 --- /dev/null +++ b/src/kernel/ipc/ipc4/handler-user-kernel.c @@ -0,0 +1,877 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang +/* + * IPC (InterProcessor Communication) provides a method of two way + * communication between the host processor and the DSP. The IPC used here + * utilises a shared mailbox and door bell between the host and DSP. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include "../audio/copier/ipcgtw_copier.h" + +/* Command format errors during fuzzing are reported for virtually all + * commands, and the resulting flood of logging becomes a severe + * performance penalty (i.e. we get a lot less fuzzing done per CPU + * cycle). + */ +#ifdef CONFIG_ARCH_POSIX_LIBFUZZER +#define ipc_cmd_err(...) +#else +#define ipc_cmd_err(...) tr_err(__VA_ARGS__) +#endif + +/* Kernel part of the IPC4 module and global message handling: the routers, + * which run in the kernel IPC thread, and the handlers that only they call. + * The part that also runs in user mode is in src/ipc/ipc4/handler-user.c + */ + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +/* Userspace message context, copied in/out by kernel IPC thread. */ +/* fw sends a fw ipc message to send the status of the last host ipc message */ +static struct ipc_msg *msg_reply; + +#ifdef CONFIG_SOF_USERSPACE_LL +/* + * Determine the target core for an IPC4 module message. + * Falls back to current core when no component is bound yet. + */ +static unsigned int ipc4_user_target_core_module(struct ipc4_message_request *ipc4) +{ + /* + * Also works for struct ipc4_module_large_config, struct ipc4_module_bind_unbind, + * struct ipc4_module_delete_instance + */ + struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; + uint32_t module_id = config->primary.r.module_id; + + if (module_id) { + uint32_t instance_id = config->primary.r.instance_id; + struct comp_dev *dev = ipc4_get_comp_dev(IPC4_COMP_ID(module_id, instance_id)); + + if (dev) + return dev->ipc_config.core; + } + + return cpu_get_id(); +} + +/* + * Handle a multi-pipeline SET_PIPELINE_STATE, where the pipelines are on + * different cores. The user-space IPC thread can't use IDC to reach the + * other cores, so the kernel runs the prepare phase on all pipelines and + * then the trigger phase on all pipelines, forwarding each step to the + * user-space IPC thread of the core, owning the pipeline. + */ +__cold static int ipc4_user_set_pipeline_state_multicore(struct ipc4_message_request *ipc4, + const uint32_t *ppl_id, + unsigned int ppl_count) +{ + static const uint32_t phases[] = { + IDC_PPL_STATE_PHASE_PREPARE, + IDC_PPL_STATE_PHASE_TRIGGER, + }; + struct ipc *ipc = ipc_get(); + struct ipc_user *pdata = ipc->ipc_user_pdata; + struct ipc4_pipeline_set_state state; + struct ipc_comp_dev *ppl_icd; + unsigned int i, j; + int ret; + + assert_can_be_cold(); + + for (j = 0; j < ARRAY_SIZE(phases); j++) { + for (i = 0; i < ppl_count; i++) { + ppl_icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, ppl_id[i], + IPC_COMP_ALL); + if (!ppl_icd) { + ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); + return IPC4_INVALID_RESOURCE_ID; + } + + /* Forward as a single-pipeline request */ + state.primary.dat = ipc4->primary.dat; + state.extension.dat = ipc4->extension.dat; + state.primary.r.ppl_id = ppl_id[i]; + state.extension.r.multi_ppl = 0; + + pdata->ppl_phase = phases[j]; + ret = ipc_user_forward_cmd(state.primary.dat, state.extension.dat, + ppl_icd->core); + pdata->ppl_phase = 0; + if (ret != 0) + return ret; + } + } + + return IPC4_SUCCESS; +} +#else /* CONFIG_SOF_USERSPACE_LL */ +__cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) +{ + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); +} + +__cold static int ipc4_delete_pipeline(struct ipc4_message_request *ipc4) +{ + struct ipc4_pipeline_delete *pipe; + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + pipe = (struct ipc4_pipeline_delete *)ipc4; + tr_dbg(&ipc_tr, "ipc4 delete pipeline %x:", (uint32_t)pipe->primary.r.instance_id); + + return ipc_pipeline_free(ipc, pipe->primary.r.instance_id); +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +__cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) +{ + assert_can_be_cold(); + +#if CONFIG_COMP_CHAIN_DMA + struct ipc_comp_dev *cdma_comp; + struct ipc *ipc = ipc_get(); + const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4; + int comp_id; + int ret; + + comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0); + cdma_comp = ipc_get_comp_by_id(ipc, comp_id); + + if (!cdma_comp) { + /* + * Nothing to do when the chainDMA is not allocated and asked to + * be freed + */ + if (!cdma->primary.r.allocate && !cdma->primary.r.enable) + return IPC4_SUCCESS; + + ret = ipc4_chain_manager_create(cdma); + if (ret < 0) + return IPC4_FAILURE; + + cdma_comp = ipc_get_comp_by_id(ipc, comp_id); + if (!cdma_comp) { + return IPC4_FAILURE; + } + + ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); + if (ret < 0) { + comp_free(cdma_comp->cd); + list_item_del(&cdma_comp->list); + rfree(cdma_comp); + return IPC4_FAILURE; + } + + return IPC4_SUCCESS; + } + + ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); + if (ret < 0) + return IPC4_INVALID_CHAIN_STATE_TRANSITION; + + return IPC4_SUCCESS; +#else /* CONFIG_COMP_CHAIN_DMA */ + return IPC4_UNAVAILABLE; +#endif /* CONFIG_COMP_CHAIN_DMA */ +} + +__cold static int ipc4_process_ipcgtw_cmd(struct ipc4_message_request *ipc4) +{ + assert_can_be_cold(); + +#if CONFIG_IPC4_GATEWAY + struct ipc *ipc = ipc_get(); + uint32_t reply_size = 0; + int err; + + err = copier_ipcgtw_process((const struct ipc4_ipcgtw_cmd *)ipc4, ipc->comp_data, + &reply_size); + /* reply size is returned in header extension dword */ + msg_reply->extension = reply_size; + + if (reply_size > 0) { + msg_reply->tx_data = ipc->comp_data; + msg_reply->tx_size = reply_size; + } + + return err < 0 ? IPC4_FAILURE : IPC4_SUCCESS; +#else /* CONFIG_IPC4_GATEWAY */ + ipc_cmd_err(&ipc_tr, "CONFIG_IPC4_GATEWAY is disabled"); + return IPC4_UNAVAILABLE; +#endif /* CONFIG_IPC4_GATEWAY */ +} + +static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) +{ +#if CONFIG_GDBSTUB + ipc_enter_gdb = true; + return IPC4_SUCCESS; +#else + return IPC4_UNAVAILABLE; +#endif +} + +int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ +#ifdef CONFIG_SOF_USERSPACE_LL + struct ipc *ipc = ipc_get(); +#endif + uint32_t type; + int ret; + + type = ipc4->primary.r.type; + msg_reply = reply; + + switch (type) { + case SOF_IPC4_GLB_BOOT_CONFIG: + case SOF_IPC4_GLB_ROM_CONTROL: + case SOF_IPC4_GLB_PERF_MEASUREMENTS_CMD: + case SOF_IPC4_GLB_LOAD_MULTIPLE_MODULES: + case SOF_IPC4_GLB_UNLOAD_MULTIPLE_MODULES: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + case SOF_IPC4_GLB_CHAIN_DMA: + ret = ipc4_process_chain_dma(ipc4); + break; + + /* pipeline settings */ + case SOF_IPC4_GLB_CREATE_PIPELINE: +#ifdef CONFIG_SOF_USERSPACE_LL + { + const struct ipc4_pipeline_create *create = + (const struct ipc4_pipeline_create *)ipc4; + + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + create->extension.r.core_id); + } +#else + ret = ipc4_new_pipeline(ipc4); +#endif + break; + case SOF_IPC4_GLB_DELETE_PIPELINE: +#ifdef CONFIG_SOF_USERSPACE_LL + { + const struct ipc4_pipeline_delete *del = (const struct ipc4_pipeline_delete *)ipc4; + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + del->primary.r.instance_id, + IPC_COMP_ALL); + + if (!ppl) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); + } +#else + ret = ipc4_delete_pipeline(ipc4); +#endif + break; + case SOF_IPC4_GLB_SET_PIPELINE_STATE: +#ifdef CONFIG_SOF_USERSPACE_LL + { + struct ipc4_pipeline_set_state state = { + .primary.dat = ipc4->primary.dat, + .extension.dat = ipc4->extension.dat, + }; + const uint32_t *ppl_id; + unsigned int ppl_count; + int id = ipc4_pipeline_id_get(ipc4, &state, &ppl_id, &ppl_count); + if (id < 0) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, id, + IPC_COMP_ALL); + + if (!ppl) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + + /* + * Validate all pipelines before any state change and check, + * if they span multiple cores, needing per-core processing + */ + bool multicore = false; + + ret = IPC4_SUCCESS; + for (unsigned int i = 1; i < ppl_count; i++) { + struct ipc_comp_dev *icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + ppl_id[i], IPC_COMP_ALL); + + if (!icd) { + ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + + if (icd->core != ppl->core) + multicore = true; + } + + if (ret != IPC4_SUCCESS) + break; + + if (multicore) + ret = ipc4_user_set_pipeline_state_multicore(ipc4, ppl_id, ppl_count); + else + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ppl->core); + } +#else /* CONFIG_SOF_USERSPACE_LL */ + ret = ipc4_set_pipeline_state(ipc4); +#endif /* CONFIG_SOF_USERSPACE_LL */ + break; + + case SOF_IPC4_GLB_GET_PIPELINE_STATE: + case SOF_IPC4_GLB_GET_PIPELINE_CONTEXT_SIZE: + case SOF_IPC4_GLB_SAVE_PIPELINE: + case SOF_IPC4_GLB_RESTORE_PIPELINE: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + case SOF_IPC4_GLB_INTERNAL_MESSAGE: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + /* Notification (FW to SW driver) */ + case SOF_IPC4_GLB_NOTIFICATION: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + case SOF_IPC4_GLB_IPCGATEWAY_CMD: + ret = ipc4_process_ipcgtw_cmd(ipc4); + break; + + case SOF_IPC4_GLB_ENTER_GDB: + ret = ipc_glb_gdb_debug(ipc4); + break; + + default: + ipc_cmd_err(&ipc_tr, "unsupported ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + } + + return ret; +} + +/* + * Ipc4 Module message <------> ipc3 module message + * init module <-------> create component + * bind modules <-------> connect components + * module set_large_config <-------> component cmd + * delete module <-------> free component + */ + +__cold static int ipc4_init_module_instance(struct ipc4_message_request *ipc4) +{ + const struct ipc4_module_init_instance *module_init = + (const struct ipc4_module_init_instance *)ipc4; + struct comp_dev *dev; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)module_init->primary.r.module_id, + (uint32_t)module_init->primary.r.instance_id); + + /* Pass IPC to target core */ + if (!cpu_is_me(module_init->extension.r.core_id)) + return ipc4_process_on_core(module_init->extension.r.core_id, false); + + dev = comp_new_ipc4(module_init); + if (!dev) { + ipc_cmd_err(&ipc_tr, "error: failed to init module %x : %x", + (uint32_t)module_init->primary.r.module_id, + (uint32_t)module_init->primary.r.instance_id); + return IPC4_MOD_NOT_INITIALIZED; + } + + return 0; +} + +#ifndef CONFIG_SOF_USERSPACE_LL +__cold static int ipc4_bind_module_instance(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x with %x : %x", + (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, + (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); + + return ipc4_comp_connect(ipc, bu); +} + +__cold static int ipc4_unbind_module_instance(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x with %x : %x", + (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, + (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); + + return ipc4_comp_disconnect(ipc, bu); +} +#endif /* !CONFIG_SOF_USERSPACE_LL */ + +#ifndef CONFIG_SOF_USERSPACE_LL +static int ipc4_set_get_config_module_instance(struct ipc4_message_request *ipc4, bool set) +{ + uint32_t reply_ext; + int ret; + + ret = ipc4_process_module_config(ipc4, set, &reply_ext); + + if (!set) + msg_reply->extension = reply_ext; + + return ret; +} +#endif /* !CONFIG_SOF_USERSPACE_LL */ + +__cold static int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_large_config_reply reply; + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + char *data = ipc_get()->comp_data; + const struct comp_driver *drv; + struct comp_dev *dev = NULL; + uint32_t data_offset; + int ret; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); + + /* get component dev for non-basefw since there is no + * component dev for basefw + */ + if (config->primary.r.module_id) { + uint32_t comp_id; + + comp_id = IPC4_COMP_ID(config->primary.r.module_id, + config->primary.r.instance_id); + dev = ipc4_get_comp_dev(comp_id); + if (!dev) + return IPC4_INVALID_RESOURCE_ID; + + drv = dev->drv; + + /* Pass IPC to target core */ + if (!cpu_is_me(dev->ipc_config.core)) + return ipc4_process_on_core(dev->ipc_config.core, false); + } else { + /* BaseFW module has only 0th instance */ + if (config->primary.r.instance_id) + return IPC4_INVALID_RESOURCE_ID; + + drv = ipc4_get_comp_drv(config->primary.r.module_id); + } + + if (!drv) + return IPC4_INVALID_RESOURCE_ID; + + if (!drv->ops.get_large_config) + return IPC4_INVALID_REQUEST; + + data_offset = config->extension.r.data_off_size; + + /* check for vendor param first */ + if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + /* data_off_size is a 20-bit host-controlled field, so it can + * claim far more than the hostbox can physically hold. + */ + if (data_offset > MAILBOX_HOSTBOX_SIZE) { + ipc_cmd_err(&ipc_tr, "data_off_size %u exceeds mailbox bound", + data_offset); + return IPC4_INVALID_CONFIG_DATA_STRUCT; + } + /* For now only vendor_config case uses payload from hostbox */ + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + config->extension.r.data_off_size); + ret = ipc4_get_vendor_config_module_instance(dev, drv, + config->extension.r.init_block, + config->extension.r.final_block, + &data_offset, data, + (const char *)MAILBOX_HOSTBOX_BASE); + } else { +#if CONFIG_LIBRARY + data += sizeof(reply); +#endif + ipc4_prepare_for_kcontrol_get(dev, config->extension.r.large_param_id, + data, data_offset); + + ret = drv->ops.get_large_config(dev, config->extension.r.large_param_id, + config->extension.r.init_block, + config->extension.r.final_block, + &data_offset, data); + } + + /* set up ipc4 error code for reply data */ + if (ret < 0) + ret = IPC4_INVALID_RESOURCE_ID; + + /* Copy host config and overwrite */ + reply.extension.dat = config->extension.dat; + reply.extension.r.data_off_size = data_offset; + + /* The last block, no more data */ + if (!config->extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) + reply.extension.r.final_block = 1; + + /* Indicate last block if error occurs */ + if (ret) + reply.extension.r.final_block = 1; + + /* no need to allocate memory for reply msg */ + if (ret) + return ret; + + /* A module must not report more reply data than the DSP mailbox can hold. */ + if (data_offset > MAILBOX_DSPBOX_SIZE) { + ipc_cmd_err(&ipc_tr, "reply size %u exceeds mailbox bound", data_offset); + return IPC4_INVALID_CONFIG_DATA_STRUCT; + } + + msg_reply->extension = reply.extension.dat; + msg_reply->tx_size = data_offset; + msg_reply->tx_data = data; + return ret; +} + +__cold static int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4) +{ + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + struct comp_dev *dev = NULL; + const struct comp_driver *drv; + int ret; + + assert_can_be_cold(); + + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + config->extension.r.data_off_size); + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); + + if (config->primary.r.module_id) { + uint32_t comp_id; + + comp_id = IPC4_COMP_ID(config->primary.r.module_id, config->primary.r.instance_id); + dev = ipc4_get_comp_dev(comp_id); + if (!dev) + return IPC4_INVALID_RESOURCE_ID; + + drv = dev->drv; + + /* Pass IPC to target core */ + if (!cpu_is_me(dev->ipc_config.core)) + return ipc4_process_on_core(dev->ipc_config.core, false); + } else { + /* BaseFW module has only 0th instance */ + if (config->primary.r.instance_id) + return IPC4_INVALID_RESOURCE_ID; + + drv = ipc4_get_comp_drv(config->primary.r.module_id); + } + + if (!drv) + return IPC4_INVALID_RESOURCE_ID; + + if (!drv->ops.set_large_config) + return IPC4_INVALID_REQUEST; + + /* check for vendor param first */ + if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + ret = ipc4_set_vendor_config_module_instance(dev, drv, + (uint32_t)config->primary.r.module_id, + (uint32_t)config->primary.r.instance_id, + config->extension.r.init_block, + config->extension.r.final_block, + config->extension.r.data_off_size, + (const char *)MAILBOX_HOSTBOX_BASE); + } else { +#if CONFIG_LIBRARY + struct ipc *ipc = ipc_get(); + const char *data = (const char *)ipc->comp_data + sizeof(*config); +#else + const char *data = (const char *)MAILBOX_HOSTBOX_BASE; +#endif + ret = drv->ops.set_large_config(dev, config->extension.r.large_param_id, + config->extension.r.init_block, config->extension.r.final_block, + config->extension.r.data_off_size, data); + if (ret < 0) { + ipc_cmd_err(&ipc_tr, "failed to set large_config_module_instance %x : %x", + (uint32_t)config->primary.r.module_id, + (uint32_t)config->primary.r.instance_id); + ret = IPC4_INVALID_RESOURCE_ID; + } + } + + return ret; +} + +#ifndef CONFIG_SOF_USERSPACE_LL +__cold static int ipc4_delete_module_instance(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_delete_instance module; + struct ipc *ipc = ipc_get(); + uint32_t comp_id; + + assert_can_be_cold(); + + int ret = memcpy_s(&module, sizeof(module), ipc4, sizeof(*ipc4)); + + if (ret < 0) + return IPC4_FAILURE; + + tr_dbg(&ipc_tr, "%x : %x", (uint32_t)module.primary.r.module_id, + (uint32_t)module.primary.r.instance_id); + + comp_id = IPC4_COMP_ID(module.primary.r.module_id, module.primary.r.instance_id); + ret = ipc_comp_free(ipc, comp_id); + if (ret < 0) { + ipc_cmd_err(&ipc_tr, "failed to delete module instance %x : %x", + (uint32_t)module.primary.r.module_id, + (uint32_t)module.primary.r.instance_id); + ret = IPC4_INVALID_RESOURCE_ID; + } + + return ret; +} +#endif /* !CONFIG_SOF_USERSPACE_LL */ + +__cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + const struct ipc4_module_init_instance *mi; + uint32_t type; + int ret; + + assert_can_be_cold(); + + type = ipc4->primary.r.type; + msg_reply = reply; + + switch (type) { + case SOF_IPC4_MOD_INIT_INSTANCE: + mi = (const struct ipc4_module_init_instance *)ipc4; + + if (cpu_is_me(mi->extension.r.core_id) && !mi->extension.r.proc_domain && + IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { +#ifdef CONFIG_SOF_USERSPACE_LL + BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= + sizeof(((struct ipc_user *)0)->init_drv_data), + "ipc_user.init_drv_data too small for driver copy"); + + /* User-space init: kernel does driver lookup only (requires + * access to IMR manifest and driver list in kernel memory). + * Component creation (drv->ops.create) runs in user thread + * so untrusted module code does not execute in kernel context. + */ + uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, + mi->primary.r.instance_id); + const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); + + if (!drv || !drv->ops.create) { + ret = IPC4_MOD_NOT_INITIALIZED; + break; + } + + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); + + if (ctx && drv->type == SOF_COMP_MODULE_ADAPTER) { + int err = ipc4_user_module_load(drv, mi); + + if (err < 0) { + ret = IPC4_MOD_NOT_INITIALIZED; + break; + } + } + + ipc_user_get()->init_drv = drv; + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + mi->extension.r.core_id); +#endif /* CONFIG_SOF_USERSPACE_LL */ + } else { + /* + * DP module creation starts running in kernel mode and + * switches to userspace later via a scheduler_dp_thread_ipc() + * call in module_init(). + */ + ret = ipc4_init_module_instance(ipc4); + } + break; + case SOF_IPC4_MOD_CONFIG_GET: +#ifdef CONFIG_SOF_USERSPACE_LL + /* Forward to user thread for privilege-separated execution */ + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); + if (!ret) { + struct ipc_user *pdata = ipc_user_get(); + + msg_reply->extension = pdata->reply_ext; + } +#else + ret = ipc4_set_get_config_module_instance(ipc4, false); +#endif + break; + case SOF_IPC4_MOD_CONFIG_SET: +#ifdef CONFIG_SOF_USERSPACE_LL + /* Forward to user thread for privilege-separated execution */ + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +#else + ret = ipc4_set_get_config_module_instance(ipc4, true); +#endif + break; + case SOF_IPC4_MOD_LARGE_CONFIG_GET: +#ifdef CONFIG_SOF_USERSPACE_LL + { + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + + if (config->primary.r.module_id) { + /* Module case: forward to user thread */ + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); + if (!ret) { + struct ipc_user *pdata = ipc_user_get(); + + msg_reply->extension = pdata->reply_ext; + msg_reply->tx_size = pdata->reply_tx_size; + msg_reply->tx_data = pdata->reply_tx_data; + } + } else { + /* Base firmware (module_id==0): keep in kernel — + * ipc4_get_comp_drv() accesses IMR manifest which + * has no user-space partition. + */ + ret = ipc4_get_large_config_module_instance(ipc4); + } + } +#else /* CONFIG_SOF_USERSPACE_LL */ + ret = ipc4_get_large_config_module_instance(ipc4); +#endif /* CONFIG_SOF_USERSPACE_LL */ + break; + case SOF_IPC4_MOD_LARGE_CONFIG_SET: +#ifdef CONFIG_SOF_USERSPACE_LL + { + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + + if (config->primary.r.module_id) { + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); + } else { + /* Base firmware: keep in kernel (IMR access) */ + ret = ipc4_set_large_config_module_instance(ipc4); + } + } +#else + ret = ipc4_set_large_config_module_instance(ipc4); +#endif + break; + case SOF_IPC4_MOD_BIND: +#ifdef CONFIG_SOF_USERSPACE_LL + /* + * bind and unbind can connect LL with DP. In that case it isn't + * immediately clear whether the handler should run in the DP + * thread context or in the LL IPC thread context. The LL IPC + * thread has access to DP modules, so we have to perform + * binding in the LL IPC thread context + */ + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +#else + ret = ipc4_bind_module_instance(ipc4); +#endif + break; + case SOF_IPC4_MOD_UNBIND: +#ifdef CONFIG_SOF_USERSPACE_LL + /* DP / LL: see comment above */ + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +#else + ret = ipc4_unbind_module_instance(ipc4); +#endif + break; + case SOF_IPC4_MOD_DELETE_INSTANCE: +#ifdef CONFIG_SOF_USERSPACE_LL + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +#else + ret = ipc4_delete_module_instance(ipc4); +#endif + break; + case SOF_IPC4_MOD_ENTER_MODULE_RESTORE: + case SOF_IPC4_MOD_EXIT_MODULE_RESTORE: + ret = IPC4_UNAVAILABLE; + break; + default: + ret = IPC4_UNKNOWN_MESSAGE_TYPE; + break; + } + + return ret; +} From ed5de8cb6db11225f2379d2138995ca85c39c5f5 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sat, 7 Nov 2026 22:25:07 +0000 Subject: [PATCH 10/19] ipc4: split the kernel part of helper.c into src/kernel/ipc/ipc4 helper.c mixes the IPC4 topology helpers, which also run in the user-space IPC thread, with code that needs kernel mode. Move the kernel part to src/kernel/ipc/ipc4/helper-kernel.c: - the component driver lookup, which reads the IMR manifest, registers loadable modules with the library manager and walks the driver list; - comp_new_ipc4(), the module creation of the kernel IPC thread (DP modules and builds without user-space LL), and the privileged part of the user-space module creation, ipc4_user_module_load(); - the IDC helpers, ipc4_process_on_core() and ipc_comp_free_remote(); - the chain DMA manager, which only the kernel CHAIN_DMA command uses. helper.c keeps the pipeline and component management, binding and the DMA config parsing. The module config parsing and the module init payload lookup, which both parts use, are no longer static and are declared in sof/ipc/topology.h, as is ipc_comp_free_remote(). helper.c no longer includes the rimage manifest, platform memory map and library manager headers. The code is unchanged, but the functions that now call across the split can no longer inline their callees: ipc_pipeline_new(), ipc_pipeline_free(), ipc4_comp_connect() and ipc4_comp_disconnect() call ipc4_process_on_core() and ipc_comp_free_remote(), and comp_new_ipc4(), ipc4_user_module_load() and ipc4_chain_manager_create() call the module config and component list helpers, instead of inlining them. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/ipc/topology.h | 5 + src/ipc/ipc4/helper.c | 410 +------------------------ src/kernel/ipc/ipc4/CMakeLists.txt | 1 + src/kernel/ipc/ipc4/helper-kernel.c | 459 ++++++++++++++++++++++++++++ 4 files changed, 471 insertions(+), 404 deletions(-) create mode 100644 src/kernel/ipc/ipc4/helper-kernel.c diff --git a/src/include/sof/ipc/topology.h b/src/include/sof/ipc/topology.h index d072abc16a24..1711a0fe3a1c 100644 --- a/src/include/sof/ipc/topology.h +++ b/src/include/sof/ipc/topology.h @@ -48,6 +48,11 @@ struct ipc_comp_dev; const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id); struct comp_dev *ipc4_get_comp_dev(uint32_t comp_id); int ipc4_add_comp_dev(struct comp_dev *dev); +struct comp_ipc_config; +int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, + const struct ipc4_module_init_instance *module_init); +unsigned char *ipc4_get_comp_new_data(void); +int ipc_comp_free_remote(struct comp_dev *dev); #ifdef CONFIG_SOF_USERSPACE_LL struct ipc4_message_request; struct comp_driver; diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index 18690cb9f28c..2996fcbf7fcb 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -29,26 +29,14 @@ #include #include -/* TODO: Remove platform-specific code, see https://github.com/thesofproject/sof/issues/7549 */ -#if defined(CONFIG_SOC_SERIES_INTEL_ADSP_ACE) || defined(CONFIG_INTEL_ADSP_CAVS) -#define RIMAGE_MANIFEST 1 -#endif - -#ifdef RIMAGE_MANIFEST -#include -#endif - #include #include -#include -#include #include #include #include #include #include #include -#include #include #include @@ -59,11 +47,9 @@ #include #include -LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); - -extern struct tr_ctx comp_tr; +/* The IPC4 helpers that run in kernel mode are in src/kernel/ipc/ipc4/helper-kernel.c */ -static const struct comp_driver *ipc4_get_drv(const void *uuid); +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); void ipc_build_stream_posn(struct sof_ipc_stream_posn *posn, uint32_t type, uint32_t id) @@ -77,7 +63,7 @@ void ipc_build_comp_event(struct sof_ipc_comp_event *event, uint32_t type, } #if CONFIG_LIBRARY -static inline unsigned char *ipc4_get_comp_new_data(void) +unsigned char *ipc4_get_comp_new_data(void) { struct ipc *ipc = ipc_get(); unsigned char *data = (unsigned char *)ipc->comp_data + @@ -85,13 +71,8 @@ static inline unsigned char *ipc4_get_comp_new_data(void) return data; } - -static const struct comp_driver *ipc4_library_get_comp_drv(unsigned char *data) -{ - return ipc4_get_drv(data); -} #else -__cold static inline unsigned char *ipc4_get_comp_new_data(void) +__cold unsigned char *ipc4_get_comp_new_data(void) { assert_can_be_cold(); @@ -99,8 +80,8 @@ __cold static inline unsigned char *ipc4_get_comp_new_data(void) } #endif /* CONFIG_LIBRARY */ -__cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, - const struct ipc4_module_init_instance *module_init) +__cold int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, + const struct ipc4_module_init_instance *module_init) { uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, module_init->primary.r.instance_id); @@ -171,112 +152,7 @@ __cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, return 0; } -/* Only called from ipc4_init_module_instance(), which is __cold */ -__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) -{ - struct comp_ipc_config ipc_config; - const struct comp_driver *drv; - struct comp_dev *dev; - unsigned char *data; - uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, - module_init->primary.r.instance_id); - - assert_can_be_cold(); - - int ret = ipc4_comp_new_config(&ipc_config, module_init); - - if (ret < 0) - return NULL; - -#ifdef CONFIG_DCACHE_LINE_SIZE - if (!IS_ENABLED(CONFIG_LIBRARY)) - sys_cache_data_invd_range((__sparse_force void __sparse_cache *) - MAILBOX_HOSTBOX_BASE, - ALIGN_UP(ipc_config.ipc_config_size, - CONFIG_DCACHE_LINE_SIZE)); -#endif - data = ipc4_get_comp_new_data(); -#if CONFIG_LIBRARY - ipc_config.ipc_config_size -= sizeof(struct sof_uuid); - drv = ipc4_library_get_comp_drv(data + ipc_config.ipc_config_size); -#else - drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); -#endif - if (!drv) - return NULL; - - if (!drv->ops.create) { - tr_err(&ipc_tr, "comp 0x%x: module %u is not instantiable", comp_id, - (uint32_t)IPC4_MOD_ID(comp_id)); - return NULL; - } - - if (drv->type == SOF_COMP_MODULE_ADAPTER) { - const struct ipc_config_process spec = { - .data = data, - .size = ipc_config.ipc_config_size, - }; - - dev = drv->ops.create(drv, &ipc_config, &spec); - } else { - dev = drv->ops.create(drv, &ipc_config, data); - } - if (!dev) - return NULL; - - list_init(&dev->bsource_list); - list_init(&dev->bsink_list); - -#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS - /* init global performance measurement */ - dev->perf_data.perf_data_item = perf_data_getnext(); - /* this can be null, just no performance measurements in this case */ - if (dev->perf_data.perf_data_item) { - dev->perf_data.perf_data_item->item.resource_id = comp_id; - if (perf_meas_get_state() != IPC4_PERF_MEASUREMENTS_DISABLED) - comp_init_performance_data(dev); - } -#endif - - ipc4_add_comp_dev(dev); - - comp_update_ibs_obs_cpc(dev); - - return dev; -} - #ifdef CONFIG_SOF_USERSPACE_LL - -int ipc4_user_module_load(const struct comp_driver *drv, - const struct ipc4_module_init_instance *mi) -{ - /* - * A large part of module instantiation is executed in the userspace - * mode, including calling the module's .init() method. But another part - * of it has to be executed in the kernel mode, including calling - * lib_manager_mod_create_priv(). This is the privileged part of module - * instantiation. - */ - struct comp_ipc_config ipc_config; - int ret = ipc4_comp_new_config(&ipc_config, mi); - - if (ret < 0) - return ret; - - const struct ipc_config_process spec = { - .data = ipc4_get_comp_new_data(), - .size = ipc_config.ipc_config_size, - }; - -#if CONFIG_DCACHE_LINE_SIZE && !CONFIG_LIBRARY - sys_cache_data_invd_range((__sparse_force void __sparse_cache *)spec.data, spec.size); -#endif - - struct userspace_context *userspace = NULL; - - return lib_manager_mod_create_priv(drv, &ipc_config, &spec, NULL, &userspace); -} - /** * comp_new_ipc4_user - Create component in user-space IPC thread context. * @@ -568,16 +444,6 @@ __cold int ipc_pipeline_new(struct ipc *ipc, ipc_pipe_new *_pipe_desc) return ipc4_create_pipeline(pipe_desc, valid_pparams ? &pparams : NULL); } -__cold static inline int ipc_comp_free_remote(struct comp_dev *dev) -{ - struct idc_msg msg = { IDC_MSG_FREE, IDC_MSG_FREE_EXT(dev->ipc_config.id), - dev->ipc_config.core,}; - - assert_can_be_cold(); - - return idc_send_msg(&msg, IDC_BLOCKING); -} - __cold static int ipc_pipeline_module_free(uint32_t pipeline_id) { struct ipc *ipc = ipc_get(); @@ -1171,76 +1037,6 @@ __cold int ipc4_comp_disconnect(struct ipc *ipc, const struct ipc4_module_bind_u return IPC4_SUCCESS; } -#if CONFIG_COMP_CHAIN_DMA -/* Only called from ipc4_process_chain_dma(), which is __cold */ -__cold int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma) -{ - const struct sof_uuid uuid = SOF_REG_UUID(chain_dma); - const struct comp_driver *drv; - struct comp_dev *dev; - - assert_can_be_cold(); - - drv = ipc4_get_drv(&uuid); - if (!drv) - return -EINVAL; - - dev = drv->ops.create(drv, NULL, cdma); - if (!dev) - return -EINVAL; - - /* differentiate instance by unique ids assignment */ - const uint32_t comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id - + IPC4_MAX_MODULE_COUNT, 0); - dev->ipc_config.id = comp_id; - dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT; - - return ipc4_add_comp_dev(dev); -} - -/* Only called from ipc4_process_chain_dma(), which is __cold */ -__cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma) -{ - const bool allocate = cdma->primary.r.allocate; - const bool enable = cdma->primary.r.enable; - struct ipc *ipc = ipc_get(); - struct ipc_comp_dev *icd; - struct list_item *clist, *_tmp; - int ret; - - assert_can_be_cold(); - - if (!dev) - return -EINVAL; - - if (allocate) { - if (enable) - ret = comp_trigger(dev, COMP_TRIGGER_START); - else - ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); - } else { - if (enable) - return -EINVAL; - - /* remove chain part */ - ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); - if (ret < 0) - return ret; - - list_for_item_safe(clist, _tmp, &ipc->comp_list) { - icd = container_of(clist, struct ipc_comp_dev, list); - if (icd->cd != dev) - continue; - list_item_del(&icd->list); - sof_heap_free(sof_sys_user_heap_get(), icd); - break; - } - comp_free(dev); - } - return ret; -} -#endif /* CONFIG_COMP_CHAIN_DMA */ - __cold static int ipc4_update_comps_direction(struct ipc *ipc, uint32_t ppl_id) { struct ipc_comp_dev *icd; @@ -1303,200 +1099,6 @@ int ipc4_pipeline_complete(struct ipc *ipc, uint32_t comp_id, uint32_t cmd) return ipc_pipeline_complete(ipc, comp_id); } -int ipc4_process_on_core(uint32_t core, bool blocking) -{ - int ret; - - ret = ipc_process_on_core(core, blocking); - switch (ret) { - case 0: - case 1: - return IPC4_SUCCESS; - case -EACCES: - return IPC4_INVALID_CORE_ID; - case -ETIME: - case -EBUSY: - return IPC4_BUSY; - default: - return IPC4_FAILURE; - } - - return IPC4_SUCCESS; -} - -__cold static const struct comp_driver *ipc4_search_for_drv(const void *uuid) -{ - struct comp_driver_list *drivers = comp_drivers_get(); - struct list_item *clist; - const struct comp_driver *drv = NULL; - struct comp_driver_info *info; - - assert_can_be_cold(); - - /* search driver list with UUID */ - list_for_item(clist, &drivers->list) { - info = container_of(clist, struct comp_driver_info, list); - if (!memcmp(info->drv->uid, uuid, UUID_SIZE)) { - tr_dbg(&comp_tr, "found type %d, uuid %pU", - info->drv->type, SOF_DRV_UID_NAME(info->drv)); - drv = info->drv; - break; - } - } - - return drv; -} - -__cold static const struct comp_driver *ipc4_get_drv(const void *uuid) -{ - const struct sof_uuid *const __maybe_unused sof_uuid = (const struct sof_uuid *)uuid; - const struct comp_driver *drv; - - assert_can_be_cold(); - - drv = ipc4_search_for_drv(uuid); - if (!drv) - tr_err(&comp_tr, - "the provided UUID (%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x) can't be found!", - sof_uuid->a, sof_uuid->b, sof_uuid->c, sof_uuid->d[0], sof_uuid->d[1], - sof_uuid->d[2], sof_uuid->d[3], sof_uuid->d[4], sof_uuid->d[5], - sof_uuid->d[6], sof_uuid->d[7]); - - return drv; -} - -#if defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) -/* - * ipc4_get_fuzzer_drv - resolve a module driver for native_sim fuzz builds. - * - * native_sim fuzz builds have no rimage manifest, so the manifest lookup in - * ipc4_get_comp_drv() can never resolve a driver and every module instance - * verb (INIT_INSTANCE, CONFIG_GET/SET, LARGE_CONFIG, BIND, UNBIND, - * DELETE_INSTANCE) would be unreachable from the fuzzer. - * - * Mirror the IPC3 whitebox hack in posix/ipc.c and treat module_id as a - * 1-based index into the runtime component driver list, counting only - * instantiable module-adapter drivers so the fuzzer's module_id space matches - * what a real signed manifest would expose. - */ -static const struct comp_driver *ipc4_get_fuzzer_drv(uint32_t module_id) -{ - static const struct sof_uuid basefw_reg_uuid = SOF_REG_UUID(basefw); - struct comp_driver_list *dlist = comp_drivers_get(); - struct list_item *iter; - uint32_t idx = 0; - - if (!module_id) - return ipc4_search_for_drv(&basefw_reg_uuid); - - list_for_item(iter, &dlist->list) { - struct comp_driver_info *inf = - container_of(iter, struct comp_driver_info, list); - - /* - * Skip query-only entries (e.g. BaseFW) that cannot be - * instantiated because they have no create op. - */ - if (!inf->drv->ops.create) - continue; - - /* - * A real signed manifest only lists module-adapter modules. - * The internal gateway drivers (SOF_COMP_HOST, SOF_COMP_DAI) - * are registered for IPC3 but are never IPC4 modules: on the - * IPC4 path they receive no params() pass and cannot be - * configured (e.g. their DMA buffer is never allocated). Skip - * non-module-adapter drivers so the fuzzer's module_id space - * matches a real manifest and cannot map to an unconfigurable - * component. - */ - if (inf->drv->type != SOF_COMP_MODULE_ADAPTER) - continue; - - if (++idx == module_id) - return inf->drv; - } - - tr_err(&comp_tr, "no instantiable driver at module_id %u (%u available)", - module_id, idx); - return NULL; -} -#endif /* defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) */ - -/* - * Called from - * - ipc4_get_large_config_module_instance() - * - ipc4_set_large_config_module_instance() - * - comp_new_ipc4() - * all of which are __cold - */ -__cold const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id) -{ - const struct sof_man_fw_desc *desc = NULL; - const struct sof_man_module *mod; - uint32_t entry_index; - - assert_can_be_cold(); - -#ifdef RIMAGE_MANIFEST - desc = (const struct sof_man_fw_desc *)IMR_BOOT_LDR_MANIFEST_BASE; -#elif defined(CONFIG_ARCH_POSIX_LIBFUZZER) - return ipc4_get_fuzzer_drv(module_id); -#else - /* Non-rimage platforms have no component facility yet. - * This needs to move to the platform layer. - */ - return NULL; -#endif - - uint32_t lib_idx = LIB_MANAGER_GET_LIB_ID(module_id); - - if (lib_idx == 0) { - /* module_id 0 is used for base fw which is in entry 1 or 2 */ - if (!module_id) - entry_index = 1 + IS_ENABLED(CONFIG_COLD_STORE_EXECUTE_DRAM); - else - entry_index = module_id; - - if (entry_index >= desc->header.num_module_entries) { - tr_err(&comp_tr, "Error: entry index %d out of bounds.", entry_index); - return NULL; - } - - mod = (const struct sof_man_module *)((const char *)desc + - SOF_MAN_MODULE_OFFSET(entry_index)); - } else { - /* Library index greater than 0 possible only when LIBRARY_MANAGER - * support enabled. - */ -#if CONFIG_LIBRARY_MANAGER - mod = lib_manager_get_module_manifest(module_id); - - if (!mod) { - tr_err(&comp_tr, "Error: Couldn't find loadable module with id %#x.", - module_id); - return NULL; - } -#else - tr_err(&comp_tr, "Error: lib index:%d, while loadable libraries are not supported!!!", - lib_idx); - return NULL; -#endif - } - -#if CONFIG_LIBRARY_MANAGER - /* Check already registered components */ - const struct comp_driver *drv = ipc4_search_for_drv(&mod->uuid); - - if (drv) - return drv; - - /* New module not registered yet. */ - lib_manager_register_module(module_id); -#endif - return ipc4_get_drv(&mod->uuid); -} - struct comp_dev *ipc4_get_comp_dev(uint32_t comp_id) { struct ipc_comp_dev *icd = ipc_get_comp_by_id(ipc_get(), comp_id); diff --git a/src/kernel/ipc/ipc4/CMakeLists.txt b/src/kernel/ipc/ipc4/CMakeLists.txt index 94c63418370c..53606a9ddda4 100644 --- a/src/kernel/ipc/ipc4/CMakeLists.txt +++ b/src/kernel/ipc/ipc4/CMakeLists.txt @@ -4,6 +4,7 @@ add_local_sources(sof handler-kernel.c handler-user-kernel.c + helper-kernel.c logging.c notification.c ) diff --git a/src/kernel/ipc/ipc4/helper-kernel.c b/src/kernel/ipc/ipc4/helper-kernel.c new file mode 100644 index 000000000000..73635753c539 --- /dev/null +++ b/src/kernel/ipc/ipc4/helper-kernel.c @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* TODO: Remove platform-specific code, see https://github.com/thesofproject/sof/issues/7549 */ +#if defined(CONFIG_SOC_SERIES_INTEL_ADSP_ACE) || defined(CONFIG_INTEL_ADSP_CAVS) +#define RIMAGE_MANIFEST 1 +#endif + +#ifdef RIMAGE_MANIFEST +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +/* Kernel part of the IPC4 helpers: the component driver lookup, which reads the + * IMR manifest and the driver list, module creation in kernel mode, IDC to other + * cores and the chain DMA manager. The part that also runs in user mode is in + * src/ipc/ipc4/helper.c + */ + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +extern struct tr_ctx comp_tr; + +static const struct comp_driver *ipc4_get_drv(const void *uuid); + +#if CONFIG_LIBRARY +static const struct comp_driver *ipc4_library_get_comp_drv(unsigned char *data) +{ + return ipc4_get_drv(data); +} +#endif /* CONFIG_LIBRARY */ + +/* Only called from ipc4_init_module_instance(), which is __cold */ +__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) +{ + struct comp_ipc_config ipc_config; + const struct comp_driver *drv; + struct comp_dev *dev; + unsigned char *data; + uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, + module_init->primary.r.instance_id); + + assert_can_be_cold(); + + int ret = ipc4_comp_new_config(&ipc_config, module_init); + + if (ret < 0) + return NULL; + +#ifdef CONFIG_DCACHE_LINE_SIZE + if (!IS_ENABLED(CONFIG_LIBRARY)) + sys_cache_data_invd_range((__sparse_force void __sparse_cache *) + MAILBOX_HOSTBOX_BASE, + ALIGN_UP(ipc_config.ipc_config_size, + CONFIG_DCACHE_LINE_SIZE)); +#endif + data = ipc4_get_comp_new_data(); +#if CONFIG_LIBRARY + ipc_config.ipc_config_size -= sizeof(struct sof_uuid); + drv = ipc4_library_get_comp_drv(data + ipc_config.ipc_config_size); +#else + drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); +#endif + if (!drv) + return NULL; + + if (!drv->ops.create) { + tr_err(&ipc_tr, "comp 0x%x: module %u is not instantiable", comp_id, + (uint32_t)IPC4_MOD_ID(comp_id)); + return NULL; + } + + if (drv->type == SOF_COMP_MODULE_ADAPTER) { + const struct ipc_config_process spec = { + .data = data, + .size = ipc_config.ipc_config_size, + }; + + dev = drv->ops.create(drv, &ipc_config, &spec); + } else { + dev = drv->ops.create(drv, &ipc_config, data); + } + if (!dev) + return NULL; + + list_init(&dev->bsource_list); + list_init(&dev->bsink_list); + +#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + /* init global performance measurement */ + dev->perf_data.perf_data_item = perf_data_getnext(); + /* this can be null, just no performance measurements in this case */ + if (dev->perf_data.perf_data_item) { + dev->perf_data.perf_data_item->item.resource_id = comp_id; + if (perf_meas_get_state() != IPC4_PERF_MEASUREMENTS_DISABLED) + comp_init_performance_data(dev); + } +#endif + + ipc4_add_comp_dev(dev); + + comp_update_ibs_obs_cpc(dev); + + return dev; +} + +#ifdef CONFIG_SOF_USERSPACE_LL +int ipc4_user_module_load(const struct comp_driver *drv, + const struct ipc4_module_init_instance *mi) +{ + /* + * A large part of module instantiation is executed in the userspace + * mode, including calling the module's .init() method. But another part + * of it has to be executed in the kernel mode, including calling + * lib_manager_mod_create_priv(). This is the privileged part of module + * instantiation. + */ + struct comp_ipc_config ipc_config; + int ret = ipc4_comp_new_config(&ipc_config, mi); + + if (ret < 0) + return ret; + + const struct ipc_config_process spec = { + .data = ipc4_get_comp_new_data(), + .size = ipc_config.ipc_config_size, + }; + +#if CONFIG_DCACHE_LINE_SIZE && !CONFIG_LIBRARY + sys_cache_data_invd_range((__sparse_force void __sparse_cache *)spec.data, spec.size); +#endif + + struct userspace_context *userspace = NULL; + + return lib_manager_mod_create_priv(drv, &ipc_config, &spec, NULL, &userspace); +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +__cold int ipc_comp_free_remote(struct comp_dev *dev) +{ + struct idc_msg msg = { IDC_MSG_FREE, IDC_MSG_FREE_EXT(dev->ipc_config.id), + dev->ipc_config.core,}; + + assert_can_be_cold(); + + return idc_send_msg(&msg, IDC_BLOCKING); +} + +#if CONFIG_COMP_CHAIN_DMA +/* Only called from ipc4_process_chain_dma(), which is __cold */ +__cold int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma) +{ + const struct sof_uuid uuid = SOF_REG_UUID(chain_dma); + const struct comp_driver *drv; + struct comp_dev *dev; + + assert_can_be_cold(); + + drv = ipc4_get_drv(&uuid); + if (!drv) + return -EINVAL; + + dev = drv->ops.create(drv, NULL, cdma); + if (!dev) + return -EINVAL; + + /* differentiate instance by unique ids assignment */ + const uint32_t comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + + IPC4_MAX_MODULE_COUNT, 0); + dev->ipc_config.id = comp_id; + dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT; + + return ipc4_add_comp_dev(dev); +} + +/* Only called from ipc4_process_chain_dma(), which is __cold */ +__cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma) +{ + const bool allocate = cdma->primary.r.allocate; + const bool enable = cdma->primary.r.enable; + struct ipc *ipc = ipc_get(); + struct ipc_comp_dev *icd; + struct list_item *clist, *_tmp; + int ret; + + assert_can_be_cold(); + + if (!dev) + return -EINVAL; + + if (allocate) { + if (enable) + ret = comp_trigger(dev, COMP_TRIGGER_START); + else + ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); + } else { + if (enable) + return -EINVAL; + + /* remove chain part */ + ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); + if (ret < 0) + return ret; + + list_for_item_safe(clist, _tmp, &ipc->comp_list) { + icd = container_of(clist, struct ipc_comp_dev, list); + if (icd->cd != dev) + continue; + list_item_del(&icd->list); + sof_heap_free(sof_sys_user_heap_get(), icd); + break; + } + comp_free(dev); + } + return ret; +} +#endif /* CONFIG_COMP_CHAIN_DMA */ + +int ipc4_process_on_core(uint32_t core, bool blocking) +{ + int ret; + + ret = ipc_process_on_core(core, blocking); + switch (ret) { + case 0: + case 1: + return IPC4_SUCCESS; + case -EACCES: + return IPC4_INVALID_CORE_ID; + case -ETIME: + case -EBUSY: + return IPC4_BUSY; + default: + return IPC4_FAILURE; + } + + return IPC4_SUCCESS; +} + +__cold static const struct comp_driver *ipc4_search_for_drv(const void *uuid) +{ + struct comp_driver_list *drivers = comp_drivers_get(); + struct list_item *clist; + const struct comp_driver *drv = NULL; + struct comp_driver_info *info; + + assert_can_be_cold(); + + /* search driver list with UUID */ + list_for_item(clist, &drivers->list) { + info = container_of(clist, struct comp_driver_info, list); + if (!memcmp(info->drv->uid, uuid, UUID_SIZE)) { + tr_dbg(&comp_tr, "found type %d, uuid %pU", + info->drv->type, SOF_DRV_UID_NAME(info->drv)); + drv = info->drv; + break; + } + } + + return drv; +} + +__cold static const struct comp_driver *ipc4_get_drv(const void *uuid) +{ + const struct sof_uuid *const __maybe_unused sof_uuid = (const struct sof_uuid *)uuid; + const struct comp_driver *drv; + + assert_can_be_cold(); + + drv = ipc4_search_for_drv(uuid); + if (!drv) + tr_err(&comp_tr, + "the provided UUID (%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x) can't be found!", + sof_uuid->a, sof_uuid->b, sof_uuid->c, sof_uuid->d[0], sof_uuid->d[1], + sof_uuid->d[2], sof_uuid->d[3], sof_uuid->d[4], sof_uuid->d[5], + sof_uuid->d[6], sof_uuid->d[7]); + + return drv; +} + +#if defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) +/* + * ipc4_get_fuzzer_drv - resolve a module driver for native_sim fuzz builds. + * + * native_sim fuzz builds have no rimage manifest, so the manifest lookup in + * ipc4_get_comp_drv() can never resolve a driver and every module instance + * verb (INIT_INSTANCE, CONFIG_GET/SET, LARGE_CONFIG, BIND, UNBIND, + * DELETE_INSTANCE) would be unreachable from the fuzzer. + * + * Mirror the IPC3 whitebox hack in posix/ipc.c and treat module_id as a + * 1-based index into the runtime component driver list, counting only + * instantiable module-adapter drivers so the fuzzer's module_id space matches + * what a real signed manifest would expose. + */ +static const struct comp_driver *ipc4_get_fuzzer_drv(uint32_t module_id) +{ + static const struct sof_uuid basefw_reg_uuid = SOF_REG_UUID(basefw); + struct comp_driver_list *dlist = comp_drivers_get(); + struct list_item *iter; + uint32_t idx = 0; + + if (!module_id) + return ipc4_search_for_drv(&basefw_reg_uuid); + + list_for_item(iter, &dlist->list) { + struct comp_driver_info *inf = + container_of(iter, struct comp_driver_info, list); + + /* + * Skip query-only entries (e.g. BaseFW) that cannot be + * instantiated because they have no create op. + */ + if (!inf->drv->ops.create) + continue; + + /* + * A real signed manifest only lists module-adapter modules. + * The internal gateway drivers (SOF_COMP_HOST, SOF_COMP_DAI) + * are registered for IPC3 but are never IPC4 modules: on the + * IPC4 path they receive no params() pass and cannot be + * configured (e.g. their DMA buffer is never allocated). Skip + * non-module-adapter drivers so the fuzzer's module_id space + * matches a real manifest and cannot map to an unconfigurable + * component. + */ + if (inf->drv->type != SOF_COMP_MODULE_ADAPTER) + continue; + + if (++idx == module_id) + return inf->drv; + } + + tr_err(&comp_tr, "no instantiable driver at module_id %u (%u available)", + module_id, idx); + return NULL; +} +#endif /* defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) */ + +/* + * Called from + * - ipc4_get_large_config_module_instance() + * - ipc4_set_large_config_module_instance() + * - comp_new_ipc4() + * all of which are __cold + */ +__cold const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id) +{ + const struct sof_man_fw_desc *desc = NULL; + const struct sof_man_module *mod; + uint32_t entry_index; + + assert_can_be_cold(); + +#ifdef RIMAGE_MANIFEST + desc = (const struct sof_man_fw_desc *)IMR_BOOT_LDR_MANIFEST_BASE; +#elif defined(CONFIG_ARCH_POSIX_LIBFUZZER) + return ipc4_get_fuzzer_drv(module_id); +#else + /* Non-rimage platforms have no component facility yet. + * This needs to move to the platform layer. + */ + return NULL; +#endif + + uint32_t lib_idx = LIB_MANAGER_GET_LIB_ID(module_id); + + if (lib_idx == 0) { + /* module_id 0 is used for base fw which is in entry 1 or 2 */ + if (!module_id) + entry_index = 1 + IS_ENABLED(CONFIG_COLD_STORE_EXECUTE_DRAM); + else + entry_index = module_id; + + if (entry_index >= desc->header.num_module_entries) { + tr_err(&comp_tr, "Error: entry index %d out of bounds.", entry_index); + return NULL; + } + + mod = (const struct sof_man_module *)((const char *)desc + + SOF_MAN_MODULE_OFFSET(entry_index)); + } else { + /* Library index greater than 0 possible only when LIBRARY_MANAGER + * support enabled. + */ +#if CONFIG_LIBRARY_MANAGER + mod = lib_manager_get_module_manifest(module_id); + + if (!mod) { + tr_err(&comp_tr, "Error: Couldn't find loadable module with id %#x.", + module_id); + return NULL; + } +#else + tr_err(&comp_tr, "Error: lib index:%d, while loadable libraries are not supported!!!", + lib_idx); + return NULL; +#endif + } + +#if CONFIG_LIBRARY_MANAGER + /* Check already registered components */ + const struct comp_driver *drv = ipc4_search_for_drv(&mod->uuid); + + if (drv) + return drv; + + /* New module not registered yet. */ + lib_manager_register_module(module_id); +#endif + return ipc4_get_drv(&mod->uuid); +} From 83761fed2ffa6ad532194d21d0025c80097abf4d Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:13:33 +0000 Subject: [PATCH 11/19] schedule: split the kernel part of zephyr_domain.c into src/kernel/schedule zephyr_domain.c mixes the LL scheduler thread function and the parts of the timer domain that run in the LL scheduler threads with the thread and timer setup, which needs kernel mode. Move the kernel part to src/kernel/schedule/zephyr_domain-kernel.c: - the LL scheduler thread stacks and the timer interrupt handler; - domain register and unregister of builds without user-space LL, which create and abort the scheduler threads and start and stop the timer; - the user-space LL thread init and free, the kernel-only per-core thread table and the thread lookups used for access grants; - the domain ops and zephyr_domain_init(). zephyr_domain.c keeps the thread function, the user-space LL register and unregister bookkeeping, the cross-core stream block and unblock and ll_sch_is_current(). The domain data structures and the functions the kernel part refers to are declared in the new scheduler-private header sof/schedule/zephyr_ll.h. zephyr_domain.c no longer includes the timer, userspace helper, platform and memory map headers. The code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/schedule/zephyr_ll.h | 61 +++ src/kernel/schedule/CMakeLists.txt | 4 + src/kernel/schedule/zephyr_domain-kernel.c | 452 ++++++++++++++++++++ src/schedule/zephyr_domain.c | 463 +-------------------- 4 files changed, 529 insertions(+), 451 deletions(-) create mode 100644 src/include/sof/schedule/zephyr_ll.h create mode 100644 src/kernel/schedule/zephyr_domain-kernel.c diff --git a/src/include/sof/schedule/zephyr_ll.h b/src/include/sof/schedule/zephyr_ll.h new file mode 100644 index 000000000000..d3d1e36ac1ee --- /dev/null +++ b/src/include/sof/schedule/zephyr_ll.h @@ -0,0 +1,61 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2019-2026 Intel Corporation. + */ + +/* + * Private data of the Zephyr LL scheduler and its timer domain, shared by + * the parts in src/schedule, which also run in the LL scheduler threads, and + * the kernel parts in src/kernel/schedule. Not for use outside the scheduler. + */ + +#ifndef __SOF_SCHEDULE_ZEPHYR_LL_H__ +#define __SOF_SCHEDULE_ZEPHYR_LL_H__ + +#include +#include +#include +#include + +struct zephyr_domain_thread { + struct k_thread *ll_thread; + struct k_sem *sem; +#ifndef CONFIG_SOF_USERSPACE_LL + struct k_thread ll_thread_obj; + struct k_sem sem_obj; +#endif + void (*handler)(void *arg); + void *arg; +}; + +struct zephyr_domain { + struct k_timer *timer; +#ifndef CONFIG_SOF_USERSPACE_LL + struct k_timer timer_obj; +#endif + struct zephyr_domain_thread domain_thread[CONFIG_CORE_COUNT]; + struct ll_schedule_domain *ll_domain; +#if CONFIG_CROSS_CORE_STREAM + atomic_t block; + struct k_mutex block_mutex; + struct k_condvar block_condvar; +#endif +}; + +/* LL scheduler thread function, p1 is the domain and p2 the core */ +void zephyr_domain_thread_fn(void *p1, void *p2, void *p3); + +#ifdef CONFIG_SOF_USERSPACE_LL +int zephyr_domain_register_user(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg); +int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, + struct task *task, uint32_t num_tasks); +#endif + +#if CONFIG_CROSS_CORE_STREAM +void zephyr_domain_block(struct ll_schedule_domain *domain); +void zephyr_domain_unblock(struct ll_schedule_domain *domain); +#endif + +#endif /* __SOF_SCHEDULE_ZEPHYR_LL_H__ */ diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index 277378449165..6b0d7011f409 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -2,6 +2,10 @@ is_zephyr(zephyr) if(zephyr) + zephyr_library_sources( + zephyr_domain-kernel.c + ) + zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_LL zephyr_ll_user.c ) diff --git a/src/kernel/schedule/zephyr_domain-kernel.c b/src/kernel/schedule/zephyr_domain-kernel.c new file mode 100644 index 000000000000..de320d93c9e4 --- /dev/null +++ b/src/kernel/schedule/zephyr_domain-kernel.c @@ -0,0 +1,452 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2019-2021 Intel Corporation. All rights reserved. +// +// Author: Tomasz Lauda + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); + +/* Kernel part of the Zephyr LL scheduler timer domain: the scheduler threads, the + * timer and its interrupt handler. The thread function and the parts that run in the + * LL scheduler threads are in src/schedule/zephyr_domain.c + */ + +/* + * Currently the Zephyr clock rate is part it's Kconfig known at build time. + * SOF on Intel CAVS platforms currently only aligns with Zephyr when both + * use the CAVS 19.2 MHz SSP clock. TODO - needs runtime alignment. + */ +#if CONFIG_XTENSA && CONFIG_CAVS && !CONFIG_INTEL_ADSP_TIMER +#error "Zephyr uses 19.2MHz clock derived from SSP which must be enabled." +#endif + +#define ZEPHYR_LL_STACK_SIZE 8192 + +#if CONFIG_SOF_USERSPACE_LL +K_THREAD_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); +#else +K_KERNEL_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); +#endif + +/* Timer callback: runs in timer IRQ context */ +static void zephyr_domain_timer_fn(struct k_timer *timer) +{ + struct zephyr_domain *zephyr_domain = k_timer_user_data_get(timer); + int core; + + /* + * A race is possible when the Zephyr LL scheduling domain is being + * unregistered while a timer IRQ is processed on a different core. Then + * the timer is removed by the former but then re-added by the latter, + * but this time with no user data and no handler set. This leads to the + * timer continuing to trigger and then leading to a lock up when it is + * registered again next time. + */ + if (!zephyr_domain) { + k_timer_stop(timer); + return; + } + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + if (dt->handler) + k_sem_give(dt->sem); + } +} + +/* The normal kernel-space implementation for register/unregister */ +#ifndef CONFIG_SOF_USERSPACE_LL + +static int zephyr_domain_register(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + char thread_name[] = "ll_thread0"; + k_tid_t thread; + k_spinlock_key_t key; + + tr_dbg(&ll_tr, "entry"); + + /* domain work only needs registered once on each core */ + if (dt->handler) + return 0; + + dt->handler = handler; + dt->arg = arg; + + /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ + k_sem_init(dt->sem, 0, 10); + + thread_name[sizeof(thread_name) - 2] = '0' + core; + + /* not allocated dynamically when LL in kernel space */ + dt->ll_thread = &dt->ll_thread_obj; + + thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, + zephyr_domain_thread_fn, zephyr_domain, INT_TO_POINTER(core), + NULL, CONFIG_LL_THREAD_PRIORITY, 0, K_FOREVER); + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, core); +#endif + k_thread_name_set(thread, thread_name); + + k_thread_start(thread); + + key = k_spin_lock(&domain->lock); + + if (!k_timer_user_data_get(zephyr_domain->timer)) { + k_timeout_t start = {0}; + + k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); + k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); + + k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); + + /* Enable the watchdog */ + watchdog_enable(core); + } + + k_spin_unlock(&domain->lock, key); + + tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", + domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); + + return 0; +} + +static int zephyr_domain_unregister(struct ll_schedule_domain *domain, + struct task *task, uint32_t num_tasks) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + k_spinlock_key_t key; + + tr_dbg(&ll_tr, "entry"); + + /* tasks still registered on this core */ + if (num_tasks) + return 0; + + key = k_spin_lock(&domain->lock); + + if (!atomic_read(&domain->total_num_tasks)) { + /* Disable the watchdog */ + watchdog_disable(core); + + k_timer_stop(zephyr_domain->timer); + k_timer_user_data_set(zephyr_domain->timer, NULL); + } + + zephyr_domain->domain_thread[core].handler = NULL; + + k_spin_unlock(&domain->lock, key); + + tr_info(&ll_tr, "domain->type %d domain->clk %d", + domain->type, domain->clk); + + /* + * If running in the context of the domain thread, k_thread_abort() will + * not return + */ + k_thread_abort(zephyr_domain->domain_thread[core].ll_thread); + + tr_dbg(&ll_tr, "exit"); + + return 0; +} + +#else /* CONFIG_SOF_USERSPACE_LL */ + +/* + * Kernel-owned per-core LL thread table. Populated from the privileged + * domain-thread init path and consulted by z_impl_zephyr_ll_task_sem_alloc() + * so that privileged code never has to traverse the user-accessible + * scheduler/domain objects to find the LL thread of a given core. + */ +static struct k_thread *ll_thread_tid[CONFIG_CORE_COUNT]; + +/* + * Privileged thread initialization for userspace LL scheduling. + * Creates the scheduling thread, sets up timer, grants access to kernel + * objects. Must be called from kernel context before any user-space + * domain_register() calls. + */ +static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, + struct task *task) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + struct zephyr_domain_thread *dt; + char thread_name[] = "ll_thread0"; + k_tid_t thread; + int core = task->core; + + tr_dbg(&ll_tr, "entry"); + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + dt = zephyr_domain->domain_thread + core; + + /* thread only needs to be created once per core */ + if (dt->ll_thread) + return 0; + + dt->handler = NULL; + dt->arg = NULL; + + dt->sem = k_object_alloc(K_OBJ_SEM); + if (!dt->sem) { + tr_err(&ll_tr, "Failed to allocate semaphore for core %d", core); + return -ENOMEM; + } + + /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ + k_sem_init(dt->sem, 0, 10); + + thread_name[sizeof(thread_name) - 2] = '0' + core; + + /* Allocate thread structure dynamically */ + dt->ll_thread = k_object_alloc(K_OBJ_THREAD); + if (!dt->ll_thread) { + tr_err(&ll_tr, "Failed to allocate thread object for core %d", core); + k_object_free(dt->sem); + dt->sem = NULL; + return -ENOMEM; + } + + thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, + zephyr_domain_thread_fn, zephyr_domain, + INT_TO_POINTER(core), NULL, CONFIG_LL_THREAD_PRIORITY, + K_USER, K_FOREVER); + + /* record in the kernel-only table for syscall-context lookups */ + ll_thread_tid[core] = dt->ll_thread; + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, core); +#endif + k_thread_name_set(thread, thread_name); + + k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); + k_thread_access_grant(thread, dt->sem, domain->lock, zephyr_domain->timer); + user_grant_dai_access_all(thread); + user_grant_dma_access_all(thread); + user_grant_uaol_access_all(thread); + tr_dbg(&ll_tr, "granted LL access to thread %p (core %d)", thread, core); + + k_thread_start(thread); + + k_mutex_lock(domain->lock, K_FOREVER); + if (!k_timer_user_data_get(zephyr_domain->timer)) { + k_timeout_t start = {0}; + + k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); + k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); + + k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); + + /* Enable the watchdog */ + watchdog_enable(core); + } + + k_mutex_unlock(domain->lock); + + tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", + domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); + + return 0; +} + +/* + * Free resources acquired by zephyr_domain_thread_init(). + * Stops the timer, aborts the scheduling thread and frees the thread object. + * Must be called from kernel context. + */ +static void zephyr_domain_thread_free(struct ll_schedule_domain *domain, + uint32_t num_tasks) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + tr_dbg(&ll_tr, "thread_free entry, core %d, num_tasks %u", core, num_tasks); + + /* Still tasks on other cores, only clean up this core's thread */ + k_mutex_lock(domain->lock, K_FOREVER); + + if (!num_tasks && !atomic_read(&domain->total_num_tasks)) { + /* Last task globally: stop the timer and watchdog */ + watchdog_disable(core); + + k_timer_stop(zephyr_domain->timer); + k_timer_user_data_set(zephyr_domain->timer, NULL); + } + + dt->handler = NULL; + dt->arg = NULL; + + k_mutex_unlock(domain->lock); + + if (dt->ll_thread) { + k_thread_abort(dt->ll_thread); + k_object_free(dt->ll_thread); + dt->ll_thread = NULL; + ll_thread_tid[core] = NULL; + } + + if (dt->sem) { + k_object_free(dt->sem); + dt->sem = NULL; + } + + tr_info(&ll_tr, "thread_free done, core %d", core); +} + +struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + tr_dbg(&ll_tr, "entry"); + + return dt->ll_thread; +} + +/* + * Return the LL scheduling thread for an explicitly given core. + * + * Reads a kernel-only table keyed by core, so it is safe to call from a + * privileged syscall context without dereferencing any user-accessible + * scheduler or domain object, and without relying on cpu_get_id(). + */ +struct k_thread *zephyr_domain_thread_tid_for_core(int core) +{ + if (core < 0 || core >= CONFIG_CORE_COUNT) + return NULL; + + return ll_thread_tid[core]; +} + +struct k_thread *zephyr_ll_domain_thread(void) +{ + struct ll_schedule_domain *ll_domain = zephyr_ll_domain(); + + if (!ll_domain) + return NULL; + + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(ll_domain); + + return zephyr_domain->domain_thread[cpu_get_id()].ll_thread; +} + +#endif /* CONFIG_SOF_USERSPACE_LL */ + +APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { +#ifdef CONFIG_SOF_USERSPACE_LL + .domain_register = zephyr_domain_register_user, + .domain_unregister = zephyr_domain_unregister_user, + .domain_thread_init = zephyr_domain_thread_init, + .domain_thread_free = zephyr_domain_thread_free, +#else + .domain_register = zephyr_domain_register, + .domain_unregister = zephyr_domain_unregister, +#endif +#if CONFIG_CROSS_CORE_STREAM + .domain_block = zephyr_domain_block, + .domain_unblock = zephyr_domain_unblock, +#endif +}; + +/* Core 0 only */ +struct ll_schedule_domain *zephyr_domain_init(int clk) +{ + struct ll_schedule_domain *domain; + struct zephyr_domain *zephyr_domain; + int core; + + domain = domain_init(SOF_SCHEDULE_LL_TIMER, clk, false, + &zephyr_domain_ops); + if (!domain) { + tr_err(&ll_tr, "domain init failed"); + return NULL; + } + +#if CONFIG_SOF_USERSPACE_LL + zephyr_domain = sof_heap_alloc(zephyr_ll_user_heap(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*zephyr_domain), sizeof(void *)); + if (zephyr_domain) + memset(zephyr_domain, 0, sizeof(*zephyr_domain)); +#else + zephyr_domain = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + sizeof(*zephyr_domain)); +#endif + if (!zephyr_domain) { + tr_err(&ll_tr, "domain allocation failed"); + k_panic(); + return NULL; + } + + zephyr_domain->ll_domain = domain; + +#if CONFIG_SOF_USERSPACE_LL + /* Allocate timer dynamically for userspace access */ + zephyr_domain->timer = k_object_alloc(K_OBJ_TIMER); + if (!zephyr_domain->timer) { + tr_err(&ll_tr, "timer allocation failed"); + k_panic(); + return NULL; + } +#else + /* not allocated dynamically when LL in kernel space */ + zephyr_domain->timer = &zephyr_domain->timer_obj; +#endif + +#if CONFIG_CROSS_CORE_STREAM + atomic_set(&zephyr_domain->block, 0); + k_mutex_init(&zephyr_domain->block_mutex); + k_condvar_init(&zephyr_domain->block_condvar); +#endif + + ll_sch_domain_set_pdata(domain, zephyr_domain); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { +#ifndef CONFIG_SOF_USERSPACE_LL + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + /* not allocated dynamically when LL in kernel space */ + dt->sem = &dt->sem_obj; +#endif + } + + return domain; +} diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index 9e10212a1cda..a26e1cea854c 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -4,17 +4,14 @@ // // Author: Tomasz Lauda -#include #include #include -#include #include -#include #include #include -#include #include #include +#include #include #include #include @@ -28,47 +25,9 @@ LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); -/* - * Currently the Zephyr clock rate is part it's Kconfig known at build time. - * SOF on Intel CAVS platforms currently only aligns with Zephyr when both - * use the CAVS 19.2 MHz SSP clock. TODO - needs runtime alignment. +/* The thread and timer setup, which runs in kernel mode, is in + * src/kernel/schedule/zephyr_domain-kernel.c */ -#if CONFIG_XTENSA && CONFIG_CAVS && !CONFIG_INTEL_ADSP_TIMER -#error "Zephyr uses 19.2MHz clock derived from SSP which must be enabled." -#endif - -#define ZEPHYR_LL_STACK_SIZE 8192 - -#if CONFIG_SOF_USERSPACE_LL -K_THREAD_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); -#else -K_KERNEL_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); -#endif - -struct zephyr_domain_thread { - struct k_thread *ll_thread; - struct k_sem *sem; -#ifndef CONFIG_SOF_USERSPACE_LL - struct k_thread ll_thread_obj; - struct k_sem sem_obj; -#endif - void (*handler)(void *arg); - void *arg; -}; - -struct zephyr_domain { - struct k_timer *timer; -#ifndef CONFIG_SOF_USERSPACE_LL - struct k_timer timer_obj; -#endif - struct zephyr_domain_thread domain_thread[CONFIG_CORE_COUNT]; - struct ll_schedule_domain *ll_domain; -#if CONFIG_CROSS_CORE_STREAM - atomic_t block; - struct k_mutex block_mutex; - struct k_condvar block_condvar; -#endif -}; /* perf measurement windows size 2^x */ #define CYCLES_WINDOW_SIZE CONFIG_SCHEDULE_LL_STATS_LOG_WINDOW_SIZE @@ -86,7 +45,7 @@ static inline void stats_report(unsigned int runs, int core, unsigned int cycles } #endif /* CONFIG_SCHEDULE_LL_STATS_LOG */ -static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) +void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) { struct zephyr_domain *zephyr_domain = p1; int core = POINTER_TO_INT(p2); @@ -155,247 +114,14 @@ static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) } } -/* Timer callback: runs in timer IRQ context */ -static void zephyr_domain_timer_fn(struct k_timer *timer) -{ - struct zephyr_domain *zephyr_domain = k_timer_user_data_get(timer); - int core; - - /* - * A race is possible when the Zephyr LL scheduling domain is being - * unregistered while a timer IRQ is processed on a different core. Then - * the timer is removed by the former but then re-added by the latter, - * but this time with no user data and no handler set. This leads to the - * timer continuing to trigger and then leading to a lock up when it is - * registered again next time. - */ - if (!zephyr_domain) { - k_timer_stop(timer); - return; - } - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - if (dt->handler) - k_sem_give(dt->sem); - } -} - -/* The normal kernel-space implementation for register/unregister */ -#ifndef CONFIG_SOF_USERSPACE_LL - -static int zephyr_domain_register(struct ll_schedule_domain *domain, - struct task *task, - void (*handler)(void *arg), void *arg) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - char thread_name[] = "ll_thread0"; - k_tid_t thread; - k_spinlock_key_t key; - - tr_dbg(&ll_tr, "entry"); - - /* domain work only needs registered once on each core */ - if (dt->handler) - return 0; - - dt->handler = handler; - dt->arg = arg; - - /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ - k_sem_init(dt->sem, 0, 10); - - thread_name[sizeof(thread_name) - 2] = '0' + core; - - /* not allocated dynamically when LL in kernel space */ - dt->ll_thread = &dt->ll_thread_obj; - - thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, - zephyr_domain_thread_fn, zephyr_domain, INT_TO_POINTER(core), - NULL, CONFIG_LL_THREAD_PRIORITY, 0, K_FOREVER); - -#ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_pin(thread, core); -#endif - k_thread_name_set(thread, thread_name); - - k_thread_start(thread); - - key = k_spin_lock(&domain->lock); - - if (!k_timer_user_data_get(zephyr_domain->timer)) { - k_timeout_t start = {0}; - - k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); - k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); - - k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); - - /* Enable the watchdog */ - watchdog_enable(core); - } - - k_spin_unlock(&domain->lock, key); - - tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", - domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); - - return 0; -} - -static int zephyr_domain_unregister(struct ll_schedule_domain *domain, - struct task *task, uint32_t num_tasks) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - k_spinlock_key_t key; - - tr_dbg(&ll_tr, "entry"); - - /* tasks still registered on this core */ - if (num_tasks) - return 0; - - key = k_spin_lock(&domain->lock); - - if (!atomic_read(&domain->total_num_tasks)) { - /* Disable the watchdog */ - watchdog_disable(core); - - k_timer_stop(zephyr_domain->timer); - k_timer_user_data_set(zephyr_domain->timer, NULL); - } - - zephyr_domain->domain_thread[core].handler = NULL; - - k_spin_unlock(&domain->lock, key); - - tr_info(&ll_tr, "domain->type %d domain->clk %d", - domain->type, domain->clk); - - /* - * If running in the context of the domain thread, k_thread_abort() will - * not return - */ - k_thread_abort(zephyr_domain->domain_thread[core].ll_thread); - - tr_dbg(&ll_tr, "exit"); - - return 0; -} - -#else /* CONFIG_SOF_USERSPACE_LL */ - -/* - * Kernel-owned per-core LL thread table. Populated from the privileged - * domain-thread init path and consulted by z_impl_zephyr_ll_task_sem_alloc() - * so that privileged code never has to traverse the user-accessible - * scheduler/domain objects to find the LL thread of a given core. - */ -static struct k_thread *ll_thread_tid[CONFIG_CORE_COUNT]; - -/* - * Privileged thread initialization for userspace LL scheduling. - * Creates the scheduling thread, sets up timer, grants access to kernel - * objects. Must be called from kernel context before any user-space - * domain_register() calls. - */ -static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, - struct task *task) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - struct zephyr_domain_thread *dt; - char thread_name[] = "ll_thread0"; - k_tid_t thread; - int core = task->core; - - tr_dbg(&ll_tr, "entry"); - - if (core >= CONFIG_CORE_COUNT) - return -EINVAL; - - dt = zephyr_domain->domain_thread + core; - - /* thread only needs to be created once per core */ - if (dt->ll_thread) - return 0; - - dt->handler = NULL; - dt->arg = NULL; - - dt->sem = k_object_alloc(K_OBJ_SEM); - if (!dt->sem) { - tr_err(&ll_tr, "Failed to allocate semaphore for core %d", core); - return -ENOMEM; - } - - /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ - k_sem_init(dt->sem, 0, 10); - - thread_name[sizeof(thread_name) - 2] = '0' + core; - - /* Allocate thread structure dynamically */ - dt->ll_thread = k_object_alloc(K_OBJ_THREAD); - if (!dt->ll_thread) { - tr_err(&ll_tr, "Failed to allocate thread object for core %d", core); - k_object_free(dt->sem); - dt->sem = NULL; - return -ENOMEM; - } - - thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, - zephyr_domain_thread_fn, zephyr_domain, - INT_TO_POINTER(core), NULL, CONFIG_LL_THREAD_PRIORITY, - K_USER, K_FOREVER); - - /* record in the kernel-only table for syscall-context lookups */ - ll_thread_tid[core] = dt->ll_thread; - -#ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_pin(thread, core); -#endif - k_thread_name_set(thread, thread_name); - - k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); - k_thread_access_grant(thread, dt->sem, domain->lock, zephyr_domain->timer); - user_grant_dai_access_all(thread); - user_grant_dma_access_all(thread); - user_grant_uaol_access_all(thread); - tr_dbg(&ll_tr, "granted LL access to thread %p (core %d)", thread, core); - - k_thread_start(thread); - - k_mutex_lock(domain->lock, K_FOREVER); - if (!k_timer_user_data_get(zephyr_domain->timer)) { - k_timeout_t start = {0}; - - k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); - k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); - - k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); - - /* Enable the watchdog */ - watchdog_enable(core); - } - - k_mutex_unlock(domain->lock); - - tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", - domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); - - return 0; -} - +#ifdef CONFIG_SOF_USERSPACE_LL /* * User-space register: bookkeeping only. The privileged thread setup has * already been done by domain_thread_init() called from kernel context. */ -static int zephyr_domain_register_user(struct ll_schedule_domain *domain, - struct task *task, - void (*handler)(void *arg), void *arg) +int zephyr_domain_register_user(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); struct zephyr_domain_thread *dt; @@ -426,8 +152,8 @@ static int zephyr_domain_register_user(struct ll_schedule_domain *domain, return 0; } -static int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, - struct task *task, uint32_t num_tasks) +int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, + struct task *task, uint32_t num_tasks) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); int core = task->core; @@ -454,94 +180,10 @@ static int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, return 0; } - -/* - * Free resources acquired by zephyr_domain_thread_init(). - * Stops the timer, aborts the scheduling thread and frees the thread object. - * Must be called from kernel context. - */ -static void zephyr_domain_thread_free(struct ll_schedule_domain *domain, - uint32_t num_tasks) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - tr_dbg(&ll_tr, "thread_free entry, core %d, num_tasks %u", core, num_tasks); - - /* Still tasks on other cores, only clean up this core's thread */ - k_mutex_lock(domain->lock, K_FOREVER); - - if (!num_tasks && !atomic_read(&domain->total_num_tasks)) { - /* Last task globally: stop the timer and watchdog */ - watchdog_disable(core); - - k_timer_stop(zephyr_domain->timer); - k_timer_user_data_set(zephyr_domain->timer, NULL); - } - - dt->handler = NULL; - dt->arg = NULL; - - k_mutex_unlock(domain->lock); - - if (dt->ll_thread) { - k_thread_abort(dt->ll_thread); - k_object_free(dt->ll_thread); - dt->ll_thread = NULL; - ll_thread_tid[core] = NULL; - } - - if (dt->sem) { - k_object_free(dt->sem); - dt->sem = NULL; - } - - tr_info(&ll_tr, "thread_free done, core %d", core); -} - -struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - tr_dbg(&ll_tr, "entry"); - - return dt->ll_thread; -} - -/* - * Return the LL scheduling thread for an explicitly given core. - * - * Reads a kernel-only table keyed by core, so it is safe to call from a - * privileged syscall context without dereferencing any user-accessible - * scheduler or domain object, and without relying on cpu_get_id(). - */ -struct k_thread *zephyr_domain_thread_tid_for_core(int core) -{ - if (core < 0 || core >= CONFIG_CORE_COUNT) - return NULL; - - return ll_thread_tid[core]; -} - -struct k_thread *zephyr_ll_domain_thread(void) -{ - struct ll_schedule_domain *ll_domain = zephyr_ll_domain(); - - if (!ll_domain) - return NULL; - - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(ll_domain); - - return zephyr_domain->domain_thread[cpu_get_id()].ll_thread; -} - #endif /* CONFIG_SOF_USERSPACE_LL */ #if CONFIG_CROSS_CORE_STREAM -static void zephyr_domain_block(struct ll_schedule_domain *domain) +void zephyr_domain_block(struct ll_schedule_domain *domain) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); @@ -552,7 +194,7 @@ static void zephyr_domain_block(struct ll_schedule_domain *domain) k_mutex_unlock(&zephyr_domain->block_mutex); } -static void zephyr_domain_unblock(struct ll_schedule_domain *domain) +void zephyr_domain_unblock(struct ll_schedule_domain *domain) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); @@ -565,87 +207,6 @@ static void zephyr_domain_unblock(struct ll_schedule_domain *domain) } #endif /* CONFIG_CROSS_CORE_STREAM */ -APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { -#ifdef CONFIG_SOF_USERSPACE_LL - .domain_register = zephyr_domain_register_user, - .domain_unregister = zephyr_domain_unregister_user, - .domain_thread_init = zephyr_domain_thread_init, - .domain_thread_free = zephyr_domain_thread_free, -#else - .domain_register = zephyr_domain_register, - .domain_unregister = zephyr_domain_unregister, -#endif -#if CONFIG_CROSS_CORE_STREAM - .domain_block = zephyr_domain_block, - .domain_unblock = zephyr_domain_unblock, -#endif -}; - -/* Core 0 only */ -struct ll_schedule_domain *zephyr_domain_init(int clk) -{ - struct ll_schedule_domain *domain; - struct zephyr_domain *zephyr_domain; - int core; - - domain = domain_init(SOF_SCHEDULE_LL_TIMER, clk, false, - &zephyr_domain_ops); - if (!domain) { - tr_err(&ll_tr, "domain init failed"); - return NULL; - } - -#if CONFIG_SOF_USERSPACE_LL - zephyr_domain = sof_heap_alloc(zephyr_ll_user_heap(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*zephyr_domain), sizeof(void *)); - if (zephyr_domain) - memset(zephyr_domain, 0, sizeof(*zephyr_domain)); -#else - zephyr_domain = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, - sizeof(*zephyr_domain)); -#endif - if (!zephyr_domain) { - tr_err(&ll_tr, "domain allocation failed"); - k_panic(); - return NULL; - } - - zephyr_domain->ll_domain = domain; - -#if CONFIG_SOF_USERSPACE_LL - /* Allocate timer dynamically for userspace access */ - zephyr_domain->timer = k_object_alloc(K_OBJ_TIMER); - if (!zephyr_domain->timer) { - tr_err(&ll_tr, "timer allocation failed"); - k_panic(); - return NULL; - } -#else - /* not allocated dynamically when LL in kernel space */ - zephyr_domain->timer = &zephyr_domain->timer_obj; -#endif - -#if CONFIG_CROSS_CORE_STREAM - atomic_set(&zephyr_domain->block, 0); - k_mutex_init(&zephyr_domain->block_mutex); - k_condvar_init(&zephyr_domain->block_condvar); -#endif - - ll_sch_domain_set_pdata(domain, zephyr_domain); - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { -#ifndef CONFIG_SOF_USERSPACE_LL - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - /* not allocated dynamically when LL in kernel space */ - dt->sem = &dt->sem_obj; -#endif - } - - return domain; -} - /* Check if currently running in the LL scheduler thread context */ bool ll_sch_is_current(void) { From f04bb227bd492df2d68f88e31097af1f2683708b Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:14:23 +0000 Subject: [PATCH 12/19] schedule: split the kernel part of zephyr_ll.c into src/kernel/schedule zephyr_ll.c is the LL scheduler, which runs in the LL scheduler threads and, with user-space LL, in user mode. Its setup and a few other parts need kernel mode. Move them to src/kernel/schedule/zephyr_ll-kernel.c: - zephyr_ll_scheduler_init() and the scheduler ops, which allocate the per-core scheduler data and lock and register the scheduler at boot; - the kernel half of the zephyr_ll_task_sem_alloc() and zephyr_ll_task_sem_free() system calls, which allocate the per-task semaphores as kernel objects and grant the LL thread access to them; - zephyr_ll_init_context(), which sets up the LL thread of a core and grants it access to the scheduler lock and the task semaphore; - user_ll_grant_access(), which grants a thread access to the scheduler lock of a core. The scheduler and per-task data structures, the per-core lock table and the scheduler ops are declared in sof/schedule/zephyr_ll.h. The kernel part is built under the same condition as zephyr_ll.c. The code is unchanged, but the compiler inlines differently in the smaller zephyr_ll.c: in builds with user-space support, zephyr_ll_task_done() now inlines the system call wrapper of a log message instead of calling an out-of-line copy of it, and the register allocation of zephyr_ll_task_free() and zephyr_ll_task_init() differs. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/schedule/zephyr_ll.h | 44 ++++ src/kernel/schedule/CMakeLists.txt | 7 + src/kernel/schedule/zephyr_ll-kernel.c | 274 +++++++++++++++++++++++ src/schedule/zephyr_ll.c | 291 ++----------------------- 4 files changed, 339 insertions(+), 277 deletions(-) create mode 100644 src/kernel/schedule/zephyr_ll-kernel.c diff --git a/src/include/sof/schedule/zephyr_ll.h b/src/include/sof/schedule/zephyr_ll.h index d3d1e36ac1ee..e86ec13731f5 100644 --- a/src/include/sof/schedule/zephyr_ll.h +++ b/src/include/sof/schedule/zephyr_ll.h @@ -12,11 +12,55 @@ #ifndef __SOF_SCHEDULE_ZEPHYR_LL_H__ #define __SOF_SCHEDULE_ZEPHYR_LL_H__ +#include #include #include #include +#include #include +/* per-scheduler data */ +struct zephyr_ll { + struct list_item tasks; /* list of ll tasks */ + unsigned int n_tasks; /* task counter */ + struct ll_schedule_domain *ll_domain; /* scheduling domain */ + unsigned int core; /* core ID of this instance */ +#if CONFIG_SOF_USERSPACE_LL + struct k_mutex *lock; /* mutex for userspace */ +#endif + struct k_heap *heap; +}; + +/* per-task scheduler data */ +struct zephyr_ll_pdata { + bool run; + bool freeing; + struct k_sem *sem_p; +#if !CONFIG_DYNAMIC_OBJECTS + struct k_sem sem; +#endif +}; + +/* LL scheduler ops, see zephyr_ll.c */ +int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start, + uint64_t period); +int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *before); +int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *after); +int zephyr_ll_task_free(void *data, struct task *task); +int zephyr_ll_task_cancel(void *data, struct task *task); +#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY +void zephyr_ll_scheduler_free(void *data, uint32_t flags); +#endif + +#if CONFIG_SOF_USERSPACE_LL +/* per-core LL scheduler locks, see zephyr_ll.c */ +extern struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; + +struct k_thread *zephyr_ll_init_context(void *data, struct task *task); +#endif + struct zephyr_domain_thread { struct k_thread *ll_thread; struct k_sem *sem; diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index 6b0d7011f409..5fb06ef0adaa 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -6,6 +6,13 @@ if(zephyr) zephyr_domain-kernel.c ) + # The kernel part of zephyr_ll.c, built with it, see src/schedule/CMakeLists.txt + if(NOT CONFIG_SOC_ACP_6_0 OR CONFIG_DMA_DOMAIN) + zephyr_library_sources( + zephyr_ll-kernel.c + ) + endif() + zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_LL zephyr_ll_user.c ) diff --git a/src/kernel/schedule/zephyr_ll-kernel.c b/src/kernel/schedule/zephyr_ll-kernel.c new file mode 100644 index 000000000000..4118733699d4 --- /dev/null +++ b/src/kernel/schedule/zephyr_ll-kernel.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Guennadi Liakhovetski + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); + +/* Kernel part of the Zephyr LL scheduler: the scheduler initialization, the task + * semaphore syscalls and the access grants to the scheduler objects. The rest is in + * src/schedule/zephyr_ll.c + */ + +#if CONFIG_DYNAMIC_OBJECTS +static struct list_item zephyr_ll_task_sem_list = LIST_INIT(zephyr_ll_task_sem_list); + +struct zephyr_ll_task_sem { + struct task *task; + struct k_sem *sem; + struct list_item list; +}; + +/* + * The syscall validators pass in pdata that they read once from user memory + * and validated, so these helpers must not read task->priv_data again. + */ +static int zephyr_ll_task_sem_alloc_pdata(struct task *task, struct zephyr_ll_pdata *pdata) +{ + struct zephyr_ll_task_sem *ts = rmalloc(SOF_MEM_FLAG_COHERENT, sizeof(*ts)); + + if (!ts) + return -ENOMEM; + + ts->sem = k_object_alloc(K_OBJ_SEM); + if (!ts->sem) { + rfree(ts); + return -ENOMEM; + } + + k_sem_init(ts->sem, 0, 1); + +#if CONFIG_SOF_USERSPACE_LL + /* + * The per-task semaphore is signalled from zephyr_ll_task_done(), + * which runs in the (unprivileged) LL scheduler thread when a task is + * freed while it is still running. k_object_alloc() only grants access + * to the calling thread (the IPC handler that creates the task), so the + * LL thread must be granted access explicitly, otherwise its + * k_sem_give() traps with a userspace permission fault. + * + * Resolve the LL thread from kernel-only per-core state keyed by the + * task's target core; never traverse the user-accessible scheduler or + * domain objects from privileged context. + */ + struct k_thread *ll_tid = zephyr_domain_thread_tid_for_core(task->core); + + if (ll_tid) + k_thread_access_grant(ll_tid, ts->sem); +#endif /* CONFIG_SOF_USERSPACE_LL */ + + ts->task = task; + pdata->sem_p = ts->sem; + /* List is protected by IPC serialization */ + list_item_append(&ts->list, &zephyr_ll_task_sem_list); + + return 0; +} + +int z_impl_zephyr_ll_task_sem_alloc(struct task *task) +{ + return zephyr_ll_task_sem_alloc_pdata(task, task->priv_data); +} + +static int zephyr_ll_task_sem_free_pdata(struct task *task, struct zephyr_ll_pdata *pdata) +{ + struct list_item *list; + struct zephyr_ll_task_sem *ts; + bool found = false; + + /* List is protected by IPC serialization */ + list_for_item(list, &zephyr_ll_task_sem_list) { + ts = container_of(list, struct zephyr_ll_task_sem, list); + if (ts->task == task) { + found = true; + break; + } + } + + if (!found) + return -ENOENT; + + if (pdata->sem_p != ts->sem) + return -EINVAL; + + list_item_del(list); + k_object_free(ts->sem); + rfree(ts); + + return 0; +} + +int z_impl_zephyr_ll_task_sem_free(struct task *task) +{ + return zephyr_ll_task_sem_free_pdata(task, task->priv_data); +} + +#ifdef CONFIG_USERSPACE +#include +/* + * Read task->priv_data exactly once. The task lives in user memory and a + * thread on another core can change the pointer after it has been checked. + * + * @return validated pdata pointer, or NULL if the task has none + */ +static struct zephyr_ll_pdata *zephyr_ll_task_pdata_vrfy(struct task *task) +{ + struct zephyr_ll_pdata *pdata; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); + K_OOPS(k_usermode_from_copy(&pdata, &task->priv_data, sizeof(pdata))); + if (pdata) + K_OOPS(K_SYSCALL_MEMORY_WRITE(pdata, sizeof(*pdata))); + + return pdata; +} + +static inline int z_vrfy_zephyr_ll_task_sem_alloc(struct task *task) +{ + struct zephyr_ll_pdata *pdata; + + if (!task) + return -EINVAL; + pdata = zephyr_ll_task_pdata_vrfy(task); + if (!pdata) + return -EINVAL; + + return zephyr_ll_task_sem_alloc_pdata(task, pdata); +} +#include + +static inline int z_vrfy_zephyr_ll_task_sem_free(struct task *task) +{ + struct zephyr_ll_pdata *pdata; + + if (!task) + return -EINVAL; + pdata = zephyr_ll_task_pdata_vrfy(task); + if (!pdata) + return -EINVAL; + + return zephyr_ll_task_sem_free_pdata(task, pdata); +} +#include +#endif /* CONFIG_USERSPACE */ +#endif /* CONFIG_DYNAMIC_OBJECTS */ + +#if CONFIG_SOF_USERSPACE_LL +struct k_thread *zephyr_ll_init_context(void *data, struct task *task) +{ + struct zephyr_ll *sch = data; + struct zephyr_ll_pdata *pdata = task->priv_data; + int ret; + + /* + * Use domain_thread_init() for privileged setup (thread creation, + * timer, access grants). domain_register() is now bookkeeping only + * and will be called later from user context when scheduling tasks. + */ + ret = domain_thread_init(sch->ll_domain, task); + if (ret < 0) { + tr_err(&ll_tr, "cannot init_context %d", ret); + return NULL; + } + + assert(!k_is_user_context()); + k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock, pdata->sem_p); + + tr_dbg(&ll_tr, "granting access to lock %p for thread %p", sch->lock, + zephyr_domain_thread_tid(sch->ll_domain)); + tr_dbg(&ll_tr, "granting access to domain lock %p for thread %p", &sch->ll_domain->lock, + zephyr_domain_thread_tid(sch->ll_domain)); + + return zephyr_domain_thread_tid(sch->ll_domain); +} + +void user_ll_grant_access(struct k_thread *thread, int core) +{ + assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] != NULL); + + k_thread_access_grant(thread, zephyr_ll_locks[core]); +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +static const struct scheduler_ops zephyr_ll_ops = { + .schedule_task = zephyr_ll_task_schedule, + .schedule_task_before = zephyr_ll_task_schedule_before, + .schedule_task_after = zephyr_ll_task_schedule_after, + .schedule_task_free = zephyr_ll_task_free, + .schedule_task_cancel = zephyr_ll_task_cancel, +#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY + .scheduler_free = zephyr_ll_scheduler_free, +#endif +#if CONFIG_SOF_USERSPACE_LL + .scheduler_init_context = zephyr_ll_init_context, +#endif +}; + +/* TODO: low-power mode clock support */ +/* Runs on each core during initialisation with the same domain argument */ +__cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) +{ + struct zephyr_ll *sch; + int core = cpu_get_id(); + struct k_heap *heap = sof_sys_heap_get(); + int flags = SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT; + + assert_can_be_cold(); + +#if CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); + flags = SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT; +#endif + tr_dbg(&ll_tr, "init on core %d", core); + + /* initialize per-core scheduler private data */ + sch = sof_heap_alloc(heap, flags, sizeof(*sch), 0); + if (!sch) { + tr_err(&ll_tr, "allocation failed"); + return -ENOMEM; + } + + memset(sch, 0, sizeof(*sch)); + + list_init(&sch->tasks); + sch->ll_domain = domain; + sch->core = core; + sch->n_tasks = 0; + sch->heap = heap; + +#if CONFIG_SOF_USERSPACE_LL + /* Allocate mutex dynamically for userspace access */ + sch->lock = k_object_alloc(K_OBJ_MUTEX); + if (!sch->lock) { + tr_err(&ll_tr, "mutex allocation failed"); + sof_heap_free(sch->heap, sch); + return -ENOMEM; + } + assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] == NULL); + zephyr_ll_locks[core] = sch->lock; + k_mutex_init(sch->lock); + + tr_dbg(&ll_tr, "ll-scheduler init done, sch %p sch->lock %p", sch, sch->lock); +#endif + + scheduler_init(domain->type, &zephyr_ll_ops, sch); + + return 0; +} diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 3edebbca5b20..b8d5906e6bbf 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -26,27 +27,9 @@ SOF_DEFINE_REG_UUID(zll_sched); /* unused with Zephyr, generates no output */ DECLARE_TR_CTX(ll_tr, SOF_UUID(zll_sched_uuid), LOG_LEVEL_INFO); -/* per-scheduler data */ -struct zephyr_ll { - struct list_item tasks; /* list of ll tasks */ - unsigned int n_tasks; /* task counter */ - struct ll_schedule_domain *ll_domain; /* scheduling domain */ - unsigned int core; /* core ID of this instance */ -#if CONFIG_SOF_USERSPACE_LL - struct k_mutex *lock; /* mutex for userspace */ -#endif - struct k_heap *heap; -}; - -/* per-task scheduler data */ -struct zephyr_ll_pdata { - bool run; - bool freeing; - struct k_sem *sem_p; -#if !CONFIG_DYNAMIC_OBJECTS - struct k_sem sem; -#endif -}; +/* The scheduler syscalls and access grants, which run in kernel mode, are in + * src/kernel/schedule/zephyr_ll-kernel.c + */ #if CONFIG_SOF_USERSPACE_LL /* @@ -54,7 +37,7 @@ struct zephyr_ll_pdata { * can read the pointer for syscalls. The actual lock object resides * in kernel memory, there are just handles! */ -static APP_SYSUSER_BSS struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; +APP_SYSUSER_BSS struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; /* * Shadow ownership tracking for the per-core LL mutex, used only to @@ -426,178 +409,35 @@ static int zephyr_ll_task_schedule_common(struct zephyr_ll *sch, struct task *ta return 0; } -static int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start, - uint64_t period) +int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start, + uint64_t period) { struct zephyr_ll *sch = data; return zephyr_ll_task_schedule_common(sch, task, start, period, NULL, false); } -static int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start, - uint64_t period, struct task *before) +int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *before) { struct zephyr_ll *sch = data; return zephyr_ll_task_schedule_common(sch, task, start, period, before, true); } -static int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start, - uint64_t period, struct task *after) +int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *after) { struct zephyr_ll *sch = data; return zephyr_ll_task_schedule_common(sch, task, start, period, after, false); } -#if CONFIG_DYNAMIC_OBJECTS -static struct list_item zephyr_ll_task_sem_list = LIST_INIT(zephyr_ll_task_sem_list); - -struct zephyr_ll_task_sem { - struct task *task; - struct k_sem *sem; - struct list_item list; -}; - -/* - * The syscall validators pass in pdata that they read once from user memory - * and validated, so these helpers must not read task->priv_data again. - */ -static int zephyr_ll_task_sem_alloc_pdata(struct task *task, struct zephyr_ll_pdata *pdata) -{ - struct zephyr_ll_task_sem *ts = rmalloc(SOF_MEM_FLAG_COHERENT, sizeof(*ts)); - - if (!ts) - return -ENOMEM; - - ts->sem = k_object_alloc(K_OBJ_SEM); - if (!ts->sem) { - rfree(ts); - return -ENOMEM; - } - - k_sem_init(ts->sem, 0, 1); - -#if CONFIG_SOF_USERSPACE_LL - /* - * The per-task semaphore is signalled from zephyr_ll_task_done(), - * which runs in the (unprivileged) LL scheduler thread when a task is - * freed while it is still running. k_object_alloc() only grants access - * to the calling thread (the IPC handler that creates the task), so the - * LL thread must be granted access explicitly, otherwise its - * k_sem_give() traps with a userspace permission fault. - * - * Resolve the LL thread from kernel-only per-core state keyed by the - * task's target core; never traverse the user-accessible scheduler or - * domain objects from privileged context. - */ - struct k_thread *ll_tid = zephyr_domain_thread_tid_for_core(task->core); - - if (ll_tid) - k_thread_access_grant(ll_tid, ts->sem); -#endif /* CONFIG_SOF_USERSPACE_LL */ - - ts->task = task; - pdata->sem_p = ts->sem; - /* List is protected by IPC serialization */ - list_item_append(&ts->list, &zephyr_ll_task_sem_list); - - return 0; -} - -int z_impl_zephyr_ll_task_sem_alloc(struct task *task) -{ - return zephyr_ll_task_sem_alloc_pdata(task, task->priv_data); -} - -static int zephyr_ll_task_sem_free_pdata(struct task *task, struct zephyr_ll_pdata *pdata) -{ - struct list_item *list; - struct zephyr_ll_task_sem *ts; - bool found = false; - - /* List is protected by IPC serialization */ - list_for_item(list, &zephyr_ll_task_sem_list) { - ts = container_of(list, struct zephyr_ll_task_sem, list); - if (ts->task == task) { - found = true; - break; - } - } - - if (!found) - return -ENOENT; - - if (pdata->sem_p != ts->sem) - return -EINVAL; - - list_item_del(list); - k_object_free(ts->sem); - rfree(ts); - - return 0; -} - -int z_impl_zephyr_ll_task_sem_free(struct task *task) -{ - return zephyr_ll_task_sem_free_pdata(task, task->priv_data); -} - -#ifdef CONFIG_USERSPACE -#include -/* - * Read task->priv_data exactly once. The task lives in user memory and a - * thread on another core can change the pointer after it has been checked. - * - * @return validated pdata pointer, or NULL if the task has none - */ -static struct zephyr_ll_pdata *zephyr_ll_task_pdata_vrfy(struct task *task) -{ - struct zephyr_ll_pdata *pdata; - - K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); - K_OOPS(k_usermode_from_copy(&pdata, &task->priv_data, sizeof(pdata))); - if (pdata) - K_OOPS(K_SYSCALL_MEMORY_WRITE(pdata, sizeof(*pdata))); - - return pdata; -} - -static inline int z_vrfy_zephyr_ll_task_sem_alloc(struct task *task) -{ - struct zephyr_ll_pdata *pdata; - - if (!task) - return -EINVAL; - pdata = zephyr_ll_task_pdata_vrfy(task); - if (!pdata) - return -EINVAL; - - return zephyr_ll_task_sem_alloc_pdata(task, pdata); -} -#include - -static inline int z_vrfy_zephyr_ll_task_sem_free(struct task *task) -{ - struct zephyr_ll_pdata *pdata; - - if (!task) - return -EINVAL; - pdata = zephyr_ll_task_pdata_vrfy(task); - if (!pdata) - return -EINVAL; - - return zephyr_ll_task_sem_free_pdata(task, pdata); -} -#include -#endif /* CONFIG_USERSPACE */ -#endif /* CONFIG_DYNAMIC_OBJECTS */ - /* * This is synchronous - after this returns the object can be destroyed! * Assertion: under Zephyr this is always called from a thread context! */ -static int zephyr_ll_task_free(void *data, struct task *task) +int zephyr_ll_task_free(void *data, struct task *task) { struct zephyr_ll *sch = data; uint32_t flags; @@ -665,7 +505,7 @@ static int zephyr_ll_task_free(void *data, struct task *task) } /* This is asynchronous */ -static int zephyr_ll_task_cancel(void *data, struct task *task) +int zephyr_ll_task_cancel(void *data, struct task *task) { struct zephyr_ll *sch = data; uint32_t flags; @@ -695,7 +535,7 @@ static int zephyr_ll_task_cancel(void *data, struct task *task) } #if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY -static void zephyr_ll_scheduler_free(void *data, uint32_t flags) +void zephyr_ll_scheduler_free(void *data, uint32_t flags) { struct zephyr_ll *sch = data; @@ -717,50 +557,6 @@ static void zephyr_ll_scheduler_free(void *data, uint32_t flags) } #endif /* CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY */ -#if CONFIG_SOF_USERSPACE_LL -struct k_thread *zephyr_ll_init_context(void *data, struct task *task) -{ - struct zephyr_ll *sch = data; - struct zephyr_ll_pdata *pdata = task->priv_data; - int ret; - - /* - * Use domain_thread_init() for privileged setup (thread creation, - * timer, access grants). domain_register() is now bookkeeping only - * and will be called later from user context when scheduling tasks. - */ - ret = domain_thread_init(sch->ll_domain, task); - if (ret < 0) { - tr_err(&ll_tr, "cannot init_context %d", ret); - return NULL; - } - - assert(!k_is_user_context()); - k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock, pdata->sem_p); - - tr_dbg(&ll_tr, "granting access to lock %p for thread %p", sch->lock, - zephyr_domain_thread_tid(sch->ll_domain)); - tr_dbg(&ll_tr, "granting access to domain lock %p for thread %p", &sch->ll_domain->lock, - zephyr_domain_thread_tid(sch->ll_domain)); - - return zephyr_domain_thread_tid(sch->ll_domain); -} -#endif /* CONFIG_SOF_USERSPACE_LL */ - -static const struct scheduler_ops zephyr_ll_ops = { - .schedule_task = zephyr_ll_task_schedule, - .schedule_task_before = zephyr_ll_task_schedule_before, - .schedule_task_after = zephyr_ll_task_schedule_after, - .schedule_task_free = zephyr_ll_task_free, - .schedule_task_cancel = zephyr_ll_task_cancel, -#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY - .scheduler_free = zephyr_ll_scheduler_free, -#endif -#if CONFIG_SOF_USERSPACE_LL - .scheduler_init_context = zephyr_ll_init_context, -#endif -}; - #if CONFIG_SOF_USERSPACE_LL struct task *zephyr_ll_task_alloc(void) { @@ -774,13 +570,6 @@ struct task *zephyr_ll_task_alloc(void) return task; } -void user_ll_grant_access(struct k_thread *thread, int core) -{ - assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] != NULL); - - k_thread_access_grant(thread, zephyr_ll_locks[core]); -} - /** * Lock the LL scheduler to prevent it from processing tasks. * @@ -866,58 +655,6 @@ int zephyr_ll_task_init(struct task *task, } EXPORT_SYMBOL(zephyr_ll_task_init); -/* TODO: low-power mode clock support */ -/* Runs on each core during initialisation with the same domain argument */ -__cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) -{ - struct zephyr_ll *sch; - int core = cpu_get_id(); - struct k_heap *heap = sof_sys_heap_get(); - int flags = SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT; - - assert_can_be_cold(); - -#if CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); - flags = SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT; -#endif - tr_dbg(&ll_tr, "init on core %d", core); - - /* initialize per-core scheduler private data */ - sch = sof_heap_alloc(heap, flags, sizeof(*sch), 0); - if (!sch) { - tr_err(&ll_tr, "allocation failed"); - return -ENOMEM; - } - - memset(sch, 0, sizeof(*sch)); - - list_init(&sch->tasks); - sch->ll_domain = domain; - sch->core = core; - sch->n_tasks = 0; - sch->heap = heap; - -#if CONFIG_SOF_USERSPACE_LL - /* Allocate mutex dynamically for userspace access */ - sch->lock = k_object_alloc(K_OBJ_MUTEX); - if (!sch->lock) { - tr_err(&ll_tr, "mutex allocation failed"); - sof_heap_free(sch->heap, sch); - return -ENOMEM; - } - assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] == NULL); - zephyr_ll_locks[core] = sch->lock; - k_mutex_init(sch->lock); - - tr_dbg(&ll_tr, "ll-scheduler init done, sch %p sch->lock %p", sch, sch->lock); -#endif - - scheduler_init(domain->type, &zephyr_ll_ops, sch); - - return 0; -} - void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, uint32_t *data_off_size) { From c1e66b984f98fc212c0b2594e23b9ca43e81460e Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:26:02 +0000 Subject: [PATCH 13/19] schedule: move the DP scheduler private header to src/include zephyr_dp_schedule.h holds the DP scheduler data structures and helpers shared by the DP scheduler sources. The kernel parts of those sources move to src/kernel/schedule in the next patches and need it too, so move it to sof/schedule/zephyr_dp_schedule.h. Add the include guard that it did not have. The generated code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/{ => include/sof}/schedule/zephyr_dp_schedule.h | 10 +++++++++- src/schedule/zephyr_dp_schedule.c | 2 +- src/schedule/zephyr_dp_schedule_application.c | 2 +- src/schedule/zephyr_dp_schedule_thread.c | 2 +- 4 files changed, 12 insertions(+), 4 deletions(-) rename src/{ => include/sof}/schedule/zephyr_dp_schedule.h (92%) diff --git a/src/schedule/zephyr_dp_schedule.h b/src/include/sof/schedule/zephyr_dp_schedule.h similarity index 92% rename from src/schedule/zephyr_dp_schedule.h rename to src/include/sof/schedule/zephyr_dp_schedule.h index 07d8ee4a4754..f84e1901d5ae 100644 --- a/src/schedule/zephyr_dp_schedule.h +++ b/src/include/sof/schedule/zephyr_dp_schedule.h @@ -6,11 +6,17 @@ */ /** - * @file src/schedule/zephyr_dp_schedule.h + * @file include/sof/schedule/zephyr_dp_schedule.h * @brief Internal Data Processing (DP) scheduler structures, memory partitions, and thread helpers. * @ingroup scheduling_api + * + * Shared by the DP scheduler sources in src/schedule and their kernel parts in + * src/kernel/schedule. Not for use outside the scheduler. */ +#ifndef __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ +#define __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ + #include #include #include @@ -102,3 +108,5 @@ void scheduler_dp_unlock(unsigned int key); * @param[in] core Target DSP core ID. */ void scheduler_dp_grant(k_tid_t thread_id, uint16_t core); + +#endif /* __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ */ diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 314f8df5bba0..2e76b214ad33 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -24,7 +24,7 @@ #include #include -#include "zephyr_dp_schedule.h" +#include #include diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 5e69d0943419..b98b9129bcae 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -29,7 +29,7 @@ #include #include -#include "zephyr_dp_schedule.h" +#include LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx dp_tr; diff --git a/src/schedule/zephyr_dp_schedule_thread.c b/src/schedule/zephyr_dp_schedule_thread.c index fa8583832a03..f742e215a750 100644 --- a/src/schedule/zephyr_dp_schedule_thread.c +++ b/src/schedule/zephyr_dp_schedule_thread.c @@ -24,7 +24,7 @@ #include #include -#include "zephyr_dp_schedule.h" +#include LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx dp_tr; From 3b03c51ad55c8f9b8a6d6289f41cb41977adfec1 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:28:55 +0000 Subject: [PATCH 14/19] schedule: split the kernel part of zephyr_dp_schedule.c into src/kernel/schedule zephyr_dp_schedule.c holds the DP scheduler code common to its thread and application variants. Part of it needs kernel mode. Move it to src/kernel/schedule/zephyr_dp_schedule-kernel.c: - the per-core DP scheduler locks, which are kernel objects, and scheduler_dp_grant(), which grants threads access to them; - the scheduler_dp_ll_tick() system call, with the description of the DP scheduling it drives; - scheduler_dp_init(), which allocates the per-core scheduler data, registers the scheduler and its ops at boot and sets up the LL tick task, together with the scheduler UUID, trace context and log module. zephyr_dp_schedule.c keeps the lock and unlock helpers, the task schedule, stop, cancel and free ops and the task info query. The locks and the ops are declared in sof/schedule/zephyr_dp_schedule.h. The code is unchanged, but z_impl_scheduler_dp_ll_tick() now calls the lock and unlock helpers instead of inlining them, and in the application variant scheduler_dp_task_free() calls scheduler_dp_task_stop() instead of inlining it. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/schedule/zephyr_dp_schedule.h | 14 + src/kernel/schedule/CMakeLists.txt | 4 + .../schedule/zephyr_dp_schedule-kernel.c | 279 ++++++++++++++++++ src/schedule/zephyr_dp_schedule.c | 254 +--------------- 4 files changed, 306 insertions(+), 245 deletions(-) create mode 100644 src/kernel/schedule/zephyr_dp_schedule-kernel.c diff --git a/src/include/sof/schedule/zephyr_dp_schedule.h b/src/include/sof/schedule/zephyr_dp_schedule.h index f84e1901d5ae..1056efbfec4c 100644 --- a/src/include/sof/schedule/zephyr_dp_schedule.h +++ b/src/include/sof/schedule/zephyr_dp_schedule.h @@ -72,6 +72,20 @@ struct task_dp_pdata { #endif }; +/** @brief Per-core DP scheduler locks, see scheduler_dp_lock(). */ +extern struct k_sem dp_lock[CONFIG_MP_MAX_NUM_CPUS]; + +/** + * @brief DP scheduler ops, see struct scheduler_ops. + */ +int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, + uint64_t period); +int scheduler_dp_task_stop(void *data, struct task *task); +int scheduler_dp_task_free(void *data, struct task *task); +#if CONFIG_SOF_USERSPACE_APPLICATION +int scheduler_dp_task_cancel(void *data, struct task *task); +#endif + /** * @brief Recalculate deadlines and evaluate readiness for all active DP tasks on an LL tick. * @param[in,out] dp_sch Pointer to per-core DP scheduler state container. diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index 5fb06ef0adaa..5c878b19f3c3 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -16,4 +16,8 @@ if(zephyr) zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_LL zephyr_ll_user.c ) + + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER + zephyr_dp_schedule-kernel.c + ) endif() diff --git a/src/kernel/schedule/zephyr_dp_schedule-kernel.c b/src/kernel/schedule/zephyr_dp_schedule-kernel.c new file mode 100644 index 000000000000..2fd035e106ea --- /dev/null +++ b/src/kernel/schedule/zephyr_dp_schedule-kernel.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2023 Intel Corporation. All rights reserved. + * + * Author: Marcin Szkudlinski + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include + +LOG_MODULE_REGISTER(dp_schedule, CONFIG_SOF_LOG_LEVEL); +SOF_DEFINE_REG_UUID(dp_sched); + +/* unused with Zephyr, generates no output */ +DECLARE_TR_CTX(dp_tr, SOF_UUID(dp_sched_uuid), LOG_LEVEL_INFO); + +/* Kernel part of the DP scheduler: the scheduler locks, their access grants, the LL + * tick system call and the scheduler initialization. The rest is in + * src/schedule/zephyr_dp_schedule.c + */ + +#define DP_LOCK_INIT(i, _) Z_SEM_INITIALIZER(dp_lock[i], 1, 1) +#define DP_LOCK_INIT_LIST LISTIFY(CONFIG_MP_MAX_NUM_CPUS, DP_LOCK_INIT, (,)) + +/* User threads don't need access to this array. Access is performed from + * the kernel space via a syscall. Array must be placed in special section + * to be qualified as initialized by the gen_kobject_list.py script. + */ +STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_lock, CONFIG_MP_MAX_NUM_CPUS) = { DP_LOCK_INIT_LIST }; + +void scheduler_dp_grant(k_tid_t thread_id, uint16_t core) +{ + if (thread_id) + k_thread_access_grant(thread_id, &dp_lock[core]); +} + +/* dummy LL task - to start LL on secondary cores */ +static enum task_state scheduler_dp_ll_tick_dummy(void *data) +{ + return SOF_TASK_STATE_RESCHEDULE; +} + +/* + * function called after every LL tick + * + * This function checks if the queued DP tasks are ready to processing (meaning + * the module run by the task has enough data at all sources and enough free space + * on all sinks) + * + * if the task becomes ready, a deadline is set allowing Zephyr to schedule threads + * in right order + * + * TODO: currently there's a limitation - DP module must be surrounded by LL modules. + * it simplifies algorithm - there's no need to browse through DP chains calculating + * deadlines for each module in function of all modules execution status. + * Now is simple - modules deadline is its start + tick time. + * + * example: + * Lets assume we do have a pipeline: + * + * LL1 -> DP1 -> LL2 -> DP2 -> LL3 -> DP3 -> LL4 + * + * all LLs starts in 1ms tick + * + * for simplification lets assume + * - all LLs are on primary core, all DPs on secondary (100% CPU is for DP) + * - context switching requires 0 cycles + * + * DP1 - starts every 1ms, needs 0.5ms to finish processing + * DP2 - starts every 2ms, needs 0.6ms to finish processing + * DP3 - starts every 10ms, needs 0.3ms to finish processing + * + * TICK0 + * only LL1 is ready to run + * LL1 processing (producing data chunk for DP1) + * + * TICK1 + * LL1 is ready to run + * DP1 is ready tu run (has data from LL1) set deadline to TICK2 + * LL1 processing (producing second data chunk for DP1) + * DP1 processing for 0.5ms (consuming first data chunk, producing data chunk for LL2) + * CPU is idle for 0.5ms + * + * TICK2 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK3 + * LL2 is ready to run + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * CPU is idle for 0.5ms + * + * TICK3 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK4 + * LL2 is ready to run + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * CPU is idle for 0.5ms + * + * TICK4 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK5 + * LL2 is ready to run + * DP2 is ready to run set deadline to TICK6 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% of second data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used + * + * !!!!!! Note here - DP1 must do before DP2 as it MUST finish in this tick. DP2 can wait + * >>>>>>> this is what we call EDF - EARIEST DEADLINE FIRST <<<<<< + * + * TICK5 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK6 + * LL2 is ready to run + * DP2 is in progress, deadline is set to TICK6 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of second data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.1ms (producing TWO data chunks for LL3) + * CPU is idle for 0.4ms (60% used) + * + * TICK6 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK7 + * LL2 is ready to run + * DP2 is ready to run set deadline to TICK8 + * LL3 is ready to run + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% of second data chunk for DP2) + * LL3 processing (producing 10% of first data chunk for DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used + * + * + * + * (........ 9 more cycles - LL3 procuces 100% of data for DP3......) + * + * + * TICK15 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK16 + * LL2 is ready to run + * DP2 is ready to run set deadline to TICK17 + * LL3 is ready to run + * DP3 is ready to run set deadline to TICK25 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% of data chunk for DP2) + * LL3 processing (producing 10% of second data chunk for DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used - + * !!! note that DP3 is ready but has no chance to get CPU in this cycle + * + * TICK16 + * LL1 is ready to run set deadline to TICK17 + * DP1 is ready tu run + * LL2 is ready to run + * DP2 is in progress, deadline is set to TICK17 + * LL3 is ready to run + * DP3 is in progress, deadline is set to TICK25 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of data chunk for DP2) + * LL3 processing (producing 10% of second data chunk for DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.1ms (producing data) + * DP3 processing for 0.2ms (producing 10 data chunks for LL4) + * 90% CPU used + * + * TICK17 + * LL1 is ready to run + * DP1 is ready tu run + * LL2 is ready to run + * DP2 is ready to run + * LL3 is ready to run + * LL4 is ready to run + * !! NOTE that DP3 is not ready - it will be ready again in TICK25 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of data chunk for DP2) + * LL3 processing (producing next 10% of second data chunk for DP3) + * LL4 processing (consuming 10% of data prepared by DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used + * + * + * Now - pipeline is in stable state, CPU used almost in 100% (it would be 100% if DP3 + * needed 1.2ms for processing - but the example would be too complicated) + */ + +void z_impl_scheduler_dp_ll_tick(void) +{ + unsigned int lock_key; +#if CONFIG_SOF_USERSPACE_LL + struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); +#else + struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); +#endif + + if (!dp_sch) + return; + + /* remember current timestamp as "NOW" */ + dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); + + lock_key = scheduler_dp_lock(cpu_get_id()); + scheduler_dp_recalculate(dp_sch); + scheduler_dp_unlock(lock_key); +} + +#ifdef CONFIG_USERSPACE +#include +void z_vrfy_scheduler_dp_ll_tick(void) +{ + z_impl_scheduler_dp_ll_tick(); +} +#include +#endif + +APP_SYSUSER_DATA static struct scheduler_ops schedule_dp_ops = { + .schedule_task = scheduler_dp_task_shedule, +#if CONFIG_SOF_USERSPACE_APPLICATION + .schedule_task_cancel = scheduler_dp_task_cancel, +#else + .schedule_task_cancel = scheduler_dp_task_stop, +#endif + .schedule_task_free = scheduler_dp_task_free, +}; + +/* Runs on each core */ +__cold int scheduler_dp_init(void) +{ + assert_can_be_cold(); + + struct scheduler_dp_data *dp_sch = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_KERNEL, sizeof(*dp_sch), 0); + if (!dp_sch) + return -ENOMEM; + + memset(dp_sch, 0, sizeof(*dp_sch)); + list_init(&dp_sch->tasks); + + scheduler_init(SOF_SCHEDULE_DP, &schedule_dp_ops, dp_sch); + + /* init src of DP tick */ + return schedule_task_init_ll(&dp_sch->ll_tick_src, + SOF_UUID(dp_sched_uuid), + SOF_SCHEDULE_LL_TIMER, + 0, scheduler_dp_ll_tick_dummy, dp_sch, + cpu_get_id(), 0); +} diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 2e76b214ad33..ac50500abd7f 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -31,21 +31,12 @@ #include #include -LOG_MODULE_REGISTER(dp_schedule, CONFIG_SOF_LOG_LEVEL); -SOF_DEFINE_REG_UUID(dp_sched); +LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); +extern struct tr_ctx dp_tr; -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(dp_tr, SOF_UUID(dp_sched_uuid), LOG_LEVEL_INFO); - -#define DP_LOCK_INIT(i, _) Z_SEM_INITIALIZER(dp_lock[i], 1, 1) -#define DP_LOCK_INIT_LIST LISTIFY(CONFIG_MP_MAX_NUM_CPUS, DP_LOCK_INIT, (,)) - -/* User threads don't need access to this array. Access is performed from - * the kernel space via a syscall. Array must be placed in special section - * to be qualified as initialized by the gen_kobject_list.py script. +/* The scheduler locks, the LL tick system call and the scheduler initialization, which + * run in kernel mode, are in src/kernel/schedule/zephyr_dp_schedule-kernel.c */ -static -STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_lock, CONFIG_MP_MAX_NUM_CPUS) = { DP_LOCK_INIT_LIST }; /* Each per-core instance of DP scheduler has separate structures; hence, locks are per-core. * @@ -62,202 +53,8 @@ void scheduler_dp_unlock(unsigned int key) k_sem_give(&dp_lock[key]); } -void scheduler_dp_grant(k_tid_t thread_id, uint16_t core) -{ - if (thread_id) - k_thread_access_grant(thread_id, &dp_lock[core]); -} - -/* dummy LL task - to start LL on secondary cores */ -static enum task_state scheduler_dp_ll_tick_dummy(void *data) -{ - return SOF_TASK_STATE_RESCHEDULE; -} - -/* - * function called after every LL tick - * - * This function checks if the queued DP tasks are ready to processing (meaning - * the module run by the task has enough data at all sources and enough free space - * on all sinks) - * - * if the task becomes ready, a deadline is set allowing Zephyr to schedule threads - * in right order - * - * TODO: currently there's a limitation - DP module must be surrounded by LL modules. - * it simplifies algorithm - there's no need to browse through DP chains calculating - * deadlines for each module in function of all modules execution status. - * Now is simple - modules deadline is its start + tick time. - * - * example: - * Lets assume we do have a pipeline: - * - * LL1 -> DP1 -> LL2 -> DP2 -> LL3 -> DP3 -> LL4 - * - * all LLs starts in 1ms tick - * - * for simplification lets assume - * - all LLs are on primary core, all DPs on secondary (100% CPU is for DP) - * - context switching requires 0 cycles - * - * DP1 - starts every 1ms, needs 0.5ms to finish processing - * DP2 - starts every 2ms, needs 0.6ms to finish processing - * DP3 - starts every 10ms, needs 0.3ms to finish processing - * - * TICK0 - * only LL1 is ready to run - * LL1 processing (producing data chunk for DP1) - * - * TICK1 - * LL1 is ready to run - * DP1 is ready tu run (has data from LL1) set deadline to TICK2 - * LL1 processing (producing second data chunk for DP1) - * DP1 processing for 0.5ms (consuming first data chunk, producing data chunk for LL2) - * CPU is idle for 0.5ms - * - * TICK2 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK3 - * LL2 is ready to run - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * CPU is idle for 0.5ms - * - * TICK3 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK4 - * LL2 is ready to run - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * CPU is idle for 0.5ms - * - * TICK4 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK5 - * LL2 is ready to run - * DP2 is ready to run set deadline to TICK6 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% of second data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - * - * !!!!!! Note here - DP1 must do before DP2 as it MUST finish in this tick. DP2 can wait - * >>>>>>> this is what we call EDF - EARIEST DEADLINE FIRST <<<<<< - * - * TICK5 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK6 - * LL2 is ready to run - * DP2 is in progress, deadline is set to TICK6 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of second data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.1ms (producing TWO data chunks for LL3) - * CPU is idle for 0.4ms (60% used) - * - * TICK6 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK7 - * LL2 is ready to run - * DP2 is ready to run set deadline to TICK8 - * LL3 is ready to run - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% of second data chunk for DP2) - * LL3 processing (producing 10% of first data chunk for DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - * - * - * - * (........ 9 more cycles - LL3 procuces 100% of data for DP3......) - * - * - * TICK15 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK16 - * LL2 is ready to run - * DP2 is ready to run set deadline to TICK17 - * LL3 is ready to run - * DP3 is ready to run set deadline to TICK25 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% of data chunk for DP2) - * LL3 processing (producing 10% of second data chunk for DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - - * !!! note that DP3 is ready but has no chance to get CPU in this cycle - * - * TICK16 - * LL1 is ready to run set deadline to TICK17 - * DP1 is ready tu run - * LL2 is ready to run - * DP2 is in progress, deadline is set to TICK17 - * LL3 is ready to run - * DP3 is in progress, deadline is set to TICK25 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of data chunk for DP2) - * LL3 processing (producing 10% of second data chunk for DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.1ms (producing data) - * DP3 processing for 0.2ms (producing 10 data chunks for LL4) - * 90% CPU used - * - * TICK17 - * LL1 is ready to run - * DP1 is ready tu run - * LL2 is ready to run - * DP2 is ready to run - * LL3 is ready to run - * LL4 is ready to run - * !! NOTE that DP3 is not ready - it will be ready again in TICK25 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of data chunk for DP2) - * LL3 processing (producing next 10% of second data chunk for DP3) - * LL4 processing (consuming 10% of data prepared by DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - * - * - * Now - pipeline is in stable state, CPU used almost in 100% (it would be 100% if DP3 - * needed 1.2ms for processing - but the example would be too complicated) - */ - -void z_impl_scheduler_dp_ll_tick(void) -{ - unsigned int lock_key; -#if CONFIG_SOF_USERSPACE_LL - struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); -#else - struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); -#endif - - if (!dp_sch) - return; - - /* remember current timestamp as "NOW" */ - dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); - - lock_key = scheduler_dp_lock(cpu_get_id()); - scheduler_dp_recalculate(dp_sch); - scheduler_dp_unlock(lock_key); -} - -#ifdef CONFIG_USERSPACE -#include -void z_vrfy_scheduler_dp_ll_tick(void) -{ - z_impl_scheduler_dp_ll_tick(); -} -#include -#endif - #if CONFIG_SOF_USERSPACE_APPLICATION -static int scheduler_dp_task_cancel(void *data, struct task *task) +int scheduler_dp_task_cancel(void *data, struct task *task) { /* Should never be called */ k_panic(); @@ -265,7 +62,7 @@ static int scheduler_dp_task_cancel(void *data, struct task *task) } #endif -static int scheduler_dp_task_stop(void *data, struct task *task) +int scheduler_dp_task_stop(void *data, struct task *task) { unsigned int lock_key; struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)data; @@ -293,7 +90,7 @@ static int scheduler_dp_task_stop(void *data, struct task *task) return 0; } -static int scheduler_dp_task_free(void *data, struct task *task) +int scheduler_dp_task_free(void *data, struct task *task) { struct task_dp_pdata *pdata = task->priv_data; int ret; @@ -317,8 +114,8 @@ static int scheduler_dp_task_free(void *data, struct task *task) return ret; } -static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, - uint64_t period) +int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, + uint64_t period) { struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)data; struct task_dp_pdata *pdata = task->priv_data; @@ -354,39 +151,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta return 0; } -APP_SYSUSER_DATA static struct scheduler_ops schedule_dp_ops = { - .schedule_task = scheduler_dp_task_shedule, -#if CONFIG_SOF_USERSPACE_APPLICATION - .schedule_task_cancel = scheduler_dp_task_cancel, -#else - .schedule_task_cancel = scheduler_dp_task_stop, -#endif - .schedule_task_free = scheduler_dp_task_free, -}; - -/* Runs on each core */ -__cold int scheduler_dp_init(void) -{ - assert_can_be_cold(); - - struct scheduler_dp_data *dp_sch = sof_heap_alloc(sof_sys_user_heap_get(), - SOF_MEM_FLAG_KERNEL, sizeof(*dp_sch), 0); - if (!dp_sch) - return -ENOMEM; - - memset(dp_sch, 0, sizeof(*dp_sch)); - list_init(&dp_sch->tasks); - - scheduler_init(SOF_SCHEDULE_DP, &schedule_dp_ops, dp_sch); - - /* init src of DP tick */ - return schedule_task_init_ll(&dp_sch->ll_tick_src, - SOF_UUID(dp_sched_uuid), - SOF_SCHEDULE_LL_TIMER, - 0, scheduler_dp_ll_tick_dummy, dp_sch, - cpu_get_id(), 0); -} - void scheduler_get_task_info_dp(struct scheduler_props *scheduler_props, uint32_t *data_off_size) { unsigned int lock_key; From de1f89c7975d68cd1c6c6235d25db29fe58d2cf2 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:29:27 +0000 Subject: [PATCH 15/19] schedule: split the kernel part of zephyr_dp_schedule_thread.c into src/kernel/schedule In the thread variant of the DP scheduler, scheduler_dp_task_init() allocates the DP thread, its stack and event, creates and pins the thread and, for user-space modules, grants it access to its objects and adds it to the module memory domain. Move it, and the kernel half of the scheduler_dp_internal_free() system call, which frees those objects, to src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c. zephyr_dp_schedule_thread.c keeps the DP thread function and the task readiness and deadline calculation. The code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/kernel/schedule/CMakeLists.txt | 5 + .../zephyr_dp_schedule_thread-kernel.c | 208 ++++++++++++++++++ src/schedule/zephyr_dp_schedule_thread.c | 177 +-------------- 3 files changed, 217 insertions(+), 173 deletions(-) create mode 100644 src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index 5c878b19f3c3..4afc98e320bf 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -20,4 +20,9 @@ if(zephyr) zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER zephyr_dp_schedule-kernel.c ) + if(NOT CONFIG_SOF_USERSPACE_APPLICATION) + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER + zephyr_dp_schedule_thread-kernel.c + ) + endif() endif() diff --git a/src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c b/src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c new file mode 100644 index 000000000000..05c85ad98b1e --- /dev/null +++ b/src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Marcin Szkudlinski + * Adrian Warecki + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include + +LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); +extern struct tr_ctx dp_tr; + +/* Kernel part of the thread variant of the DP scheduler: the DP thread creation and + * the scheduler_dp_internal_free() system call. The DP thread function is in + * src/schedule/zephyr_dp_schedule_thread.c + */ + +int scheduler_dp_task_init(struct task **task, + const struct sof_uuid_entry *uid, + const struct task_ops *ops, + struct processing_module *mod, + uint16_t core, + size_t stack_size, + uint32_t options) +{ + void __sparse_cache *p_stack = NULL; + struct k_heap *const user_heap = mod->dev->drv->user_heap; + + /* memory allocation helper structure */ + struct { + struct task task; /* keep first, used for freeing below */ + struct task_dp_pdata pdata; + } *task_memory; + + int ret; + + /* must be called on the same core the task will be binded to */ + assert(cpu_get_id() == core); + + /* + * allocate memory + * to avoid multiple malloc operations allocate all required memory as a single structure + * and return pointer to task_memory->task + * As the structure contains zephyr kernel specific data, it must be located in + * shared, non cached memory + */ + task_memory = sof_heap_alloc(user_heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*task_memory), 0); + if (!task_memory) { + tr_err(&dp_tr, "memory alloc failed"); + return -ENOMEM; + } + + memset(task_memory, 0, sizeof(*task_memory)); + /* allocate stack - must be aligned and cached so a separate alloc */ + p_stack = user_stack_allocate(stack_size, options); + if (!p_stack) { + tr_err(&dp_tr, "stack alloc failed"); + ret = -ENOMEM; + goto err; + } + + /* internal SOF task init */ + ret = schedule_task_init(&task_memory->task, uid, SOF_SCHEDULE_DP, 0, ops->run, + mod, core, options); + if (ret < 0) { + tr_err(&dp_tr, "schedule_task_init failed"); + goto err; + } + + struct task_dp_pdata *pdata = &task_memory->pdata; + + /* Point to event_struct event for kernel threads synchronization */ + /* It will be overwritten for K_USER threads to dynamic ones. */ + pdata->event = &pdata->event_struct; + pdata->thread = &pdata->thread_struct; + +#ifdef CONFIG_USERSPACE + if (options & K_USER) { + pdata->event = k_object_alloc(K_OBJ_EVENT); + if (!pdata->event) { + tr_err(&dp_tr, "Event object allocation failed"); + ret = -ENOMEM; + goto err; + } + + pdata->thread = k_object_alloc(K_OBJ_THREAD); + if (!pdata->thread) { + tr_err(&dp_tr, "Thread object allocation failed"); + ret = -ENOMEM; + goto err; + } + } +#endif /* CONFIG_USERSPACE */ + + /* initialize other task structures */ + task_memory->task.ops.complete = ops->complete; + task_memory->task.ops.get_deadline = ops->get_deadline; + task_memory->task.state = SOF_TASK_STATE_INIT; + task_memory->task.core = core; + task_memory->task.priv_data = pdata; + k_event_init(pdata->event); + + /* success, fill the structures */ + pdata->p_stack = p_stack; + pdata->mod = mod; + + /* create a zephyr thread for the task */ + pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)p_stack, + stack_size, dp_thread_fn, &task_memory->task, NULL, NULL, + CONFIG_DP_THREAD_PRIORITY, task_memory->task.flags, + K_FOREVER); + +#ifdef CONFIG_SCHED_CPU_MASK + /* pin the thread to specific core */ + ret = k_thread_cpu_pin(pdata->thread_id, core); + if (ret < 0) { + tr_err(&dp_tr, "zephyr task pin to core failed"); + goto e_thread; + } +#endif + +#ifdef CONFIG_USERSPACE + if (options & K_USER) { + k_thread_access_grant(pdata->thread_id, pdata->event); + scheduler_dp_grant(pdata->thread_id, core); + + ret = k_mem_domain_add_thread(pdata->mod->user_ctx->comp_dom, pdata->thread_id); + if (ret < 0) { + tr_err(&dp_tr, "k_mem_domain_add_thread() failed %d", ret); + goto e_thread; + } + +#if IS_ENABLED(CONFIG_SOF_USERSPACE_MOD_IPC_BY_DP_THREAD) + pdata->ipc_work_item = userspace_proxy_register_ipc_handler(mod, pdata->event); + assert(pdata->ipc_work_item); +#endif + } +#endif /* CONFIG_USERSPACE */ + + /* start the thread, it should immediately stop at an event */ + k_thread_start(pdata->thread_id); + + /* success, fill output parameter */ + *task = &task_memory->task; + return 0; + +e_thread: + k_thread_abort(pdata->thread_id); +err: + /* cleanup - free all allocated resources */ + if (user_stack_free((__sparse_force void *)p_stack)) + tr_err(&dp_tr, "user_stack_free failed!"); + + /* k_object_free looks for a pointer in the list, any invalid value can be passed */ + k_object_free(task_memory->pdata.event); + k_object_free(task_memory->pdata.thread); + sof_heap_free(user_heap, task_memory); + return ret; +} + +void z_impl_scheduler_dp_internal_free(struct task *task) +{ + struct task_dp_pdata *pdata = task->priv_data; + +#ifdef CONFIG_USERSPACE + if (pdata->event != &pdata->event_struct) + k_object_free(pdata->event); + if (pdata->thread != &pdata->thread_struct) + k_object_free(pdata->thread); +#endif + + /* task is the first member in task_memory above */ + sof_heap_free(pdata->mod->dev->drv->user_heap, task); +} + +#ifdef CONFIG_USERSPACE +#include + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + /* + * With the thread DP scheduler variant scheduler_dp_internal_free() is + * never called from the userspace context + */ + K_OOPS(true); +} +#include +#endif diff --git a/src/schedule/zephyr_dp_schedule_thread.c b/src/schedule/zephyr_dp_schedule_thread.c index f742e215a750..9c6db72feacd 100644 --- a/src/schedule/zephyr_dp_schedule_thread.c +++ b/src/schedule/zephyr_dp_schedule_thread.c @@ -29,6 +29,10 @@ LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx dp_tr; +/* The DP thread creation and the scheduler_dp_internal_free() system call, which run + * in kernel mode, are in src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c + */ + /* Go through all DP tasks and recalculate their readiness and deadlines * NOT REENTRANT, should be called with scheduler_dp_lock() */ @@ -197,176 +201,3 @@ void dp_thread_fn(void *p1, void *p2, void *p3) if (task->state == SOF_TASK_STATE_COMPLETED) task_complete(task); } - -int scheduler_dp_task_init(struct task **task, - const struct sof_uuid_entry *uid, - const struct task_ops *ops, - struct processing_module *mod, - uint16_t core, - size_t stack_size, - uint32_t options) -{ - void __sparse_cache *p_stack = NULL; - struct k_heap *const user_heap = mod->dev->drv->user_heap; - - /* memory allocation helper structure */ - struct { - struct task task; /* keep first, used for freeing below */ - struct task_dp_pdata pdata; - } *task_memory; - - int ret; - - /* must be called on the same core the task will be binded to */ - assert(cpu_get_id() == core); - - /* - * allocate memory - * to avoid multiple malloc operations allocate all required memory as a single structure - * and return pointer to task_memory->task - * As the structure contains zephyr kernel specific data, it must be located in - * shared, non cached memory - */ - task_memory = sof_heap_alloc(user_heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*task_memory), 0); - if (!task_memory) { - tr_err(&dp_tr, "memory alloc failed"); - return -ENOMEM; - } - - memset(task_memory, 0, sizeof(*task_memory)); - /* allocate stack - must be aligned and cached so a separate alloc */ - p_stack = user_stack_allocate(stack_size, options); - if (!p_stack) { - tr_err(&dp_tr, "stack alloc failed"); - ret = -ENOMEM; - goto err; - } - - /* internal SOF task init */ - ret = schedule_task_init(&task_memory->task, uid, SOF_SCHEDULE_DP, 0, ops->run, - mod, core, options); - if (ret < 0) { - tr_err(&dp_tr, "schedule_task_init failed"); - goto err; - } - - struct task_dp_pdata *pdata = &task_memory->pdata; - - /* Point to event_struct event for kernel threads synchronization */ - /* It will be overwritten for K_USER threads to dynamic ones. */ - pdata->event = &pdata->event_struct; - pdata->thread = &pdata->thread_struct; - -#ifdef CONFIG_USERSPACE - if (options & K_USER) { - pdata->event = k_object_alloc(K_OBJ_EVENT); - if (!pdata->event) { - tr_err(&dp_tr, "Event object allocation failed"); - ret = -ENOMEM; - goto err; - } - - pdata->thread = k_object_alloc(K_OBJ_THREAD); - if (!pdata->thread) { - tr_err(&dp_tr, "Thread object allocation failed"); - ret = -ENOMEM; - goto err; - } - } -#endif /* CONFIG_USERSPACE */ - - /* initialize other task structures */ - task_memory->task.ops.complete = ops->complete; - task_memory->task.ops.get_deadline = ops->get_deadline; - task_memory->task.state = SOF_TASK_STATE_INIT; - task_memory->task.core = core; - task_memory->task.priv_data = pdata; - k_event_init(pdata->event); - - /* success, fill the structures */ - pdata->p_stack = p_stack; - pdata->mod = mod; - - /* create a zephyr thread for the task */ - pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)p_stack, - stack_size, dp_thread_fn, &task_memory->task, NULL, NULL, - CONFIG_DP_THREAD_PRIORITY, task_memory->task.flags, - K_FOREVER); - -#ifdef CONFIG_SCHED_CPU_MASK - /* pin the thread to specific core */ - ret = k_thread_cpu_pin(pdata->thread_id, core); - if (ret < 0) { - tr_err(&dp_tr, "zephyr task pin to core failed"); - goto e_thread; - } -#endif - -#ifdef CONFIG_USERSPACE - if (options & K_USER) { - k_thread_access_grant(pdata->thread_id, pdata->event); - scheduler_dp_grant(pdata->thread_id, core); - - ret = k_mem_domain_add_thread(pdata->mod->user_ctx->comp_dom, pdata->thread_id); - if (ret < 0) { - tr_err(&dp_tr, "k_mem_domain_add_thread() failed %d", ret); - goto e_thread; - } - -#if IS_ENABLED(CONFIG_SOF_USERSPACE_MOD_IPC_BY_DP_THREAD) - pdata->ipc_work_item = userspace_proxy_register_ipc_handler(mod, pdata->event); - assert(pdata->ipc_work_item); -#endif - } -#endif /* CONFIG_USERSPACE */ - - /* start the thread, it should immediately stop at an event */ - k_thread_start(pdata->thread_id); - - /* success, fill output parameter */ - *task = &task_memory->task; - return 0; - -e_thread: - k_thread_abort(pdata->thread_id); -err: - /* cleanup - free all allocated resources */ - if (user_stack_free((__sparse_force void *)p_stack)) - tr_err(&dp_tr, "user_stack_free failed!"); - - /* k_object_free looks for a pointer in the list, any invalid value can be passed */ - k_object_free(task_memory->pdata.event); - k_object_free(task_memory->pdata.thread); - sof_heap_free(user_heap, task_memory); - return ret; -} - -void z_impl_scheduler_dp_internal_free(struct task *task) -{ - struct task_dp_pdata *pdata = task->priv_data; - -#ifdef CONFIG_USERSPACE - if (pdata->event != &pdata->event_struct) - k_object_free(pdata->event); - if (pdata->thread != &pdata->thread_struct) - k_object_free(pdata->thread); -#endif - - /* task is the first member in task_memory above */ - sof_heap_free(pdata->mod->dev->drv->user_heap, task); -} - -#ifdef CONFIG_USERSPACE -#include - -void z_vrfy_scheduler_dp_internal_free(struct task *task) -{ - /* - * With the thread DP scheduler variant scheduler_dp_internal_free() is - * never called from the userspace context - */ - K_OOPS(true); -} -#include -#endif From b1ca6ca65cece509e913fa8eb5282d28da711b88 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:30:22 +0000 Subject: [PATCH 16/19] schedule: split the kernel part of zephyr_dp_schedule_application.c into src/kernel/schedule In the application variant of the DP scheduler, scheduler_dp_task_init() allocates the DP thread and its kernel objects, sets up the memory domain of the module, grants the DP, IPC and LL threads access, and records the kernel objects of the task in a kernel-only list. Move it to src/kernel/schedule/zephyr_dp_schedule_application-kernel.c, with: - the kernel object record list and its helpers; - the memory domain pool and teardown; - the kernel half of the scheduler_dp_internal_free() system call and its verification of the caller; - the per-core DP thread synchronization semaphores, which are kernel objects. zephyr_dp_schedule_application.c keeps the IPC packing and unpacking, scheduler_dp_thread_ipc(), the task readiness and deadline calculation and the DP thread function. The task memory layout and the packed IPC structure, which both parts use, and the semaphores are declared in sof/schedule/zephyr_dp_schedule.h. The code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/schedule/zephyr_dp_schedule.h | 57 ++ src/kernel/schedule/CMakeLists.txt | 6 +- .../zephyr_dp_schedule_application-kernel.c | 533 +++++++++++++++++ src/schedule/zephyr_dp_schedule_application.c | 542 +----------------- 4 files changed, 598 insertions(+), 540 deletions(-) create mode 100644 src/kernel/schedule/zephyr_dp_schedule_application-kernel.c diff --git a/src/include/sof/schedule/zephyr_dp_schedule.h b/src/include/sof/schedule/zephyr_dp_schedule.h index 1056efbfec4c..582193a6a816 100644 --- a/src/include/sof/schedule/zephyr_dp_schedule.h +++ b/src/include/sof/schedule/zephyr_dp_schedule.h @@ -18,10 +18,14 @@ #define __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ #include +#include #include #include #include +#include +#include + #include #include @@ -72,6 +76,59 @@ struct task_dp_pdata { #endif }; +#if CONFIG_SOF_USERSPACE_APPLICATION +/** @brief Per-core DP thread start and IPC completion semaphores. */ +extern struct k_sem dp_sync[CONFIG_CORE_COUNT]; + +struct ipc4_flat { + unsigned int cmd; + int ret; + union { + struct { + struct ipc4_module_bind_unbind bu; + enum bind_type type; + } bind; + struct { + unsigned int trigger_cmd; + enum ipc4_pipeline_state state; + int n_sources; + int n_sinks; + struct sof_source *source[CONFIG_MODULE_MAX_CONNECTIONS]; + struct sof_sink *sink[CONFIG_MODULE_MAX_CONNECTIONS]; + } pipeline_state; + /* + * SOF_IPC4_MOD_INIT_INSTANCE: a by-value copy of the caller's + * ext_data. The original lives on the calling thread's stack, + * which the DP thread's memory domain doesn't have access to. + * The pointers it contains reference the IPC mailbox, which stays + * valid and is covered by the DP thread's SOF_DP_PART_CFG partition. + */ + struct module_ext_init_data init_instance; + struct { + uint32_t param_id; + enum module_cfg_fragment_position position; + uint32_t data_offset_size; + size_t fragment_size; + const char *data; + } set_config; + struct { + uint32_t param_id; + uint32_t data_offset_size; + size_t fragment_size; + char *data; + } get_config; + }; +}; + +/* memory allocation helper structure */ +struct scheduler_dp_task_memory { + struct task task; + struct task_dp_pdata pdata; + struct comp_driver drv; + struct ipc4_flat flat; +}; + +#endif /* CONFIG_SOF_USERSPACE_APPLICATION */ /** @brief Per-core DP scheduler locks, see scheduler_dp_lock(). */ extern struct k_sem dp_lock[CONFIG_MP_MAX_NUM_CPUS]; diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index 4afc98e320bf..f49fc2d5f869 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -20,7 +20,11 @@ if(zephyr) zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER zephyr_dp_schedule-kernel.c ) - if(NOT CONFIG_SOF_USERSPACE_APPLICATION) + if(CONFIG_SOF_USERSPACE_APPLICATION) + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER + zephyr_dp_schedule_application-kernel.c + ) + else() zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER zephyr_dp_schedule_thread-kernel.c ) diff --git a/src/kernel/schedule/zephyr_dp_schedule_application-kernel.c b/src/kernel/schedule/zephyr_dp_schedule_application-kernel.c new file mode 100644 index 000000000000..395f1f5129be --- /dev/null +++ b/src/kernel/schedule/zephyr_dp_schedule_application-kernel.c @@ -0,0 +1,533 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Marcin Szkudlinski + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include + +LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); +extern struct tr_ctx dp_tr; + +/* Kernel part of the application variant of the DP scheduler: the DP thread creation, + * its memory domain, the kernel-only record of its kernel objects and the + * scheduler_dp_internal_free() system call. The DP thread function and the IPC + * forwarding to it are in src/schedule/zephyr_dp_schedule_application.c + */ + +static struct objpool_head dp_mdom_head = {.list = LIST_INIT(dp_mdom_head.list)}; + +/* Synchronization semaphore for the scheduler thread to wait for DP startup */ +#define DP_SYNC_INIT(i, _) Z_SEM_INITIALIZER(dp_sync[i], 0, 1) +#define DP_SYNC_INIT_LIST LISTIFY(CONFIG_CORE_COUNT, DP_SYNC_INIT, (,)) +STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_sync, CONFIG_CORE_COUNT) = { DP_SYNC_INIT_LIST }; + +/* + * Kernel-only record of the kernel objects owned by a DP task. + * + * struct scheduler_dp_task_memory lives in the module's heap, which is mapped + * into the DP thread's memory domain, and is also writable by the LL and IPC + * user threads. Freeing must therefore never use the pointers stored there: + * scheduler_dp_internal_free() takes them from this record instead. The task + * pointer is only used as a lookup key. + */ +struct scheduler_dp_task_kobj { + struct list_item list; + struct task *task; + struct processing_module *mod; + struct k_event *event; + struct k_thread *thread; + struct k_mem_domain *mdom; + struct k_mem_partition mpart[SOF_DP_PART_TYPE_COUNT]; + uint32_t comp_id; + uint32_t gen; /* unique per record, see dp_task_kobj_take() */ +}; + +static struct list_item dp_task_kobj_list = LIST_INIT(dp_task_kobj_list); +static struct k_spinlock dp_task_kobj_lock; +/* generation of the last added record, under dp_task_kobj_lock */ +static uint32_t dp_task_kobj_gen; + +static void dp_task_kobj_add(struct scheduler_dp_task_kobj *kobj) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + + kobj->gen = ++dp_task_kobj_gen; + list_item_append(&kobj->list, &dp_task_kobj_list); + k_spin_unlock(&dp_task_kobj_lock, key); +} + +/* Find the record for @task, caller holds dp_task_kobj_lock */ +static struct scheduler_dp_task_kobj *dp_task_kobj_find(const struct task *task) +{ + struct scheduler_dp_task_kobj *kobj; + struct list_item *item; + + list_for_item(item, &dp_task_kobj_list) { + kobj = container_of(item, struct scheduler_dp_task_kobj, list); + if (kobj->task == task) + return kobj; + } + + return NULL; +} + +/* Copy the record for @task to @copy without unlinking it */ +static bool dp_task_kobj_peek(const struct task *task, struct scheduler_dp_task_kobj *copy) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); + + if (kobj) + *copy = *kobj; + + k_spin_unlock(&dp_task_kobj_lock, key); + + return kobj != NULL; +} + +/* + * Find the record for @task and unlink it, so it can only be freed once. + * If @expected is given, only take the record it was copied from, so a + * record replaced since @expected was peeked is left alone. + */ +static struct scheduler_dp_task_kobj * +dp_task_kobj_take(const struct task *task, const struct scheduler_dp_task_kobj *expected) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); + + /* + * Compare the generation, not the object addresses: the task could + * have been freed and recreated at the same addresses meanwhile. + */ + if (kobj && expected && kobj->gen != expected->gen) + kobj = NULL; + + if (kobj) + list_item_del(&kobj->list); + + k_spin_unlock(&dp_task_kobj_lock, key); + + return kobj; +} + +/* + * Safe to call with partial successful initialisation, + * k_mem_domain_remove_partition() then just returns -ENOENT + */ +static void scheduler_dp_domain_free(struct processing_module *pmod, struct k_mem_domain *mdom, + struct k_mem_partition *mpart, uint32_t comp_id) +{ + llext_manager_rm_domain(comp_id, mdom); + + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP); + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP_CACHE); + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG); + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG_CACHE); + + if (mdom->num_partitions) { + /* Violation: all partitions should be freed by now */ + unsigned int n = mdom->num_partitions; + + LOG_WRN("%u domain partition(s) not freed, force-freeing them all", n); + + for (unsigned int i = 0; i < arch_mem_domain_max_partitions_get() && n; i++) { + struct k_mem_partition *dpart = &mdom->partitions[i]; + + if (dpart->size) { + k_mem_domain_remove_partition(mdom, dpart); + n--; + } + } + } + + /* All partitions removed, the domain can be freed now */ + pmod->mdom = NULL; + k_mem_domain_deinit(mdom); + objpool_free(&dp_mdom_head, mdom); +} + +/* + * Free the kernel objects of @task, using only the kernel-only record. + * + * @return false if @task has no record (unknown task or already freed) + */ +static bool scheduler_dp_internal_free_kobj(struct task *task, + const struct scheduler_dp_task_kobj *expected) +{ + struct scheduler_dp_task_kobj *kobj = dp_task_kobj_take(task, expected); + + if (!kobj) + return false; + + /* normally already terminated by scheduler_dp_task_free(), no-op then */ + k_thread_abort(kobj->thread); + k_object_free(kobj->event); + k_object_free(kobj->thread); + scheduler_dp_domain_free(kobj->mod, kobj->mdom, kobj->mpart, kobj->comp_id); + + mod_free(kobj->mod, container_of(task, struct scheduler_dp_task_memory, task)); + rfree(kobj); + + return true; +} + +void z_impl_scheduler_dp_internal_free(struct task *task) +{ + if (!scheduler_dp_internal_free_kobj(task, NULL)) + tr_err(&dp_tr, "unknown DP task %p", task); +} + +#ifdef CONFIG_THREAD_NAME +static void scheduler_dp_thread_name_set(k_tid_t thread_id, struct processing_module *mod) +{ + char name[CONFIG_THREAD_MAX_NAME_LEN]; + + snprintf(name, sizeof(name), "DP:%#x", mod->dev->ipc_config.id); + + k_thread_name_set(thread_id, name); +} +#else +/* k_thread_name_set() is a no-op so skip constructing a thread name */ +#define scheduler_dp_thread_name_set(x, y) +#endif + +/* Called only in IPC context in kernel mode (this can change) */ +int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, + const struct task_ops *ops, struct processing_module *mod, + uint16_t core, size_t stack_size, uint32_t options) +{ + k_thread_stack_t *p_stack; + struct scheduler_dp_task_memory *task_memory; + uint8_t *ipc_config_data = NULL; + k_tid_t thread_id; + uint32_t comp_id; + + int ret; + + /* must be called on the same core the task will be bound to */ + assert(cpu_get_id() == core); + assert(!k_is_user_context()); + + /* + * allocate memory + * to avoid multiple malloc operations allocate all required memory as a single structure + * and return pointer to task_memory->task + * As the structure contains zephyr kernel specific data, it must be located in + * shared, non cached memory + */ + task_memory = mod_alloc_ext(mod, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*task_memory), 0); + if (!task_memory) { + tr_err(&dp_tr, "memory alloc failed"); + return -ENOMEM; + } + + memset(task_memory, 0, sizeof(*task_memory)); + + const struct comp_driver *drv = mod->dev->drv; + + task_memory->drv = *mod->dev->drv; + mod->dev->drv = &task_memory->drv; + + /* allocate stack - must be aligned and cached so a separate alloc */ + p_stack = user_stack_allocate(stack_size, options); + if (!p_stack) { + tr_err(&dp_tr, "stack alloc failed"); + ret = -ENOMEM; + goto e_tmem; + } + + struct task *ptask = &task_memory->task; + + /* internal SOF task init */ + ret = schedule_task_init(ptask, uid, SOF_SCHEDULE_DP, 0, ops->run, mod, core, options); + if (ret < 0) { + tr_err(&dp_tr, "schedule_task_init failed"); + goto e_stack; + } + + /* + * task_memory, including pdata, is in the module heap, which user + * threads can write while this function runs. Keep everything the + * kernel uses here or at teardown in kobj and locals, and only mirror + * it into pdata for the DP thread. + */ + struct task_dp_pdata *pdata = &task_memory->pdata; + struct scheduler_dp_task_kobj *kobj = rmalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + sizeof(*kobj)); + + if (!kobj) { + tr_err(&dp_tr, "task record alloc failed"); + ret = -ENOMEM; + goto e_stack; + } + + kobj->task = ptask; + kobj->mod = mod; + comp_id = mod->dev->ipc_config.id; + kobj->comp_id = comp_id; + + if (mod->dev->drv->adapter_ops->get_configuration) { + ipc_config_data = mod_alloc_ext(mod, SOF_MEM_FLAG_USER, SOF_IPC_MSG_MAX_SIZE, 0); + if (!ipc_config_data) { + tr_err(&dp_tr, "Failed to allocate memory for get-config"); + ret = -ENOMEM; + goto e_kobj; + } + } + pdata->ipc_config_data = ipc_config_data; + + pdata->flat = &task_memory->flat; + + kobj->event = k_object_alloc(K_OBJ_EVENT); + if (!kobj->event) { + tr_err(&dp_tr, "Event object allocation failed"); + ret = -ENOMEM; + goto e_config; + } + + pdata->event = kobj->event; + + kobj->thread = k_object_alloc(K_OBJ_THREAD); + if (!kobj->thread) { + tr_err(&dp_tr, "Thread object allocation failed"); + ret = -ENOMEM; + goto e_event; + } + memset(&kobj->thread->arch, 0, sizeof(kobj->thread->arch)); + pdata->thread = kobj->thread; + + /* success, fill the structures */ + pdata->p_stack = p_stack; + pdata->mod = mod; + + /* initialize other task structures */ + ptask->ops.complete = ops->complete; + ptask->ops.get_deadline = ops->get_deadline; + ptask->priv_data = pdata; + list_init(&ptask->list); + + /* create a zephyr thread for the task */ + thread_id = k_thread_create(kobj->thread, p_stack, + stack_size, dp_thread_fn, ptask, NULL, NULL, + CONFIG_DP_THREAD_PRIORITY, ptask->flags, K_FOREVER); + pdata->thread_id = thread_id; + scheduler_dp_thread_name_set(thread_id, mod); + + unsigned int pidx; + size_t size; + uintptr_t start; + bool on_pool = false; + +#ifdef CONFIG_SCHED_CPU_MASK + /* pin the thread to specific core */ + ret = k_thread_cpu_pin(thread_id, core); + if (ret < 0) { + tr_err(&dp_tr, "zephyr task pin to core failed"); + goto e_thread; + } +#endif + + k_thread_access_grant(thread_id, kobj->event, &dp_sync[core]); + scheduler_dp_grant(thread_id, core); +#if CONFIG_SOF_USERSPACE_LL + struct k_thread *thread_ipc = ipc_thread_user(core); + + k_thread_access_grant(thread_ipc, kobj->event, thread_id, p_stack, &dp_sync[core]); + scheduler_dp_grant(thread_ipc, core); + scheduler_dp_grant(zephyr_ll_domain_thread(), core); +#endif + + struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), + SOF_MEM_FLAG_COHERENT); + + if (!mdom) { + tr_err(&dp_tr, "objpool allocation failed"); + ret = -ENOMEM; + goto e_thread; + } + + on_pool = true; + + mod->mdom = mdom; + kobj->mdom = mdom; + + if (!mdom->arch.ptables) { + ret = k_mem_domain_init(mdom, 0, NULL); + if (ret < 0) + goto e_thread; + } + + /* Module heap partition */ + mod_heap_info(mod, &size, &start); + kobj->mpart[SOF_DP_PART_HEAP] = (struct k_mem_partition){ + .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)start), + .size = size, + .attr = K_MEM_PARTITION_P_RW_U_RW, + }; + kobj->mpart[SOF_DP_PART_HEAP_CACHE] = (struct k_mem_partition){ + .start = start, + .size = size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + /* Host mailbox partition for additional IPC parameters: read-only */ + kobj->mpart[SOF_DP_PART_CFG] = (struct k_mem_partition){ + .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)MAILBOX_HOSTBOX_BASE), + .size = MAILBOX_HOSTBOX_SIZE, + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + kobj->mpart[SOF_DP_PART_CFG_CACHE] = (struct k_mem_partition){ + .start = (uintptr_t)MAILBOX_HOSTBOX_BASE, + .size = MAILBOX_HOSTBOX_SIZE, + .attr = K_MEM_PARTITION_P_RO_U_RO | XTENSA_MMU_CACHED_WB, + }; + + for (pidx = 0; pidx < SOF_DP_PART_TYPE_COUNT; pidx++) { + ret = k_mem_domain_add_partition(mdom, kobj->mpart + pidx); + if (ret < 0) + goto e_dom; + } + + memcpy(pdata->mpart, kobj->mpart, sizeof(pdata->mpart)); + + ret = llext_manager_add_domain(comp_id, mdom); + if (ret < 0) { + tr_err(&dp_tr, "failed to add LLEXT to domain %d", ret); + goto e_dom; + } + + /* + * Keep this call last, able to fail, otherwise domain will be removed + * before its thread + */ + ret = k_mem_domain_add_thread(mdom, thread_id); + if (ret < 0) { + tr_err(&dp_tr, "failed to add thread to domain %d", ret); + goto e_dom; + } + + /* publish the record before the thread can run */ + dp_task_kobj_add(kobj); + + /* start the thread, it should immediately stop at the semaphore */ + k_event_init(kobj->event); + k_thread_start(thread_id); + + /* ptask points into task_memory, so only publish it once it cannot be freed */ + *task = ptask; + + return 0; + +e_dom: + scheduler_dp_domain_free(mod, mdom, kobj->mpart, comp_id); + on_pool = false; +e_thread: + if (on_pool) + objpool_free(&dp_mdom_head, mdom); + k_thread_abort(thread_id); + k_object_free(kobj->thread); +e_event: + k_object_free(kobj->event); +e_config: + mod_free(mod, ipc_config_data); +e_kobj: + rfree(kobj); +e_stack: + user_stack_free(p_stack); +e_tmem: + /* the copy lives in task_memory, so stop pointing at it before freeing */ + mod->dev->drv = drv; + mod_free(mod, task_memory); + return ret; +} + +#ifdef CONFIG_USERSPACE +#include + +/* + * mod_free() follows the module's resource and allocator pointers in + * supervisor mode. They are user-writable, so at least check that they + * point to memory the caller can access. + */ +static void scheduler_dp_mod_vrfy(struct processing_module *mod) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod->dev, sizeof(*mod->dev))); + K_OOPS(K_SYSCALL_MEMORY_READ(mod->dev->drv, sizeof(*mod->dev->drv))); + + struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(alloc, sizeof(*alloc))); + if (alloc->heap) { + size_t h_size = 0; + uintptr_t h_start; + + mod_heap_info(mod, &h_size, &h_start); + if (h_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); + } + if (alloc->vreg) { + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(alloc->vreg, &vr_size, &vr_start); + K_OOPS(K_SYSCALL_MEMORY_WRITE(vr_start, vr_size)); + } +} + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + struct scheduler_dp_task_kobj kobj; + + /* + * Do not read task->priv_data: the kernel objects come from the + * kernel-only record, and an unknown task pointer is rejected. + */ + K_OOPS(!dp_task_kobj_peek(task, &kobj)); + + /* + * A DP thread holds its own thread and event objects, so it would pass + * the checks below for its own task. Freeing would then abort the + * caller after the record is unlinked, leaking everything it holds. + */ + K_OOPS(kobj.thread == k_current_get()); + + /* + * The caller must have been granted the task's objects. Check before + * unlinking the record, so an unauthorized call leaves it intact. + */ + K_OOPS(K_SYSCALL_OBJ(kobj.event, K_OBJ_EVENT)); + K_OOPS(K_SYSCALL_OBJ_INIT(kobj.thread, K_OBJ_THREAD)); + scheduler_dp_mod_vrfy(kobj.mod); + + K_OOPS(!scheduler_dp_internal_free_kobj(task, &kobj)); +} +#include +#endif /* CONFIG_USERSPACE */ diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index b98b9129bcae..bb9cf29b87a3 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -34,52 +34,9 @@ LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx dp_tr; -static struct objpool_head dp_mdom_head = {.list = LIST_INIT(dp_mdom_head.list)}; - -/* Synchronization semaphore for the scheduler thread to wait for DP startup */ -#define DP_SYNC_INIT(i, _) Z_SEM_INITIALIZER(dp_sync[i], 0, 1) -#define DP_SYNC_INIT_LIST LISTIFY(CONFIG_CORE_COUNT, DP_SYNC_INIT, (,)) -static STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_sync, CONFIG_CORE_COUNT) = { DP_SYNC_INIT_LIST }; - -struct ipc4_flat { - unsigned int cmd; - int ret; - union { - struct { - struct ipc4_module_bind_unbind bu; - enum bind_type type; - } bind; - struct { - unsigned int trigger_cmd; - enum ipc4_pipeline_state state; - int n_sources; - int n_sinks; - struct sof_source *source[CONFIG_MODULE_MAX_CONNECTIONS]; - struct sof_sink *sink[CONFIG_MODULE_MAX_CONNECTIONS]; - } pipeline_state; - /* - * SOF_IPC4_MOD_INIT_INSTANCE: a by-value copy of the caller's - * ext_data. The original lives on the calling thread's stack, - * which the DP thread's memory domain doesn't have access to. - * The pointers it contains reference the IPC mailbox, which stays - * valid and is covered by the DP thread's SOF_DP_PART_CFG partition. - */ - struct module_ext_init_data init_instance; - struct { - uint32_t param_id; - enum module_cfg_fragment_position position; - uint32_t data_offset_size; - size_t fragment_size; - const char *data; - } set_config; - struct { - uint32_t param_id; - uint32_t data_offset_size; - size_t fragment_size; - char *data; - } get_config; - }; -}; +/* The DP thread creation and the scheduler_dp_internal_free() system call, which run + * in kernel mode, are in src/kernel/schedule/zephyr_dp_schedule_application-kernel.c + */ /* Pack IPC input data */ static int ipc_thread_flatten(unsigned int cmd, const union scheduler_dp_thread_ipc_param *param, @@ -142,104 +99,6 @@ static int ipc_thread_flatten(unsigned int cmd, const union scheduler_dp_thread_ return 0; } -/* memory allocation helper structure */ -struct scheduler_dp_task_memory { - struct task task; - struct task_dp_pdata pdata; - struct comp_driver drv; - struct ipc4_flat flat; -}; - -/* - * Kernel-only record of the kernel objects owned by a DP task. - * - * struct scheduler_dp_task_memory lives in the module's heap, which is mapped - * into the DP thread's memory domain, and is also writable by the LL and IPC - * user threads. Freeing must therefore never use the pointers stored there: - * scheduler_dp_internal_free() takes them from this record instead. The task - * pointer is only used as a lookup key. - */ -struct scheduler_dp_task_kobj { - struct list_item list; - struct task *task; - struct processing_module *mod; - struct k_event *event; - struct k_thread *thread; - struct k_mem_domain *mdom; - struct k_mem_partition mpart[SOF_DP_PART_TYPE_COUNT]; - uint32_t comp_id; - uint32_t gen; /* unique per record, see dp_task_kobj_take() */ -}; - -static struct list_item dp_task_kobj_list = LIST_INIT(dp_task_kobj_list); -static struct k_spinlock dp_task_kobj_lock; -/* generation of the last added record, under dp_task_kobj_lock */ -static uint32_t dp_task_kobj_gen; - -static void dp_task_kobj_add(struct scheduler_dp_task_kobj *kobj) -{ - k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); - - kobj->gen = ++dp_task_kobj_gen; - list_item_append(&kobj->list, &dp_task_kobj_list); - k_spin_unlock(&dp_task_kobj_lock, key); -} - -/* Find the record for @task, caller holds dp_task_kobj_lock */ -static struct scheduler_dp_task_kobj *dp_task_kobj_find(const struct task *task) -{ - struct scheduler_dp_task_kobj *kobj; - struct list_item *item; - - list_for_item(item, &dp_task_kobj_list) { - kobj = container_of(item, struct scheduler_dp_task_kobj, list); - if (kobj->task == task) - return kobj; - } - - return NULL; -} - -/* Copy the record for @task to @copy without unlinking it */ -static bool dp_task_kobj_peek(const struct task *task, struct scheduler_dp_task_kobj *copy) -{ - k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); - struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); - - if (kobj) - *copy = *kobj; - - k_spin_unlock(&dp_task_kobj_lock, key); - - return kobj != NULL; -} - -/* - * Find the record for @task and unlink it, so it can only be freed once. - * If @expected is given, only take the record it was copied from, so a - * record replaced since @expected was peeked is left alone. - */ -static struct scheduler_dp_task_kobj * -dp_task_kobj_take(const struct task *task, const struct scheduler_dp_task_kobj *expected) -{ - k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); - struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); - - /* - * Compare the generation, not the object addresses: the task could - * have been freed and recreated at the same addresses meanwhile. - */ - if (kobj && expected && kobj->gen != expected->gen) - kobj = NULL; - - if (kobj) - list_item_del(&kobj->list); - - k_spin_unlock(&dp_task_kobj_lock, key); - - return kobj; -} - /* Unpack IPC data and execute a callback */ static void ipc_thread_unflatten_run(struct processing_module *pmod, struct ipc4_flat *flat) { @@ -529,398 +388,3 @@ void dp_thread_fn(void *p1, void *p2, void *p3) if (task->state == SOF_TASK_STATE_COMPLETED) task_complete(task); } - -/* - * Safe to call with partial successful initialisation, - * k_mem_domain_remove_partition() then just returns -ENOENT - */ -static void scheduler_dp_domain_free(struct processing_module *pmod, struct k_mem_domain *mdom, - struct k_mem_partition *mpart, uint32_t comp_id) -{ - llext_manager_rm_domain(comp_id, mdom); - - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP); - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP_CACHE); - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG); - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG_CACHE); - - if (mdom->num_partitions) { - /* Violation: all partitions should be freed by now */ - unsigned int n = mdom->num_partitions; - - LOG_WRN("%u domain partition(s) not freed, force-freeing them all", n); - - for (unsigned int i = 0; i < arch_mem_domain_max_partitions_get() && n; i++) { - struct k_mem_partition *dpart = &mdom->partitions[i]; - - if (dpart->size) { - k_mem_domain_remove_partition(mdom, dpart); - n--; - } - } - } - - /* All partitions removed, the domain can be freed now */ - pmod->mdom = NULL; - k_mem_domain_deinit(mdom); - objpool_free(&dp_mdom_head, mdom); -} - -/* - * Free the kernel objects of @task, using only the kernel-only record. - * - * @return false if @task has no record (unknown task or already freed) - */ -static bool scheduler_dp_internal_free_kobj(struct task *task, - const struct scheduler_dp_task_kobj *expected) -{ - struct scheduler_dp_task_kobj *kobj = dp_task_kobj_take(task, expected); - - if (!kobj) - return false; - - /* normally already terminated by scheduler_dp_task_free(), no-op then */ - k_thread_abort(kobj->thread); - k_object_free(kobj->event); - k_object_free(kobj->thread); - scheduler_dp_domain_free(kobj->mod, kobj->mdom, kobj->mpart, kobj->comp_id); - - mod_free(kobj->mod, container_of(task, struct scheduler_dp_task_memory, task)); - rfree(kobj); - - return true; -} - -void z_impl_scheduler_dp_internal_free(struct task *task) -{ - if (!scheduler_dp_internal_free_kobj(task, NULL)) - tr_err(&dp_tr, "unknown DP task %p", task); -} - -#ifdef CONFIG_THREAD_NAME -static void scheduler_dp_thread_name_set(k_tid_t thread_id, struct processing_module *mod) -{ - char name[CONFIG_THREAD_MAX_NAME_LEN]; - - snprintf(name, sizeof(name), "DP:%#x", mod->dev->ipc_config.id); - - k_thread_name_set(thread_id, name); -} -#else -/* k_thread_name_set() is a no-op so skip constructing a thread name */ -#define scheduler_dp_thread_name_set(x, y) -#endif - -/* Called only in IPC context in kernel mode (this can change) */ -int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, - const struct task_ops *ops, struct processing_module *mod, - uint16_t core, size_t stack_size, uint32_t options) -{ - k_thread_stack_t *p_stack; - struct scheduler_dp_task_memory *task_memory; - uint8_t *ipc_config_data = NULL; - k_tid_t thread_id; - uint32_t comp_id; - - int ret; - - /* must be called on the same core the task will be bound to */ - assert(cpu_get_id() == core); - assert(!k_is_user_context()); - - /* - * allocate memory - * to avoid multiple malloc operations allocate all required memory as a single structure - * and return pointer to task_memory->task - * As the structure contains zephyr kernel specific data, it must be located in - * shared, non cached memory - */ - task_memory = mod_alloc_ext(mod, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*task_memory), 0); - if (!task_memory) { - tr_err(&dp_tr, "memory alloc failed"); - return -ENOMEM; - } - - memset(task_memory, 0, sizeof(*task_memory)); - - const struct comp_driver *drv = mod->dev->drv; - - task_memory->drv = *mod->dev->drv; - mod->dev->drv = &task_memory->drv; - - /* allocate stack - must be aligned and cached so a separate alloc */ - p_stack = user_stack_allocate(stack_size, options); - if (!p_stack) { - tr_err(&dp_tr, "stack alloc failed"); - ret = -ENOMEM; - goto e_tmem; - } - - struct task *ptask = &task_memory->task; - - /* internal SOF task init */ - ret = schedule_task_init(ptask, uid, SOF_SCHEDULE_DP, 0, ops->run, mod, core, options); - if (ret < 0) { - tr_err(&dp_tr, "schedule_task_init failed"); - goto e_stack; - } - - /* - * task_memory, including pdata, is in the module heap, which user - * threads can write while this function runs. Keep everything the - * kernel uses here or at teardown in kobj and locals, and only mirror - * it into pdata for the DP thread. - */ - struct task_dp_pdata *pdata = &task_memory->pdata; - struct scheduler_dp_task_kobj *kobj = rmalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, - sizeof(*kobj)); - - if (!kobj) { - tr_err(&dp_tr, "task record alloc failed"); - ret = -ENOMEM; - goto e_stack; - } - - kobj->task = ptask; - kobj->mod = mod; - comp_id = mod->dev->ipc_config.id; - kobj->comp_id = comp_id; - - if (mod->dev->drv->adapter_ops->get_configuration) { - ipc_config_data = mod_alloc_ext(mod, SOF_MEM_FLAG_USER, SOF_IPC_MSG_MAX_SIZE, 0); - if (!ipc_config_data) { - tr_err(&dp_tr, "Failed to allocate memory for get-config"); - ret = -ENOMEM; - goto e_kobj; - } - } - pdata->ipc_config_data = ipc_config_data; - - pdata->flat = &task_memory->flat; - - kobj->event = k_object_alloc(K_OBJ_EVENT); - if (!kobj->event) { - tr_err(&dp_tr, "Event object allocation failed"); - ret = -ENOMEM; - goto e_config; - } - - pdata->event = kobj->event; - - kobj->thread = k_object_alloc(K_OBJ_THREAD); - if (!kobj->thread) { - tr_err(&dp_tr, "Thread object allocation failed"); - ret = -ENOMEM; - goto e_event; - } - memset(&kobj->thread->arch, 0, sizeof(kobj->thread->arch)); - pdata->thread = kobj->thread; - - /* success, fill the structures */ - pdata->p_stack = p_stack; - pdata->mod = mod; - - /* initialize other task structures */ - ptask->ops.complete = ops->complete; - ptask->ops.get_deadline = ops->get_deadline; - ptask->priv_data = pdata; - list_init(&ptask->list); - - /* create a zephyr thread for the task */ - thread_id = k_thread_create(kobj->thread, p_stack, - stack_size, dp_thread_fn, ptask, NULL, NULL, - CONFIG_DP_THREAD_PRIORITY, ptask->flags, K_FOREVER); - pdata->thread_id = thread_id; - scheduler_dp_thread_name_set(thread_id, mod); - - unsigned int pidx; - size_t size; - uintptr_t start; - bool on_pool = false; - -#ifdef CONFIG_SCHED_CPU_MASK - /* pin the thread to specific core */ - ret = k_thread_cpu_pin(thread_id, core); - if (ret < 0) { - tr_err(&dp_tr, "zephyr task pin to core failed"); - goto e_thread; - } -#endif - - k_thread_access_grant(thread_id, kobj->event, &dp_sync[core]); - scheduler_dp_grant(thread_id, core); -#if CONFIG_SOF_USERSPACE_LL - struct k_thread *thread_ipc = ipc_thread_user(core); - - k_thread_access_grant(thread_ipc, kobj->event, thread_id, p_stack, &dp_sync[core]); - scheduler_dp_grant(thread_ipc, core); - scheduler_dp_grant(zephyr_ll_domain_thread(), core); -#endif - - struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), - SOF_MEM_FLAG_COHERENT); - - if (!mdom) { - tr_err(&dp_tr, "objpool allocation failed"); - ret = -ENOMEM; - goto e_thread; - } - - on_pool = true; - - mod->mdom = mdom; - kobj->mdom = mdom; - - if (!mdom->arch.ptables) { - ret = k_mem_domain_init(mdom, 0, NULL); - if (ret < 0) - goto e_thread; - } - - /* Module heap partition */ - mod_heap_info(mod, &size, &start); - kobj->mpart[SOF_DP_PART_HEAP] = (struct k_mem_partition){ - .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)start), - .size = size, - .attr = K_MEM_PARTITION_P_RW_U_RW, - }; - kobj->mpart[SOF_DP_PART_HEAP_CACHE] = (struct k_mem_partition){ - .start = start, - .size = size, - .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, - }; - /* Host mailbox partition for additional IPC parameters: read-only */ - kobj->mpart[SOF_DP_PART_CFG] = (struct k_mem_partition){ - .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)MAILBOX_HOSTBOX_BASE), - .size = MAILBOX_HOSTBOX_SIZE, - .attr = K_MEM_PARTITION_P_RO_U_RO, - }; - kobj->mpart[SOF_DP_PART_CFG_CACHE] = (struct k_mem_partition){ - .start = (uintptr_t)MAILBOX_HOSTBOX_BASE, - .size = MAILBOX_HOSTBOX_SIZE, - .attr = K_MEM_PARTITION_P_RO_U_RO | XTENSA_MMU_CACHED_WB, - }; - - for (pidx = 0; pidx < SOF_DP_PART_TYPE_COUNT; pidx++) { - ret = k_mem_domain_add_partition(mdom, kobj->mpart + pidx); - if (ret < 0) - goto e_dom; - } - - memcpy(pdata->mpart, kobj->mpart, sizeof(pdata->mpart)); - - ret = llext_manager_add_domain(comp_id, mdom); - if (ret < 0) { - tr_err(&dp_tr, "failed to add LLEXT to domain %d", ret); - goto e_dom; - } - - /* - * Keep this call last, able to fail, otherwise domain will be removed - * before its thread - */ - ret = k_mem_domain_add_thread(mdom, thread_id); - if (ret < 0) { - tr_err(&dp_tr, "failed to add thread to domain %d", ret); - goto e_dom; - } - - /* publish the record before the thread can run */ - dp_task_kobj_add(kobj); - - /* start the thread, it should immediately stop at the semaphore */ - k_event_init(kobj->event); - k_thread_start(thread_id); - - /* ptask points into task_memory, so only publish it once it cannot be freed */ - *task = ptask; - - return 0; - -e_dom: - scheduler_dp_domain_free(mod, mdom, kobj->mpart, comp_id); - on_pool = false; -e_thread: - if (on_pool) - objpool_free(&dp_mdom_head, mdom); - k_thread_abort(thread_id); - k_object_free(kobj->thread); -e_event: - k_object_free(kobj->event); -e_config: - mod_free(mod, ipc_config_data); -e_kobj: - rfree(kobj); -e_stack: - user_stack_free(p_stack); -e_tmem: - /* the copy lives in task_memory, so stop pointing at it before freeing */ - mod->dev->drv = drv; - mod_free(mod, task_memory); - return ret; -} - -#ifdef CONFIG_USERSPACE -#include - -/* - * mod_free() follows the module's resource and allocator pointers in - * supervisor mode. They are user-writable, so at least check that they - * point to memory the caller can access. - */ -static void scheduler_dp_mod_vrfy(struct processing_module *mod) -{ - K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(mod->dev, sizeof(*mod->dev))); - K_OOPS(K_SYSCALL_MEMORY_READ(mod->dev->drv, sizeof(*mod->dev->drv))); - - struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; - - K_OOPS(K_SYSCALL_MEMORY_WRITE(alloc, sizeof(*alloc))); - if (alloc->heap) { - size_t h_size = 0; - uintptr_t h_start; - - mod_heap_info(mod, &h_size, &h_start); - if (h_size) - K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); - } - if (alloc->vreg) { - size_t vr_size = 0; - uintptr_t vr_start; - - vregion_mem_info(alloc->vreg, &vr_size, &vr_start); - K_OOPS(K_SYSCALL_MEMORY_WRITE(vr_start, vr_size)); - } -} - -void z_vrfy_scheduler_dp_internal_free(struct task *task) -{ - struct scheduler_dp_task_kobj kobj; - - /* - * Do not read task->priv_data: the kernel objects come from the - * kernel-only record, and an unknown task pointer is rejected. - */ - K_OOPS(!dp_task_kobj_peek(task, &kobj)); - - /* - * A DP thread holds its own thread and event objects, so it would pass - * the checks below for its own task. Freeing would then abort the - * caller after the record is unlinked, leaking everything it holds. - */ - K_OOPS(kobj.thread == k_current_get()); - - /* - * The caller must have been granted the task's objects. Check before - * unlinking the record, so an unauthorized call leaves it intact. - */ - K_OOPS(K_SYSCALL_OBJ(kobj.event, K_OBJ_EVENT)); - K_OOPS(K_SYSCALL_OBJ_INIT(kobj.thread, K_OBJ_THREAD)); - scheduler_dp_mod_vrfy(kobj.mod); - - K_OOPS(!scheduler_dp_internal_free_kobj(task, &kobj)); -} -#include -#endif /* CONFIG_USERSPACE */ From 2d1547b7b17e8341d28ff3d4671f526df5e578a7 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:31:13 +0000 Subject: [PATCH 17/19] schedule: move the TWB scheduler to src/kernel/schedule The thread-with-budget scheduler is kernel-only code. It serializes with irq_lock(), including in its thread function, changes the priority and time slice of its threads, whose budget callback runs in interrupt context, and allocates the thread stacks itself. Move it to src/kernel/schedule, which builds it into sof_kernel, and update the scheduler README. The code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/kernel/schedule/CMakeLists.txt | 4 ++++ src/{ => kernel}/schedule/zephyr_twb_schedule.c | 0 src/schedule/CMakeLists.txt | 4 ---- src/schedule/README.md | 2 +- 4 files changed, 5 insertions(+), 5 deletions(-) rename src/{ => kernel}/schedule/zephyr_twb_schedule.c (100%) diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index f49fc2d5f869..a0a65021878d 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -29,4 +29,8 @@ if(zephyr) zephyr_dp_schedule_thread-kernel.c ) endif() + + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_TWB_SCHEDULER + zephyr_twb_schedule.c + ) endif() diff --git a/src/schedule/zephyr_twb_schedule.c b/src/kernel/schedule/zephyr_twb_schedule.c similarity index 100% rename from src/schedule/zephyr_twb_schedule.c rename to src/kernel/schedule/zephyr_twb_schedule.c diff --git a/src/schedule/CMakeLists.txt b/src/schedule/CMakeLists.txt index bc14c6ecff46..ee9ceb32ef9d 100644 --- a/src/schedule/CMakeLists.txt +++ b/src/schedule/CMakeLists.txt @@ -50,8 +50,4 @@ else() ) endif() -zephyr_library_sources_ifdef(CONFIG_ZEPHYR_TWB_SCHEDULER - zephyr_twb_schedule.c -) - endif() diff --git a/src/schedule/README.md b/src/schedule/README.md index 0bef23248837..937f9034dc09 100644 --- a/src/schedule/README.md +++ b/src/schedule/README.md @@ -139,7 +139,7 @@ stateDiagram-v2 ## 3. TWB (Thread With Budget) Scheduler -The TWB scheduler (`zephyr_twb_schedule.c`) provides execution budget limits for specific tasks to prevent them from starving the CPU. This is useful for intensive workloads that shouldn't disrupt the overall systemic low-latency chain. +The TWB scheduler (`src/kernel/schedule/zephyr_twb_schedule.c`) provides execution budget limits for specific tasks to prevent them from starving the CPU. This is useful for intensive workloads that shouldn't disrupt the overall systemic low-latency chain. ### Architecture From 24dad1c48d96716a7b48982aa24a55d0a484a246 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:32:06 +0000 Subject: [PATCH 18/19] schedule: split the scheduler registration out of schedule.c scheduler_init() and scheduler_register() allocate and register the schedulers at boot, on the kernel heap or, for the user-space schedulers with user-space LL, on the system user heap, and panic on failure. Move them to src/kernel/schedule/schedule-kernel.c. schedule.c keeps schedule_task_init() and the task info query. scheduler_is_user(), which both parts use, becomes a static inline in sof/schedule/schedule.h. The code is unchanged. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/schedule/schedule.h | 15 +++++ src/kernel/schedule/CMakeLists.txt | 1 + src/kernel/schedule/schedule-kernel.c | 81 +++++++++++++++++++++++++++ src/schedule/schedule.c | 67 +--------------------- 4 files changed, 100 insertions(+), 64 deletions(-) create mode 100644 src/kernel/schedule/schedule-kernel.c diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index f5f646cfbc50..362d15fe948d 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -235,6 +235,21 @@ struct schedulers **arch_user_schedulers_get(void); */ struct schedulers **arch_user_schedulers_get_for_core(int core); +/** + * @brief Check whether a scheduler type is managed in user space. + * @param[in] type Scheduler type identifier (SOF_SCHEDULE_*). + * @return true for the LL timer and DP schedulers. + */ +static inline bool scheduler_is_user(int type) +{ + /* + * currently only LL managed in user-space, but longterm + * goal is to move all audio application level scheduling + * to user-space and only keep Zephyr scheduler logic in + * kernel + */ + return type == SOF_SCHEDULE_LL_TIMER || type == SOF_SCHEDULE_DP; +} /** * @brief Lookup scheduler private context data by scheduler type from a schedulers container. * @param[in] schedulers Schedulers list container. diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index a0a65021878d..8711fde99b95 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -3,6 +3,7 @@ is_zephyr(zephyr) if(zephyr) zephyr_library_sources( + schedule-kernel.c zephyr_domain-kernel.c ) diff --git a/src/kernel/schedule/schedule-kernel.c b/src/kernel/schedule/schedule-kernel.c new file mode 100644 index 000000000000..bdefa7f35567 --- /dev/null +++ b/src/kernel/schedule/schedule-kernel.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2019 Intel Corporation. All rights reserved. +// +// Author: Bartosz Kokoszko + +/* Generic scheduler */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(schedule, CONFIG_SOF_LOG_LEVEL); + +extern struct tr_ctx sch_tr; + +/* Kernel part of the generic scheduler: scheduler registration. The rest is in + * src/schedule/schedule.c + */ + +__cold static void scheduler_register(struct schedule_data *scheduler) +{ + struct schedulers **sch = arch_schedulers_get(); + struct k_heap *heap = NULL; + + assert_can_be_cold(); + + if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(scheduler->type)) { + sch = arch_user_schedulers_get(); + heap = sof_sys_user_heap_get(); + } + + if (!*sch) { + /* init schedulers list */ + *sch = sof_heap_alloc(heap, 0, sizeof(**sch), 0); + if (!*sch) { + tr_err(&sch_tr, "allocation failed"); + return; + } + memset(*sch, 0, sizeof(**sch)); + list_init(&(*sch)->list); + } + + list_item_append(&scheduler->list, &(*sch)->list); +} + +__cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data) +{ + struct schedule_data *sch; + struct k_heap *heap = NULL; + + assert_can_be_cold(); + + if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(type)) + heap = sof_sys_user_heap_get(); + + if (!ops || !ops->schedule_task || !ops->schedule_task_cancel || + !ops->schedule_task_free) + return; + + sch = sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, sizeof(*sch), 0); + if (!sch) { + tr_err(&sch_tr, "allocation failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + memset(sch, 0, sizeof(*sch)); + list_init(&sch->list); + sch->type = type; + sch->ops = ops; + sch->data = data; + + scheduler_register(sch); +} diff --git a/src/schedule/schedule.c b/src/schedule/schedule.c index fd3349a14902..62438be7c663 100644 --- a/src/schedule/schedule.c +++ b/src/schedule/schedule.c @@ -25,16 +25,9 @@ SOF_DEFINE_REG_UUID(schedule); /* unused with Zephyr, generates no output */ DECLARE_TR_CTX(sch_tr, SOF_UUID(schedule_uuid), LOG_LEVEL_INFO); -static inline bool scheduler_is_user(int type) -{ - /* - * currently only LL managed in user-space, but longterm - * goal is to move all audio application level scheduling - * to user-space and only keep Zephyr scheduler logic in - * kernel - */ - return type == SOF_SCHEDULE_LL_TIMER || type == SOF_SCHEDULE_DP; -} +/* Scheduler registration, which runs in kernel mode, is in + * src/kernel/schedule/schedule-kernel.c + */ int schedule_task_init(struct task *task, const struct sof_uuid_entry *uid, uint16_t type, @@ -84,60 +77,6 @@ int schedule_task_init(struct task *task, return 0; } -__cold static void scheduler_register(struct schedule_data *scheduler) -{ - struct schedulers **sch = arch_schedulers_get(); - struct k_heap *heap = NULL; - - assert_can_be_cold(); - - if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(scheduler->type)) { - sch = arch_user_schedulers_get(); - heap = sof_sys_user_heap_get(); - } - - if (!*sch) { - /* init schedulers list */ - *sch = sof_heap_alloc(heap, 0, sizeof(**sch), 0); - if (!*sch) { - tr_err(&sch_tr, "allocation failed"); - return; - } - memset(*sch, 0, sizeof(**sch)); - list_init(&(*sch)->list); - } - - list_item_append(&scheduler->list, &(*sch)->list); -} - -__cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data) -{ - struct schedule_data *sch; - struct k_heap *heap = NULL; - - assert_can_be_cold(); - - if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(type)) - heap = sof_sys_user_heap_get(); - - if (!ops || !ops->schedule_task || !ops->schedule_task_cancel || - !ops->schedule_task_free) - return; - - sch = sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, sizeof(*sch), 0); - if (!sch) { - tr_err(&sch_tr, "allocation failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - memset(sch, 0, sizeof(*sch)); - list_init(&sch->list); - sch->type = type; - sch->ops = ops; - sch->data = data; - - scheduler_register(sch); -} - /* Locks for the list here should be held by the caller, * as different schedulers use different locks */ From 477d8ee00ef1948943b28c6c2b9e822edb10b835 Mon Sep 17 00:00:00 2001 From: Liam Girdwood Date: Sun, 8 Nov 2026 14:32:44 +0000 Subject: [PATCH 19/19] docs: schedule: describe the kernel parts of the schedulers Update the scheduler README for the split of the scheduler sources: the kernel parts are in src/kernel/schedule, the TWB scheduler is there in full, and the shared data structures are in sof/schedule/zephyr_ll.h and sof/schedule/zephyr_dp_schedule.h. Point the syscall table of the user-space README at the kernel files. Co-Authored-By: Claude Opus 5.5 Signed-off-by: Liam Girdwood --- src/include/sof/userspace/README.md | 2 +- src/schedule/README.md | 17 ++++++++++------- 2 files changed, 11 insertions(+), 8 deletions(-) diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index 143f2ec8b2ee..02db15ed391b 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -124,7 +124,7 @@ syscall by subsystem: | CPU | `cpu_get_id` | `sof/lib/cpu.h` | `zephyr/syscall/cpu.c` / `zephyr/lib/cpu.c` | | Module management | `mod_balloc_align`, `mod_alloc_ext`, `mod_free`, `mod_free_all`, `mod_data_blob_handler_new`, `mod_fast_get` | `sof/audio/module_adapter/module/generic.h` | scattered: `audio/module_adapter/module/generic.c` | | Library manager | `lib_manager_free_module` | `sof/lib_manager.h` | scattered: `kernel/library_manager/lib_manager.c` | -| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `schedule/zephyr_ll.c`, `schedule/zephyr_dp_schedule*.c` | +| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `kernel/schedule/zephyr_ll-kernel.c`, `kernel/schedule/zephyr_dp_schedule*-kernel.c` | | Debug | `debug_stream_slot_send_record` | `user/debug_stream_slot.h` | scattered: `debug/debug_stream/debug_stream_slot.c` | The `zephyr/syscall/*.c` files hold the validators in one place per subsystem; the rows marked diff --git a/src/schedule/README.md b/src/schedule/README.md index 937f9034dc09..d2a86f6bf55c 100644 --- a/src/schedule/README.md +++ b/src/schedule/README.md @@ -1,12 +1,15 @@ # SOF Scheduling Architecture -> **Runs in:** Boundary / bridge — this directory owns the LL/DP schedulers and their thread + -> memory-domain setup. `zephyr_ll.c` and `zephyr_domain.c` are kernel-side; the LL user memory -> domain and heap are set up by `src/kernel/schedule/zephyr_ll_user.c`. -> `zephyr_dp_schedule_thread.c` is boundary code that can create and manage both kernel and -> `K_USER` DP threads depending on configuration; `zephyr_dp_schedule_application.c` is the -> DP application-mode path. This directory hosts the `zephyr_ll_task_sem_*` and -> `scheduler_dp_*` syscalls. See [User/kernel split](../include/sof/userspace/README.md). +> **Runs in:** Boundary / bridge — this directory holds the parts of the LL and DP schedulers +> that run in the scheduler and IPC threads, which are user threads with user-space LL. Their +> kernel parts are in `src/kernel/schedule`, one `*-kernel.c` file per source here: scheduler +> registration and initialization, the LL thread and timer setup and the timer interrupt, the +> DP thread creation and memory domains, the access grants, and the kernel halves of the +> `zephyr_ll_task_sem_*` and `scheduler_dp_*` syscalls. The LL user memory domain and heap are +> set up by `src/kernel/schedule/zephyr_ll_user.c`, and the TWB scheduler, which is kernel-only, +> is `src/kernel/schedule/zephyr_twb_schedule.c`. The scheduler data structures shared by both +> parts are in `sof/schedule/zephyr_ll.h` and `sof/schedule/zephyr_dp_schedule.h`. See +> [User/kernel split](../include/sof/userspace/README.md). This directory (`src/schedule`) contains the Sound Open Firmware (SOF) scheduling infrastructure, deeply integrated with the underlying Zephyr RTOS. SOF utilizes a multi-tiered scheduling approach to cater to different real-time constraints, ranging from hard real-time, low-latency requirements to more relaxed, compute-intensive data processing tasks.