diff --git a/AGENTS.md b/AGENTS.md index 6a2f4cdfee5b..3eb2efa4321b 100644 --- a/AGENTS.md +++ b/AGENTS.md @@ -29,12 +29,11 @@ memory-partition markers and Kconfig hierarchy are documented in [`src/include/sof/userspace/README.md`](src/include/sof/userspace/README.md). When adding or moving code, follow it: -* **Place code in the right tier.** OS/HAL/boot/inter-core → a kernel-only directory - (`kernel/`, which is built into the `sof_kernel` library, and the directories not yet - moved there: `arch/`, `platform/`, `trace/`); audio - application logic → `audio/`; reusable pure code → a shared-library directory (`lib/`, - `math/`, `module/`) kept free of privileged operations. Each top-level directory's - `README.md` states its tier in a "Runs in:" banner — keep it accurate. +* **Place code in the right tier.** OS/HAL/boot/inter-core → the kernel-only directory + (`kernel/`, which is built into the `sof_kernel` library); audio application logic → + `audio/`; reusable pure code → a shared-library directory (`lib/`, `math/`, `module/`) kept + free of privileged operations. Each top-level directory's `README.md` states its tier in a + "Runs in:" banner — keep it accurate. * **Cross the boundary only via a syscall.** Add a `__syscall` declaration, put the `z_vrfy_` validator in the boundary tier (preferably `zephyr/syscall/.c`), validate every user-supplied argument, and add the call to diff --git a/CMakeLists.txt b/CMakeLists.txt index 1fe6aa02b614..1af14e4b4089 100644 --- a/CMakeLists.txt +++ b/CMakeLists.txt @@ -144,6 +144,6 @@ endif() if(BUILD_UNIT_TESTS) enable_testing() - add_subdirectory(src/arch/${ARCH}) + add_subdirectory(src/kernel/arch/${ARCH}) add_subdirectory(test) endif() diff --git a/CODEOWNERS b/CODEOWNERS index 543435d5f3c3..e06147afb8cb 100644 --- a/CODEOWNERS +++ b/CODEOWNERS @@ -47,9 +47,9 @@ src/audio/module_adapter/* @ranj063 @jxstelter src/audio/google/* @thesofproject/google # platforms -src/platform/imx8/** @dbaluta -src/platform/amd/** @bhiregoudar @sunilkumardommati -src/platform/mtk/** @yaochunhung @kuanhsuncheng +src/kernel/platform/imx8/** @dbaluta +src/kernel/platform/amd/** @bhiregoudar @sunilkumardommati +src/kernel/platform/mtk/** @yaochunhung @kuanhsuncheng # drivers src/drivers/intel/dmic.c @singalsu diff --git a/app/CMakeLists.txt b/app/CMakeLists.txt index fe9b208fe99f..5537caecda09 100644 --- a/app/CMakeLists.txt +++ b/app/CMakeLists.txt @@ -27,6 +27,6 @@ target_sources(app PRIVATE ) zephyr_library_include_directories(app PUBLIC - ${CMAKE_CURRENT_SOURCE_DIR}/../src/arch/xtensa/include + ${CMAKE_CURRENT_SOURCE_DIR}/../src/kernel/arch/xtensa/include ${CMAKE_CURRENT_SOURCE_DIR}/../src/include ) diff --git a/app/boards/acp_7_0_adsp.conf b/app/boards/acp_7_0_adsp.conf index 5eb2ff9f7b95..96ffd80506c2 100644 --- a/app/boards/acp_7_0_adsp.conf +++ b/app/boards/acp_7_0_adsp.conf @@ -18,7 +18,6 @@ CONFIG_DMA_AMD_ACP_TDM=y CONFIG_DAI_AMD_TDM=y CONFIG_AMS=n CONFIG_WRAP_ACTUAL_POSITION=y -CONFIG_TRACE=n CONFIG_COMP_VOLUME=y CONFIG_COMP_SRC=n CONFIG_COMP_FIR=n diff --git a/app/boards/acp_7_x_adsp.conf b/app/boards/acp_7_x_adsp.conf index 44847f4c77de..36af59b502f5 100644 --- a/app/boards/acp_7_x_adsp.conf +++ b/app/boards/acp_7_x_adsp.conf @@ -19,7 +19,6 @@ CONFIG_DAI_AMD_SDW=y CONFIG_DAI_AMD_TDM=y CONFIG_AMS=n CONFIG_WRAP_ACTUAL_POSITION=y -CONFIG_TRACE=n CONFIG_COMP_VOLUME=y CONFIG_COMP_SRC=n CONFIG_COMP_FIR=n diff --git a/app/boards/imx8mp_evk_mimx8ml8_adsp.conf b/app/boards/imx8mp_evk_mimx8ml8_adsp.conf index da9e56d759a4..aee1c653d332 100644 --- a/app/boards/imx8mp_evk_mimx8ml8_adsp.conf +++ b/app/boards/imx8mp_evk_mimx8ml8_adsp.conf @@ -5,7 +5,6 @@ CONFIG_KPB_FORCE_COPY_TYPE_NORMAL=n CONFIG_DMA=y CONFIG_DMA_NXP_SDMA=y CONFIG_DAI_NXP_MICFIL=y -CONFIG_TRACE=n CONFIG_SHARED_INTERRUPTS=y CONFIG_ZEPHYR_NATIVE_DRIVERS=y CONFIG_SAI_HAS_MCLK_CONFIG_OPTION=y diff --git a/app/boards/intel_adsp_ace40_nvl.conf b/app/boards/intel_adsp_ace40_nvl.conf index 4576e5a45210..bc62ec76e877 100644 --- a/app/boards/intel_adsp_ace40_nvl.conf +++ b/app/boards/intel_adsp_ace40_nvl.conf @@ -31,7 +31,6 @@ CONFIG_LIBRARY_BUILD_LIB=y CONFIG_LIBRARY_DEFAULT_MODULAR=y # SOF / logging -CONFIG_TRACE=n CONFIG_SOF_LOG_LEVEL_INF=y # Zephyr / OS features diff --git a/app/boards/intel_adsp_ace40_nvls.conf b/app/boards/intel_adsp_ace40_nvls.conf index 67cdb0e2cb86..0013a290bc4a 100644 --- a/app/boards/intel_adsp_ace40_nvls.conf +++ b/app/boards/intel_adsp_ace40_nvls.conf @@ -31,7 +31,6 @@ CONFIG_LIBRARY_BUILD_LIB=y CONFIG_LIBRARY_DEFAULT_MODULAR=y # SOF / logging -CONFIG_TRACE=n CONFIG_SOF_LOG_LEVEL_INF=y # Zephyr / OS features diff --git a/app/prj.conf b/app/prj.conf index 5c22e19d6e0e..b989ac6d1ac4 100644 --- a/app/prj.conf +++ b/app/prj.conf @@ -15,7 +15,6 @@ CONFIG_HAVE_AGENT=n CONFIG_MINIMAL_LIBC=y CONFIG_ZEPHYR_LOG=y -CONFIG_TRACE=n CONFIG_LOG=y CONFIG_LOG_PRINTK=y # Log processing is offloaded to a low-priority thread. diff --git a/doc/sof.doxygen.in b/doc/sof.doxygen.in index 02750d0f324f..c4d6e196c18a 100644 --- a/doc/sof.doxygen.in +++ b/doc/sof.doxygen.in @@ -45,7 +45,6 @@ PREDEFINED = "__attribute__(x)= " \ "CONFIG_FORMAT_S16LE=1" \ "CONFIG_FORMAT_S32LE=1" \ "CONFIG_FORMAT_S24LE=1" \ - "CONFIG_TRACE=1" \ "CONFIG_IPC4=1" \ "CONFIG_IPC3=1" \ "CONFIG_IPC_MAJOR_4=1" \ diff --git a/posix/include/rtos/sof.h b/posix/include/rtos/sof.h index d6aa0fce66db..3487e0d11d32 100644 --- a/posix/include/rtos/sof.h +++ b/posix/include/rtos/sof.h @@ -17,7 +17,6 @@ struct clock_info; struct comp_driver_list; struct dai_info; struct dma_info; -struct dma_trace_data; struct ipc; struct ll_schedule_domain; struct mm; @@ -27,7 +26,6 @@ struct notify_data; struct pm_runtime_data; struct sa; struct timer; -struct trace; struct pipeline_posn; struct probe_pdata; @@ -49,11 +47,7 @@ struct sof { /* system agent */ struct sa *sa; - /* DMA for Trace*/ - struct dma_trace_data *dmat; - /* generic trace structure */ - struct trace *trace; /* platform clock information */ struct clock_info *clocks; diff --git a/scripts/cmake/defconfigs.cmake b/scripts/cmake/defconfigs.cmake index 9bdbd23539a8..84d7eb0fe752 100644 --- a/scripts/cmake/defconfigs.cmake +++ b/scripts/cmake/defconfigs.cmake @@ -1,7 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause # Looks for defconfig files in arch directories where kconfig.cmake looks too. -set(DEFCONFIGS_DIRECTORY "${PROJECT_SOURCE_DIR}/src/arch/${ARCH}/configs") +set(DEFCONFIGS_DIRECTORY "${PROJECT_SOURCE_DIR}/src/kernel/arch/${ARCH}/configs") file(GLOB DEFCONFIG_PATHS "${DEFCONFIGS_DIRECTORY}/*_defconfig") # Adds dependency on defconfigs directory diff --git a/scripts/cmake/kconfig.cmake b/scripts/cmake/kconfig.cmake index f885598387bf..7d3dd4481b18 100644 --- a/scripts/cmake/kconfig.cmake +++ b/scripts/cmake/kconfig.cmake @@ -13,14 +13,14 @@ if(NOT INIT_CONFIG_found) set(INIT_CONFIG "initial.config" CACHE STRING "Initial .config file") # - ".' is the top source directory. -# - "src/arch/host/configs" holds the library (testbench) configurations +# - "src/kernel/arch/host/configs" holds the library (testbench) configurations # and is also used by defconfigs.cmake. # - "test/cmocka/configs" holds the unit test configurations. # - First found wins. # - If two directories ever use the same defconfig name then a full # path must be used, e.g.: -DINIT_CONFIG=test/cmocka/configs/collision_defconfig -set(init_config_search_list "." "src/arch/host/configs" "test/cmocka/configs") +set(init_config_search_list "." "src/kernel/arch/host/configs" "test/cmocka/configs") find_file(INIT_CONFIG_found NAMES ${INIT_CONFIG} @@ -46,7 +46,7 @@ message(FATAL_ERROR "Initial configuration missing, no ${INIT_CONFIG} found. " "Provide a ${PROJECT_SOURCE_DIR}/initial.config file or specify some " "other -DINIT_CONFIG=location relative to '${PROJECT_SOURCE_DIR}/' or " - "'${PROJECT_SOURCE_DIR}/src/arch/*/configs/'" + "'${PROJECT_SOURCE_DIR}/src/kernel/arch/*/configs/'" ) endif() diff --git a/scripts/kernel_user_partition.txt b/scripts/kernel_user_partition.txt index b5d6cda1c33f..39d5e1191366 100644 --- a/scripts/kernel_user_partition.txt +++ b/scripts/kernel_user_partition.txt @@ -40,13 +40,12 @@ src/include/sof/drivers/* src/include/sof/platform.h src/include/sof/llext_manager.h src/include/sof/ipc/driver.h -src/include/sof/trace/dma-trace.h src/include/sof/lib/agent.h src/include/sof/lib/cpu-clk-manager.h src/include/sof/lib/mailbox.h src/include/sof/lib/perf_cnt.h src/include/sof/lib/watchdog.h -src/platform/* +src/kernel/platform/* zephyr/include/rtos/clk.h zephyr/include/rtos/idc.h zephyr/include/rtos/interrupt.h diff --git a/scripts/rebuild-testbench.sh b/scripts/rebuild-testbench.sh index 996d16f45a8c..578632dfa1cb 100755 --- a/scripts/rebuild-testbench.sh +++ b/scripts/rebuild-testbench.sh @@ -116,7 +116,7 @@ For temporary, interactive Kconfiguration use: $BUILD_BACKEND -C $TESTBENCH_DIR/$BUILD_DIR_NAME/sof_ep/build/ menuconfig -Permanent configuration is "src/arch/host/configs/library_defconfig". +Permanent configuration is "src/kernel/arch/host/configs/library_defconfig". For instant, incremental build: diff --git a/src/CMakeLists.txt b/src/CMakeLists.txt index a4d31797de8f..aa5c682264c4 100644 --- a/src/CMakeLists.txt +++ b/src/CMakeLists.txt @@ -1,7 +1,5 @@ # SPDX-License-Identifier: BSD-3-Clause -add_subdirectory(platform) -add_subdirectory(arch) add_subdirectory(ipc) add_subdirectory(audio) add_subdirectory(lib) @@ -14,6 +12,3 @@ endif() add_subdirectory(kernel) -if (CONFIG_TRACE) - add_subdirectory(trace) -endif() diff --git a/src/Kconfig b/src/Kconfig index 6dbd3c396c04..d008574a9684 100644 --- a/src/Kconfig +++ b/src/Kconfig @@ -1,13 +1,11 @@ # SPDX-License-Identifier: BSD-3-Clause -rsource "arch/Kconfig" +rsource "kernel/arch/Kconfig" -rsource "platform/Kconfig" +rsource "kernel/platform/Kconfig" rsource "audio/Kconfig" -rsource "trace/Kconfig" - rsource "samples/Kconfig" rsource "schedule/Kconfig" diff --git a/src/audio/Kconfig b/src/audio/Kconfig index dbd12f1626b2..fecbf0d25afa 100644 --- a/src/audio/Kconfig +++ b/src/audio/Kconfig @@ -422,14 +422,6 @@ config PCM_REMAPPING_CONVERTERS Enable conversion functions that perform both format conversion and channel remapping simultaneously. -config TRACE_CHANNEL - int "TRACE DMA Channel configuration" - default 0 - help - This option is used for custom configuration of trace dma channel other - than zero dma channel.Define macro DMA_TRACE_CHANNEL in platform to - configure the required channel number. - config WRAP_ACTUAL_POSITION bool "Buffer wrapping" help diff --git a/src/audio/README.md b/src/audio/README.md index 73a48ec7520e..39cc910c4dfa 100644 --- a/src/audio/README.md +++ b/src/audio/README.md @@ -15,5 +15,5 @@ modules in user context. When adding a component here, keep it inside the application tier: allocate through the module allocation syscalls (`mod_*`), mark any globals shared with the kernel using the partition -markers, and never call kernel-only subsystems (`kernel/drivers/`, `platform/`, `kernel/idc/`) +markers, and never call kernel-only subsystems (`kernel/drivers/`, `kernel/platform/`, `kernel/idc/`) directly. diff --git a/src/audio/module_adapter/module/waves/waves.c b/src/audio/module_adapter/module/waves/waves.c index 208bb7e010ff..b77dcdcee88d 100644 --- a/src/audio/module_adapter/module/waves/waves.c +++ b/src/audio/module_adapter/module/waves/waves.c @@ -432,31 +432,6 @@ static int waves_effect_revision(struct processing_module *mod) return -EINVAL; } -#if CONFIG_TRACEV - if (revision_len) { - const uint32_t *ptr = (uint32_t *)revision; - uint32_t len = revision_len / sizeof(uint32_t); - uint32_t idx = 0; - - /* get requests from codec_adapter are not supported - * printing strings is not supported - * so dumping revision string to trace log as ascii values - * if simply write a for loop here then depending on trace filtering settings - * some parts of revision might not be printed - this is highly unwanted - */ - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - dump_hex(ptr, idx, len); - } -#endif - comp_info(dev, "done"); return 0; } @@ -528,20 +503,6 @@ static int waves_effect_message(struct processing_module *mod, void *data, uint3 } } -#if CONFIG_TRACEV - /* at time of writing codec adapter does not support getting something from codec - * so response is stored to internal structure and dumped into trace messages - */ - if (response_size) { - uint32_t idx; - uint32_t len = response_size / sizeof(uint32_t); - const uint32_t *ptr = (uint32_t *)waves_codec->response; - - for (idx = 0; idx < len; ) - dump_hex(ptr, idx, len); - } -#endif - return 0; } diff --git a/src/debug/panic.c b/src/debug/panic.c index 0c56886e386a..7f163905ee83 100644 --- a/src/debug/panic.c +++ b/src/debug/panic.c @@ -87,10 +87,6 @@ void panic_dump(uint32_t p, struct sof_ipc_panic_info *panic_info, dump_panicinfo(ext_offset, panic_info); ext_offset += sizeof(struct sof_ipc_panic_info); -#if CONFIG_TRACE - trace_flush_dma_to_mbox(); -#endif - /* Dump stack frames and override panic code 'p' if ext_offset is * off stack_limit. Find stack_ptr. */ diff --git a/src/include/ipc4/handler.h b/src/include/ipc4/handler.h index 3335c108fa29..5fc9f749e7fb 100644 --- a/src/include/ipc4/handler.h +++ b/src/include/ipc4/handler.h @@ -12,7 +12,10 @@ #ifndef __SOF_IPC4_HANDLER_H__ #define __SOF_IPC4_HANDLER_H__ +struct comp_dev; +struct comp_driver; struct ipc4_message_request; +struct ipc4_pipeline_set_state; /** * \brief Processes an IPC4 userspace module message. @@ -69,6 +72,73 @@ int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, struct ipc_ */ int ipc4_set_pipeline_state(struct ipc4_message_request *ipc4); +/** + * \brief Get the first pipeline ID of a SET_PIPELINE_STATE message. + * \ingroup ipc4_envelope_api + * @param[in] ipc4 IPC4 message request structure. + * @param[in] state Copy of the SET_PIPELINE_STATE message header. + * @param[out] ppl_id Receives the pipeline ID array of a multi-pipeline + * message, NULL otherwise (may be NULL). + * @param[out] ppl_count Receives the number of pipelines (may be NULL). + * @return The first pipeline ID, or a negative error code. + */ +int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, + struct ipc4_pipeline_set_state *state, + const uint32_t **ppl_id, unsigned int *ppl_count); + +/** + * \brief Copy the kcontrol payload header of a LARGE_CONFIG_GET from the hostbox. + * \ingroup ipc4_envelope_api + * @param[in] dev Target component, NULL for the base firmware. + * @param[in] param_id Large config parameter ID. + * @param[out] data_out Reply data buffer. + * @param[in] data_size Size of the payload in the hostbox. + */ +void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t param_id, + char *data_out, uint32_t data_size); + +/** + * \brief Process a vendor config LARGE_CONFIG_GET. + * \ingroup ipc4_envelope_api + * @param[in] dev Target component, NULL for the base firmware. + * @param[in] drv Component driver. + * @param[in] init_block True for the first block. + * @param[in] final_block True for the last block. + * @param[in,out] data_off_size Payload size in, reply data size out. + * @param[out] data_out Reply data buffer. + * @param[in] data_in Request payload. + * @return IPC4 status code (IPC4_SUCCESS on success). + */ +int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, + const struct comp_driver *drv, + bool init_block, + bool final_block, + uint32_t *data_off_size, + char *data_out, + const char *data_in); + +/** + * \brief Process a vendor config LARGE_CONFIG_SET. + * \ingroup ipc4_envelope_api + * @param[in] dev Target component, NULL for the base firmware. + * @param[in] drv Component driver. + * @param[in] module_id Module ID, for error messages. + * @param[in] instance_id Instance ID, for error messages. + * @param[in] init_block True for the first block. + * @param[in] final_block True for the last block. + * @param[in] data_off_size Payload size. + * @param[in] data Request payload. + * @return IPC4 status code (IPC4_SUCCESS on success). + */ +int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, + const struct comp_driver *drv, + uint32_t module_id, + uint32_t instance_id, + bool init_block, + bool final_block, + uint32_t data_off_size, + const char *data); + /** * \brief Complete an IPC compound message transaction and signal waiting threads. * \ingroup ipc4_envelope_api diff --git a/src/include/sof/debug/debug.h b/src/include/sof/debug/debug.h index fa3fa990b2b4..3e656e43dc87 100644 --- a/src/include/sof/debug/debug.h +++ b/src/include/sof/debug/debug.h @@ -143,34 +143,5 @@ ((n & 0x00FF0000) >> 8) | \ ((n & 0xFF000000) >> 24); }) -#if CONFIG_TRACE -/* dump up to 4 32-bit words into trace starting in ptr, idx is incremented inside - * unconditionally swaps endianness of data - * todo: use cpu_to_endianness so trace data will be independent of platform - */ -#define dump_hex(ptr, idx, len) \ - do { \ - __typeof__(idx) __i = (idx); \ - __typeof__(ptr) __p = (ptr) + __i; \ - __typeof__(idx) __l = (len) - __i; \ - __typeof__(idx) __n = __l > 4 ? 4 : __l; \ - if (__n == 4) { \ - comp_info(dev, "%08x%08x%08x%08x", bswap32(__p[0]), bswap32(__p[1]), \ - bswap32(__p[2]), bswap32(__p[3])); \ - } else if (__n == 3) { \ - comp_info(dev, "%08x%08x%08x", bswap32(__p[0]), bswap32(__p[1]), \ - bswap32(__p[2])); \ - } else if (__n == 2) { \ - comp_info(dev, "%08x%08x", bswap32(__p[0]), bswap32(__p[1])); \ - } else if (__n == 1) { \ - comp_info(dev, "%08x", bswap32(__p[0])); \ - } \ - idx += __n; \ - } while (0) -#else - -#define dump_hex() do {} while (0) - -#endif #endif /* __SOF_DEBUG_DEBUG_H__ */ diff --git a/src/include/sof/ipc/common.h b/src/include/sof/ipc/common.h index 696cb553661d..88bd598a9f08 100644 --- a/src/include/sof/ipc/common.h +++ b/src/include/sof/ipc/common.h @@ -213,12 +213,6 @@ int ipc_process_host_buffer(struct ipc *ipc, struct dma_sg_elem_array *elem_array, uint32_t *ring_size); -/** - * \brief Send DMA trace host buffer position to host. - * @return 0 on success. - */ -int ipc_dma_trace_send_position(void); - /** * \brief send a IPC buffer status notify message */ @@ -333,6 +327,19 @@ void ipc_complete_cmd(struct ipc *ipc); extern bool ipc_enter_gdb; #ifdef CONFIG_SOF_USERSPACE_LL +/* Events that wake the user-space IPC thread */ +#define IPC_USER_EVENT_CMD BIT(0) +#define IPC_USER_EVENT_STOP BIT(1) + +/** + * @brief User-space IPC thread entry point. + * + * @param p1 struct ipc_user shared with the kernel + * @param p2 Core ID + * @param p3 Unused + */ +void ipc_user_thread_fn(void *p1, void *p2, void *p3); + /** * @brief Forward an IPC command to the user-space thread and wait for it. * diff --git a/src/include/sof/ipc/msg.h b/src/include/sof/ipc/msg.h index 943faf6df813..528026783924 100644 --- a/src/include/sof/ipc/msg.h +++ b/src/include/sof/ipc/msg.h @@ -138,16 +138,4 @@ void ipc_build_stream_posn(struct sof_ipc_stream_posn *posn, uint32_t type, void ipc_build_comp_event(struct sof_ipc_comp_event *event, uint32_t type, uint32_t id); -/** - * \brief Check if trace buffer is ready for transmission. - * @param[in,out] avail Data available in trace buffer - */ -bool ipc_trigger_trace_xfer(uint32_t avail); - -/** - * \brief Build trace position IPC message. - * @param[in,out] posn Trace position message - */ -void ipc_build_trace_posn(struct sof_ipc_dma_trace_posn *posn); - #endif diff --git a/src/include/sof/ipc/topology.h b/src/include/sof/ipc/topology.h index d072abc16a24..1711a0fe3a1c 100644 --- a/src/include/sof/ipc/topology.h +++ b/src/include/sof/ipc/topology.h @@ -48,6 +48,11 @@ struct ipc_comp_dev; const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id); struct comp_dev *ipc4_get_comp_dev(uint32_t comp_id); int ipc4_add_comp_dev(struct comp_dev *dev); +struct comp_ipc_config; +int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, + const struct ipc4_module_init_instance *module_init); +unsigned char *ipc4_get_comp_new_data(void); +int ipc_comp_free_remote(struct comp_dev *dev); #ifdef CONFIG_SOF_USERSPACE_LL struct ipc4_message_request; struct comp_driver; diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index f5f646cfbc50..362d15fe948d 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -235,6 +235,21 @@ struct schedulers **arch_user_schedulers_get(void); */ struct schedulers **arch_user_schedulers_get_for_core(int core); +/** + * @brief Check whether a scheduler type is managed in user space. + * @param[in] type Scheduler type identifier (SOF_SCHEDULE_*). + * @return true for the LL timer and DP schedulers. + */ +static inline bool scheduler_is_user(int type) +{ + /* + * currently only LL managed in user-space, but longterm + * goal is to move all audio application level scheduling + * to user-space and only keep Zephyr scheduler logic in + * kernel + */ + return type == SOF_SCHEDULE_LL_TIMER || type == SOF_SCHEDULE_DP; +} /** * @brief Lookup scheduler private context data by scheduler type from a schedulers container. * @param[in] schedulers Schedulers list container. diff --git a/src/schedule/zephyr_dp_schedule.h b/src/include/sof/schedule/zephyr_dp_schedule.h similarity index 63% rename from src/schedule/zephyr_dp_schedule.h rename to src/include/sof/schedule/zephyr_dp_schedule.h index 07d8ee4a4754..582193a6a816 100644 --- a/src/schedule/zephyr_dp_schedule.h +++ b/src/include/sof/schedule/zephyr_dp_schedule.h @@ -6,16 +6,26 @@ */ /** - * @file src/schedule/zephyr_dp_schedule.h + * @file include/sof/schedule/zephyr_dp_schedule.h * @brief Internal Data Processing (DP) scheduler structures, memory partitions, and thread helpers. * @ingroup scheduling_api + * + * Shared by the DP scheduler sources in src/schedule and their kernel parts in + * src/kernel/schedule. Not for use outside the scheduler. */ +#ifndef __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ +#define __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ + #include +#include #include #include #include +#include +#include + #include #include @@ -66,6 +76,73 @@ struct task_dp_pdata { #endif }; +#if CONFIG_SOF_USERSPACE_APPLICATION +/** @brief Per-core DP thread start and IPC completion semaphores. */ +extern struct k_sem dp_sync[CONFIG_CORE_COUNT]; + +struct ipc4_flat { + unsigned int cmd; + int ret; + union { + struct { + struct ipc4_module_bind_unbind bu; + enum bind_type type; + } bind; + struct { + unsigned int trigger_cmd; + enum ipc4_pipeline_state state; + int n_sources; + int n_sinks; + struct sof_source *source[CONFIG_MODULE_MAX_CONNECTIONS]; + struct sof_sink *sink[CONFIG_MODULE_MAX_CONNECTIONS]; + } pipeline_state; + /* + * SOF_IPC4_MOD_INIT_INSTANCE: a by-value copy of the caller's + * ext_data. The original lives on the calling thread's stack, + * which the DP thread's memory domain doesn't have access to. + * The pointers it contains reference the IPC mailbox, which stays + * valid and is covered by the DP thread's SOF_DP_PART_CFG partition. + */ + struct module_ext_init_data init_instance; + struct { + uint32_t param_id; + enum module_cfg_fragment_position position; + uint32_t data_offset_size; + size_t fragment_size; + const char *data; + } set_config; + struct { + uint32_t param_id; + uint32_t data_offset_size; + size_t fragment_size; + char *data; + } get_config; + }; +}; + +/* memory allocation helper structure */ +struct scheduler_dp_task_memory { + struct task task; + struct task_dp_pdata pdata; + struct comp_driver drv; + struct ipc4_flat flat; +}; + +#endif /* CONFIG_SOF_USERSPACE_APPLICATION */ +/** @brief Per-core DP scheduler locks, see scheduler_dp_lock(). */ +extern struct k_sem dp_lock[CONFIG_MP_MAX_NUM_CPUS]; + +/** + * @brief DP scheduler ops, see struct scheduler_ops. + */ +int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, + uint64_t period); +int scheduler_dp_task_stop(void *data, struct task *task); +int scheduler_dp_task_free(void *data, struct task *task); +#if CONFIG_SOF_USERSPACE_APPLICATION +int scheduler_dp_task_cancel(void *data, struct task *task); +#endif + /** * @brief Recalculate deadlines and evaluate readiness for all active DP tasks on an LL tick. * @param[in,out] dp_sch Pointer to per-core DP scheduler state container. @@ -102,3 +179,5 @@ void scheduler_dp_unlock(unsigned int key); * @param[in] core Target DSP core ID. */ void scheduler_dp_grant(k_tid_t thread_id, uint16_t core); + +#endif /* __SOF_SCHEDULE_ZEPHYR_DP_SCHEDULE_H__ */ diff --git a/src/include/sof/schedule/zephyr_ll.h b/src/include/sof/schedule/zephyr_ll.h new file mode 100644 index 000000000000..e86ec13731f5 --- /dev/null +++ b/src/include/sof/schedule/zephyr_ll.h @@ -0,0 +1,105 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2019-2026 Intel Corporation. + */ + +/* + * Private data of the Zephyr LL scheduler and its timer domain, shared by + * the parts in src/schedule, which also run in the LL scheduler threads, and + * the kernel parts in src/kernel/schedule. Not for use outside the scheduler. + */ + +#ifndef __SOF_SCHEDULE_ZEPHYR_LL_H__ +#define __SOF_SCHEDULE_ZEPHYR_LL_H__ + +#include +#include +#include +#include +#include +#include + +/* per-scheduler data */ +struct zephyr_ll { + struct list_item tasks; /* list of ll tasks */ + unsigned int n_tasks; /* task counter */ + struct ll_schedule_domain *ll_domain; /* scheduling domain */ + unsigned int core; /* core ID of this instance */ +#if CONFIG_SOF_USERSPACE_LL + struct k_mutex *lock; /* mutex for userspace */ +#endif + struct k_heap *heap; +}; + +/* per-task scheduler data */ +struct zephyr_ll_pdata { + bool run; + bool freeing; + struct k_sem *sem_p; +#if !CONFIG_DYNAMIC_OBJECTS + struct k_sem sem; +#endif +}; + +/* LL scheduler ops, see zephyr_ll.c */ +int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start, + uint64_t period); +int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *before); +int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *after); +int zephyr_ll_task_free(void *data, struct task *task); +int zephyr_ll_task_cancel(void *data, struct task *task); +#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY +void zephyr_ll_scheduler_free(void *data, uint32_t flags); +#endif + +#if CONFIG_SOF_USERSPACE_LL +/* per-core LL scheduler locks, see zephyr_ll.c */ +extern struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; + +struct k_thread *zephyr_ll_init_context(void *data, struct task *task); +#endif + +struct zephyr_domain_thread { + struct k_thread *ll_thread; + struct k_sem *sem; +#ifndef CONFIG_SOF_USERSPACE_LL + struct k_thread ll_thread_obj; + struct k_sem sem_obj; +#endif + void (*handler)(void *arg); + void *arg; +}; + +struct zephyr_domain { + struct k_timer *timer; +#ifndef CONFIG_SOF_USERSPACE_LL + struct k_timer timer_obj; +#endif + struct zephyr_domain_thread domain_thread[CONFIG_CORE_COUNT]; + struct ll_schedule_domain *ll_domain; +#if CONFIG_CROSS_CORE_STREAM + atomic_t block; + struct k_mutex block_mutex; + struct k_condvar block_condvar; +#endif +}; + +/* LL scheduler thread function, p1 is the domain and p2 the core */ +void zephyr_domain_thread_fn(void *p1, void *p2, void *p3); + +#ifdef CONFIG_SOF_USERSPACE_LL +int zephyr_domain_register_user(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg); +int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, + struct task *task, uint32_t num_tasks); +#endif + +#if CONFIG_CROSS_CORE_STREAM +void zephyr_domain_block(struct ll_schedule_domain *domain); +void zephyr_domain_unblock(struct ll_schedule_domain *domain); +#endif + +#endif /* __SOF_SCHEDULE_ZEPHYR_LL_H__ */ diff --git a/src/include/sof/trace/dma-trace.h b/src/include/sof/trace/dma-trace.h deleted file mode 100644 index 018576cd7ca2..000000000000 --- a/src/include/sof/trace/dma-trace.h +++ /dev/null @@ -1,172 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * - * Copyright(c) 2016 Intel Corporation. All rights reserved. - * - * Author: Yan Wang - */ - -/** - * \file include/sof/trace/dma-trace.h - * \brief Direct Memory Access (DMA) trace buffer management and host streaming. - * \ingroup trace_logging_api - */ - -#ifndef __SOF_TRACE_DMA_TRACE_H__ -#define __SOF_TRACE_DMA_TRACE_H__ - -#include -#include -#include -#include -#include -#include - -struct ipc_msg; -struct sof; - -/** - * \brief Internal ring buffer descriptor for DSP trace packets. - * \ingroup trace_logging_api - */ -struct dma_trace_buf { - void *w_ptr; /**< Buffer write pointer where new trace packets are appended */ - void *r_ptr; /**< Buffer read pointer tracking data transferred by DMA */ - void *addr; /**< Base address of the circular trace buffer */ - void *end_addr; /**< Upper boundary address of the trace buffer */ - uint32_t size; /**< Total size of the buffer in bytes */ - uint32_t avail; /**< Number of readable bytes currently available */ -}; - -/** - * \brief Main DMA trace context and state machine descriptor. - * \ingroup trace_logging_api - */ -struct dma_trace_data { - struct dma_sg_config config; /**< Scatter-gather configuration for host DMA */ - struct dma_trace_buf dmatb; /**< Internal circular buffer tracking read/write pointers */ - struct dma_copy dc; /**< DMA copy engine instance */ - struct sof_ipc_dma_trace_posn posn; /**< Position report payload sent to host */ - struct ipc_msg *msg; /**< IPC message used to report buffer position */ - uint32_t host_size; /**< Total size of host-allocated trace buffer */ - struct task dmat_work; /**< Periodic scheduling task driving DMA transfers */ - uint32_t enabled; /**< 1 if DMA trace streaming is active, 0 otherwise */ - uint32_t copy_in_progress; /**< 1 if a DMA copy operation is currently underway */ - uint32_t stream_tag; /**< Assigned host DMA stream tag */ - uint32_t active_stream_tag; /**< Current active stream tag */ - uint32_t dma_copy_align; /**< Minimal byte alignment required for DMA transfer chunks */ - uint32_t dropped_entries; /**< Cumulative counter of dropped entries due to buffer overflow */ - struct k_spinlock lock; /**< Spinlock synchronizing buffer writes and DMA triggers */ - uint64_t time_delta; /**< Offset between host wallclock and DSP hardware timer */ -}; - -/** - * \brief Early initialization of the DMA trace subsystem during bootstrap. - * \ingroup trace_logging_api - * @param[in,out] sof Global SOF system context. - * @return 0 on success, or negative error code on failure. - */ -int dma_trace_init_early(struct sof *sof); - -/** - * \brief Complete initialization of DMA trace once host channel is negotiated. - * \ingroup trace_logging_api - * @param[in,out] d DMA trace data context. - * @return 0 on success, or negative error code on failure. - */ -int dma_trace_init_complete(struct dma_trace_data *d); - -/** - * \brief Configure the host-side destination scatter-gather buffer for DMA trace. - * \ingroup trace_logging_api - * @param[in,out] d DMA trace data context. - * @param[in] elem_array Array of scatter-gather buffer page elements. - * @param[in] host_size Total size in bytes of the host buffer. - * @return 0 on success, or negative error code on failure. - */ -int dma_trace_host_buffer(struct dma_trace_data *d, - struct dma_sg_elem_array *elem_array, - uint32_t host_size); - -/** - * \brief Enable DMA trace streaming to the host. - * \ingroup trace_logging_api - * @param[in,out] d DMA trace data context. - * @return 0 on success, or negative error code on failure. - */ -int dma_trace_enable(struct dma_trace_data *d); - -/** - * \brief Disable DMA trace streaming and cancel pending periodic tasks. - * \ingroup trace_logging_api - * @param[in,out] d DMA trace data context. - */ -void dma_trace_disable(struct dma_trace_data *d); - -/** - * \brief Flush residual trace data from local DSP buffer to destination memory or mailbox. - * \ingroup trace_logging_api - * @param[out] destination Pointer to destination memory buffer, or NULL for default mailbox. - */ -void dma_trace_flush(void *destination); - -/** - * \brief Globally unpause and resume trace event logging. - * \ingroup trace_logging_api - */ -void dma_trace_on(void); - -/** - * \brief Globally pause trace event logging. - * \ingroup trace_logging_api - */ -void dma_trace_off(void); - -/** - * \brief Append a binary trace event to the circular buffer. - * \ingroup trace_logging_api - * @param[in] e Pointer to binary event packet. - * @param[in] size Packet size in bytes. - */ -void dtrace_event(const char *e, uint32_t size); - -/** - * \brief Append a binary trace event to the buffer from atomic or interrupt context. - * \ingroup trace_logging_api - * @param[in] e Pointer to binary event packet. - * @param[in] length Size of packet in bytes. - */ -void dtrace_event_atomic(const char *e, uint32_t length); - -/** - * \brief Check whether DMA trace subsystem has been initialized. - * \ingroup trace_logging_api - * @param[in] d DMA trace context. - * @return True if initialized with valid buffer memory, false otherwise. - */ -static inline bool dma_trace_initialized(const struct dma_trace_data *d) -{ - return d && d->dmatb.addr; -} - -/** - * \brief Retrieve global DMA trace data context from system runtime. - * \ingroup trace_logging_api - * @return Pointer to struct dma_trace_data. - */ -static inline struct dma_trace_data *dma_trace_data_get(void) -{ - return sof_get()->dmat; -} - -/** - * \brief Calculate contiguous bytes remaining until the end of the circular buffer. - * \ingroup trace_logging_api - * @param[in] buffer Pointer to circular trace buffer descriptor. - * @return Contiguous byte margin before write wrap-around. - */ -static inline uint32_t dtrace_calc_buf_margin(struct dma_trace_buf *buffer) -{ - return (char *)buffer->end_addr - (char *)buffer->w_ptr; -} - -#endif /* __SOF_TRACE_DMA_TRACE_H__ */ diff --git a/src/include/sof/trace/trace-host.h b/src/include/sof/trace/trace-host.h new file mode 100644 index 000000000000..bb3298ccf45d --- /dev/null +++ b/src/include/sof/trace/trace-host.h @@ -0,0 +1,75 @@ +/* SPDX-License-Identifier: BSD-3-Clause + * + * Copyright(c) 2016,2024 Intel Corporation. + */ + +/** + * \file include/sof/trace/trace-host.h + * \brief Trace output of the host builds (testbench, host library). + * \ingroup trace_logging_api + * + * Messages are printed to stderr when their level is at or below + * host_trace_level, which the testbench sets with its -d option. + */ + +#ifndef __SOF_TRACE_TRACE_HOST_H__ +#define __SOF_TRACE_TRACE_HOST_H__ + +#include +#include +#include +#include /* LOG_LEVEL_... */ + +/* trace level used on host configurations */ +extern int host_trace_level; + +char *get_trace_class(uint32_t trace_class); + +#define _log_message(level, ctx, id_1, id_2, format, ...) \ +do { \ + (void)ctx; \ + (void)id_1; \ + (void)id_2; \ + struct timeval tv; \ + char slevel[5][4] = {"dis", "err", "wrn", "inf", "dbg"}; \ + char *msg = "(%s:%d) " format; \ + if (level <= host_trace_level) { \ + gettimeofday(&tv, NULL); \ + fprintf(stderr, "trace: [%ld.%6.6ld] <%s> ", tv.tv_sec, tv.tv_usec, \ + slevel[level]); \ + fprintf(stderr, msg, strrchr(__FILE__, '/') + 1, \ + __LINE__, ##__VA_ARGS__); \ + fprintf(stderr, "\n"); \ + } \ +} while (0) + +/* tracing from device (component, pipeline, dai, ...) */ + +#define trace_dev_err(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ + _log_message(LOG_LEVEL_CRITICAL, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) + +#define trace_dev_warn(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ + _log_message(LOG_LEVEL_WARNING, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) + +#define trace_dev_info(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ + _log_message(LOG_LEVEL_INFO, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) + +#define trace_dev_dbg(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ + _log_message(LOG_LEVEL_VERBOSE, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) + +/* tracing from infrastructure part */ + +#define tr_err(ctx, fmt, ...) \ + _log_message(LOG_LEVEL_CRITICAL, ctx, _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) +#define tr_warn(ctx, fmt, ...) \ + _log_message(LOG_LEVEL_WARNING, ctx, _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) +#define tr_info(ctx, fmt, ...) \ + _log_message(LOG_LEVEL_INFO, ctx, _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) +#define tr_dbg(ctx, fmt, ...) \ + _log_message(LOG_LEVEL_VERBOSE, ctx, _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) + +#endif /* __SOF_TRACE_TRACE_HOST_H__ */ diff --git a/src/include/sof/trace/trace-soflogger.h b/src/include/sof/trace/trace-soflogger.h deleted file mode 100644 index b8b46a0f4dab..000000000000 --- a/src/include/sof/trace/trace-soflogger.h +++ /dev/null @@ -1,476 +0,0 @@ -/* SPDX-License-Identifier: BSD-3-Clause - * - * Copyright(c) 2016,2024 Intel Corporation. - */ - -/** - * \file include/sof/trace/trace-soflogger.h - * \brief Dictionary-based trace logging engine, log filtering, and mailbox trace fallback. - * \ingroup trace_logging_api - */ - -#ifndef __SOF_TRACE_TRACE_SOFLOGGER_H__ -#define __SOF_TRACE_TRACE_SOFLOGGER_H__ - -#if !CONFIG_LIBRARY -#include -#endif - -#if CONFIG_LIBRARY -#include -#endif - -struct sof; -struct trace; -struct tr_ctx; - -/** - * \brief Runtime trace filter criteria for components and pipelines. - * \ingroup trace_logging_api - */ -struct trace_filter { - uint32_t uuid_id; /**< Component UUID type identifier, or 0 for wildcard */ - int32_t comp_id; /**< Component ID, or -1 for wildcard */ - int32_t pipe_id; /**< Pipeline ID, or -1 for wildcard */ - int32_t log_level; /**< Target log level threshold (LOG_LEVEL_*) */ -}; - -/** The start of this linker output MUST match the 'ldc_entry_header' - * struct defined in the logger program running in user space. - */ -#define _DECLARE_LOG_ENTRY(lvl, format, comp_class, n_params) \ - __section(".static_log." #lvl) \ - static const struct { \ - uint32_t level; \ - uint32_t component_class; \ - uint32_t params_num; \ - uint32_t line_idx; \ - uint32_t file_name_len; \ - uint32_t text_len; \ - const char file_name[sizeof(RELATIVE_FILE)]; \ - const char text[sizeof(format)]; \ - } log_entry = { \ - lvl, \ - comp_class, \ - n_params, \ - __LINE__, \ - sizeof(RELATIVE_FILE), \ - sizeof(format), \ - RELATIVE_FILE, \ - format \ - } - -#define _TRACE_EVENT_MAX_ARGUMENT_COUNT 4 - -/** - * \brief Retrieve global trace subsystem context from SOF runtime. - * \ingroup trace_logging_api - * @return Pointer to struct trace. - */ -static inline struct trace *trace_get(void) -{ - return sof_get()->trace; -} - -#include -#include /* LOG_LEVEL_... */ - -/* - * trace_event macro definition - * - * trace_event() macro is used for logging events that occur at runtime. - * It comes in 2 main flavours, atomic and non-atomic. Depending of definitions - * above, it might also propagate log messages to mbox if desired. - * - * First argument is always class of event being logged, as defined in - * user/trace.h - TRACE_CLASS_* (deprecated - do not use). - * Second argument is string literal in printf format, followed by up to 4 - * parameters (uint32_t), that are used to expand into string fromat when - * parsing log data. - * - * All compile-time accessible data (verbosity, class, source file name, line - * index and string literal) are linked into .static_log_entries section - * of binary and then extracted by smex, so they do not contribute to loadable - * image size. This way more elaborate log messages are possible and encouraged, - * for better debugging experience, without worrying about runtime performance. - */ - -/* Map the different trace_xxxx_with_ids(... ) levels to the - * _trace_event_with_ids(level_xxxx, ...) macro shared across log - * levels. - */ -#define trace_event_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_event_with_ids(LOG_LEVEL_INFO, class, ctx, id_1, id_2, \ - format, ##__VA_ARGS__) - -#define trace_event_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_event_atomic_with_ids(LOG_LEVEL_INFO, class, ctx, id_1, id_2, \ - format, ##__VA_ARGS__) - -#define trace_warn_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_event_with_ids(LOG_LEVEL_WARNING, class, ctx, id_1, id_2, \ - format, ##__VA_ARGS__) - -#define trace_warn_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_event_atomic_with_ids(LOG_LEVEL_WARNING, class, \ - ctx, id_1, id_2, \ - format, ##__VA_ARGS__) - -/** - * \brief Flush buffered trace data from DMA buffer to Mailbox Window. - * \ingroup trace_logging_api - */ -void trace_flush_dma_to_mbox(void); - -/** - * \brief Globally resume trace event logging. - * \ingroup trace_logging_api - */ -void trace_on(void); - -/** - * \brief Globally pause trace event logging. - * \ingroup trace_logging_api - */ -void trace_off(void); - -/** - * \brief Initialize the trace subsystem and allocate global trace buffers. - * \ingroup trace_logging_api - * @param[in,out] sof Global SOF runtime context. - */ -void trace_init(struct sof *sof); - -/** - * \brief Function pointer type for formatted trace logger backends. - * \ingroup trace_logging_api - * @param[in] send_atomic True if logging from atomic or interrupt context. - * @param[in] log_entry Pointer to static log entry descriptor. - * @param[in] ctx Pointer to caller trace context. - * @param[in] lvl Severity log level (LOG_LEVEL_*). - * @param[in] id_1 Primary component/pipe identifier. - * @param[in] id_2 Secondary component/subid identifier. - * @param[in] arg_count Number of variable arguments. - * @param[in] args Variable arguments list. - */ -typedef void (*log_func_t)(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list args); - -/** - * \brief Emit a filtered dictionary trace message. - * \ingroup trace_logging_api - * @param[in] send_atomic True if logging from atomic context. - * @param[in] log_entry Pointer to static log entry. - * @param[in] ctx Trace context. - * @param[in] lvl Log level. - * @param[in] id_1 Primary identifier. - * @param[in] id_2 Secondary identifier. - * @param[in] arg_count Argument count. - * @param[in] args Argument list. - */ -void trace_log_filtered(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list args); - -/** - * \brief Emit an unfiltered dictionary trace message directly to buffers. - * \ingroup trace_logging_api - * @param[in] send_atomic True if logging from atomic context. - * @param[in] log_entry Pointer to static log entry. - * @param[in] ctx Trace context. - * @param[in] lvl Log level. - * @param[in] id_1 Primary identifier. - * @param[in] id_2 Secondary identifier. - * @param[in] arg_count Argument count. - * @param[in] args Argument list. - */ -void trace_log_unfiltered(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list args); - -/** - * \brief Populate an IPC trace filter element array. - * \ingroup trace_logging_api - * @param[in,out] elem Pointer to current output filter element. - * @param[in] end Boundary pointer of element array. - * @param[in] filter Source trace filter configuration. - * @return Pointer to next available filter element slot. - */ -struct sof_ipc_trace_filter_elem *trace_filter_fill(struct sof_ipc_trace_filter_elem *elem, - struct sof_ipc_trace_filter_elem *end, - struct trace_filter *filter); - -/** - * \brief Update runtime trace filtering configuration from IPC payload. - * \ingroup trace_logging_api - * @param[in] elem Trace filter descriptor to apply. - * @return 0 on success, or negative error code on failure. - */ -int trace_filter_update(const struct trace_filter *elem); - -#define _trace_event_with_ids(lvl, class, ctx, id_1, id_2, format, ...) \ - _log_message(trace_log_filtered, false, lvl, class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -#define _trace_event_atomic_with_ids(lvl, class, ctx, id_1, id_2, format, ...) \ - _log_message(trace_log_filtered, true, lvl, class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -/** - * \brief Appends one SOF dictionary entry and log statement to shared memory mtrace. - * \ingroup trace_logging_api - * @param atomic_context Take the trace->lock if false. - * @param log_entry_pointer Dictionary index produced by _DECLARE_LOG_ENTRY macro. - * @param n_args Number of variable arguments. - */ -void mtrace_dict_entry(bool atomic_context, uint32_t log_entry_pointer, int n_args, ...); - -/** - * \brief Posts a fully prepared log header and log entry packet to mtrace buffer. - * \ingroup trace_logging_api - * @param complete_packet Pointer to prepared log packet buffer. - * @param length Packet size in bytes. - */ -void mtrace_event(const char *complete_packet, uint32_t length); - -#ifdef CONFIG_TRACEM /* Send everything to shared memory too */ -# ifdef __ZEPHYR__ -/* We don't use Zephyr's dictionary yet so there's not enough space for - * DEBUG messages - */ -# define MTRACE_DUPLICATION_LEVEL LOG_LEVEL_INFO -# else -# define MTRACE_DUPLICATION_LEVEL LOG_LEVEL_DEBUG -# endif -#else /* copy only ERRORS */ -# define MTRACE_DUPLICATION_LEVEL LOG_LEVEL_ERROR -#endif /* CONFIG_TRACEM */ - -/** - * \brief Internal dictionary logger formatting and dispatch helper. - * \ingroup trace_logging_api - * @param sofdict_logf Backend log emission callback function. - * @param atomic True if executed in atomic context. - * @param log_entry Pointer to static log entry. - * @param ctx Trace context. - * @param lvl Log severity level. - * @param id_1 Primary identifier. - * @param id_2 Secondary identifier. - * @param arg_count Number of format arguments. - */ -void _log_sofdict(log_func_t sofdict_logf, bool atomic, const void *log_entry, - const struct tr_ctx *ctx, const uint32_t lvl, - uint32_t id_1, uint32_t id_2, int arg_count, ...); - -/* _log_message() */ - -#ifdef CONFIG_LIBRARY - -#include - -/* trace level used on host configurations */ -extern int host_trace_level; - -char *get_trace_class(uint32_t trace_class); -#define _log_message(ignored_log_func, atomic, level, comp_class, ctx, id_1, id_2, format, ...) \ -do { \ - (void)ctx; \ - (void)id_1; \ - (void)id_2; \ - struct timeval tv; \ - char slevel[5][4] = {"dis", "err", "wrn", "inf", "dbg"}; \ - char *msg = "(%s:%d) " format; \ - if (level <= host_trace_level) { \ - gettimeofday(&tv, NULL); \ - fprintf(stderr, "trace: [%ld.%6.6ld] <%s> ", tv.tv_sec, tv.tv_usec, \ - slevel[level]); \ - fprintf(stderr, msg, strrchr(__FILE__, '/') + 1, \ - __LINE__, ##__VA_ARGS__); \ - fprintf(stderr, "\n"); \ - } \ -} while (0) - -#define trace_point(x) do {} while (0) - -#else /* CONFIG_LIBRARY */ - -#define trace_point(x) platform_trace_point(x) - -#define BASE_LOG_ASSERT_FAIL_MSG \ -unsupported_amount_of_params_in_trace_event\ -_thrown_from_macro_BASE_LOG_in_trace_h - -#define CT_ASSERT(COND, MESSAGE) \ - ((void)sizeof(char[1 - 2 * !(COND)])) - -#define trace_check_size_uint32(a) \ - CT_ASSERT(sizeof(a) <= sizeof(uint32_t), "error: trace argument is bigger than a uint32_t"); - -#define STATIC_ASSERT_ARG_SIZE(...) \ - META_MAP(1, trace_check_size_uint32, __VA_ARGS__) - -/** _log_message is where the memory-saving dictionary magic described - * above happens: the "format" string argument is moved to a special - * linker section and replaced by a &log_entry pointer to it. This must - * be a macro for the source location to be meaningful. - */ -#define _log_message(log_func, atomic, lvl, comp_class, ctx, id_1, id_2, format, ...) \ -do { \ - _DECLARE_LOG_ENTRY(lvl, format, comp_class, \ - META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__)); \ - STATIC_ASSERT_ARG_SIZE(__VA_ARGS__); \ - STATIC_ASSERT(_TRACE_EVENT_MAX_ARGUMENT_COUNT >= \ - META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__), \ - BASE_LOG_ASSERT_FAIL_MSG \ - ); \ - _log_sofdict(log_func, atomic, &log_entry, ctx, lvl, id_1, id_2, \ - META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__), ##__VA_ARGS__); \ - _log_nodict(atomic, META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__), \ - lvl, format, ##__VA_ARGS__); \ -} while (0) - -#ifdef __ZEPHYR__ -/* Just like XTOS, only the most urgent messages go to limited - * shared memory. - */ -#define _log_nodict(atomic, arg_count, lvl, format, ...) \ -do { \ - if ((lvl) <= MTRACE_DUPLICATION_LEVEL) \ - printk("%llu " format "\n", k_cycle_get_64(), \ - ##__VA_ARGS__); \ -} while (0) -#else -#define _log_nodict(atomic, n_args, lvl, format, ...) -#endif - -#endif /* CONFIG_LIBRARY */ - -#if CONFIG_TRACEV -/* Enable tr_dbg() statements by defining tracev_...() */ -#define tracev_event_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_event_with_ids(LOG_LEVEL_VERBOSE, class, \ - ctx, id_1, id_2, \ - format, ##__VA_ARGS__) - -#define tracev_event_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_event_atomic_with_ids(LOG_LEVEL_VERBOSE, class, \ - ctx, id_1, id_2, \ - format, ##__VA_ARGS__) - -#else /* CONFIG_TRACEV */ -#define tracev_event_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) -#define tracev_event_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -#endif /* CONFIG_TRACEV */ - -/* The _error_ level has 2, 1 or 0 backends depending on Kconfig */ -#if CONFIG_TRACEE -/* LOG_LEVEL_CRITICAL messages are duplicated to the mail box */ -#define _trace_error_with_ids(class, ctx, id_1, id_2, format, ...) \ - _log_message(trace_log_filtered, true, LOG_LEVEL_CRITICAL, class, ctx, id_1, \ - id_2, format, ##__VA_ARGS__) -#define trace_error_with_ids(class, ctx, id_1, id_2, format, ...) \ - _trace_error_with_ids(class, ctx, id_1, id_2, format, ##__VA_ARGS__) -#define trace_error_atomic_with_ids(...) trace_error_with_ids(__VA_ARGS__) - -#elif CONFIG_TRACE -/* Goes to trace_log_filtered() too but with a downgraded, LOG_INFO level */ -#define trace_error_with_ids(...) trace_event_with_ids(__VA_ARGS__) -#define trace_error_atomic_with_ids(...) \ - trace_event_atomic_with_ids(__VA_ARGS__) - -#endif /* CONFIG_TRACEE, CONFIG_TRACE */ - -/* tracing from device (component, pipeline, dai, ...) */ - -/** \brief Trace from a device on err level. - * - * @param get_ctx_m Macro that can retrieve trace context from dev - * @param get_id_m Macro that can retrieve device's id0 from the dev - * @param get_subid_m Macro that can retrieve device's id1 from the dev - * @param dev Device - * @param fmt Format followed by parameters - * @param ... Parameters - */ -#define trace_dev_err(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_error_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) - -/** \brief Trace from a device on warning level. */ -#define trace_dev_warn(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_warn_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) - -/** \brief Trace from a device on info level. */ -#define trace_dev_info(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_event_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) - -/** \brief Trace from a device on dbg level. */ -#define trace_dev_dbg(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - tracev_event_with_ids(_TRACE_INV_CLASS, \ - get_ctx_m(dev), get_id_m(dev), \ - get_subid_m(dev), fmt, ##__VA_ARGS__) - -/* tracing from infrastructure part */ - -#define tr_err_atomic(ctx, fmt, ...) \ - trace_error_atomic_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, \ - fmt, ##__VA_ARGS__) - -#define tr_warn_atomic(ctx, fmt, ...) \ - trace_warn_atomic_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, \ - fmt, ##__VA_ARGS__) - -#define tr_info_atomic(ctx, fmt, ...) \ - trace_event_atomic_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, \ - fmt, ##__VA_ARGS__) - -#define tr_dbg_atomic(ctx, fmt, ...) \ - tracev_event_atomic_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, \ - fmt, ##__VA_ARGS__) - -#define tr_err(ctx, fmt, ...) \ - trace_error_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) - -#define tr_warn(ctx, fmt, ...) \ - trace_warn_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) - -#define tr_info(ctx, fmt, ...) \ - trace_event_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) - -/* tracev_ output depends on CONFIG_TRACEV=y */ -#define tr_dbg(ctx, fmt, ...) \ - tracev_event_with_ids(_TRACE_INV_CLASS, ctx, \ - _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) - -/** Direct, low-level access to mbox / shared memory logging when DMA - * tracing is either not initialized yet or disabled or found broken for - * any reason. - * To keep it simpler than and with minimal dependencies on - * the huge number of lines above, this does not check arguments at compile - * time. - * There is neither log level filtering, throttling or any other - * advanced feature. - */ -#define mtrace_printf(log_level, format_str, ...) \ - do { \ - STATIC_ASSERT(META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__) \ - <= _TRACE_EVENT_MAX_ARGUMENT_COUNT, \ - too_many_mtrace_printf_arguments); \ - _DECLARE_LOG_ENTRY(log_level, format_str, _TRACE_INV_CLASS, \ - META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__)); \ - mtrace_dict_entry(true, (uint32_t)&log_entry, \ - META_COUNT_VARAGS_BEFORE_COMPILE(__VA_ARGS__), \ - ##__VA_ARGS__); \ - } while (0) - -#endif diff --git a/src/include/sof/trace/trace.h b/src/include/sof/trace/trace.h index 0639ac4adbd9..188796d24606 100644 --- a/src/include/sof/trace/trace.h +++ b/src/include/sof/trace/trace.h @@ -45,9 +45,9 @@ #define tr_info(ctx, fmt, ...) LOG_INF(fmt, ##__VA_ARGS__) #define tr_dbg(ctx, fmt, ...) LOG_DBG(fmt, ##__VA_ARGS__) -#elif CONFIG_TRACE +#elif CONFIG_LIBRARY && !defined(__ZEPHYR__) -#include "trace-soflogger.h" +#include "trace-host.h" #else @@ -60,62 +60,23 @@ #define tr_dbg(ctx, fmt, ...) \ trace_unused(_TRACE_INV_CLASS, ctx, _TRACE_INV_ID, _TRACE_INV_ID, fmt, ##__VA_ARGS__) -#endif - -#ifndef CONFIG_TRACE - -struct trace_filter; -struct sof; - -#define trace_event_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) -#define trace_event_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -#define trace_warn_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) -#define trace_warn_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -#define tracev_event_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) - -#define trace_point(x) do {} while (0) - -static inline void trace_flush_dma_to_mbox(void) { } -static inline void trace_on(void) { } -static inline void trace_off(void) { } -static inline void trace_init(struct sof *sof) { } -static inline int trace_filter_update(const struct trace_filter *filter) - { return 0; } - -#define trace_error_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) -#define trace_error_atomic_with_ids(class, ctx, id_1, id_2, format, ...) \ - trace_unused(class, ctx, id_1, id_2, format, ##__VA_ARGS__) - #define trace_dev_err(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_event_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) + trace_unused(_TRACE_INV_CLASS, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) #define trace_dev_warn(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_event_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) + trace_unused(_TRACE_INV_CLASS, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) #define trace_dev_info(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_event_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) + trace_unused(_TRACE_INV_CLASS, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) #define trace_dev_dbg(get_ctx_m, get_id_m, get_subid_m, dev, fmt, ...) \ - trace_event_with_ids(_TRACE_INV_CLASS, get_ctx_m(dev), \ - get_id_m(dev), get_subid_m(dev), \ - fmt, ##__VA_ARGS__) + trace_unused(_TRACE_INV_CLASS, get_ctx_m(dev), get_id_m(dev), \ + get_subid_m(dev), fmt, ##__VA_ARGS__) -static inline void mtrace_printf(int log_level, const char *format_str, ...) -{ -}; +#endif -#endif /* !CONFIG_TRACE */ +/* Boot progress markers; no longer reported anywhere */ +#define trace_point(x) do {} while (0) /** Default value when there is no specific pipeline, dev, dai, etc. */ #define _TRACE_INV_ID -1 @@ -134,12 +95,6 @@ struct tr_ctx { uint32_t level; /**< Default log level */ }; -#if defined(UNIT_TEST) || !defined(CONFIG_TRACE) -#define TRACE_CONTEXT_SECTION -#else -#define TRACE_CONTEXT_SECTION __section(".trace_ctx") -#endif - /** * Declares trace context. * @param ctx_name (Symbol) name. @@ -147,7 +102,7 @@ struct tr_ctx { * @param default_log_level Default log level. */ #define DECLARE_TR_CTX(ctx_name, uuid, default_log_level) \ - struct tr_ctx ctx_name TRACE_CONTEXT_SECTION = { \ + struct tr_ctx ctx_name = { \ .uuid_p = uuid, \ .level = default_log_level, \ } diff --git a/src/include/sof/userspace/README.md b/src/include/sof/userspace/README.md index d342bbc941f7..02db15ed391b 100644 --- a/src/include/sof/userspace/README.md +++ b/src/include/sof/userspace/README.md @@ -74,7 +74,7 @@ its subdirectories span all tiers. | Tier | Meaning | Directories | |------|---------|-------------| -| **Kernel-only** | OS integration, HAL, boot, inter-core. Never executes in a user thread. Must not be reached directly from user code — only via a syscall. | `kernel/` (see [its README](../../../kernel/README.md)), `arch/`, `platform/`, `trace/` * | +| **Kernel-only** | OS integration, HAL, boot, inter-core. Never executes in a user thread. Must not be reached directly from user code — only via a syscall. | `kernel/` (see [its README](../../../kernel/README.md)) | | **Boundary / bridge** | Owns the syscalls and the thread / memory-domain setup. Split internally: some translation units run kernel-side, some user-side (e.g. `handler-kernel.c` vs `handler-user.c`). | `schedule/`, `ipc/`, `debug/debug_stream/` *, and (in the Zephyr integration tree) `zephyr/syscall/` | | **Application** | Audio pipeline / component logic that runs in user threads under userspace configs. | `audio/` | | **Shared library** | Context-agnostic code linked into whichever context calls it. Must stay "user-safe": no privileged operations (no direct MMIO, no spinlocks held across syscalls, no raw kernel object access) so it is safe when pulled into a user build. | `math/`, `module/` | @@ -82,9 +82,9 @@ its subdirectories span all tiers. Directories marked * do not yet have a "Runs in:" README banner. -Kernel-only code is being moved under `src/kernel/`, which Zephyr builds into a library of its -own, `sof_kernel`, instead of `modules_sof`. The other kernel-only directories in the table -follow one at a time. +Kernel-only code lives under `src/kernel/`, which Zephyr builds into a library of its own, +`sof_kernel`, instead of `modules_sof`. Source files that mix kernel and user code are split, +and their kernel halves moved there, separately. Classification is grep-verifiable — see [Verifying the taxonomy](#verifying-the-taxonomy). @@ -124,7 +124,7 @@ syscall by subsystem: | CPU | `cpu_get_id` | `sof/lib/cpu.h` | `zephyr/syscall/cpu.c` / `zephyr/lib/cpu.c` | | Module management | `mod_balloc_align`, `mod_alloc_ext`, `mod_free`, `mod_free_all`, `mod_data_blob_handler_new`, `mod_fast_get` | `sof/audio/module_adapter/module/generic.h` | scattered: `audio/module_adapter/module/generic.c` | | Library manager | `lib_manager_free_module` | `sof/lib_manager.h` | scattered: `kernel/library_manager/lib_manager.c` | -| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `schedule/zephyr_ll.c`, `schedule/zephyr_dp_schedule*.c` | +| Scheduling | `zephyr_ll_task_sem_alloc`, `zephyr_ll_task_sem_free`, `scheduler_dp_internal_free`, `scheduler_dp_ll_tick` | `sof/schedule/ll_schedule_domain.h`, `sof/schedule/dp_schedule.h` | scattered: `kernel/schedule/zephyr_ll-kernel.c`, `kernel/schedule/zephyr_dp_schedule*-kernel.c` | | Debug | `debug_stream_slot_send_record` | `user/debug_stream_slot.h` | scattered: `debug/debug_stream/debug_stream_slot.c` | The `zephyr/syscall/*.c` files hold the validators in one place per subsystem; the rows marked diff --git a/src/ipc/CMakeLists.txt b/src/ipc/CMakeLists.txt index f12ac465acd0..3608369cda7d 100644 --- a/src/ipc/CMakeLists.txt +++ b/src/ipc/CMakeLists.txt @@ -13,14 +13,6 @@ add_local_sources(sof is_zephyr(zephyr) if(zephyr) ### Zephyr ### -# dma-copy only used for dma-trace -zephyr_library_sources_ifdef(CONFIG_TRACE dma-copy.c) -else() ### Not Zephyr ### - -if (CONFIG_TRACE) - add_local_sources(sof - dma-copy.c) -endif() endif() # Zephyr diff --git a/src/ipc/README.md b/src/ipc/README.md index 844c13a84f1b..c240cc54a50e 100644 --- a/src/ipc/README.md +++ b/src/ipc/README.md @@ -45,7 +45,7 @@ graph TD ## Processing Flow -When the host writes a command to the IPC mailbox and triggers an interrupt, the hardware-specific driver (`src/platform/...`) catches the IRQ and eventually calls down into the IPC framework. +When the host writes a command to the IPC mailbox and triggers an interrupt, the hardware-specific driver (`src/kernel/platform/...`) catches the IRQ and eventually calls down into the IPC framework. Different RTOS environments (Zephyr vs. bare metal SOF native) handle the thread handoff differently. In Zephyr, this leverages the `k_work` queues heavily for `ipc_work_handler`. diff --git a/src/ipc/dma-copy.c b/src/ipc/dma-copy.c deleted file mode 100644 index b7f55a90292e..000000000000 --- a/src/ipc/dma-copy.c +++ /dev/null @@ -1,159 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause -// -// Copyright(c) 2016 Intel Corporation. All rights reserved. -// -// Author: Liam Girdwood -// Keyon Jie - -#include -#include -#include -#include -#include -#include - -#include -#include -#include - -/* tracing */ -LOG_MODULE_REGISTER(dma_copy, CONFIG_SOF_LOG_LEVEL); - -SOF_DEFINE_REG_UUID(dma_copy); - -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(dmacpy_tr, SOF_UUID(dma_copy_uuid), LOG_LEVEL_INFO); - -static struct dma_sg_elem *sg_get_elem_at(struct dma_sg_config *host_sg, - int32_t *offset) -{ - struct dma_sg_elem *host_sg_elem; - int i; - int32_t _offset = *offset; - - /* find host element with host_offset */ - for (i = 0; i < host_sg->elem_array.count; i++) { - - host_sg_elem = host_sg->elem_array.elems + i; - - /* is offset in this elem ? */ - if (_offset >= 0 && _offset < host_sg_elem->size) { - *offset = _offset; - return host_sg_elem; - } - - _offset -= host_sg_elem->size; - } - - /* host offset in beyond end of SG buffer */ - tr_err(&dmacpy_tr, "host offset in beyond end of SG buffer"); - return NULL; -} - -/* Copy DSP memory to host memory. - * Copies DSP memory to host in a single PAGE_SIZE or smaller block. Does not - * waits/sleeps and can be used in IRQ context. - */ - -static int -dma_copy_to_host_flags(struct dma_copy *dc, struct dma_sg_config *host_sg, - int32_t host_offset, void *local_ptr, int32_t size, - uint32_t flags) -{ - struct dma_sg_config config; - struct dma_sg_elem *host_sg_elem; - struct dma_sg_elem local_sg_elem; - int32_t err; - int32_t offset = host_offset; - - if (size <= 0) - return 0; - - /* find host element with host_offset */ - host_sg_elem = sg_get_elem_at(host_sg, &offset); - if (!host_sg_elem) - return -EINVAL; - - /* set up DMA configuration */ - config.direction = DMA_DIR_LMEM_TO_HMEM; - config.src_width = sizeof(uint32_t); - config.dest_width = sizeof(uint32_t); - config.cyclic = 0; - config.irq_disabled = false; - dma_sg_init(&config.elem_array); - - /* configure local DMA elem */ - local_sg_elem.dest = host_sg_elem->dest + offset; - local_sg_elem.src = (uintptr_t)local_ptr; - if (size >= HOST_PAGE_SIZE - offset) - local_sg_elem.size = HOST_PAGE_SIZE - offset; - else - local_sg_elem.size = size; - - config.elem_array.elems = &local_sg_elem; - config.elem_array.count = 1; - - /* start the DMA */ - err = dma_set_config_legacy(dc->chan, &config); - if (err < 0) - return err; - - err = dma_copy_legacy(dc->chan, local_sg_elem.size, - flags); - if (err < 0) - return err; - - /* bytes copied */ - return local_sg_elem.size; -} - -int dma_copy_to_host(struct dma_copy *dc, struct dma_sg_config *host_sg, - int32_t host_offset, void *local_ptr, int32_t size) -{ - return dma_copy_to_host_flags(dc, host_sg, host_offset, local_ptr, size, - DMA_COPY_ONE_SHOT | DMA_COPY_BLOCKING); -} - -int dma_copy_to_host_nowait(struct dma_copy *dc, struct dma_sg_config *host_sg, - int32_t host_offset, void *local_ptr, int32_t size) -{ - return dma_copy_to_host_flags(dc, host_sg, host_offset, local_ptr, size, - DMA_COPY_ONE_SHOT); -} - -int dma_copy_new(struct dma_copy *dc) -{ - uint32_t dir, cap, dev; - - /* request HDA DMA in the dir LMEM->HMEM with shared access */ - dir = DMA_DIR_LMEM_TO_HMEM; - dev = DMA_DEV_HOST; - cap = 0; - dc->dmac = dma_get(dir, cap, dev, DMA_ACCESS_SHARED); - if (!dc->dmac) { - tr_err(&dmacpy_tr, "dc->dmac = NULL"); - return -ENODEV; - } - - /* get DMA channel from DMAC0 */ - dc->chan = dma_channel_get_legacy(dc->dmac, CONFIG_TRACE_CHANNEL); - if (!dc->chan) { - tr_err(&dmacpy_tr, "dc->chan is NULL"); - return -ENODEV; - } - - return 0; -} - -int dma_copy_set_stream_tag(struct dma_copy *dc, uint32_t stream_tag) -{ - /* get DMA channel from DMAC */ - dc->chan = dma_channel_get_legacy(dc->dmac, stream_tag - 1); - if (!dc->chan) { - tr_err(&dmacpy_tr, "dc->chan is NULL"); - return -EINVAL; - } - - return 0; -} - diff --git a/src/ipc/ipc-common.c b/src/ipc/ipc-common.c index bbc4a9b8a7d3..6c8126f51a7a 100644 --- a/src/ipc/ipc-common.c +++ b/src/ipc/ipc-common.c @@ -48,23 +48,9 @@ #include -LOG_MODULE_REGISTER(ipc, CONFIG_SOF_LOG_LEVEL); +/* The IPC code that runs in kernel mode is in src/kernel/ipc/ipc-common-kernel.c */ -SOF_DEFINE_REG_UUID(ipc); - -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(ipc_tr, SOF_UUID(ipc_uuid), LOG_LEVEL_INFO); - -#ifdef CONFIG_SOF_USERSPACE_LL -K_APPMEM_PARTITION_DEFINE(ipc_context_part); - -K_APP_BMEM(ipc_context_part) static struct ipc ipc_context; - -struct ipc *ipc_get(void) -{ - return &ipc_context; -} -#endif +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); struct ipc_msg *ipc_msg_w_ext_init(struct k_heap *heap, uint32_t header, uint32_t extension, uint32_t size) @@ -97,46 +83,6 @@ struct ipc_msg *ipc_msg_w_ext_init(struct k_heap *heap, uint32_t header, return msg; } -int ipc_process_on_core(uint32_t core, bool blocking) -{ - struct ipc *ipc = ipc_get(); - struct idc_msg msg = { .header = IDC_MSG_IPC, .core = core, }; - int ret; - - /* check if requested core is enabled */ - if (!cpu_is_core_enabled(core)) { - tr_err(&ipc_tr, "core #%d is disabled", core); - return -EACCES; - } - -#if CONFIG_IPC_MAJOR_3 - /* The other core will write there its response */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - ((struct sof_ipc_cmd_hdr *)ipc->comp_data)->size); -#endif - - /* - * If the primary core is waiting for secondary cores to complete, it - * will also reply to the host - */ - if (!blocking) { - k_spinlock_key_t key; - - ipc->core = core; - key = k_spin_lock(&ipc->lock); - ipc->task_mask |= IPC_TASK_SECONDARY_CORE; - k_spin_unlock(&ipc->lock, key); - } - - /* send IDC message */ - ret = idc_send_msg(&msg, blocking ? IDC_BLOCKING : IDC_NON_BLOCKING); - if (ret < 0) - return ret; - - /* reply written by other core */ - return 1; -} - /* * Components, buffers and pipelines are stored in the same lists, hence * type and ID have to be used for the identification. @@ -198,301 +144,7 @@ struct ipc_comp_dev *ipc_get_ppl_comp(struct ipc *ipc, uint32_t pipeline_id, int return next_ppl_icd; } -void ipc_send_queued_msg(void) -{ - struct ipc *ipc = ipc_get(); - struct ipc_msg *msg; - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - - if (ipc->pm_prepare_D3) - goto out; - - /* any messages to send ? */ - if (list_is_empty(&ipc->msg_list)) - goto out; - - msg = list_first_item(&ipc->msg_list, struct ipc_msg, - list); - - if (ipc_platform_send_msg(msg) == 0) { - /* Remove the message from the list if it has been successfully sent. */ - list_item_del(&msg->list); - /* Invoke a callback to notify that the message has been sent. */ - if (msg->callback) - msg->callback(msg); -#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS - /* Increment performance counters */ - io_perf_monitor_update_data(ipc->io_perf_out_msg_count, 1); -#endif - } -out: - k_spin_unlock(&ipc->lock, key); -} - -#ifdef __ZEPHYR__ -static K_THREAD_STACK_DEFINE(ipc_send_wq_stack, CONFIG_STACK_SIZE_IPC_TX); - -/* - * Kernel-only: these must not live in struct ipc, which user threads can - * write with CONFIG_SOF_USERSPACE_LL. The work queue embeds the k_thread of - * a supervisor thread and the work item holds the handler it calls. - */ -static struct k_work_q ipc_send_wq; -static struct k_work_delayable ipc_send_work; - -struct k_work_q *ipc_send_wq_get(void) -{ - return &ipc_send_wq; -} -#endif - -static void schedule_ipc_worker(void) -{ - /* - * note: in XTOS builds, this is handled in - * task_main_primary_core() - */ -#ifdef __ZEPHYR__ - k_work_schedule_for_queue(&ipc_send_wq, &ipc_send_work, K_USEC(IPC_PERIOD_USEC)); -#endif -} - -__cold void ipc_msg_send_direct(struct ipc_msg *msg, void *data) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - int ret; - - assert_can_be_cold(); - - key = k_spin_lock(&ipc->lock); - - /* copy mailbox data to message if not already copied */ - if (data && msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE && - msg->tx_data != data) { - ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); - assert(!ret); - } - - ipc_platform_send_msg_direct(msg); - - k_spin_unlock(&ipc->lock, key); -} - -void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - int ret; - - key = k_spin_lock(&ipc->lock); - - /* copy mailbox data to message if not already copied */ - if (data && (msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE) && - msg->tx_data != data) { - ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); - assert(!ret); - } - - /* - * note: This function can be executed in LL or EDF context, from any core. - * In Zephyr builds, there is IPC queue that is always handled by the primary core, - * whereas submitting to the queue is allowed from any core. Therefore disable option - * of sending IPC immediately by any context/core to secure IPC registers/mailbox - * access. - */ -#ifndef __ZEPHYR__ - /* try to send critical notifications right away */ - if (high_priority) { - ret = ipc_platform_send_msg(msg); - if (!ret) { - k_spin_unlock(&ipc->lock, key); - return; - } - } -#endif - /* add to queue unless already there */ - if (list_is_empty(&msg->list)) { - if (high_priority) - list_item_prepend(&msg->list, &ipc->msg_list); - else - list_item_append(&msg->list, &ipc->msg_list); - } - - schedule_ipc_worker(); - - k_spin_unlock(&ipc->lock, key); -} -EXPORT_SYMBOL(ipc_msg_send); - -#ifdef CONFIG_USERSPACE -void z_vrfy_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) -{ - K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); - - K_OOPS(K_SYSCALL_VERIFY(msg->tx_size <= SOF_IPC_MSG_MAX_SIZE)); - - K_OOPS(K_SYSCALL_VERIFY(list_is_empty(&msg->list))); - - if (msg->tx_size > 0) { - K_OOPS(K_SYSCALL_VERIFY(msg->tx_data != NULL)); - K_OOPS(K_SYSCALL_MEMORY_WRITE(msg->tx_data, msg->tx_size)); - } - - /* If data != NULL and tx_size > 0, verify the data buffer */ - if (data && msg->tx_size > 0) - K_OOPS(K_SYSCALL_MEMORY_READ(data, msg->tx_size)); - - z_impl_ipc_msg_send(msg, data, high_priority); -} -#include -#endif /* CONFIG_USERSPACE */ - -void z_impl_ipc_msg_list_remove(struct ipc_msg *msg) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - list_item_del(&msg->list); - k_spin_unlock(&ipc->lock, key); -} - -#ifdef CONFIG_USERSPACE -void z_vrfy_ipc_msg_list_remove(struct ipc_msg *msg) -{ - struct ipc *ipc = ipc_get(); - struct list_item *mlist, *_mlist; - bool found = false; - - K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); - - /* - * special case: empty list was passed. we can't trust where - * list->prev points to, so do not pass to - * z_impl_ipc_msg_list_remove(), but handle here - */ - if (list_is_empty(&msg->list)) - return; - - list_for_item_safe(mlist, _mlist, &ipc->msg_list) { - if (mlist == &msg->list) { - found = true; - break; - } - } - K_OOPS(K_SYSCALL_VERIFY(found)); - z_impl_ipc_msg_list_remove(msg); -} -#include -#endif /* CONFIG_USERSPACE */ - -#ifdef __ZEPHYR__ -static void ipc_work_handler(struct k_work *work) -{ - struct ipc *ipc = ipc_get(); - k_spinlock_key_t key; - - ipc_send_queued_msg(); - - key = k_spin_lock(&ipc->lock); - - if (!list_is_empty(&ipc->msg_list) && !ipc->pm_prepare_D3) - schedule_ipc_worker(); - - k_spin_unlock(&ipc->lock, key); -} -#endif - -void ipc_schedule_process(struct ipc *ipc) -{ -#if CONFIG_TWB_IPC_TASK - schedule_task(ipc->ipc_task, 0, IPC_PERIOD_USEC); -#else - schedule_task(&ipc->ipc_task, 0, IPC_PERIOD_USEC); -#endif -} - #ifdef CONFIG_SOF_USERSPACE_LL -/* Generic user-space IPC handling thread infrastructure. Protocol-specific - * command interpretation lives in ipc_user_thread_dispatch(), implemented by - * the active IPC major (see src/ipc/ipc4/). - */ - -#define IPC_USER_EVENT_CMD BIT(0) -#define IPC_USER_EVENT_STOP BIT(1) - -SOF_DEFINE_REG_UUID(sec_core_init); - -static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, - CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); - -/* - * Kernel-only copies of the IPC user thread state. struct ipc_user and the - * ipc->ipc_user_pdata pointer are both writable by user threads, so kernel - * code must use these instead of reading kernel object pointers from there. - */ -static struct ipc_user *ipc_user_kptr; -static struct k_sem *ipc_user_ksem; -static struct k_event *ipc_user_kevent[CONFIG_CORE_COUNT]; -static struct k_thread *ipc_user_kthread[CONFIG_CORE_COUNT]; -static bool ipc_user_kinit_needed[CONFIG_CORE_COUNT]; - -/** - * @brief Forward an IPC command to the user-space thread. - * - * Called from kernel context (IPC EDF task) to forward the message words - * to the user-space thread for processing. Sets IPC_TASK_IN_THREAD in - * task_mask so the host is not signaled until the user thread completes. - * Blocks until the user thread finishes processing and returns the result. - * - * @param primary Primary message word - * @param extension Extension message word - * @return Result from user thread processing - */ -int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core) -{ - struct ipc *ipc = ipc_get(); - struct ipc_user *pdata = ipc_user_kptr; - k_spinlock_key_t key; - int ret; - - if (core >= CONFIG_CORE_COUNT) - return -EINVAL; - - LOG_DBG("IPC: forward cmd %08x", primary); - - /* Copy message words — original buffer may be reused */ - pdata->ipc_msg_pri = primary; - pdata->ipc_msg_ext = extension; - pdata->ipc = ipc; - - /* Prevent host completion until user thread finishes */ - key = k_spin_lock(&ipc->lock); - ipc->task_mask |= IPC_TASK_IN_THREAD; - k_spin_unlock(&ipc->lock, key); - - /* Wake the user thread */ - k_event_set(ipc_user_kevent[core], IPC_USER_EVENT_CMD); - - /* Wait for user thread to complete */ - ret = k_sem_take(ipc_user_ksem, K_MSEC(100)); - if (ret) { - LOG_ERR("IPC user: sem error %d\n", ret); - /* fall through to complete the cmd */ - } - - /* Clear the task mask bit and check for completion */ - key = k_spin_lock(&ipc->lock); - ipc->task_mask &= ~IPC_TASK_IN_THREAD; - ipc_complete_cmd(ipc); - k_spin_unlock(&ipc->lock, key); - - return !ret ? pdata->result : ret; -} - /** * @brief Protocol-specific dispatch of a forwarded IPC command. * @@ -509,7 +161,7 @@ __weak int ipc_user_thread_dispatch(struct ipc_user *ipc_user) * User-space IPC thread entry point. p1 points to the ipc_user context * shared with the kernel launcher. */ -static void ipc_user_thread_fn(void *p1, void *p2, void *p3) +void ipc_user_thread_fn(void *p1, void *p2, void *p3) { struct ipc_user *ipc_user = p1; unsigned int core = POINTER_TO_UINT(p2); @@ -540,410 +192,4 @@ static void ipc_user_thread_fn(void *p1, void *p2, void *p3) break; } } - -__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) -{ - char thread_name[] = "ll_user0"; - struct k_thread *thread; - struct k_event *event; - int ret; - - assert_can_be_cold(); - - /* - * Build the objects in locals and publish them to struct ipc_user only - * at the end. When a secondary core is brought up, user threads on - * other cores are already running and can overwrite struct ipc_user, - * so its fields must not be read back here. - */ - event = k_object_alloc(K_OBJ_EVENT); - if (!event) { - LOG_ERR("user IPC event alloc failed"); - return -ENOMEM; - } - k_event_init(event); - - thread = k_object_alloc(K_OBJ_THREAD); - if (!thread) { - LOG_ERR("user IPC thread alloc failed"); - ret = -ENOMEM; - goto e_event; - } - - k_thread_create(thread, ipc_user_stack[core], - CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, - ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, - -1, K_USER, K_FOREVER); - - k_thread_cpu_pin(thread, core); - thread_name[sizeof(thread_name) - 2] = '0' + core; - k_thread_name_set(thread, thread_name); - - /* - * Each userspace IPC thread must be able to wait on its private event - * and signal completion on the primary core semaphore - */ - k_thread_access_grant(thread, ipc_user_ksem, event); - user_grant_dai_access_all(thread); - user_grant_dma_access_all(thread); - k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); - user_ll_grant_access(thread, core); - pipeline_posn_grant_access(thread); - - ipc_user_kevent[core] = event; - ipc_user_kthread[core] = thread; - - /* for ipc_user_thread_fn() and ipc_thread_user() in user context */ - ipc_user->event[core] = event; - ipc_user->thread[core] = thread; - - return 0; - -e_event: - k_object_free(event); - - return ret; -} - -__cold int ipc_user_init_secondary(unsigned int core) -{ - struct ipc_user *ipc_user = ipc_user_kptr; - struct k_thread *audio_thread; - int ret; - - if (core >= CONFIG_CORE_COUNT) - return -EINVAL; - - ret = ipc_user_init_thread(ipc_user, core); - - if (ret < 0) - return ret; - - assert_can_be_cold(); - - k_thread_start(ipc_user_kthread[core]); - - struct task *task = zephyr_ll_task_alloc(); - - if (!task) { - LOG_ERR("user LL task allocation failed"); - k_panic(); - } - - schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, core, 0); - - audio_thread = scheduler_init_context(task); - if (!audio_thread) { - LOG_ERR("user LL thread init failed"); - k_panic(); - } - - k_thread_access_grant(ipc_user_kthread[core], audio_thread); - ipc_user->audio_thread[core] = audio_thread; - - return 0; -} - -struct k_thread *ipc_thread_user(unsigned int core) -{ - if (core >= CONFIG_CORE_COUNT) - return NULL; - - /* user threads can not read the kernel-only copy */ - if (k_is_user_context()) - return ipc_get()->ipc_user_pdata->thread[core]; - - return ipc_user_kthread[core]; -} - -struct ipc_user *ipc_user_get(void) -{ - return ipc_user_kptr; -} - -void ipc_user_wait_core_init(unsigned int core) -{ - if (core >= CONFIG_CORE_COUNT) - return; - - if (ipc_user_kinit_needed[core]) { - /* wait for IPC thread (ipc_user_thread_fn()) */ - k_sem_take(ipc_user_ksem, K_FOREVER); - ipc_user_kinit_needed[core] = false; - } -} - -/* Primary core only */ -__cold static void ipc_user_init(void) -{ - struct ipc *ipc = ipc_get(); - struct k_thread *audio_thread; - struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ipc_user), 0); - int ret, core; - - if (!ipc_user) { - LOG_ERR("user IPC pdata alloc failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - - /* sof_heap_alloc() does not zero, start with clean state (e.g. ppl_phase) */ - memset(ipc_user, 0, sizeof(*ipc_user)); - - assert_can_be_cold(); - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { - if (core != PLATFORM_PRIMARY_CORE_ID) - ipc_user_kinit_needed[core] = true; - } - - ipc_user_ksem = k_object_alloc(K_OBJ_SEM); - if (!ipc_user_ksem) { - LOG_ERR("user IPC sem alloc failed"); - k_panic(); - } - ipc_user_kptr = ipc_user; - ipc_user->sem = ipc_user_ksem; - - ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &ipc_context_part); - if (ret < 0) - LOG_WRN("ipc context partition add failed: %d", ret); - - /* - * Grant user-space read access to the .coldrodata section in IMR. The - * prepare path walks component code that may reference __cold_rodata - * data. - * - * .cold text is deliberately NOT added as a partition: Zephyr already - * maps it executable in the shared ring. - */ -#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM - extern char __coldrodata_start[]; - extern char _imr_end[]; - - if (&_imr_end[0] > &__coldrodata_start[0]) { - struct k_mem_partition cold_part = { - .start = (uintptr_t)__coldrodata_start, - .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, - CONFIG_MMU_PAGE_SIZE), - .attr = K_MEM_PARTITION_P_RO_U_RO, - }; - - ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); - if (ret < 0) - LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", - cold_part.size, cold_part.start, ret); - } -#endif /* CONFIG_COLD_STORE_EXECUTE_DRAM */ - - k_sem_init(ipc_user_ksem, 0, 1); - - ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); - if (ret < 0) { - LOG_ERR("user IPC thread initialization failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - - ret = user_access_to_mailbox(zephyr_ll_mem_domain(), - ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); - if (ret < 0) { - LOG_ERR("ipc user: mailbox access grant failed: %d", ret); - sof_panic(SOF_IPC_PANIC_IPC); - } - - /* Store references in ipc struct so kernel handler can forward commands */ - ipc->ipc_user_pdata = ipc_user; - - struct task *task = zephyr_ll_task_alloc(); - - if (!task) { - LOG_ERR("task allocation failed"); - k_panic(); - } - - schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, - 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); - audio_thread = scheduler_init_context(task); - if (!audio_thread) { - LOG_ERR("user LL thread init failed"); - k_panic(); - } - - /* Grant ipc_user thread permission on the audio thread object. - * Needed so user-space dai_common_new() can call - * k_thread_access_grant(audio_thread, dai_mutex) from user context. - */ - k_thread_access_grant(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID], audio_thread); - ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; - - k_thread_start(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); - - /* Wait for user thread startup — consumes the initial k_sem_give from thread */ - k_sem_take(ipc_user_ksem, K_FOREVER); -} -#else /* CONFIG_SOF_USERSPACE_LL */ -static void ipc_user_init(void) -{ -} #endif /* CONFIG_SOF_USERSPACE_LL */ - -__cold int ipc_init(struct sof *sof) -{ - struct k_heap *heap; - struct ipc *ipc; - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "entry"); - -#ifdef CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); - - ipc = ipc_get(); - memset(ipc, 0, sizeof(*ipc)); - ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*ipc->ll_alloc), 0); - if (!ipc->ll_alloc) { - tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc"); - sof_panic(SOF_IPC_PANIC_IPC); - } - ipc->ll_alloc->heap = heap; - ipc->ll_alloc->vreg = NULL; -#else - heap = NULL; - - /* init ipc data */ - ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*ipc)); - if (!ipc) { - tr_err(&ipc_tr, "Unable to allocate IPC data"); - return -ENOMEM; - } - sof->ipc = ipc; -#endif /* CONFIG_SOF_USERSPACE_LL */ - - ipc->comp_data = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - SOF_IPC_MSG_MAX_SIZE, 0); - if (!ipc->comp_data) { - tr_err(&ipc_tr, "Unable to allocate IPC component data"); - sof_panic(SOF_IPC_PANIC_IPC); - } - memset(ipc->comp_data, 0, SOF_IPC_MSG_MAX_SIZE); - - k_spinlock_init(&ipc->lock); - list_init(&ipc->msg_list); - list_init(&ipc->comp_list); - -#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS - struct io_perf_data_item init_data = {IO_PERF_IPC_ID, - cpu_get_id(), - IO_PERF_INPUT_DIRECTION, - IO_PERF_POWERED_UP_ENABLED, - IO_PERF_D0IX_POWER_MODE, - 0, 0, 0 }; - io_perf_monitor_init_data(&ipc->io_perf_in_msg_count, &init_data); - init_data.direction = IO_PERF_OUTPUT_DIRECTION; - io_perf_monitor_init_data(&ipc->io_perf_out_msg_count, &init_data); -#endif - -#if CONFIG_SOF_BOOT_TEST_STANDALONE - LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); - return 0; -#endif - -#ifdef __ZEPHYR__ - k_tid_t thread; - - k_work_queue_init(&ipc_send_wq); - k_work_queue_start(&ipc_send_wq, ipc_send_wq_stack, - K_THREAD_STACK_SIZEOF(ipc_send_wq_stack), 1, NULL); - - thread = k_work_queue_thread_get(&ipc_send_wq); - - k_thread_suspend(thread); - -#ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_pin(thread, PLATFORM_PRIMARY_CORE_ID); -#endif - k_thread_name_set(thread, "ipc_send_wq"); - - k_thread_resume(thread); - - k_work_init_delayable(&ipc_send_work, ipc_work_handler); -#endif /* __ZEPHYR__ */ - - ipc_user_init(); - - return platform_ipc_init(ipc); -} - -/* Locking: call with ipc->lock held and interrupts disabled */ -void ipc_complete_cmd(struct ipc *ipc) -{ - /* - * We have up to three contexts, attempting to complete IPC processing: - * the original IPC EDF task, the IDC EDF task on a secondary core, or - * an LL pipeline thread, running either on the primary or one of - * secondary cores. All these three contexts execute asynchronously. It - * is important to only signal the host that the IPC processing has - * completed after *all* tasks have completed. Therefore only the last - * context should do that. We accomplish this by setting IPC_TASK_* bits - * in ipc->task_mask for each used IPC context and by clearing them when - * each of those contexts completes. Only when the mask is 0 we can - * signal the host. - */ - if (ipc->task_mask) - return; - - ipc_platform_complete_cmd(ipc); -} - -bool ipc_enter_gdb; - -__attribute__((weak)) void ipc_platform_wait_ack(struct ipc *ipc) -{ - k_msleep(1); -} - -static void ipc_complete_task(void *data) -{ - struct ipc *ipc = data; - k_spinlock_key_t key; - - key = k_spin_lock(&ipc->lock); - ipc->task_mask &= ~IPC_TASK_INLINE; - ipc_complete_cmd(ipc); - k_spin_unlock(&ipc->lock, key); -#if CONFIG_GDBSTUB - if (ipc_enter_gdb) { - ipc_enter_gdb = false; - ipc_platform_wait_ack(ipc); - gdb_init(); - } -#endif -} - -static enum task_state ipc_do_cmd(void *data) -{ - struct ipc *ipc = data; - -#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS - /* Increment performance counters */ - io_perf_monitor_update_data(ipc->io_perf_in_msg_count, 1); -#endif - - /* - * 32-bit writes are atomic and at the moment no IPC processing is - * taking place, so, no need for a lock. - */ - ipc->task_mask = IPC_TASK_INLINE; - - return ipc_platform_do_cmd(ipc); -} - -struct task_ops ipc_task_ops = { - .run = ipc_do_cmd, - .complete = ipc_complete_task, - .get_deadline = ipc_task_deadline, -}; diff --git a/src/ipc/ipc4/README.md b/src/ipc/ipc4/README.md index 237a7c1ecb36..70ee4d606af2 100644 --- a/src/ipc/ipc4/README.md +++ b/src/ipc/ipc4/README.md @@ -2,9 +2,10 @@ > **Runs in:** Boundary / bridge — `handler-user.c` and `notification-user.c` run in the user > thread that services modules. Their kernel counterparts, `handler-kernel.c` (the kernel -> dispatcher) and `notification.c`, are in `src/kernel/ipc/ipc4`, together with the -> `z_vrfy_`/`z_impl_` pairs for the compound-message, reply and notification syscalls. See -> [User/kernel split](../../include/sof/userspace/README.md). +> dispatcher), `handler-user-kernel.c` (the global and module message routers, which forward +> commands to the user thread) and `notification.c`, are in `src/kernel/ipc/ipc4`, together +> with the `z_vrfy_`/`z_impl_` pairs for the compound-message, reply and notification +> syscalls. See [User/kernel split](../../include/sof/userspace/README.md). This directory holds the handlers and topology parsing logic for Inter-Processor Communication Version 4. IPC4 introduces a significantly denser, compound-command structure heavily based around the concept of "pipelines" and dynamic "modules" rather than static DSP stream roles. diff --git a/src/ipc/ipc4/handler-user.c b/src/ipc/ipc4/handler-user.c index 0af33fe7fd1c..58df68018242 100644 --- a/src/ipc/ipc4/handler-user.c +++ b/src/ipc/ipc4/handler-user.c @@ -44,8 +44,6 @@ #include #include -#include "../audio/copier/ipcgtw_copier.h" - /* Command format errors during fuzzing are reported for virtually all * commands, and the resulting flood of logging becomes a severe * performance penalty (i.e. we get a lot less fuzzing done per CPU @@ -57,11 +55,11 @@ #define ipc_cmd_err(...) tr_err(__VA_ARGS__) #endif -LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); +/* The IPC4 routers, which run in kernel mode, are in + * src/kernel/ipc/ipc4/handler-user-kernel.c + */ -/* Userspace message context, copied in/out by kernel IPC thread. */ -/* fw sends a fw ipc message to send the status of the last host ipc message */ -static struct ipc_msg *msg_reply; +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); #if CONFIG_LIBRARY @@ -90,54 +88,6 @@ static inline const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data( /* * Global IPC Operations. */ -#ifdef CONFIG_SOF_USERSPACE_LL -/* - * Determine the target core for an IPC4 module message. - * Falls back to current core when no component is bound yet. - */ -static unsigned int ipc4_user_target_core_module(struct ipc4_message_request *ipc4) -{ - /* - * Also works for struct ipc4_module_large_config, struct ipc4_module_bind_unbind, - * struct ipc4_module_delete_instance - */ - struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; - uint32_t module_id = config->primary.r.module_id; - - if (module_id) { - uint32_t instance_id = config->primary.r.instance_id; - struct comp_dev *dev = ipc4_get_comp_dev(IPC4_COMP_ID(module_id, instance_id)); - - if (dev) - return dev->ipc_config.core; - } - - return cpu_get_id(); -} -#else /* CONFIG_SOF_USERSPACE_LL */ -__cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) -{ - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); -} - -__cold static int ipc4_delete_pipeline(struct ipc4_message_request *ipc4) -{ - struct ipc4_pipeline_delete *pipe; - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - pipe = (struct ipc4_pipeline_delete *)ipc4; - tr_dbg(&ipc_tr, "ipc4 delete pipeline %x:", (uint32_t)pipe->primary.r.instance_id); - - return ipc_pipeline_free(ipc, pipe->primary.r.instance_id); -} -#endif /* CONFIG_SOF_USERSPACE_LL */ - static int ipc4_pcm_params(struct ipc_comp_dev *pcm_dev) { int err, reset_err; @@ -450,9 +400,9 @@ __cold const struct ipc4_pipeline_set_state_data *ipc4_get_pipeline_data_wrapper return ipc4_get_pipeline_data(); } -static int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, - struct ipc4_pipeline_set_state *state, - const uint32_t **ppl_id, unsigned int *ppl_count) +int ipc4_pipeline_id_get(struct ipc4_message_request *ipc4, + struct ipc4_pipeline_set_state *state, + const uint32_t **ppl_id, unsigned int *ppl_count) { if (!state->extension.r.multi_ppl) { if (ppl_count) @@ -648,364 +598,8 @@ static int ipc4_set_pipeline_state_phase(struct ipc4_message_request *ipc4, uint return IPC4_SUCCESS; } - -/* - * Handle a multi-pipeline SET_PIPELINE_STATE, where the pipelines are on - * different cores. The user-space IPC thread can't use IDC to reach the - * other cores, so the kernel runs the prepare phase on all pipelines and - * then the trigger phase on all pipelines, forwarding each step to the - * user-space IPC thread of the core, owning the pipeline. - */ -__cold static int ipc4_user_set_pipeline_state_multicore(struct ipc4_message_request *ipc4, - const uint32_t *ppl_id, - unsigned int ppl_count) -{ - static const uint32_t phases[] = { - IDC_PPL_STATE_PHASE_PREPARE, - IDC_PPL_STATE_PHASE_TRIGGER, - }; - struct ipc *ipc = ipc_get(); - struct ipc_user *pdata = ipc->ipc_user_pdata; - struct ipc4_pipeline_set_state state; - struct ipc_comp_dev *ppl_icd; - unsigned int i, j; - int ret; - - assert_can_be_cold(); - - for (j = 0; j < ARRAY_SIZE(phases); j++) { - for (i = 0; i < ppl_count; i++) { - ppl_icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, ppl_id[i], - IPC_COMP_ALL); - if (!ppl_icd) { - ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); - return IPC4_INVALID_RESOURCE_ID; - } - - /* Forward as a single-pipeline request */ - state.primary.dat = ipc4->primary.dat; - state.extension.dat = ipc4->extension.dat; - state.primary.r.ppl_id = ppl_id[i]; - state.extension.r.multi_ppl = 0; - - pdata->ppl_phase = phases[j]; - ret = ipc_user_forward_cmd(state.primary.dat, state.extension.dat, - ppl_icd->core); - pdata->ppl_phase = 0; - if (ret != 0) - return ret; - } - } - - return IPC4_SUCCESS; -} #endif /* CONFIG_SOF_USERSPACE_LL */ -__cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) -{ - assert_can_be_cold(); - -#if CONFIG_COMP_CHAIN_DMA - struct ipc_comp_dev *cdma_comp; - struct ipc *ipc = ipc_get(); - const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4; - int comp_id; - int ret; - - comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0); - cdma_comp = ipc_get_comp_by_id(ipc, comp_id); - - if (!cdma_comp) { - /* - * Nothing to do when the chainDMA is not allocated and asked to - * be freed - */ - if (!cdma->primary.r.allocate && !cdma->primary.r.enable) - return IPC4_SUCCESS; - - ret = ipc4_chain_manager_create(cdma); - if (ret < 0) - return IPC4_FAILURE; - - cdma_comp = ipc_get_comp_by_id(ipc, comp_id); - if (!cdma_comp) { - return IPC4_FAILURE; - } - - ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); - if (ret < 0) { - comp_free(cdma_comp->cd); - list_item_del(&cdma_comp->list); - rfree(cdma_comp); - return IPC4_FAILURE; - } - - return IPC4_SUCCESS; - } - - ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); - if (ret < 0) - return IPC4_INVALID_CHAIN_STATE_TRANSITION; - - return IPC4_SUCCESS; -#else /* CONFIG_COMP_CHAIN_DMA */ - return IPC4_UNAVAILABLE; -#endif /* CONFIG_COMP_CHAIN_DMA */ -} - -__cold static int ipc4_process_ipcgtw_cmd(struct ipc4_message_request *ipc4) -{ - assert_can_be_cold(); - -#if CONFIG_IPC4_GATEWAY - struct ipc *ipc = ipc_get(); - uint32_t reply_size = 0; - int err; - - err = copier_ipcgtw_process((const struct ipc4_ipcgtw_cmd *)ipc4, ipc->comp_data, - &reply_size); - /* reply size is returned in header extension dword */ - msg_reply->extension = reply_size; - - if (reply_size > 0) { - msg_reply->tx_data = ipc->comp_data; - msg_reply->tx_size = reply_size; - } - - return err < 0 ? IPC4_FAILURE : IPC4_SUCCESS; -#else /* CONFIG_IPC4_GATEWAY */ - ipc_cmd_err(&ipc_tr, "CONFIG_IPC4_GATEWAY is disabled"); - return IPC4_UNAVAILABLE; -#endif /* CONFIG_IPC4_GATEWAY */ -} - -static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) -{ -#if CONFIG_GDBSTUB - ipc_enter_gdb = true; - return IPC4_SUCCESS; -#else - return IPC4_UNAVAILABLE; -#endif -} - -int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, - struct ipc_msg *reply) -{ -#ifdef CONFIG_SOF_USERSPACE_LL - struct ipc *ipc = ipc_get(); -#endif - uint32_t type; - int ret; - - type = ipc4->primary.r.type; - msg_reply = reply; - - switch (type) { - case SOF_IPC4_GLB_BOOT_CONFIG: - case SOF_IPC4_GLB_ROM_CONTROL: - case SOF_IPC4_GLB_PERF_MEASUREMENTS_CMD: - case SOF_IPC4_GLB_LOAD_MULTIPLE_MODULES: - case SOF_IPC4_GLB_UNLOAD_MULTIPLE_MODULES: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - case SOF_IPC4_GLB_CHAIN_DMA: - ret = ipc4_process_chain_dma(ipc4); - break; - - /* pipeline settings */ - case SOF_IPC4_GLB_CREATE_PIPELINE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_pipeline_create *create = - (const struct ipc4_pipeline_create *)ipc4; - - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - create->extension.r.core_id); - } -#else - ret = ipc4_new_pipeline(ipc4); -#endif - break; - case SOF_IPC4_GLB_DELETE_PIPELINE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_pipeline_delete *del = (const struct ipc4_pipeline_delete *)ipc4; - struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, - del->primary.r.instance_id, - IPC_COMP_ALL); - - if (!ppl) { - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); - } -#else - ret = ipc4_delete_pipeline(ipc4); -#endif - break; - case SOF_IPC4_GLB_SET_PIPELINE_STATE: -#ifdef CONFIG_SOF_USERSPACE_LL - { - struct ipc4_pipeline_set_state state = { - .primary.dat = ipc4->primary.dat, - .extension.dat = ipc4->extension.dat, - }; - const uint32_t *ppl_id; - unsigned int ppl_count; - int id = ipc4_pipeline_id_get(ipc4, &state, &ppl_id, &ppl_count); - if (id < 0) { - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - - struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, id, - IPC_COMP_ALL); - - if (!ppl) { - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - - /* - * Validate all pipelines before any state change and check, - * if they span multiple cores, needing per-core processing - */ - bool multicore = false; - - ret = IPC4_SUCCESS; - for (unsigned int i = 1; i < ppl_count; i++) { - struct ipc_comp_dev *icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, - ppl_id[i], IPC_COMP_ALL); - - if (!icd) { - ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); - ret = IPC4_INVALID_RESOURCE_ID; - break; - } - - if (icd->core != ppl->core) - multicore = true; - } - - if (ret != IPC4_SUCCESS) - break; - - if (multicore) - ret = ipc4_user_set_pipeline_state_multicore(ipc4, ppl_id, ppl_count); - else - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ppl->core); - } -#else /* CONFIG_SOF_USERSPACE_LL */ - ret = ipc4_set_pipeline_state(ipc4); -#endif /* CONFIG_SOF_USERSPACE_LL */ - break; - - case SOF_IPC4_GLB_GET_PIPELINE_STATE: - case SOF_IPC4_GLB_GET_PIPELINE_CONTEXT_SIZE: - case SOF_IPC4_GLB_SAVE_PIPELINE: - case SOF_IPC4_GLB_RESTORE_PIPELINE: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - case SOF_IPC4_GLB_INTERNAL_MESSAGE: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - /* Notification (FW to SW driver) */ - case SOF_IPC4_GLB_NOTIFICATION: - ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - - case SOF_IPC4_GLB_IPCGATEWAY_CMD: - ret = ipc4_process_ipcgtw_cmd(ipc4); - break; - - case SOF_IPC4_GLB_ENTER_GDB: - ret = ipc_glb_gdb_debug(ipc4); - break; - - default: - ipc_cmd_err(&ipc_tr, "unsupported ipc message type %d", type); - ret = IPC4_UNAVAILABLE; - break; - } - - return ret; -} - -/* - * Ipc4 Module message <------> ipc3 module message - * init module <-------> create component - * bind modules <-------> connect components - * module set_large_config <-------> component cmd - * delete module <-------> free component - */ - -__cold static int ipc4_init_module_instance(struct ipc4_message_request *ipc4) -{ - const struct ipc4_module_init_instance *module_init = - (const struct ipc4_module_init_instance *)ipc4; - struct comp_dev *dev; - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)module_init->primary.r.module_id, - (uint32_t)module_init->primary.r.instance_id); - - /* Pass IPC to target core */ - if (!cpu_is_me(module_init->extension.r.core_id)) - return ipc4_process_on_core(module_init->extension.r.core_id, false); - - dev = comp_new_ipc4(module_init); - if (!dev) { - ipc_cmd_err(&ipc_tr, "error: failed to init module %x : %x", - (uint32_t)module_init->primary.r.module_id, - (uint32_t)module_init->primary.r.instance_id); - return IPC4_MOD_NOT_INITIALIZED; - } - - return 0; -} - -#ifndef CONFIG_SOF_USERSPACE_LL -__cold static int ipc4_bind_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x with %x : %x", - (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, - (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); - - return ipc4_comp_connect(ipc, bu); -} - -__cold static int ipc4_unbind_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; - struct ipc *ipc = ipc_get(); - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x with %x : %x", - (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, - (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); - - return ipc4_comp_disconnect(ipc, bu); -} -#endif /* !CONFIG_SOF_USERSPACE_LL */ - /** * @brief Process MOD_CONFIG_GET or MOD_CONFIG_SET in any execution context. * @@ -1077,23 +671,8 @@ __cold int ipc4_process_module_config(struct ipc4_message_request *ipc4, return ret; } -#ifndef CONFIG_SOF_USERSPACE_LL -static int ipc4_set_get_config_module_instance(struct ipc4_message_request *ipc4, bool set) -{ - uint32_t reply_ext; - int ret; - - ret = ipc4_process_module_config(ipc4, set, &reply_ext); - - if (!set) - msg_reply->extension = reply_ext; - - return ret; -} -#endif /* !CONFIG_SOF_USERSPACE_LL */ - -__cold static void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t param_id, - char *data_out, uint32_t data_size) +__cold void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t param_id, + char *data_out, uint32_t data_size) { const char *hostbox; @@ -1122,13 +701,13 @@ __cold static void ipc4_prepare_for_kcontrol_get(struct comp_dev *dev, uint8_t p } } -__cold static int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, - const struct comp_driver *drv, - bool init_block, - bool final_block, - uint32_t *data_off_size, - char *data_out, - const char *data_in) +__cold int ipc4_get_vendor_config_module_instance(struct comp_dev *dev, + const struct comp_driver *drv, + bool init_block, + bool final_block, + uint32_t *data_off_size, + char *data_out, + const char *data_in) { const struct sof_tl * const input_tl = (struct sof_tl *)data_in; int ret; @@ -1324,119 +903,7 @@ __cold static int ipc4_process_large_config_get(struct ipc4_message_request *ipc } #endif -__cold static int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_large_config_reply reply; - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - char *data = ipc_get()->comp_data; - const struct comp_driver *drv; - struct comp_dev *dev = NULL; - uint32_t data_offset; - int ret; - - assert_can_be_cold(); - - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); - - /* get component dev for non-basefw since there is no - * component dev for basefw - */ - if (config->primary.r.module_id) { - uint32_t comp_id; - - comp_id = IPC4_COMP_ID(config->primary.r.module_id, - config->primary.r.instance_id); - dev = ipc4_get_comp_dev(comp_id); - if (!dev) - return IPC4_INVALID_RESOURCE_ID; - - drv = dev->drv; - - /* Pass IPC to target core */ - if (!cpu_is_me(dev->ipc_config.core)) - return ipc4_process_on_core(dev->ipc_config.core, false); - } else { - /* BaseFW module has only 0th instance */ - if (config->primary.r.instance_id) - return IPC4_INVALID_RESOURCE_ID; - - drv = ipc4_get_comp_drv(config->primary.r.module_id); - } - - if (!drv) - return IPC4_INVALID_RESOURCE_ID; - - if (!drv->ops.get_large_config) - return IPC4_INVALID_REQUEST; - - data_offset = config->extension.r.data_off_size; - - /* check for vendor param first */ - if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { - /* data_off_size is a 20-bit host-controlled field, so it can - * claim far more than the hostbox can physically hold. - */ - if (data_offset > MAILBOX_HOSTBOX_SIZE) { - ipc_cmd_err(&ipc_tr, "data_off_size %u exceeds mailbox bound", - data_offset); - return IPC4_INVALID_CONFIG_DATA_STRUCT; - } - /* For now only vendor_config case uses payload from hostbox */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - config->extension.r.data_off_size); - ret = ipc4_get_vendor_config_module_instance(dev, drv, - config->extension.r.init_block, - config->extension.r.final_block, - &data_offset, data, - (const char *)MAILBOX_HOSTBOX_BASE); - } else { -#if CONFIG_LIBRARY - data += sizeof(reply); -#endif - ipc4_prepare_for_kcontrol_get(dev, config->extension.r.large_param_id, - data, data_offset); - - ret = drv->ops.get_large_config(dev, config->extension.r.large_param_id, - config->extension.r.init_block, - config->extension.r.final_block, - &data_offset, data); - } - - /* set up ipc4 error code for reply data */ - if (ret < 0) - ret = IPC4_INVALID_RESOURCE_ID; - - /* Copy host config and overwrite */ - reply.extension.dat = config->extension.dat; - reply.extension.r.data_off_size = data_offset; - - /* The last block, no more data */ - if (!config->extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) - reply.extension.r.final_block = 1; - - /* Indicate last block if error occurs */ - if (ret) - reply.extension.r.final_block = 1; - - /* no need to allocate memory for reply msg */ - if (ret) - return ret; - - /* A module must not report more reply data than the DSP mailbox can hold. */ - if (data_offset > MAILBOX_DSPBOX_SIZE) { - ipc_cmd_err(&ipc_tr, "reply size %u exceeds mailbox bound", data_offset); - return IPC4_INVALID_CONFIG_DATA_STRUCT; - } - - msg_reply->extension = reply.extension.dat; - msg_reply->tx_size = data_offset; - msg_reply->tx_data = data; - return ret; -} - -__cold static int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, +__cold int ipc4_set_vendor_config_module_instance(struct comp_dev *dev, const struct comp_driver *drv, uint32_t module_id, uint32_t instance_id, @@ -1586,283 +1053,6 @@ __cold int ipc4_process_large_config_set(struct ipc4_message_request *ipc4) return ret; } -__cold static int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4) -{ - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - struct comp_dev *dev = NULL; - const struct comp_driver *drv; - int ret; - - assert_can_be_cold(); - - dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, - config->extension.r.data_off_size); - tr_dbg(&ipc_tr, "%x : %x", - (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); - - if (config->primary.r.module_id) { - uint32_t comp_id; - - comp_id = IPC4_COMP_ID(config->primary.r.module_id, config->primary.r.instance_id); - dev = ipc4_get_comp_dev(comp_id); - if (!dev) - return IPC4_INVALID_RESOURCE_ID; - - drv = dev->drv; - - /* Pass IPC to target core */ - if (!cpu_is_me(dev->ipc_config.core)) - return ipc4_process_on_core(dev->ipc_config.core, false); - } else { - /* BaseFW module has only 0th instance */ - if (config->primary.r.instance_id) - return IPC4_INVALID_RESOURCE_ID; - - drv = ipc4_get_comp_drv(config->primary.r.module_id); - } - - if (!drv) - return IPC4_INVALID_RESOURCE_ID; - - if (!drv->ops.set_large_config) - return IPC4_INVALID_REQUEST; - - /* check for vendor param first */ - if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { - ret = ipc4_set_vendor_config_module_instance(dev, drv, - (uint32_t)config->primary.r.module_id, - (uint32_t)config->primary.r.instance_id, - config->extension.r.init_block, - config->extension.r.final_block, - config->extension.r.data_off_size, - (const char *)MAILBOX_HOSTBOX_BASE); - } else { -#if CONFIG_LIBRARY - struct ipc *ipc = ipc_get(); - const char *data = (const char *)ipc->comp_data + sizeof(*config); -#else - const char *data = (const char *)MAILBOX_HOSTBOX_BASE; -#endif - ret = drv->ops.set_large_config(dev, config->extension.r.large_param_id, - config->extension.r.init_block, config->extension.r.final_block, - config->extension.r.data_off_size, data); - if (ret < 0) { - ipc_cmd_err(&ipc_tr, "failed to set large_config_module_instance %x : %x", - (uint32_t)config->primary.r.module_id, - (uint32_t)config->primary.r.instance_id); - ret = IPC4_INVALID_RESOURCE_ID; - } - } - - return ret; -} - -#ifndef CONFIG_SOF_USERSPACE_LL -__cold static int ipc4_delete_module_instance(struct ipc4_message_request *ipc4) -{ - struct ipc4_module_delete_instance module; - struct ipc *ipc = ipc_get(); - uint32_t comp_id; - - assert_can_be_cold(); - - int ret = memcpy_s(&module, sizeof(module), ipc4, sizeof(*ipc4)); - - if (ret < 0) - return IPC4_FAILURE; - - tr_dbg(&ipc_tr, "%x : %x", (uint32_t)module.primary.r.module_id, - (uint32_t)module.primary.r.instance_id); - - comp_id = IPC4_COMP_ID(module.primary.r.module_id, module.primary.r.instance_id); - ret = ipc_comp_free(ipc, comp_id); - if (ret < 0) { - ipc_cmd_err(&ipc_tr, "failed to delete module instance %x : %x", - (uint32_t)module.primary.r.module_id, - (uint32_t)module.primary.r.instance_id); - ret = IPC4_INVALID_RESOURCE_ID; - } - - return ret; -} -#endif /* !CONFIG_SOF_USERSPACE_LL */ - -__cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, - struct ipc_msg *reply) -{ - const struct ipc4_module_init_instance *mi; - uint32_t type; - int ret; - - assert_can_be_cold(); - - type = ipc4->primary.r.type; - msg_reply = reply; - - switch (type) { - case SOF_IPC4_MOD_INIT_INSTANCE: - mi = (const struct ipc4_module_init_instance *)ipc4; - - if (cpu_is_me(mi->extension.r.core_id) && !mi->extension.r.proc_domain && - IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { -#ifdef CONFIG_SOF_USERSPACE_LL - BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= - sizeof(((struct ipc_user *)0)->init_drv_data), - "ipc_user.init_drv_data too small for driver copy"); - - /* User-space init: kernel does driver lookup only (requires - * access to IMR manifest and driver list in kernel memory). - * Component creation (drv->ops.create) runs in user thread - * so untrusted module code does not execute in kernel context. - */ - uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, - mi->primary.r.instance_id); - const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); - - if (!drv || !drv->ops.create) { - ret = IPC4_MOD_NOT_INITIALIZED; - break; - } - - struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); - - if (ctx && drv->type == SOF_COMP_MODULE_ADAPTER) { - int err = ipc4_user_module_load(drv, mi); - - if (err < 0) { - ret = IPC4_MOD_NOT_INITIALIZED; - break; - } - } - - ipc_user_get()->init_drv = drv; - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - mi->extension.r.core_id); -#endif /* CONFIG_SOF_USERSPACE_LL */ - } else { - /* - * DP module creation starts running in kernel mode and - * switches to userspace later via a scheduler_dp_thread_ipc() - * call in module_init(). - */ - ret = ipc4_init_module_instance(ipc4); - } - break; - case SOF_IPC4_MOD_CONFIG_GET: -#ifdef CONFIG_SOF_USERSPACE_LL - /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); - if (!ret) { - struct ipc_user *pdata = ipc_user_get(); - - msg_reply->extension = pdata->reply_ext; - } -#else - ret = ipc4_set_get_config_module_instance(ipc4, false); -#endif - break; - case SOF_IPC4_MOD_CONFIG_SET: -#ifdef CONFIG_SOF_USERSPACE_LL - /* Forward to user thread for privilege-separated execution */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_set_get_config_module_instance(ipc4, true); -#endif - break; - case SOF_IPC4_MOD_LARGE_CONFIG_GET: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - - if (config->primary.r.module_id) { - /* Module case: forward to user thread */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); - if (!ret) { - struct ipc_user *pdata = ipc_user_get(); - - msg_reply->extension = pdata->reply_ext; - msg_reply->tx_size = pdata->reply_tx_size; - msg_reply->tx_data = pdata->reply_tx_data; - } - } else { - /* Base firmware (module_id==0): keep in kernel — - * ipc4_get_comp_drv() accesses IMR manifest which - * has no user-space partition. - */ - ret = ipc4_get_large_config_module_instance(ipc4); - } - } -#else /* CONFIG_SOF_USERSPACE_LL */ - ret = ipc4_get_large_config_module_instance(ipc4); -#endif /* CONFIG_SOF_USERSPACE_LL */ - break; - case SOF_IPC4_MOD_LARGE_CONFIG_SET: -#ifdef CONFIG_SOF_USERSPACE_LL - { - const struct ipc4_module_large_config *config = - (const struct ipc4_module_large_config *)ipc4; - - if (config->primary.r.module_id) { - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); - } else { - /* Base firmware: keep in kernel (IMR access) */ - ret = ipc4_set_large_config_module_instance(ipc4); - } - } -#else - ret = ipc4_set_large_config_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_BIND: -#ifdef CONFIG_SOF_USERSPACE_LL - /* - * bind and unbind can connect LL with DP. In that case it isn't - * immediately clear whether the handler should run in the DP - * thread context or in the LL IPC thread context. The LL IPC - * thread has access to DP modules, so we have to perform - * binding in the LL IPC thread context - */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_bind_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_UNBIND: -#ifdef CONFIG_SOF_USERSPACE_LL - /* DP / LL: see comment above */ - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_unbind_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_DELETE_INSTANCE: -#ifdef CONFIG_SOF_USERSPACE_LL - ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, - ipc4_user_target_core_module(ipc4)); -#else - ret = ipc4_delete_module_instance(ipc4); -#endif - break; - case SOF_IPC4_MOD_ENTER_MODULE_RESTORE: - case SOF_IPC4_MOD_EXIT_MODULE_RESTORE: - ret = IPC4_UNAVAILABLE; - break; - default: - ret = IPC4_UNKNOWN_MESSAGE_TYPE; - break; - } - - return ret; -} - #ifdef CONFIG_SOF_USERSPACE_LL /* * IPC4 implementation of the user-thread dispatch hook. Runs in the diff --git a/src/ipc/ipc4/helper.c b/src/ipc/ipc4/helper.c index d5a0e095b3f4..2996fcbf7fcb 100644 --- a/src/ipc/ipc4/helper.c +++ b/src/ipc/ipc4/helper.c @@ -29,26 +29,14 @@ #include #include -/* TODO: Remove platform-specific code, see https://github.com/thesofproject/sof/issues/7549 */ -#if defined(CONFIG_SOC_SERIES_INTEL_ADSP_ACE) || defined(CONFIG_INTEL_ADSP_CAVS) -#define RIMAGE_MANIFEST 1 -#endif - -#ifdef RIMAGE_MANIFEST -#include -#endif - #include #include -#include -#include #include #include #include #include #include #include -#include #include #include @@ -59,11 +47,9 @@ #include #include -LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); - -extern struct tr_ctx comp_tr; +/* The IPC4 helpers that run in kernel mode are in src/kernel/ipc/ipc4/helper-kernel.c */ -static const struct comp_driver *ipc4_get_drv(const void *uuid); +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); void ipc_build_stream_posn(struct sof_ipc_stream_posn *posn, uint32_t type, uint32_t id) @@ -76,19 +62,8 @@ void ipc_build_comp_event(struct sof_ipc_comp_event *event, uint32_t type, { } -bool ipc_trigger_trace_xfer(uint32_t avail) -{ - return avail >= DMA_TRACE_LOCAL_SIZE / 2; -} - -void ipc_build_trace_posn(struct sof_ipc_dma_trace_posn *posn) -{ - posn->rhdr.hdr.cmd = SOF_IPC4_NOTIF_HEADER(SOF_IPC4_NOTIFY_LOG_BUFFER_STATUS); - posn->rhdr.hdr.size = 0; -} - #if CONFIG_LIBRARY -static inline unsigned char *ipc4_get_comp_new_data(void) +unsigned char *ipc4_get_comp_new_data(void) { struct ipc *ipc = ipc_get(); unsigned char *data = (unsigned char *)ipc->comp_data + @@ -96,13 +71,8 @@ static inline unsigned char *ipc4_get_comp_new_data(void) return data; } - -static const struct comp_driver *ipc4_library_get_comp_drv(unsigned char *data) -{ - return ipc4_get_drv(data); -} #else -__cold static inline unsigned char *ipc4_get_comp_new_data(void) +__cold unsigned char *ipc4_get_comp_new_data(void) { assert_can_be_cold(); @@ -110,8 +80,8 @@ __cold static inline unsigned char *ipc4_get_comp_new_data(void) } #endif /* CONFIG_LIBRARY */ -__cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, - const struct ipc4_module_init_instance *module_init) +__cold int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, + const struct ipc4_module_init_instance *module_init) { uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, module_init->primary.r.instance_id); @@ -182,112 +152,7 @@ __cold static int ipc4_comp_new_config(struct comp_ipc_config *ipc_config, return 0; } -/* Only called from ipc4_init_module_instance(), which is __cold */ -__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) -{ - struct comp_ipc_config ipc_config; - const struct comp_driver *drv; - struct comp_dev *dev; - unsigned char *data; - uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, - module_init->primary.r.instance_id); - - assert_can_be_cold(); - - int ret = ipc4_comp_new_config(&ipc_config, module_init); - - if (ret < 0) - return NULL; - -#ifdef CONFIG_DCACHE_LINE_SIZE - if (!IS_ENABLED(CONFIG_LIBRARY)) - sys_cache_data_invd_range((__sparse_force void __sparse_cache *) - MAILBOX_HOSTBOX_BASE, - ALIGN_UP(ipc_config.ipc_config_size, - CONFIG_DCACHE_LINE_SIZE)); -#endif - data = ipc4_get_comp_new_data(); -#if CONFIG_LIBRARY - ipc_config.ipc_config_size -= sizeof(struct sof_uuid); - drv = ipc4_library_get_comp_drv(data + ipc_config.ipc_config_size); -#else - drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); -#endif - if (!drv) - return NULL; - - if (!drv->ops.create) { - tr_err(&ipc_tr, "comp 0x%x: module %u is not instantiable", comp_id, - (uint32_t)IPC4_MOD_ID(comp_id)); - return NULL; - } - - if (drv->type == SOF_COMP_MODULE_ADAPTER) { - const struct ipc_config_process spec = { - .data = data, - .size = ipc_config.ipc_config_size, - }; - - dev = drv->ops.create(drv, &ipc_config, &spec); - } else { - dev = drv->ops.create(drv, &ipc_config, data); - } - if (!dev) - return NULL; - - list_init(&dev->bsource_list); - list_init(&dev->bsink_list); - -#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS - /* init global performance measurement */ - dev->perf_data.perf_data_item = perf_data_getnext(); - /* this can be null, just no performance measurements in this case */ - if (dev->perf_data.perf_data_item) { - dev->perf_data.perf_data_item->item.resource_id = comp_id; - if (perf_meas_get_state() != IPC4_PERF_MEASUREMENTS_DISABLED) - comp_init_performance_data(dev); - } -#endif - - ipc4_add_comp_dev(dev); - - comp_update_ibs_obs_cpc(dev); - - return dev; -} - #ifdef CONFIG_SOF_USERSPACE_LL - -int ipc4_user_module_load(const struct comp_driver *drv, - const struct ipc4_module_init_instance *mi) -{ - /* - * A large part of module instantiation is executed in the userspace - * mode, including calling the module's .init() method. But another part - * of it has to be executed in the kernel mode, including calling - * lib_manager_mod_create_priv(). This is the privileged part of module - * instantiation. - */ - struct comp_ipc_config ipc_config; - int ret = ipc4_comp_new_config(&ipc_config, mi); - - if (ret < 0) - return ret; - - const struct ipc_config_process spec = { - .data = ipc4_get_comp_new_data(), - .size = ipc_config.ipc_config_size, - }; - -#if CONFIG_DCACHE_LINE_SIZE && !CONFIG_LIBRARY - sys_cache_data_invd_range((__sparse_force void __sparse_cache *)spec.data, spec.size); -#endif - - struct userspace_context *userspace = NULL; - - return lib_manager_mod_create_priv(drv, &ipc_config, &spec, NULL, &userspace); -} - /** * comp_new_ipc4_user - Create component in user-space IPC thread context. * @@ -579,16 +444,6 @@ __cold int ipc_pipeline_new(struct ipc *ipc, ipc_pipe_new *_pipe_desc) return ipc4_create_pipeline(pipe_desc, valid_pparams ? &pparams : NULL); } -__cold static inline int ipc_comp_free_remote(struct comp_dev *dev) -{ - struct idc_msg msg = { IDC_MSG_FREE, IDC_MSG_FREE_EXT(dev->ipc_config.id), - dev->ipc_config.core,}; - - assert_can_be_cold(); - - return idc_send_msg(&msg, IDC_BLOCKING); -} - __cold static int ipc_pipeline_module_free(uint32_t pipeline_id) { struct ipc *ipc = ipc_get(); @@ -1182,76 +1037,6 @@ __cold int ipc4_comp_disconnect(struct ipc *ipc, const struct ipc4_module_bind_u return IPC4_SUCCESS; } -#if CONFIG_COMP_CHAIN_DMA -/* Only called from ipc4_process_chain_dma(), which is __cold */ -__cold int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma) -{ - const struct sof_uuid uuid = SOF_REG_UUID(chain_dma); - const struct comp_driver *drv; - struct comp_dev *dev; - - assert_can_be_cold(); - - drv = ipc4_get_drv(&uuid); - if (!drv) - return -EINVAL; - - dev = drv->ops.create(drv, NULL, cdma); - if (!dev) - return -EINVAL; - - /* differentiate instance by unique ids assignment */ - const uint32_t comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id - + IPC4_MAX_MODULE_COUNT, 0); - dev->ipc_config.id = comp_id; - dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT; - - return ipc4_add_comp_dev(dev); -} - -/* Only called from ipc4_process_chain_dma(), which is __cold */ -__cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma) -{ - const bool allocate = cdma->primary.r.allocate; - const bool enable = cdma->primary.r.enable; - struct ipc *ipc = ipc_get(); - struct ipc_comp_dev *icd; - struct list_item *clist, *_tmp; - int ret; - - assert_can_be_cold(); - - if (!dev) - return -EINVAL; - - if (allocate) { - if (enable) - ret = comp_trigger(dev, COMP_TRIGGER_START); - else - ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); - } else { - if (enable) - return -EINVAL; - - /* remove chain part */ - ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); - if (ret < 0) - return ret; - - list_for_item_safe(clist, _tmp, &ipc->comp_list) { - icd = container_of(clist, struct ipc_comp_dev, list); - if (icd->cd != dev) - continue; - list_item_del(&icd->list); - sof_heap_free(sof_sys_user_heap_get(), icd); - break; - } - comp_free(dev); - } - return ret; -} -#endif /* CONFIG_COMP_CHAIN_DMA */ - __cold static int ipc4_update_comps_direction(struct ipc *ipc, uint32_t ppl_id) { struct ipc_comp_dev *icd; @@ -1314,200 +1099,6 @@ int ipc4_pipeline_complete(struct ipc *ipc, uint32_t comp_id, uint32_t cmd) return ipc_pipeline_complete(ipc, comp_id); } -int ipc4_process_on_core(uint32_t core, bool blocking) -{ - int ret; - - ret = ipc_process_on_core(core, blocking); - switch (ret) { - case 0: - case 1: - return IPC4_SUCCESS; - case -EACCES: - return IPC4_INVALID_CORE_ID; - case -ETIME: - case -EBUSY: - return IPC4_BUSY; - default: - return IPC4_FAILURE; - } - - return IPC4_SUCCESS; -} - -__cold static const struct comp_driver *ipc4_search_for_drv(const void *uuid) -{ - struct comp_driver_list *drivers = comp_drivers_get(); - struct list_item *clist; - const struct comp_driver *drv = NULL; - struct comp_driver_info *info; - - assert_can_be_cold(); - - /* search driver list with UUID */ - list_for_item(clist, &drivers->list) { - info = container_of(clist, struct comp_driver_info, list); - if (!memcmp(info->drv->uid, uuid, UUID_SIZE)) { - tr_dbg(&comp_tr, "found type %d, uuid %pU", - info->drv->type, SOF_DRV_UID_NAME(info->drv)); - drv = info->drv; - break; - } - } - - return drv; -} - -__cold static const struct comp_driver *ipc4_get_drv(const void *uuid) -{ - const struct sof_uuid *const __maybe_unused sof_uuid = (const struct sof_uuid *)uuid; - const struct comp_driver *drv; - - assert_can_be_cold(); - - drv = ipc4_search_for_drv(uuid); - if (!drv) - tr_err(&comp_tr, - "the provided UUID (%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x) can't be found!", - sof_uuid->a, sof_uuid->b, sof_uuid->c, sof_uuid->d[0], sof_uuid->d[1], - sof_uuid->d[2], sof_uuid->d[3], sof_uuid->d[4], sof_uuid->d[5], - sof_uuid->d[6], sof_uuid->d[7]); - - return drv; -} - -#if defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) -/* - * ipc4_get_fuzzer_drv - resolve a module driver for native_sim fuzz builds. - * - * native_sim fuzz builds have no rimage manifest, so the manifest lookup in - * ipc4_get_comp_drv() can never resolve a driver and every module instance - * verb (INIT_INSTANCE, CONFIG_GET/SET, LARGE_CONFIG, BIND, UNBIND, - * DELETE_INSTANCE) would be unreachable from the fuzzer. - * - * Mirror the IPC3 whitebox hack in posix/ipc.c and treat module_id as a - * 1-based index into the runtime component driver list, counting only - * instantiable module-adapter drivers so the fuzzer's module_id space matches - * what a real signed manifest would expose. - */ -static const struct comp_driver *ipc4_get_fuzzer_drv(uint32_t module_id) -{ - static const struct sof_uuid basefw_reg_uuid = SOF_REG_UUID(basefw); - struct comp_driver_list *dlist = comp_drivers_get(); - struct list_item *iter; - uint32_t idx = 0; - - if (!module_id) - return ipc4_search_for_drv(&basefw_reg_uuid); - - list_for_item(iter, &dlist->list) { - struct comp_driver_info *inf = - container_of(iter, struct comp_driver_info, list); - - /* - * Skip query-only entries (e.g. BaseFW) that cannot be - * instantiated because they have no create op. - */ - if (!inf->drv->ops.create) - continue; - - /* - * A real signed manifest only lists module-adapter modules. - * The internal gateway drivers (SOF_COMP_HOST, SOF_COMP_DAI) - * are registered for IPC3 but are never IPC4 modules: on the - * IPC4 path they receive no params() pass and cannot be - * configured (e.g. their DMA buffer is never allocated). Skip - * non-module-adapter drivers so the fuzzer's module_id space - * matches a real manifest and cannot map to an unconfigurable - * component. - */ - if (inf->drv->type != SOF_COMP_MODULE_ADAPTER) - continue; - - if (++idx == module_id) - return inf->drv; - } - - tr_err(&comp_tr, "no instantiable driver at module_id %u (%u available)", - module_id, idx); - return NULL; -} -#endif /* defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) */ - -/* - * Called from - * - ipc4_get_large_config_module_instance() - * - ipc4_set_large_config_module_instance() - * - comp_new_ipc4() - * all of which are __cold - */ -__cold const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id) -{ - const struct sof_man_fw_desc *desc = NULL; - const struct sof_man_module *mod; - uint32_t entry_index; - - assert_can_be_cold(); - -#ifdef RIMAGE_MANIFEST - desc = (const struct sof_man_fw_desc *)IMR_BOOT_LDR_MANIFEST_BASE; -#elif defined(CONFIG_ARCH_POSIX_LIBFUZZER) - return ipc4_get_fuzzer_drv(module_id); -#else - /* Non-rimage platforms have no component facility yet. - * This needs to move to the platform layer. - */ - return NULL; -#endif - - uint32_t lib_idx = LIB_MANAGER_GET_LIB_ID(module_id); - - if (lib_idx == 0) { - /* module_id 0 is used for base fw which is in entry 1 or 2 */ - if (!module_id) - entry_index = 1 + IS_ENABLED(CONFIG_COLD_STORE_EXECUTE_DRAM); - else - entry_index = module_id; - - if (entry_index >= desc->header.num_module_entries) { - tr_err(&comp_tr, "Error: entry index %d out of bounds.", entry_index); - return NULL; - } - - mod = (const struct sof_man_module *)((const char *)desc + - SOF_MAN_MODULE_OFFSET(entry_index)); - } else { - /* Library index greater than 0 possible only when LIBRARY_MANAGER - * support enabled. - */ -#if CONFIG_LIBRARY_MANAGER - mod = lib_manager_get_module_manifest(module_id); - - if (!mod) { - tr_err(&comp_tr, "Error: Couldn't find loadable module with id %#x.", - module_id); - return NULL; - } -#else - tr_err(&comp_tr, "Error: lib index:%d, while loadable libraries are not supported!!!", - lib_idx); - return NULL; -#endif - } - -#if CONFIG_LIBRARY_MANAGER - /* Check already registered components */ - const struct comp_driver *drv = ipc4_search_for_drv(&mod->uuid); - - if (drv) - return drv; - - /* New module not registered yet. */ - lib_manager_register_module(module_id); -#endif - return ipc4_get_drv(&mod->uuid); -} - struct comp_dev *ipc4_get_comp_dev(uint32_t comp_id) { struct ipc_comp_dev *icd = ipc_get_comp_by_id(ipc_get(), comp_id); diff --git a/src/kernel/CMakeLists.txt b/src/kernel/CMakeLists.txt index b14881ad3b7a..633434c4a1fe 100644 --- a/src/kernel/CMakeLists.txt +++ b/src/kernel/CMakeLists.txt @@ -21,6 +21,10 @@ if(zephyr) add_subdirectory(logging) add_subdirectory(probe) add_subdirectory(schedule) +else() + # Only built for the host + add_subdirectory(arch) + add_subdirectory(platform) endif() # Built for Zephyr and for the host diff --git a/src/kernel/Kconfig b/src/kernel/Kconfig index f92df62d911a..537ad2eda22c 100644 --- a/src/kernel/Kconfig +++ b/src/kernel/Kconfig @@ -10,4 +10,6 @@ rsource "idc/Kconfig" rsource "library_manager/Kconfig" +rsource "logging/Kconfig" + rsource "probe/Kconfig" diff --git a/src/arch/CMakeLists.txt b/src/kernel/arch/CMakeLists.txt similarity index 100% rename from src/arch/CMakeLists.txt rename to src/kernel/arch/CMakeLists.txt diff --git a/src/arch/Kconfig b/src/kernel/arch/Kconfig similarity index 100% rename from src/arch/Kconfig rename to src/kernel/arch/Kconfig diff --git a/src/arch/README.md b/src/kernel/arch/README.md similarity index 72% rename from src/arch/README.md rename to src/kernel/arch/README.md index df7999635033..c69391224118 100644 --- a/src/arch/README.md +++ b/src/kernel/arch/README.md @@ -1,8 +1,8 @@ -# Architecture Support (`src/arch`) +# Architecture Support (`src/kernel/arch`) > **Runs in:** Kernel-only — CPU-architecture primitives (Xtensa) execute in supervisor context > and are never linked into a user thread. See -> [User/kernel split](../include/sof/userspace/README.md). +> [User/kernel split](../../include/sof/userspace/README.md). This directory contains architecture-specific code. It sits below the RTOS and audio application and has no user-space entry points. diff --git a/src/arch/host/CMakeLists.txt b/src/kernel/arch/host/CMakeLists.txt similarity index 90% rename from src/arch/host/CMakeLists.txt rename to src/kernel/arch/host/CMakeLists.txt index 75314f00f556..bc700f509094 100644 --- a/src/arch/host/CMakeLists.txt +++ b/src/kernel/arch/host/CMakeLists.txt @@ -1,8 +1,8 @@ # SPDX-License-Identifier: BSD-3-Clause # includes -target_include_directories(sof_public_headers INTERFACE ${PROJECT_SOURCE_DIR}/src/arch/host/include) -target_include_directories(sof_public_headers INTERFACE ${PROJECT_SOURCE_DIR}/src/platform/library/include) +target_include_directories(sof_public_headers INTERFACE ${PROJECT_SOURCE_DIR}/src/kernel/arch/host/include) +target_include_directories(sof_public_headers INTERFACE ${PROJECT_SOURCE_DIR}/src/kernel/platform/library/include) include(CheckCCompilerFlag) if (NOT CONFIG_SOF_ZEPHYR_STRICT_HEADERS) diff --git a/src/arch/host/configs/library_defconfig b/src/kernel/arch/host/configs/library_defconfig similarity index 98% rename from src/arch/host/configs/library_defconfig rename to src/kernel/arch/host/configs/library_defconfig index 3a72dc85c79a..a7328c55a916 100644 --- a/src/arch/host/configs/library_defconfig +++ b/src/kernel/arch/host/configs/library_defconfig @@ -32,5 +32,4 @@ CONFIG_IPC_MAJOR_3=n CONFIG_IPC_MAJOR_4=y CONFIG_LIBRARY=y CONFIG_MATH_IIR_DF2T=y -CONFIG_TRACEV=y CONFIG_XT_RUN=y diff --git a/src/arch/host/include/arch/atomic.h b/src/kernel/arch/host/include/arch/atomic.h similarity index 100% rename from src/arch/host/include/arch/atomic.h rename to src/kernel/arch/host/include/arch/atomic.h diff --git a/src/arch/host/include/arch/compiler_info.h b/src/kernel/arch/host/include/arch/compiler_info.h similarity index 100% rename from src/arch/host/include/arch/compiler_info.h rename to src/kernel/arch/host/include/arch/compiler_info.h diff --git a/src/arch/host/include/arch/debug/backtrace.h b/src/kernel/arch/host/include/arch/debug/backtrace.h similarity index 100% rename from src/arch/host/include/arch/debug/backtrace.h rename to src/kernel/arch/host/include/arch/debug/backtrace.h diff --git a/src/arch/host/include/arch/debug/panic.h b/src/kernel/arch/host/include/arch/debug/panic.h similarity index 100% rename from src/arch/host/include/arch/debug/panic.h rename to src/kernel/arch/host/include/arch/debug/panic.h diff --git a/src/arch/host/include/arch/drivers/idc.h b/src/kernel/arch/host/include/arch/drivers/idc.h similarity index 100% rename from src/arch/host/include/arch/drivers/idc.h rename to src/kernel/arch/host/include/arch/drivers/idc.h diff --git a/src/arch/host/include/arch/drivers/interrupt.h b/src/kernel/arch/host/include/arch/drivers/interrupt.h similarity index 100% rename from src/arch/host/include/arch/drivers/interrupt.h rename to src/kernel/arch/host/include/arch/drivers/interrupt.h diff --git a/src/arch/host/include/arch/drivers/timer.h b/src/kernel/arch/host/include/arch/drivers/timer.h similarity index 100% rename from src/arch/host/include/arch/drivers/timer.h rename to src/kernel/arch/host/include/arch/drivers/timer.h diff --git a/src/arch/host/include/arch/lib/cache.h b/src/kernel/arch/host/include/arch/lib/cache.h similarity index 100% rename from src/arch/host/include/arch/lib/cache.h rename to src/kernel/arch/host/include/arch/lib/cache.h diff --git a/src/arch/host/include/arch/lib/cpu.h b/src/kernel/arch/host/include/arch/lib/cpu.h similarity index 100% rename from src/arch/host/include/arch/lib/cpu.h rename to src/kernel/arch/host/include/arch/lib/cpu.h diff --git a/src/arch/host/include/arch/lib/wait.h b/src/kernel/arch/host/include/arch/lib/wait.h similarity index 100% rename from src/arch/host/include/arch/lib/wait.h rename to src/kernel/arch/host/include/arch/lib/wait.h diff --git a/src/arch/host/include/arch/schedule/task.h b/src/kernel/arch/host/include/arch/schedule/task.h similarity index 100% rename from src/arch/host/include/arch/schedule/task.h rename to src/kernel/arch/host/include/arch/schedule/task.h diff --git a/src/arch/host/include/arch/sof.h b/src/kernel/arch/host/include/arch/sof.h similarity index 100% rename from src/arch/host/include/arch/sof.h rename to src/kernel/arch/host/include/arch/sof.h diff --git a/src/arch/host/include/arch/string.h b/src/kernel/arch/host/include/arch/string.h similarity index 100% rename from src/arch/host/include/arch/string.h rename to src/kernel/arch/host/include/arch/string.h diff --git a/src/arch/host/lib/CMakeLists.txt b/src/kernel/arch/host/lib/CMakeLists.txt similarity index 100% rename from src/arch/host/lib/CMakeLists.txt rename to src/kernel/arch/host/lib/CMakeLists.txt diff --git a/src/arch/host/lib/ams.c b/src/kernel/arch/host/lib/ams.c similarity index 100% rename from src/arch/host/lib/ams.c rename to src/kernel/arch/host/lib/ams.c diff --git a/src/arch/host/lib/notifier.c b/src/kernel/arch/host/lib/notifier.c similarity index 100% rename from src/arch/host/lib/notifier.c rename to src/kernel/arch/host/lib/notifier.c diff --git a/src/arch/xtensa/Kconfig b/src/kernel/arch/xtensa/Kconfig similarity index 100% rename from src/arch/xtensa/Kconfig rename to src/kernel/arch/xtensa/Kconfig diff --git a/src/arch/xtensa/drivers/cache_attr.c b/src/kernel/arch/xtensa/drivers/cache_attr.c similarity index 100% rename from src/arch/xtensa/drivers/cache_attr.c rename to src/kernel/arch/xtensa/drivers/cache_attr.c diff --git a/src/arch/xtensa/include/arch/drivers/interrupt.h b/src/kernel/arch/xtensa/include/arch/drivers/interrupt.h similarity index 100% rename from src/arch/xtensa/include/arch/drivers/interrupt.h rename to src/kernel/arch/xtensa/include/arch/drivers/interrupt.h diff --git a/src/kernel/init/README.md b/src/kernel/init/README.md index b20d490550f2..f1e811f847c6 100644 --- a/src/kernel/init/README.md +++ b/src/kernel/init/README.md @@ -47,7 +47,7 @@ sequenceDiagram ``` 1. **Context Initialization**: Sets up the global `struct sof` context. -2. **Logging and Tracing**: Initializes Zephyr's logging timestamps and SOF's DMA trace infrastructure (`trace_init()`), printing the firmware version and ABI banner. +2. **Logging**: Initializes Zephyr's logging timestamps and prints the firmware version and ABI banner. 3. **System Subsystems**: - Initializes the system notifier (`init_system_notify()`) for inter-component messaging. - Sets up runtime power management (`pm_runtime_init()`). diff --git a/src/kernel/init/init.c b/src/kernel/init/init.c index a9acfed5e9ac..20f0f1b2fe1f 100644 --- a/src/kernel/init/init.c +++ b/src/kernel/init/init.c @@ -221,11 +221,6 @@ __cold static int primary_core_init(int argc, char *argv[], struct sof *sof) sys_clock_hw_cycles_per_sec()); #endif -#if CONFIG_TRACE - trace_point(TRACE_BOOT_SYS_TRACES); - trace_init(sof); -#endif - print_version_banner(); trace_point(TRACE_BOOT_SYS_NOTIFIER); diff --git a/src/kernel/ipc/CMakeLists.txt b/src/kernel/ipc/CMakeLists.txt index 0f9d63c0aee6..a3e7c61638c6 100644 --- a/src/kernel/ipc/CMakeLists.txt +++ b/src/kernel/ipc/CMakeLists.txt @@ -7,7 +7,7 @@ elseif (CONFIG_IPC_MAJOR_4) endif() # Files common to Zephyr and the host library -add_local_sources(sof notification_pool.c) +add_local_sources(sof ipc-common-kernel.c notification_pool.c) is_zephyr(zephyr) if(zephyr) diff --git a/src/kernel/ipc/ipc-common-kernel.c b/src/kernel/ipc/ipc-common-kernel.c new file mode 100644 index 000000000000..618ef83cc4a2 --- /dev/null +++ b/src/kernel/ipc/ipc-common-kernel.c @@ -0,0 +1,810 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2016 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Keyon Jie + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#ifdef __ZEPHYR__ +#include +#include +#endif + +#ifdef CONFIG_SOF_USERSPACE_LL +#include +#endif + +#include + +/* Kernel part of the common IPC code; the part that also runs in user mode is in + * src/ipc/ipc-common.c + */ + +LOG_MODULE_REGISTER(ipc, CONFIG_SOF_LOG_LEVEL); + +SOF_DEFINE_REG_UUID(ipc); + +/* unused with Zephyr, generates no output */ +DECLARE_TR_CTX(ipc_tr, SOF_UUID(ipc_uuid), LOG_LEVEL_INFO); + +#ifdef CONFIG_SOF_USERSPACE_LL +K_APPMEM_PARTITION_DEFINE(ipc_context_part); + +K_APP_BMEM(ipc_context_part) static struct ipc ipc_context; + +struct ipc *ipc_get(void) +{ + return &ipc_context; +} +#endif + +int ipc_process_on_core(uint32_t core, bool blocking) +{ + struct ipc *ipc = ipc_get(); + struct idc_msg msg = { .header = IDC_MSG_IPC, .core = core, }; + int ret; + + /* check if requested core is enabled */ + if (!cpu_is_core_enabled(core)) { + tr_err(&ipc_tr, "core #%d is disabled", core); + return -EACCES; + } + +#if CONFIG_IPC_MAJOR_3 + /* The other core will write there its response */ + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + ((struct sof_ipc_cmd_hdr *)ipc->comp_data)->size); +#endif + + /* + * If the primary core is waiting for secondary cores to complete, it + * will also reply to the host + */ + if (!blocking) { + k_spinlock_key_t key; + + ipc->core = core; + key = k_spin_lock(&ipc->lock); + ipc->task_mask |= IPC_TASK_SECONDARY_CORE; + k_spin_unlock(&ipc->lock, key); + } + + /* send IDC message */ + ret = idc_send_msg(&msg, blocking ? IDC_BLOCKING : IDC_NON_BLOCKING); + if (ret < 0) + return ret; + + /* reply written by other core */ + return 1; +} + +void ipc_send_queued_msg(void) +{ + struct ipc *ipc = ipc_get(); + struct ipc_msg *msg; + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + + if (ipc->pm_prepare_D3) + goto out; + + /* any messages to send ? */ + if (list_is_empty(&ipc->msg_list)) + goto out; + + msg = list_first_item(&ipc->msg_list, struct ipc_msg, + list); + + if (ipc_platform_send_msg(msg) == 0) { + /* Remove the message from the list if it has been successfully sent. */ + list_item_del(&msg->list); + /* Invoke a callback to notify that the message has been sent. */ + if (msg->callback) + msg->callback(msg); +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + /* Increment performance counters */ + io_perf_monitor_update_data(ipc->io_perf_out_msg_count, 1); +#endif + } +out: + k_spin_unlock(&ipc->lock, key); +} + +#ifdef __ZEPHYR__ +static K_THREAD_STACK_DEFINE(ipc_send_wq_stack, CONFIG_STACK_SIZE_IPC_TX); + +/* + * Kernel-only: these must not live in struct ipc, which user threads can + * write with CONFIG_SOF_USERSPACE_LL. The work queue embeds the k_thread of + * a supervisor thread and the work item holds the handler it calls. + */ +static struct k_work_q ipc_send_wq; +static struct k_work_delayable ipc_send_work; + +struct k_work_q *ipc_send_wq_get(void) +{ + return &ipc_send_wq; +} +#endif + +static void schedule_ipc_worker(void) +{ + /* + * note: in XTOS builds, this is handled in + * task_main_primary_core() + */ +#ifdef __ZEPHYR__ + k_work_schedule_for_queue(&ipc_send_wq, &ipc_send_work, K_USEC(IPC_PERIOD_USEC)); +#endif +} + +__cold void ipc_msg_send_direct(struct ipc_msg *msg, void *data) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + int ret; + + assert_can_be_cold(); + + key = k_spin_lock(&ipc->lock); + + /* copy mailbox data to message if not already copied */ + if (data && msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE && + msg->tx_data != data) { + ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); + assert(!ret); + } + + ipc_platform_send_msg_direct(msg); + + k_spin_unlock(&ipc->lock, key); +} + +void z_impl_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + int ret; + + key = k_spin_lock(&ipc->lock); + + /* copy mailbox data to message if not already copied */ + if (data && (msg->tx_size > 0 && msg->tx_size <= SOF_IPC_MSG_MAX_SIZE) && + msg->tx_data != data) { + ret = memcpy_s(msg->tx_data, msg->tx_size, data, msg->tx_size); + assert(!ret); + } + + /* + * note: This function can be executed in LL or EDF context, from any core. + * In Zephyr builds, there is IPC queue that is always handled by the primary core, + * whereas submitting to the queue is allowed from any core. Therefore disable option + * of sending IPC immediately by any context/core to secure IPC registers/mailbox + * access. + */ +#ifndef __ZEPHYR__ + /* try to send critical notifications right away */ + if (high_priority) { + ret = ipc_platform_send_msg(msg); + if (!ret) { + k_spin_unlock(&ipc->lock, key); + return; + } + } +#endif + /* add to queue unless already there */ + if (list_is_empty(&msg->list)) { + if (high_priority) + list_item_prepend(&msg->list, &ipc->msg_list); + else + list_item_append(&msg->list, &ipc->msg_list); + } + + schedule_ipc_worker(); + + k_spin_unlock(&ipc->lock, key); +} +EXPORT_SYMBOL(ipc_msg_send); + +#ifdef CONFIG_USERSPACE +void z_vrfy_ipc_msg_send(struct ipc_msg *msg, void *data, bool high_priority) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + + K_OOPS(K_SYSCALL_VERIFY(msg->tx_size <= SOF_IPC_MSG_MAX_SIZE)); + + K_OOPS(K_SYSCALL_VERIFY(list_is_empty(&msg->list))); + + if (msg->tx_size > 0) { + K_OOPS(K_SYSCALL_VERIFY(msg->tx_data != NULL)); + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg->tx_data, msg->tx_size)); + } + + /* If data != NULL and tx_size > 0, verify the data buffer */ + if (data && msg->tx_size > 0) + K_OOPS(K_SYSCALL_MEMORY_READ(data, msg->tx_size)); + + z_impl_ipc_msg_send(msg, data, high_priority); +} +#include +#endif /* CONFIG_USERSPACE */ + +void z_impl_ipc_msg_list_remove(struct ipc_msg *msg) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + list_item_del(&msg->list); + k_spin_unlock(&ipc->lock, key); +} + +#ifdef CONFIG_USERSPACE +void z_vrfy_ipc_msg_list_remove(struct ipc_msg *msg) +{ + struct ipc *ipc = ipc_get(); + struct list_item *mlist, *_mlist; + bool found = false; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(msg, sizeof(*msg))); + + /* + * special case: empty list was passed. we can't trust where + * list->prev points to, so do not pass to + * z_impl_ipc_msg_list_remove(), but handle here + */ + if (list_is_empty(&msg->list)) + return; + + list_for_item_safe(mlist, _mlist, &ipc->msg_list) { + if (mlist == &msg->list) { + found = true; + break; + } + } + K_OOPS(K_SYSCALL_VERIFY(found)); + z_impl_ipc_msg_list_remove(msg); +} +#include +#endif /* CONFIG_USERSPACE */ + +#ifdef __ZEPHYR__ +static void ipc_work_handler(struct k_work *work) +{ + struct ipc *ipc = ipc_get(); + k_spinlock_key_t key; + + ipc_send_queued_msg(); + + key = k_spin_lock(&ipc->lock); + + if (!list_is_empty(&ipc->msg_list) && !ipc->pm_prepare_D3) + schedule_ipc_worker(); + + k_spin_unlock(&ipc->lock, key); +} +#endif + +void ipc_schedule_process(struct ipc *ipc) +{ +#if CONFIG_TWB_IPC_TASK + schedule_task(ipc->ipc_task, 0, IPC_PERIOD_USEC); +#else + schedule_task(&ipc->ipc_task, 0, IPC_PERIOD_USEC); +#endif +} + +#ifdef CONFIG_SOF_USERSPACE_LL +/* Generic user-space IPC handling thread infrastructure. Protocol-specific + * command interpretation lives in ipc_user_thread_dispatch(), implemented by + * the active IPC major (see src/ipc/ipc4/). + */ + +SOF_DEFINE_REG_UUID(sec_core_init); + +static K_THREAD_STACK_ARRAY_DEFINE(ipc_user_stack, CONFIG_CORE_COUNT, + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE); + +/* + * Kernel-only copies of the IPC user thread state. struct ipc_user and the + * ipc->ipc_user_pdata pointer are both writable by user threads, so kernel + * code must use these instead of reading kernel object pointers from there. + */ +static struct ipc_user *ipc_user_kptr; +static struct k_sem *ipc_user_ksem; +static struct k_event *ipc_user_kevent[CONFIG_CORE_COUNT]; +static struct k_thread *ipc_user_kthread[CONFIG_CORE_COUNT]; +static bool ipc_user_kinit_needed[CONFIG_CORE_COUNT]; + +/** + * @brief Forward an IPC command to the user-space thread. + * + * Called from kernel context (IPC EDF task) to forward the message words + * to the user-space thread for processing. Sets IPC_TASK_IN_THREAD in + * task_mask so the host is not signaled until the user thread completes. + * Blocks until the user thread finishes processing and returns the result. + * + * @param primary Primary message word + * @param extension Extension message word + * @return Result from user thread processing + */ +int ipc_user_forward_cmd(uint32_t primary, uint32_t extension, unsigned int core) +{ + struct ipc *ipc = ipc_get(); + struct ipc_user *pdata = ipc_user_kptr; + k_spinlock_key_t key; + int ret; + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + LOG_DBG("IPC: forward cmd %08x", primary); + + /* Copy message words — original buffer may be reused */ + pdata->ipc_msg_pri = primary; + pdata->ipc_msg_ext = extension; + pdata->ipc = ipc; + + /* Prevent host completion until user thread finishes */ + key = k_spin_lock(&ipc->lock); + ipc->task_mask |= IPC_TASK_IN_THREAD; + k_spin_unlock(&ipc->lock, key); + + /* Wake the user thread */ + k_event_set(ipc_user_kevent[core], IPC_USER_EVENT_CMD); + + /* Wait for user thread to complete */ + ret = k_sem_take(ipc_user_ksem, K_MSEC(100)); + if (ret) { + LOG_ERR("IPC user: sem error %d\n", ret); + /* fall through to complete the cmd */ + } + + /* Clear the task mask bit and check for completion */ + key = k_spin_lock(&ipc->lock); + ipc->task_mask &= ~IPC_TASK_IN_THREAD; + ipc_complete_cmd(ipc); + k_spin_unlock(&ipc->lock, key); + + return !ret ? pdata->result : ret; +} + +__cold static int ipc_user_init_thread(struct ipc_user *ipc_user, unsigned int core) +{ + char thread_name[] = "ll_user0"; + struct k_thread *thread; + struct k_event *event; + int ret; + + assert_can_be_cold(); + + /* + * Build the objects in locals and publish them to struct ipc_user only + * at the end. When a secondary core is brought up, user threads on + * other cores are already running and can overwrite struct ipc_user, + * so its fields must not be read back here. + */ + event = k_object_alloc(K_OBJ_EVENT); + if (!event) { + LOG_ERR("user IPC event alloc failed"); + return -ENOMEM; + } + k_event_init(event); + + thread = k_object_alloc(K_OBJ_THREAD); + if (!thread) { + LOG_ERR("user IPC thread alloc failed"); + ret = -ENOMEM; + goto e_event; + } + + k_thread_create(thread, ipc_user_stack[core], + CONFIG_SOF_IPC_USER_THREAD_STACK_SIZE, + ipc_user_thread_fn, ipc_user, UINT_TO_POINTER(core), NULL, + -1, K_USER, K_FOREVER); + + k_thread_cpu_pin(thread, core); + thread_name[sizeof(thread_name) - 2] = '0' + core; + k_thread_name_set(thread, thread_name); + + /* + * Each userspace IPC thread must be able to wait on its private event + * and signal completion on the primary core semaphore + */ + k_thread_access_grant(thread, ipc_user_ksem, event); + user_grant_dai_access_all(thread); + user_grant_dma_access_all(thread); + k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); + user_ll_grant_access(thread, core); + pipeline_posn_grant_access(thread); + + ipc_user_kevent[core] = event; + ipc_user_kthread[core] = thread; + + /* for ipc_user_thread_fn() and ipc_thread_user() in user context */ + ipc_user->event[core] = event; + ipc_user->thread[core] = thread; + + return 0; + +e_event: + k_object_free(event); + + return ret; +} + +__cold int ipc_user_init_secondary(unsigned int core) +{ + struct ipc_user *ipc_user = ipc_user_kptr; + struct k_thread *audio_thread; + int ret; + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + ret = ipc_user_init_thread(ipc_user, core); + + if (ret < 0) + return ret; + + assert_can_be_cold(); + + k_thread_start(ipc_user_kthread[core]); + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("user LL task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(sec_core_init_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, core, 0); + + audio_thread = scheduler_init_context(task); + if (!audio_thread) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + k_thread_access_grant(ipc_user_kthread[core], audio_thread); + ipc_user->audio_thread[core] = audio_thread; + + return 0; +} + +struct k_thread *ipc_thread_user(unsigned int core) +{ + if (core >= CONFIG_CORE_COUNT) + return NULL; + + /* user threads can not read the kernel-only copy */ + if (k_is_user_context()) + return ipc_get()->ipc_user_pdata->thread[core]; + + return ipc_user_kthread[core]; +} + +struct ipc_user *ipc_user_get(void) +{ + return ipc_user_kptr; +} + +void ipc_user_wait_core_init(unsigned int core) +{ + if (core >= CONFIG_CORE_COUNT) + return; + + if (ipc_user_kinit_needed[core]) { + /* wait for IPC thread (ipc_user_thread_fn()) */ + k_sem_take(ipc_user_ksem, K_FOREVER); + ipc_user_kinit_needed[core] = false; + } +} + +/* Primary core only */ +__cold static void ipc_user_init(void) +{ + struct ipc *ipc = ipc_get(); + struct k_thread *audio_thread; + struct ipc_user *ipc_user = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*ipc_user), 0); + int ret, core; + + if (!ipc_user) { + LOG_ERR("user IPC pdata alloc failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + + /* sof_heap_alloc() does not zero, start with clean state (e.g. ppl_phase) */ + memset(ipc_user, 0, sizeof(*ipc_user)); + + assert_can_be_cold(); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + if (core != PLATFORM_PRIMARY_CORE_ID) + ipc_user_kinit_needed[core] = true; + } + + ipc_user_ksem = k_object_alloc(K_OBJ_SEM); + if (!ipc_user_ksem) { + LOG_ERR("user IPC sem alloc failed"); + k_panic(); + } + ipc_user_kptr = ipc_user; + ipc_user->sem = ipc_user_ksem; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &ipc_context_part); + if (ret < 0) + LOG_WRN("ipc context partition add failed: %d", ret); + + /* + * Grant user-space read access to the .coldrodata section in IMR. The + * prepare path walks component code that may reference __cold_rodata + * data. + * + * .cold text is deliberately NOT added as a partition: Zephyr already + * maps it executable in the shared ring. + */ +#ifdef CONFIG_COLD_STORE_EXECUTE_DRAM + extern char __coldrodata_start[]; + extern char _imr_end[]; + + if (&_imr_end[0] > &__coldrodata_start[0]) { + struct k_mem_partition cold_part = { + .start = (uintptr_t)__coldrodata_start, + .size = ALIGN_UP((uintptr_t)_imr_end - (uintptr_t)__coldrodata_start, + CONFIG_MMU_PAGE_SIZE), + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + + ret = k_mem_domain_add_partition(zephyr_ll_mem_domain(), &cold_part); + if (ret < 0) + LOG_WRN("cold rodata partition %#zx @ %#lx add failed: %d", + cold_part.size, cold_part.start, ret); + } +#endif /* CONFIG_COLD_STORE_EXECUTE_DRAM */ + + k_sem_init(ipc_user_ksem, 0, 1); + + ret = ipc_user_init_thread(ipc_user, PLATFORM_PRIMARY_CORE_ID); + if (ret < 0) { + LOG_ERR("user IPC thread initialization failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + + ret = user_access_to_mailbox(zephyr_ll_mem_domain(), + ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); + if (ret < 0) { + LOG_ERR("ipc user: mailbox access grant failed: %d", ret); + sof_panic(SOF_IPC_PANIC_IPC); + } + + /* Store references in ipc struct so kernel handler can forward commands */ + ipc->ipc_user_pdata = ipc_user; + + struct task *task = zephyr_ll_task_alloc(); + + if (!task) { + LOG_ERR("task allocation failed"); + k_panic(); + } + + schedule_task_init_ll(task, SOF_UUID(ipc_uuid), SOF_SCHEDULE_LL_TIMER, + 0, NULL, NULL, PLATFORM_PRIMARY_CORE_ID, 0); + audio_thread = scheduler_init_context(task); + if (!audio_thread) { + LOG_ERR("user LL thread init failed"); + k_panic(); + } + + /* Grant ipc_user thread permission on the audio thread object. + * Needed so user-space dai_common_new() can call + * k_thread_access_grant(audio_thread, dai_mutex) from user context. + */ + k_thread_access_grant(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID], audio_thread); + ipc_user->audio_thread[PLATFORM_PRIMARY_CORE_ID] = audio_thread; + + k_thread_start(ipc_user_kthread[PLATFORM_PRIMARY_CORE_ID]); + + /* Wait for user thread startup — consumes the initial k_sem_give from thread */ + k_sem_take(ipc_user_ksem, K_FOREVER); +} +#else /* CONFIG_SOF_USERSPACE_LL */ +static void ipc_user_init(void) +{ +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +__cold int ipc_init(struct sof *sof) +{ + struct k_heap *heap; + struct ipc *ipc; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "entry"); + +#ifdef CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); + + ipc = ipc_get(); + memset(ipc, 0, sizeof(*ipc)); + ipc->ll_alloc = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*ipc->ll_alloc), 0); + if (!ipc->ll_alloc) { + tr_err(&ipc_tr, "Unable to allocate IPC ll_alloc"); + sof_panic(SOF_IPC_PANIC_IPC); + } + ipc->ll_alloc->heap = heap; + ipc->ll_alloc->vreg = NULL; +#else + heap = NULL; + + /* init ipc data */ + ipc = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*ipc)); + if (!ipc) { + tr_err(&ipc_tr, "Unable to allocate IPC data"); + return -ENOMEM; + } + sof->ipc = ipc; +#endif /* CONFIG_SOF_USERSPACE_LL */ + + ipc->comp_data = sof_heap_alloc(heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + SOF_IPC_MSG_MAX_SIZE, 0); + if (!ipc->comp_data) { + tr_err(&ipc_tr, "Unable to allocate IPC component data"); + sof_panic(SOF_IPC_PANIC_IPC); + } + memset(ipc->comp_data, 0, SOF_IPC_MSG_MAX_SIZE); + + k_spinlock_init(&ipc->lock); + list_init(&ipc->msg_list); + list_init(&ipc->comp_list); + +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + struct io_perf_data_item init_data = {IO_PERF_IPC_ID, + cpu_get_id(), + IO_PERF_INPUT_DIRECTION, + IO_PERF_POWERED_UP_ENABLED, + IO_PERF_D0IX_POWER_MODE, + 0, 0, 0 }; + io_perf_monitor_init_data(&ipc->io_perf_in_msg_count, &init_data); + init_data.direction = IO_PERF_OUTPUT_DIRECTION; + io_perf_monitor_init_data(&ipc->io_perf_out_msg_count, &init_data); +#endif + +#if CONFIG_SOF_BOOT_TEST_STANDALONE + LOG_INF("SOF_BOOT_TEST_STANDALONE, disabling IPC."); + return 0; +#endif + +#ifdef __ZEPHYR__ + k_tid_t thread; + + k_work_queue_init(&ipc_send_wq); + k_work_queue_start(&ipc_send_wq, ipc_send_wq_stack, + K_THREAD_STACK_SIZEOF(ipc_send_wq_stack), 1, NULL); + + thread = k_work_queue_thread_get(&ipc_send_wq); + + k_thread_suspend(thread); + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, PLATFORM_PRIMARY_CORE_ID); +#endif + k_thread_name_set(thread, "ipc_send_wq"); + + k_thread_resume(thread); + + k_work_init_delayable(&ipc_send_work, ipc_work_handler); +#endif /* __ZEPHYR__ */ + + ipc_user_init(); + + return platform_ipc_init(ipc); +} + +/* Locking: call with ipc->lock held and interrupts disabled */ +void ipc_complete_cmd(struct ipc *ipc) +{ + /* + * We have up to three contexts, attempting to complete IPC processing: + * the original IPC EDF task, the IDC EDF task on a secondary core, or + * an LL pipeline thread, running either on the primary or one of + * secondary cores. All these three contexts execute asynchronously. It + * is important to only signal the host that the IPC processing has + * completed after *all* tasks have completed. Therefore only the last + * context should do that. We accomplish this by setting IPC_TASK_* bits + * in ipc->task_mask for each used IPC context and by clearing them when + * each of those contexts completes. Only when the mask is 0 we can + * signal the host. + */ + if (ipc->task_mask) + return; + + ipc_platform_complete_cmd(ipc); +} + +bool ipc_enter_gdb; + +__attribute__((weak)) void ipc_platform_wait_ack(struct ipc *ipc) +{ + k_msleep(1); +} + +static void ipc_complete_task(void *data) +{ + struct ipc *ipc = data; + k_spinlock_key_t key; + + key = k_spin_lock(&ipc->lock); + ipc->task_mask &= ~IPC_TASK_INLINE; + ipc_complete_cmd(ipc); + k_spin_unlock(&ipc->lock, key); +#if CONFIG_GDBSTUB + if (ipc_enter_gdb) { + ipc_enter_gdb = false; + ipc_platform_wait_ack(ipc); + gdb_init(); + } +#endif +} + +static enum task_state ipc_do_cmd(void *data) +{ + struct ipc *ipc = data; + +#ifdef CONFIG_SOF_TELEMETRY_IO_PERFORMANCE_MEASUREMENTS + /* Increment performance counters */ + io_perf_monitor_update_data(ipc->io_perf_in_msg_count, 1); +#endif + + /* + * 32-bit writes are atomic and at the moment no IPC processing is + * taking place, so, no need for a lock. + */ + ipc->task_mask = IPC_TASK_INLINE; + + return ipc_platform_do_cmd(ipc); +} + +struct task_ops ipc_task_ops = { + .run = ipc_do_cmd, + .complete = ipc_complete_task, + .get_deadline = ipc_task_deadline, +}; diff --git a/src/kernel/ipc/ipc3/handler.c b/src/kernel/ipc/ipc3/handler.c index 38694e10984d..58dc12eee1ec 100644 --- a/src/kernel/ipc/ipc3/handler.c +++ b/src/kernel/ipc/ipc3/handler.c @@ -35,9 +35,6 @@ #include #include #include -#if CONFIG_TRACE -#include -#endif #include #include #include @@ -759,10 +756,6 @@ static int ipc_pm_gate(uint32_t header) tr_info(&ipc_tr, "ipc: pm gate flags 0x%x", pm_gate.flags); - /* pause dma trace firstly if needed */ - if (pm_gate.flags & SOF_PM_NO_TRACE) - trace_off(); - if (pm_gate.flags & SOF_PM_PPG) { pm_runtime_disable(PM_RUNTIME_DSP, PLATFORM_PRIMARY_CORE_ID); } else { @@ -775,10 +768,6 @@ static int ipc_pm_gate(uint32_t header) pm_runtime_enable(PM_RUNTIME_DSP, PLATFORM_PRIMARY_CORE_ID); } - /* resume dma trace if needed */ - if (!(pm_gate.flags & SOF_PM_NO_TRACE)) - trace_on(); - return 0; } @@ -806,158 +795,6 @@ static int ipc_glb_pm_message(uint32_t header) } } -#if CONFIG_TRACE -/* - * Debug IPC Operations. - */ -static void ipc_dma_trace_free(uint32_t header) -{ - struct dma_trace_data *dmat = dma_trace_data_get(); - - dma_trace_disable(dmat); -} - -static int ipc_dma_trace_config(uint32_t header) -{ -#if CONFIG_HOST_PTABLE - struct dma_sg_elem_array elem_array; - uint32_t ring_size; -#endif - struct dma_trace_data *dmat = dma_trace_data_get(); - struct ipc *ipc = ipc_get(); - struct sof_ipc_dma_trace_params_ext params; - int err; - - if (!dmat) { - mtrace_printf(LOG_LEVEL_ERROR, - "failed: dmat not initialized"); - return -ENOMEM; - } - - /* copy message with ABI safe method */ - IPC_COPY_CMD(params, ipc->comp_data); - - if (iCS(header) == SOF_IPC_TRACE_DMA_PARAMS_EXT) - /* As version 5.12 Linux sends the monotonic - * ktime_get(). Search for - * "SOF_IPC_TRACE_DMA_PARAMS_EXT" in your particular - * kernel version. - */ - dmat->time_delta = k_ns_to_cyc_near64(params.timestamp_ns) - sof_cycle_get_64(); - else - dmat->time_delta = 0; - -#if CONFIG_HOST_PTABLE - err = ipc_process_host_buffer(ipc, ¶ms.buffer, - SOF_IPC_STREAM_CAPTURE, - &elem_array, - &ring_size); - if (err < 0) - goto processing_error; - - err = dma_trace_host_buffer(dmat, &elem_array, ring_size); - if (err < 0) { - ipc_cmd_err(&ipc_tr, "ipc: trace failed to set host buffers %d", - err); - goto error; - } -#else - /* stream tag of capture stream for DMA trace */ - dmat->stream_tag = params.stream_tag; - - /* host buffer size for DMA trace */ - dmat->host_size = params.buffer.size; -#endif /* CONFIG_HOST_PTABLE */ - - err = dma_trace_enable(dmat); - if (err < 0) { - ipc_cmd_err(&ipc_tr, "ipc: failed to enable trace %d", err); - goto error; - } - - return 0; - -error: -#if CONFIG_HOST_PTABLE - dma_sg_free(NULL, &elem_array); - -processing_error: -#endif - - return err; -} - -static int ipc_trace_filter_update(uint32_t header) -{ - struct sof_ipc_trace_filter_elem *next_elem; - struct sof_ipc_trace_filter_elem *elem; - struct sof_ipc_trace_filter_elem *end; - struct sof_ipc_trace_filter *packet; - struct trace_filter filter; - struct ipc *ipc = ipc_get(); - int ret = 0; - int cnt; - - packet = ipc->comp_data; - - /* validation, packet->hdr.size has already been compared with SOF_IPC_MSG_MAX_SIZE */ - if (sizeof(*packet) + sizeof(*elem) * packet->elem_cnt != packet->hdr.size) { - ipc_cmd_err(&ipc_tr, "trace_filter_update failed, elem_cnt %d is inconsistent with hdr.size %d", - packet->elem_cnt, packet->hdr.size); - return -EINVAL; - } - - tr_info(&ipc_tr, "ipc: trace_filter_update received, size %d elems", - packet->elem_cnt); - - elem = packet->elems; - end = &packet->elems[packet->elem_cnt]; - - /* read each filter set and update selected components trace settings */ - while (elem != end) { - next_elem = trace_filter_fill(elem, end, &filter); - if (!next_elem) - return -EINVAL; - - cnt = trace_filter_update(&filter); - if (cnt < 0) { - ipc_cmd_err(&ipc_tr, "trace_filter_update failed for UUID key 0x%X, comp %d.%d and log level %d", - filter.uuid_id, filter.pipe_id, filter.comp_id, - filter.log_level); - ret = cnt; - } else { - tr_info(&ipc_tr, "trace_filter_update for UUID key 0x%X, comp %d.%d affected %d components", - filter.uuid_id, filter.pipe_id, filter.comp_id, - cnt); - } - - elem = next_elem; - } - - return ret; -} - -static int ipc_glb_trace_message(uint32_t header) -{ - uint32_t cmd = iCS(header); - - tr_info(&ipc_tr, "ipc: debug cmd 0x%x", cmd); - - switch (cmd) { - case SOF_IPC_TRACE_DMA_PARAMS: - case SOF_IPC_TRACE_DMA_PARAMS_EXT: - return ipc_dma_trace_config(header); - case SOF_IPC_TRACE_DMA_FREE: - ipc_dma_trace_free(header); - return 0; - case SOF_IPC_TRACE_FILTER_UPDATE: - return ipc_trace_filter_update(header); - default: - ipc_cmd_err(&ipc_tr, "ipc: unknown debug cmd 0x%x", cmd); - return -EINVAL; - } -} -#else /* CONFIG_TRACE */ static int ipc_glb_trace_message(uint32_t header) { /* Return success, as the protocol provides no way to inform @@ -966,7 +803,6 @@ static int ipc_glb_trace_message(uint32_t header) */ return 0; } -#endif /* CONFIG_TRACE */ static int ipc_glb_gdb_debug(uint32_t header) { diff --git a/src/kernel/ipc/ipc3/helper.c b/src/kernel/ipc/ipc3/helper.c index 2fb7b52e1f4a..76303be2c267 100644 --- a/src/kernel/ipc/ipc3/helper.c +++ b/src/kernel/ipc/ipc3/helper.c @@ -61,18 +61,6 @@ void ipc_build_comp_event(struct sof_ipc_comp_event *event, uint32_t type, event->src_comp_id = id; } -bool ipc_trigger_trace_xfer(uint32_t avail) -{ - return true; -} - -void ipc_build_trace_posn(struct sof_ipc_dma_trace_posn *posn) -{ - posn->rhdr.hdr.cmd = SOF_IPC_GLB_TRACE_MSG | - SOF_IPC_TRACE_DMA_POSITION; - posn->rhdr.hdr.size = sizeof(*posn); -} - static const struct comp_driver *get_drv(struct sof_ipc_comp *comp) { struct comp_driver_list *drivers = comp_drivers_get(); diff --git a/src/kernel/ipc/ipc4/CMakeLists.txt b/src/kernel/ipc/ipc4/CMakeLists.txt index 53a37d553675..53606a9ddda4 100644 --- a/src/kernel/ipc/ipc4/CMakeLists.txt +++ b/src/kernel/ipc/ipc4/CMakeLists.txt @@ -3,6 +3,8 @@ # Files common to Zephyr and the host library add_local_sources(sof handler-kernel.c + handler-user-kernel.c + helper-kernel.c logging.c notification.c ) diff --git a/src/kernel/ipc/ipc4/handler-user-kernel.c b/src/kernel/ipc/ipc4/handler-user-kernel.c new file mode 100644 index 000000000000..ebf9f406dfe2 --- /dev/null +++ b/src/kernel/ipc/ipc4/handler-user-kernel.c @@ -0,0 +1,877 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang +/* + * IPC (InterProcessor Communication) provides a method of two way + * communication between the host processor and the DSP. The IPC used here + * utilises a shared mailbox and door bell between the host and DSP. + * + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include + +#include +#include +#include +#include + +#include "../audio/copier/ipcgtw_copier.h" + +/* Command format errors during fuzzing are reported for virtually all + * commands, and the resulting flood of logging becomes a severe + * performance penalty (i.e. we get a lot less fuzzing done per CPU + * cycle). + */ +#ifdef CONFIG_ARCH_POSIX_LIBFUZZER +#define ipc_cmd_err(...) +#else +#define ipc_cmd_err(...) tr_err(__VA_ARGS__) +#endif + +/* Kernel part of the IPC4 module and global message handling: the routers, + * which run in the kernel IPC thread, and the handlers that only they call. + * The part that also runs in user mode is in src/ipc/ipc4/handler-user.c + */ + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +/* Userspace message context, copied in/out by kernel IPC thread. */ +/* fw sends a fw ipc message to send the status of the last host ipc message */ +static struct ipc_msg *msg_reply; + +#ifdef CONFIG_SOF_USERSPACE_LL +/* + * Determine the target core for an IPC4 module message. + * Falls back to current core when no component is bound yet. + */ +static unsigned int ipc4_user_target_core_module(struct ipc4_message_request *ipc4) +{ + /* + * Also works for struct ipc4_module_large_config, struct ipc4_module_bind_unbind, + * struct ipc4_module_delete_instance + */ + struct ipc4_module_config *config = (struct ipc4_module_config *)ipc4; + uint32_t module_id = config->primary.r.module_id; + + if (module_id) { + uint32_t instance_id = config->primary.r.instance_id; + struct comp_dev *dev = ipc4_get_comp_dev(IPC4_COMP_ID(module_id, instance_id)); + + if (dev) + return dev->ipc_config.core; + } + + return cpu_get_id(); +} + +/* + * Handle a multi-pipeline SET_PIPELINE_STATE, where the pipelines are on + * different cores. The user-space IPC thread can't use IDC to reach the + * other cores, so the kernel runs the prepare phase on all pipelines and + * then the trigger phase on all pipelines, forwarding each step to the + * user-space IPC thread of the core, owning the pipeline. + */ +__cold static int ipc4_user_set_pipeline_state_multicore(struct ipc4_message_request *ipc4, + const uint32_t *ppl_id, + unsigned int ppl_count) +{ + static const uint32_t phases[] = { + IDC_PPL_STATE_PHASE_PREPARE, + IDC_PPL_STATE_PHASE_TRIGGER, + }; + struct ipc *ipc = ipc_get(); + struct ipc_user *pdata = ipc->ipc_user_pdata; + struct ipc4_pipeline_set_state state; + struct ipc_comp_dev *ppl_icd; + unsigned int i, j; + int ret; + + assert_can_be_cold(); + + for (j = 0; j < ARRAY_SIZE(phases); j++) { + for (i = 0; i < ppl_count; i++) { + ppl_icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, ppl_id[i], + IPC_COMP_ALL); + if (!ppl_icd) { + ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); + return IPC4_INVALID_RESOURCE_ID; + } + + /* Forward as a single-pipeline request */ + state.primary.dat = ipc4->primary.dat; + state.extension.dat = ipc4->extension.dat; + state.primary.r.ppl_id = ppl_id[i]; + state.extension.r.multi_ppl = 0; + + pdata->ppl_phase = phases[j]; + ret = ipc_user_forward_cmd(state.primary.dat, state.extension.dat, + ppl_icd->core); + pdata->ppl_phase = 0; + if (ret != 0) + return ret; + } + } + + return IPC4_SUCCESS; +} +#else /* CONFIG_SOF_USERSPACE_LL */ +__cold static int ipc4_new_pipeline(struct ipc4_message_request *ipc4) +{ + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + return ipc_pipeline_new(ipc, (ipc_pipe_new *)ipc4); +} + +__cold static int ipc4_delete_pipeline(struct ipc4_message_request *ipc4) +{ + struct ipc4_pipeline_delete *pipe; + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + pipe = (struct ipc4_pipeline_delete *)ipc4; + tr_dbg(&ipc_tr, "ipc4 delete pipeline %x:", (uint32_t)pipe->primary.r.instance_id); + + return ipc_pipeline_free(ipc, pipe->primary.r.instance_id); +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +__cold static int ipc4_process_chain_dma(struct ipc4_message_request *ipc4) +{ + assert_can_be_cold(); + +#if CONFIG_COMP_CHAIN_DMA + struct ipc_comp_dev *cdma_comp; + struct ipc *ipc = ipc_get(); + const struct ipc4_chain_dma *cdma = (const struct ipc4_chain_dma *)ipc4; + int comp_id; + int ret; + + comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT, 0); + cdma_comp = ipc_get_comp_by_id(ipc, comp_id); + + if (!cdma_comp) { + /* + * Nothing to do when the chainDMA is not allocated and asked to + * be freed + */ + if (!cdma->primary.r.allocate && !cdma->primary.r.enable) + return IPC4_SUCCESS; + + ret = ipc4_chain_manager_create(cdma); + if (ret < 0) + return IPC4_FAILURE; + + cdma_comp = ipc_get_comp_by_id(ipc, comp_id); + if (!cdma_comp) { + return IPC4_FAILURE; + } + + ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); + if (ret < 0) { + comp_free(cdma_comp->cd); + list_item_del(&cdma_comp->list); + rfree(cdma_comp); + return IPC4_FAILURE; + } + + return IPC4_SUCCESS; + } + + ret = ipc4_chain_dma_state(cdma_comp->cd, cdma); + if (ret < 0) + return IPC4_INVALID_CHAIN_STATE_TRANSITION; + + return IPC4_SUCCESS; +#else /* CONFIG_COMP_CHAIN_DMA */ + return IPC4_UNAVAILABLE; +#endif /* CONFIG_COMP_CHAIN_DMA */ +} + +__cold static int ipc4_process_ipcgtw_cmd(struct ipc4_message_request *ipc4) +{ + assert_can_be_cold(); + +#if CONFIG_IPC4_GATEWAY + struct ipc *ipc = ipc_get(); + uint32_t reply_size = 0; + int err; + + err = copier_ipcgtw_process((const struct ipc4_ipcgtw_cmd *)ipc4, ipc->comp_data, + &reply_size); + /* reply size is returned in header extension dword */ + msg_reply->extension = reply_size; + + if (reply_size > 0) { + msg_reply->tx_data = ipc->comp_data; + msg_reply->tx_size = reply_size; + } + + return err < 0 ? IPC4_FAILURE : IPC4_SUCCESS; +#else /* CONFIG_IPC4_GATEWAY */ + ipc_cmd_err(&ipc_tr, "CONFIG_IPC4_GATEWAY is disabled"); + return IPC4_UNAVAILABLE; +#endif /* CONFIG_IPC4_GATEWAY */ +} + +static int ipc_glb_gdb_debug(struct ipc4_message_request *ipc4) +{ +#if CONFIG_GDBSTUB + ipc_enter_gdb = true; + return IPC4_SUCCESS; +#else + return IPC4_UNAVAILABLE; +#endif +} + +int ipc4_user_process_glb_message(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ +#ifdef CONFIG_SOF_USERSPACE_LL + struct ipc *ipc = ipc_get(); +#endif + uint32_t type; + int ret; + + type = ipc4->primary.r.type; + msg_reply = reply; + + switch (type) { + case SOF_IPC4_GLB_BOOT_CONFIG: + case SOF_IPC4_GLB_ROM_CONTROL: + case SOF_IPC4_GLB_PERF_MEASUREMENTS_CMD: + case SOF_IPC4_GLB_LOAD_MULTIPLE_MODULES: + case SOF_IPC4_GLB_UNLOAD_MULTIPLE_MODULES: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + case SOF_IPC4_GLB_CHAIN_DMA: + ret = ipc4_process_chain_dma(ipc4); + break; + + /* pipeline settings */ + case SOF_IPC4_GLB_CREATE_PIPELINE: +#ifdef CONFIG_SOF_USERSPACE_LL + { + const struct ipc4_pipeline_create *create = + (const struct ipc4_pipeline_create *)ipc4; + + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + create->extension.r.core_id); + } +#else + ret = ipc4_new_pipeline(ipc4); +#endif + break; + case SOF_IPC4_GLB_DELETE_PIPELINE: +#ifdef CONFIG_SOF_USERSPACE_LL + { + const struct ipc4_pipeline_delete *del = (const struct ipc4_pipeline_delete *)ipc4; + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + del->primary.r.instance_id, + IPC_COMP_ALL); + + if (!ppl) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, ppl->core); + } +#else + ret = ipc4_delete_pipeline(ipc4); +#endif + break; + case SOF_IPC4_GLB_SET_PIPELINE_STATE: +#ifdef CONFIG_SOF_USERSPACE_LL + { + struct ipc4_pipeline_set_state state = { + .primary.dat = ipc4->primary.dat, + .extension.dat = ipc4->extension.dat, + }; + const uint32_t *ppl_id; + unsigned int ppl_count; + int id = ipc4_pipeline_id_get(ipc4, &state, &ppl_id, &ppl_count); + if (id < 0) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + + struct ipc_comp_dev *ppl = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, id, + IPC_COMP_ALL); + + if (!ppl) { + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + + /* + * Validate all pipelines before any state change and check, + * if they span multiple cores, needing per-core processing + */ + bool multicore = false; + + ret = IPC4_SUCCESS; + for (unsigned int i = 1; i < ppl_count; i++) { + struct ipc_comp_dev *icd = ipc_get_comp_by_ppl_id(ipc, COMP_TYPE_PIPELINE, + ppl_id[i], IPC_COMP_ALL); + + if (!icd) { + ipc_cmd_err(&ipc_tr, "ipc: comp %d not found", ppl_id[i]); + ret = IPC4_INVALID_RESOURCE_ID; + break; + } + + if (icd->core != ppl->core) + multicore = true; + } + + if (ret != IPC4_SUCCESS) + break; + + if (multicore) + ret = ipc4_user_set_pipeline_state_multicore(ipc4, ppl_id, ppl_count); + else + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ppl->core); + } +#else /* CONFIG_SOF_USERSPACE_LL */ + ret = ipc4_set_pipeline_state(ipc4); +#endif /* CONFIG_SOF_USERSPACE_LL */ + break; + + case SOF_IPC4_GLB_GET_PIPELINE_STATE: + case SOF_IPC4_GLB_GET_PIPELINE_CONTEXT_SIZE: + case SOF_IPC4_GLB_SAVE_PIPELINE: + case SOF_IPC4_GLB_RESTORE_PIPELINE: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + case SOF_IPC4_GLB_INTERNAL_MESSAGE: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + /* Notification (FW to SW driver) */ + case SOF_IPC4_GLB_NOTIFICATION: + ipc_cmd_err(&ipc_tr, "not implemented ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + + case SOF_IPC4_GLB_IPCGATEWAY_CMD: + ret = ipc4_process_ipcgtw_cmd(ipc4); + break; + + case SOF_IPC4_GLB_ENTER_GDB: + ret = ipc_glb_gdb_debug(ipc4); + break; + + default: + ipc_cmd_err(&ipc_tr, "unsupported ipc message type %d", type); + ret = IPC4_UNAVAILABLE; + break; + } + + return ret; +} + +/* + * Ipc4 Module message <------> ipc3 module message + * init module <-------> create component + * bind modules <-------> connect components + * module set_large_config <-------> component cmd + * delete module <-------> free component + */ + +__cold static int ipc4_init_module_instance(struct ipc4_message_request *ipc4) +{ + const struct ipc4_module_init_instance *module_init = + (const struct ipc4_module_init_instance *)ipc4; + struct comp_dev *dev; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)module_init->primary.r.module_id, + (uint32_t)module_init->primary.r.instance_id); + + /* Pass IPC to target core */ + if (!cpu_is_me(module_init->extension.r.core_id)) + return ipc4_process_on_core(module_init->extension.r.core_id, false); + + dev = comp_new_ipc4(module_init); + if (!dev) { + ipc_cmd_err(&ipc_tr, "error: failed to init module %x : %x", + (uint32_t)module_init->primary.r.module_id, + (uint32_t)module_init->primary.r.instance_id); + return IPC4_MOD_NOT_INITIALIZED; + } + + return 0; +} + +#ifndef CONFIG_SOF_USERSPACE_LL +__cold static int ipc4_bind_module_instance(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x with %x : %x", + (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, + (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); + + return ipc4_comp_connect(ipc, bu); +} + +__cold static int ipc4_unbind_module_instance(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_bind_unbind *bu = (struct ipc4_module_bind_unbind *)ipc4; + struct ipc *ipc = ipc_get(); + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x with %x : %x", + (uint32_t)bu->primary.r.module_id, (uint32_t)bu->primary.r.instance_id, + (uint32_t)bu->extension.r.dst_module_id, (uint32_t)bu->extension.r.dst_instance_id); + + return ipc4_comp_disconnect(ipc, bu); +} +#endif /* !CONFIG_SOF_USERSPACE_LL */ + +#ifndef CONFIG_SOF_USERSPACE_LL +static int ipc4_set_get_config_module_instance(struct ipc4_message_request *ipc4, bool set) +{ + uint32_t reply_ext; + int ret; + + ret = ipc4_process_module_config(ipc4, set, &reply_ext); + + if (!set) + msg_reply->extension = reply_ext; + + return ret; +} +#endif /* !CONFIG_SOF_USERSPACE_LL */ + +__cold static int ipc4_get_large_config_module_instance(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_large_config_reply reply; + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + char *data = ipc_get()->comp_data; + const struct comp_driver *drv; + struct comp_dev *dev = NULL; + uint32_t data_offset; + int ret; + + assert_can_be_cold(); + + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); + + /* get component dev for non-basefw since there is no + * component dev for basefw + */ + if (config->primary.r.module_id) { + uint32_t comp_id; + + comp_id = IPC4_COMP_ID(config->primary.r.module_id, + config->primary.r.instance_id); + dev = ipc4_get_comp_dev(comp_id); + if (!dev) + return IPC4_INVALID_RESOURCE_ID; + + drv = dev->drv; + + /* Pass IPC to target core */ + if (!cpu_is_me(dev->ipc_config.core)) + return ipc4_process_on_core(dev->ipc_config.core, false); + } else { + /* BaseFW module has only 0th instance */ + if (config->primary.r.instance_id) + return IPC4_INVALID_RESOURCE_ID; + + drv = ipc4_get_comp_drv(config->primary.r.module_id); + } + + if (!drv) + return IPC4_INVALID_RESOURCE_ID; + + if (!drv->ops.get_large_config) + return IPC4_INVALID_REQUEST; + + data_offset = config->extension.r.data_off_size; + + /* check for vendor param first */ + if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + /* data_off_size is a 20-bit host-controlled field, so it can + * claim far more than the hostbox can physically hold. + */ + if (data_offset > MAILBOX_HOSTBOX_SIZE) { + ipc_cmd_err(&ipc_tr, "data_off_size %u exceeds mailbox bound", + data_offset); + return IPC4_INVALID_CONFIG_DATA_STRUCT; + } + /* For now only vendor_config case uses payload from hostbox */ + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + config->extension.r.data_off_size); + ret = ipc4_get_vendor_config_module_instance(dev, drv, + config->extension.r.init_block, + config->extension.r.final_block, + &data_offset, data, + (const char *)MAILBOX_HOSTBOX_BASE); + } else { +#if CONFIG_LIBRARY + data += sizeof(reply); +#endif + ipc4_prepare_for_kcontrol_get(dev, config->extension.r.large_param_id, + data, data_offset); + + ret = drv->ops.get_large_config(dev, config->extension.r.large_param_id, + config->extension.r.init_block, + config->extension.r.final_block, + &data_offset, data); + } + + /* set up ipc4 error code for reply data */ + if (ret < 0) + ret = IPC4_INVALID_RESOURCE_ID; + + /* Copy host config and overwrite */ + reply.extension.dat = config->extension.dat; + reply.extension.r.data_off_size = data_offset; + + /* The last block, no more data */ + if (!config->extension.r.final_block && data_offset < SOF_IPC_MSG_MAX_SIZE) + reply.extension.r.final_block = 1; + + /* Indicate last block if error occurs */ + if (ret) + reply.extension.r.final_block = 1; + + /* no need to allocate memory for reply msg */ + if (ret) + return ret; + + /* A module must not report more reply data than the DSP mailbox can hold. */ + if (data_offset > MAILBOX_DSPBOX_SIZE) { + ipc_cmd_err(&ipc_tr, "reply size %u exceeds mailbox bound", data_offset); + return IPC4_INVALID_CONFIG_DATA_STRUCT; + } + + msg_reply->extension = reply.extension.dat; + msg_reply->tx_size = data_offset; + msg_reply->tx_data = data; + return ret; +} + +__cold static int ipc4_set_large_config_module_instance(struct ipc4_message_request *ipc4) +{ + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + struct comp_dev *dev = NULL; + const struct comp_driver *drv; + int ret; + + assert_can_be_cold(); + + dcache_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_HOSTBOX_BASE, + config->extension.r.data_off_size); + tr_dbg(&ipc_tr, "%x : %x", + (uint32_t)config->primary.r.module_id, (uint32_t)config->primary.r.instance_id); + + if (config->primary.r.module_id) { + uint32_t comp_id; + + comp_id = IPC4_COMP_ID(config->primary.r.module_id, config->primary.r.instance_id); + dev = ipc4_get_comp_dev(comp_id); + if (!dev) + return IPC4_INVALID_RESOURCE_ID; + + drv = dev->drv; + + /* Pass IPC to target core */ + if (!cpu_is_me(dev->ipc_config.core)) + return ipc4_process_on_core(dev->ipc_config.core, false); + } else { + /* BaseFW module has only 0th instance */ + if (config->primary.r.instance_id) + return IPC4_INVALID_RESOURCE_ID; + + drv = ipc4_get_comp_drv(config->primary.r.module_id); + } + + if (!drv) + return IPC4_INVALID_RESOURCE_ID; + + if (!drv->ops.set_large_config) + return IPC4_INVALID_REQUEST; + + /* check for vendor param first */ + if (config->extension.r.large_param_id == VENDOR_CONFIG_PARAM) { + ret = ipc4_set_vendor_config_module_instance(dev, drv, + (uint32_t)config->primary.r.module_id, + (uint32_t)config->primary.r.instance_id, + config->extension.r.init_block, + config->extension.r.final_block, + config->extension.r.data_off_size, + (const char *)MAILBOX_HOSTBOX_BASE); + } else { +#if CONFIG_LIBRARY + struct ipc *ipc = ipc_get(); + const char *data = (const char *)ipc->comp_data + sizeof(*config); +#else + const char *data = (const char *)MAILBOX_HOSTBOX_BASE; +#endif + ret = drv->ops.set_large_config(dev, config->extension.r.large_param_id, + config->extension.r.init_block, config->extension.r.final_block, + config->extension.r.data_off_size, data); + if (ret < 0) { + ipc_cmd_err(&ipc_tr, "failed to set large_config_module_instance %x : %x", + (uint32_t)config->primary.r.module_id, + (uint32_t)config->primary.r.instance_id); + ret = IPC4_INVALID_RESOURCE_ID; + } + } + + return ret; +} + +#ifndef CONFIG_SOF_USERSPACE_LL +__cold static int ipc4_delete_module_instance(struct ipc4_message_request *ipc4) +{ + struct ipc4_module_delete_instance module; + struct ipc *ipc = ipc_get(); + uint32_t comp_id; + + assert_can_be_cold(); + + int ret = memcpy_s(&module, sizeof(module), ipc4, sizeof(*ipc4)); + + if (ret < 0) + return IPC4_FAILURE; + + tr_dbg(&ipc_tr, "%x : %x", (uint32_t)module.primary.r.module_id, + (uint32_t)module.primary.r.instance_id); + + comp_id = IPC4_COMP_ID(module.primary.r.module_id, module.primary.r.instance_id); + ret = ipc_comp_free(ipc, comp_id); + if (ret < 0) { + ipc_cmd_err(&ipc_tr, "failed to delete module instance %x : %x", + (uint32_t)module.primary.r.module_id, + (uint32_t)module.primary.r.instance_id); + ret = IPC4_INVALID_RESOURCE_ID; + } + + return ret; +} +#endif /* !CONFIG_SOF_USERSPACE_LL */ + +__cold int ipc4_user_process_module_message(struct ipc4_message_request *ipc4, + struct ipc_msg *reply) +{ + const struct ipc4_module_init_instance *mi; + uint32_t type; + int ret; + + assert_can_be_cold(); + + type = ipc4->primary.r.type; + msg_reply = reply; + + switch (type) { + case SOF_IPC4_MOD_INIT_INSTANCE: + mi = (const struct ipc4_module_init_instance *)ipc4; + + if (cpu_is_me(mi->extension.r.core_id) && !mi->extension.r.proc_domain && + IS_ENABLED(CONFIG_SOF_USERSPACE_LL)) { +#ifdef CONFIG_SOF_USERSPACE_LL + BUILD_ASSERT(sizeof(struct comp_driver) + sizeof(struct tr_ctx) <= + sizeof(((struct ipc_user *)0)->init_drv_data), + "ipc_user.init_drv_data too small for driver copy"); + + /* User-space init: kernel does driver lookup only (requires + * access to IMR manifest and driver list in kernel memory). + * Component creation (drv->ops.create) runs in user thread + * so untrusted module code does not execute in kernel context. + */ + uint32_t comp_id = IPC4_COMP_ID(mi->primary.r.module_id, + mi->primary.r.instance_id); + const struct comp_driver *drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); + + if (!drv || !drv->ops.create) { + ret = IPC4_MOD_NOT_INITIALIZED; + break; + } + + struct lib_manager_mod_ctx *ctx = lib_manager_get_mod_ctx(comp_id); + + if (ctx && drv->type == SOF_COMP_MODULE_ADAPTER) { + int err = ipc4_user_module_load(drv, mi); + + if (err < 0) { + ret = IPC4_MOD_NOT_INITIALIZED; + break; + } + } + + ipc_user_get()->init_drv = drv; + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + mi->extension.r.core_id); +#endif /* CONFIG_SOF_USERSPACE_LL */ + } else { + /* + * DP module creation starts running in kernel mode and + * switches to userspace later via a scheduler_dp_thread_ipc() + * call in module_init(). + */ + ret = ipc4_init_module_instance(ipc4); + } + break; + case SOF_IPC4_MOD_CONFIG_GET: +#ifdef CONFIG_SOF_USERSPACE_LL + /* Forward to user thread for privilege-separated execution */ + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); + if (!ret) { + struct ipc_user *pdata = ipc_user_get(); + + msg_reply->extension = pdata->reply_ext; + } +#else + ret = ipc4_set_get_config_module_instance(ipc4, false); +#endif + break; + case SOF_IPC4_MOD_CONFIG_SET: +#ifdef CONFIG_SOF_USERSPACE_LL + /* Forward to user thread for privilege-separated execution */ + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +#else + ret = ipc4_set_get_config_module_instance(ipc4, true); +#endif + break; + case SOF_IPC4_MOD_LARGE_CONFIG_GET: +#ifdef CONFIG_SOF_USERSPACE_LL + { + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + + if (config->primary.r.module_id) { + /* Module case: forward to user thread */ + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); + if (!ret) { + struct ipc_user *pdata = ipc_user_get(); + + msg_reply->extension = pdata->reply_ext; + msg_reply->tx_size = pdata->reply_tx_size; + msg_reply->tx_data = pdata->reply_tx_data; + } + } else { + /* Base firmware (module_id==0): keep in kernel — + * ipc4_get_comp_drv() accesses IMR manifest which + * has no user-space partition. + */ + ret = ipc4_get_large_config_module_instance(ipc4); + } + } +#else /* CONFIG_SOF_USERSPACE_LL */ + ret = ipc4_get_large_config_module_instance(ipc4); +#endif /* CONFIG_SOF_USERSPACE_LL */ + break; + case SOF_IPC4_MOD_LARGE_CONFIG_SET: +#ifdef CONFIG_SOF_USERSPACE_LL + { + const struct ipc4_module_large_config *config = + (const struct ipc4_module_large_config *)ipc4; + + if (config->primary.r.module_id) { + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); + } else { + /* Base firmware: keep in kernel (IMR access) */ + ret = ipc4_set_large_config_module_instance(ipc4); + } + } +#else + ret = ipc4_set_large_config_module_instance(ipc4); +#endif + break; + case SOF_IPC4_MOD_BIND: +#ifdef CONFIG_SOF_USERSPACE_LL + /* + * bind and unbind can connect LL with DP. In that case it isn't + * immediately clear whether the handler should run in the DP + * thread context or in the LL IPC thread context. The LL IPC + * thread has access to DP modules, so we have to perform + * binding in the LL IPC thread context + */ + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +#else + ret = ipc4_bind_module_instance(ipc4); +#endif + break; + case SOF_IPC4_MOD_UNBIND: +#ifdef CONFIG_SOF_USERSPACE_LL + /* DP / LL: see comment above */ + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +#else + ret = ipc4_unbind_module_instance(ipc4); +#endif + break; + case SOF_IPC4_MOD_DELETE_INSTANCE: +#ifdef CONFIG_SOF_USERSPACE_LL + ret = ipc_user_forward_cmd(ipc4->primary.dat, ipc4->extension.dat, + ipc4_user_target_core_module(ipc4)); +#else + ret = ipc4_delete_module_instance(ipc4); +#endif + break; + case SOF_IPC4_MOD_ENTER_MODULE_RESTORE: + case SOF_IPC4_MOD_EXIT_MODULE_RESTORE: + ret = IPC4_UNAVAILABLE; + break; + default: + ret = IPC4_UNKNOWN_MESSAGE_TYPE; + break; + } + + return ret; +} diff --git a/src/kernel/ipc/ipc4/helper-kernel.c b/src/kernel/ipc/ipc4/helper-kernel.c new file mode 100644 index 000000000000..73635753c539 --- /dev/null +++ b/src/kernel/ipc/ipc4/helper-kernel.c @@ -0,0 +1,459 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Liam Girdwood +// Author: Rander Wang + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +/* TODO: Remove platform-specific code, see https://github.com/thesofproject/sof/issues/7549 */ +#if defined(CONFIG_SOC_SERIES_INTEL_ADSP_ACE) || defined(CONFIG_INTEL_ADSP_CAVS) +#define RIMAGE_MANIFEST 1 +#endif + +#ifdef RIMAGE_MANIFEST +#include +#endif + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include + +/* Kernel part of the IPC4 helpers: the component driver lookup, which reads the + * IMR manifest and the driver list, module creation in kernel mode, IDC to other + * cores and the chain DMA manager. The part that also runs in user mode is in + * src/ipc/ipc4/helper.c + */ + +LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); + +extern struct tr_ctx comp_tr; + +static const struct comp_driver *ipc4_get_drv(const void *uuid); + +#if CONFIG_LIBRARY +static const struct comp_driver *ipc4_library_get_comp_drv(unsigned char *data) +{ + return ipc4_get_drv(data); +} +#endif /* CONFIG_LIBRARY */ + +/* Only called from ipc4_init_module_instance(), which is __cold */ +__cold struct comp_dev *comp_new_ipc4(const struct ipc4_module_init_instance *module_init) +{ + struct comp_ipc_config ipc_config; + const struct comp_driver *drv; + struct comp_dev *dev; + unsigned char *data; + uint32_t comp_id = IPC4_COMP_ID(module_init->primary.r.module_id, + module_init->primary.r.instance_id); + + assert_can_be_cold(); + + int ret = ipc4_comp_new_config(&ipc_config, module_init); + + if (ret < 0) + return NULL; + +#ifdef CONFIG_DCACHE_LINE_SIZE + if (!IS_ENABLED(CONFIG_LIBRARY)) + sys_cache_data_invd_range((__sparse_force void __sparse_cache *) + MAILBOX_HOSTBOX_BASE, + ALIGN_UP(ipc_config.ipc_config_size, + CONFIG_DCACHE_LINE_SIZE)); +#endif + data = ipc4_get_comp_new_data(); +#if CONFIG_LIBRARY + ipc_config.ipc_config_size -= sizeof(struct sof_uuid); + drv = ipc4_library_get_comp_drv(data + ipc_config.ipc_config_size); +#else + drv = ipc4_get_comp_drv(IPC4_MOD_ID(comp_id)); +#endif + if (!drv) + return NULL; + + if (!drv->ops.create) { + tr_err(&ipc_tr, "comp 0x%x: module %u is not instantiable", comp_id, + (uint32_t)IPC4_MOD_ID(comp_id)); + return NULL; + } + + if (drv->type == SOF_COMP_MODULE_ADAPTER) { + const struct ipc_config_process spec = { + .data = data, + .size = ipc_config.ipc_config_size, + }; + + dev = drv->ops.create(drv, &ipc_config, &spec); + } else { + dev = drv->ops.create(drv, &ipc_config, data); + } + if (!dev) + return NULL; + + list_init(&dev->bsource_list); + list_init(&dev->bsink_list); + +#ifdef CONFIG_SOF_TELEMETRY_PERFORMANCE_MEASUREMENTS + /* init global performance measurement */ + dev->perf_data.perf_data_item = perf_data_getnext(); + /* this can be null, just no performance measurements in this case */ + if (dev->perf_data.perf_data_item) { + dev->perf_data.perf_data_item->item.resource_id = comp_id; + if (perf_meas_get_state() != IPC4_PERF_MEASUREMENTS_DISABLED) + comp_init_performance_data(dev); + } +#endif + + ipc4_add_comp_dev(dev); + + comp_update_ibs_obs_cpc(dev); + + return dev; +} + +#ifdef CONFIG_SOF_USERSPACE_LL +int ipc4_user_module_load(const struct comp_driver *drv, + const struct ipc4_module_init_instance *mi) +{ + /* + * A large part of module instantiation is executed in the userspace + * mode, including calling the module's .init() method. But another part + * of it has to be executed in the kernel mode, including calling + * lib_manager_mod_create_priv(). This is the privileged part of module + * instantiation. + */ + struct comp_ipc_config ipc_config; + int ret = ipc4_comp_new_config(&ipc_config, mi); + + if (ret < 0) + return ret; + + const struct ipc_config_process spec = { + .data = ipc4_get_comp_new_data(), + .size = ipc_config.ipc_config_size, + }; + +#if CONFIG_DCACHE_LINE_SIZE && !CONFIG_LIBRARY + sys_cache_data_invd_range((__sparse_force void __sparse_cache *)spec.data, spec.size); +#endif + + struct userspace_context *userspace = NULL; + + return lib_manager_mod_create_priv(drv, &ipc_config, &spec, NULL, &userspace); +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +__cold int ipc_comp_free_remote(struct comp_dev *dev) +{ + struct idc_msg msg = { IDC_MSG_FREE, IDC_MSG_FREE_EXT(dev->ipc_config.id), + dev->ipc_config.core,}; + + assert_can_be_cold(); + + return idc_send_msg(&msg, IDC_BLOCKING); +} + +#if CONFIG_COMP_CHAIN_DMA +/* Only called from ipc4_process_chain_dma(), which is __cold */ +__cold int ipc4_chain_manager_create(const struct ipc4_chain_dma *cdma) +{ + const struct sof_uuid uuid = SOF_REG_UUID(chain_dma); + const struct comp_driver *drv; + struct comp_dev *dev; + + assert_can_be_cold(); + + drv = ipc4_get_drv(&uuid); + if (!drv) + return -EINVAL; + + dev = drv->ops.create(drv, NULL, cdma); + if (!dev) + return -EINVAL; + + /* differentiate instance by unique ids assignment */ + const uint32_t comp_id = IPC4_COMP_ID(cdma->primary.r.host_dma_id + + IPC4_MAX_MODULE_COUNT, 0); + dev->ipc_config.id = comp_id; + dev->ipc_config.pipeline_id = cdma->primary.r.host_dma_id + IPC4_MAX_MODULE_COUNT; + + return ipc4_add_comp_dev(dev); +} + +/* Only called from ipc4_process_chain_dma(), which is __cold */ +__cold int ipc4_chain_dma_state(struct comp_dev *dev, const struct ipc4_chain_dma *cdma) +{ + const bool allocate = cdma->primary.r.allocate; + const bool enable = cdma->primary.r.enable; + struct ipc *ipc = ipc_get(); + struct ipc_comp_dev *icd; + struct list_item *clist, *_tmp; + int ret; + + assert_can_be_cold(); + + if (!dev) + return -EINVAL; + + if (allocate) { + if (enable) + ret = comp_trigger(dev, COMP_TRIGGER_START); + else + ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); + } else { + if (enable) + return -EINVAL; + + /* remove chain part */ + ret = comp_trigger(dev, COMP_TRIGGER_PAUSE); + if (ret < 0) + return ret; + + list_for_item_safe(clist, _tmp, &ipc->comp_list) { + icd = container_of(clist, struct ipc_comp_dev, list); + if (icd->cd != dev) + continue; + list_item_del(&icd->list); + sof_heap_free(sof_sys_user_heap_get(), icd); + break; + } + comp_free(dev); + } + return ret; +} +#endif /* CONFIG_COMP_CHAIN_DMA */ + +int ipc4_process_on_core(uint32_t core, bool blocking) +{ + int ret; + + ret = ipc_process_on_core(core, blocking); + switch (ret) { + case 0: + case 1: + return IPC4_SUCCESS; + case -EACCES: + return IPC4_INVALID_CORE_ID; + case -ETIME: + case -EBUSY: + return IPC4_BUSY; + default: + return IPC4_FAILURE; + } + + return IPC4_SUCCESS; +} + +__cold static const struct comp_driver *ipc4_search_for_drv(const void *uuid) +{ + struct comp_driver_list *drivers = comp_drivers_get(); + struct list_item *clist; + const struct comp_driver *drv = NULL; + struct comp_driver_info *info; + + assert_can_be_cold(); + + /* search driver list with UUID */ + list_for_item(clist, &drivers->list) { + info = container_of(clist, struct comp_driver_info, list); + if (!memcmp(info->drv->uid, uuid, UUID_SIZE)) { + tr_dbg(&comp_tr, "found type %d, uuid %pU", + info->drv->type, SOF_DRV_UID_NAME(info->drv)); + drv = info->drv; + break; + } + } + + return drv; +} + +__cold static const struct comp_driver *ipc4_get_drv(const void *uuid) +{ + const struct sof_uuid *const __maybe_unused sof_uuid = (const struct sof_uuid *)uuid; + const struct comp_driver *drv; + + assert_can_be_cold(); + + drv = ipc4_search_for_drv(uuid); + if (!drv) + tr_err(&comp_tr, + "the provided UUID (%08x-%04x-%04x-%02x%02x-%02x%02x%02x%02x%02x%02x) can't be found!", + sof_uuid->a, sof_uuid->b, sof_uuid->c, sof_uuid->d[0], sof_uuid->d[1], + sof_uuid->d[2], sof_uuid->d[3], sof_uuid->d[4], sof_uuid->d[5], + sof_uuid->d[6], sof_uuid->d[7]); + + return drv; +} + +#if defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) +/* + * ipc4_get_fuzzer_drv - resolve a module driver for native_sim fuzz builds. + * + * native_sim fuzz builds have no rimage manifest, so the manifest lookup in + * ipc4_get_comp_drv() can never resolve a driver and every module instance + * verb (INIT_INSTANCE, CONFIG_GET/SET, LARGE_CONFIG, BIND, UNBIND, + * DELETE_INSTANCE) would be unreachable from the fuzzer. + * + * Mirror the IPC3 whitebox hack in posix/ipc.c and treat module_id as a + * 1-based index into the runtime component driver list, counting only + * instantiable module-adapter drivers so the fuzzer's module_id space matches + * what a real signed manifest would expose. + */ +static const struct comp_driver *ipc4_get_fuzzer_drv(uint32_t module_id) +{ + static const struct sof_uuid basefw_reg_uuid = SOF_REG_UUID(basefw); + struct comp_driver_list *dlist = comp_drivers_get(); + struct list_item *iter; + uint32_t idx = 0; + + if (!module_id) + return ipc4_search_for_drv(&basefw_reg_uuid); + + list_for_item(iter, &dlist->list) { + struct comp_driver_info *inf = + container_of(iter, struct comp_driver_info, list); + + /* + * Skip query-only entries (e.g. BaseFW) that cannot be + * instantiated because they have no create op. + */ + if (!inf->drv->ops.create) + continue; + + /* + * A real signed manifest only lists module-adapter modules. + * The internal gateway drivers (SOF_COMP_HOST, SOF_COMP_DAI) + * are registered for IPC3 but are never IPC4 modules: on the + * IPC4 path they receive no params() pass and cannot be + * configured (e.g. their DMA buffer is never allocated). Skip + * non-module-adapter drivers so the fuzzer's module_id space + * matches a real manifest and cannot map to an unconfigurable + * component. + */ + if (inf->drv->type != SOF_COMP_MODULE_ADAPTER) + continue; + + if (++idx == module_id) + return inf->drv; + } + + tr_err(&comp_tr, "no instantiable driver at module_id %u (%u available)", + module_id, idx); + return NULL; +} +#endif /* defined(CONFIG_ARCH_POSIX_LIBFUZZER) && !defined(RIMAGE_MANIFEST) */ + +/* + * Called from + * - ipc4_get_large_config_module_instance() + * - ipc4_set_large_config_module_instance() + * - comp_new_ipc4() + * all of which are __cold + */ +__cold const struct comp_driver *ipc4_get_comp_drv(uint32_t module_id) +{ + const struct sof_man_fw_desc *desc = NULL; + const struct sof_man_module *mod; + uint32_t entry_index; + + assert_can_be_cold(); + +#ifdef RIMAGE_MANIFEST + desc = (const struct sof_man_fw_desc *)IMR_BOOT_LDR_MANIFEST_BASE; +#elif defined(CONFIG_ARCH_POSIX_LIBFUZZER) + return ipc4_get_fuzzer_drv(module_id); +#else + /* Non-rimage platforms have no component facility yet. + * This needs to move to the platform layer. + */ + return NULL; +#endif + + uint32_t lib_idx = LIB_MANAGER_GET_LIB_ID(module_id); + + if (lib_idx == 0) { + /* module_id 0 is used for base fw which is in entry 1 or 2 */ + if (!module_id) + entry_index = 1 + IS_ENABLED(CONFIG_COLD_STORE_EXECUTE_DRAM); + else + entry_index = module_id; + + if (entry_index >= desc->header.num_module_entries) { + tr_err(&comp_tr, "Error: entry index %d out of bounds.", entry_index); + return NULL; + } + + mod = (const struct sof_man_module *)((const char *)desc + + SOF_MAN_MODULE_OFFSET(entry_index)); + } else { + /* Library index greater than 0 possible only when LIBRARY_MANAGER + * support enabled. + */ +#if CONFIG_LIBRARY_MANAGER + mod = lib_manager_get_module_manifest(module_id); + + if (!mod) { + tr_err(&comp_tr, "Error: Couldn't find loadable module with id %#x.", + module_id); + return NULL; + } +#else + tr_err(&comp_tr, "Error: lib index:%d, while loadable libraries are not supported!!!", + lib_idx); + return NULL; +#endif + } + +#if CONFIG_LIBRARY_MANAGER + /* Check already registered components */ + const struct comp_driver *drv = ipc4_search_for_drv(&mod->uuid); + + if (drv) + return drv; + + /* New module not registered yet. */ + lib_manager_register_module(module_id); +#endif + return ipc4_get_drv(&mod->uuid); +} diff --git a/src/kernel/logging/Kconfig b/src/kernel/logging/Kconfig new file mode 100644 index 000000000000..80900349ec9e --- /dev/null +++ b/src/kernel/logging/Kconfig @@ -0,0 +1,40 @@ +# SPDX-License-Identifier: BSD-3-Clause + +config LOG_BACKEND_SOF_PROBE + bool "Logging backend with SOF probes" + depends on LOG + select LOG_OUTPUT + help + Enable backend for log output via SOF probe interface. + Probe interface allows to transmit logs and PCM data, muxed over + a shared DMA channel. + Logging is enabled by setting up a probe point with + probe purpose value of PROBE_PURPOSE_LOGGING. + +config LOG_BACKEND_SOF_PROBE_OUTPUT_AUTO_ENABLE + bool "Automatically enable probes logging when probe dma is started" + depends on LOG_BACKEND_SOF_PROBE + help + Automatically enable logging as soon as probes DMA + starts. This option simplifies probes logging back + end usage as there is no need to enable logs after + every boot. The probes DMA is still needed before the + log output can be enabled. + +config LOG_BACKEND_SOF_PROBE_OUTPUT_DICTIONARY + bool "Dictionary" + select LOG_DICTIONARY_SUPPORT + help + Set output format of the SOF probe logging backend to + Zephyr log dictionary. + The resulting log file can be parsed with tools in + zephyr/scripts/logging/dictionary + A dictionary for the binary is created during build. + +config LOG_BACKEND_SOF_PROBE_OUTPUT + int "Set logging output format" + default 1 if LOG_MIPI_SYST_ENABLE + default 2 if LOG_BACKEND_SOF_PROBE_OUTPUT_DICTIONARY + default 0 # Text encoding is default + help + Set the logging format for SOF probe output. diff --git a/src/platform/CMakeLists.txt b/src/kernel/platform/CMakeLists.txt similarity index 100% rename from src/platform/CMakeLists.txt rename to src/kernel/platform/CMakeLists.txt diff --git a/src/platform/Kconfig b/src/kernel/platform/Kconfig similarity index 100% rename from src/platform/Kconfig rename to src/kernel/platform/Kconfig diff --git a/src/platform/README.md b/src/kernel/platform/README.md similarity index 81% rename from src/platform/README.md rename to src/kernel/platform/README.md index da20eb6569d2..b3aa7014290a 100644 --- a/src/platform/README.md +++ b/src/kernel/platform/README.md @@ -1,8 +1,8 @@ -# Platform Support (`src/platform`) +# Platform Support (`src/kernel/platform`) > **Runs in:** Kernel-only — platform bring-up, memory maps, interrupt wiring and IPC mailbox > windows are supervisor-context concerns and never run in a user thread. See -> [User/kernel split](../include/sof/userspace/README.md). +> [User/kernel split](../../include/sof/userspace/README.md). This directory contains SoC/platform-specific integration (e.g. Intel ACE, MediaTek, i.MX), including the Kconfig that selects the build target such as `PANTHERLAKE` (PTL). It configures diff --git a/src/platform/ace30/include/platform/lib/clk.h b/src/kernel/platform/ace30/include/platform/lib/clk.h similarity index 100% rename from src/platform/ace30/include/platform/lib/clk.h rename to src/kernel/platform/ace30/include/platform/lib/clk.h diff --git a/src/platform/ace30/include/platform/lib/mailbox.h b/src/kernel/platform/ace30/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/ace30/include/platform/lib/mailbox.h rename to src/kernel/platform/ace30/include/platform/lib/mailbox.h diff --git a/src/platform/ace30/include/platform/lib/memory.h b/src/kernel/platform/ace30/include/platform/lib/memory.h similarity index 100% rename from src/platform/ace30/include/platform/lib/memory.h rename to src/kernel/platform/ace30/include/platform/lib/memory.h diff --git a/src/platform/ace30/include/platform/platform.h b/src/kernel/platform/ace30/include/platform/platform.h similarity index 100% rename from src/platform/ace30/include/platform/platform.h rename to src/kernel/platform/ace30/include/platform/platform.h diff --git a/src/platform/ace30/include/platform/trace/trace.h b/src/kernel/platform/ace30/include/platform/trace/trace.h similarity index 100% rename from src/platform/ace30/include/platform/trace/trace.h rename to src/kernel/platform/ace30/include/platform/trace/trace.h diff --git a/src/platform/ace30/lib/clk.c b/src/kernel/platform/ace30/lib/clk.c similarity index 100% rename from src/platform/ace30/lib/clk.c rename to src/kernel/platform/ace30/lib/clk.c diff --git a/src/platform/amd/acp_7_0/acp_7_0.x.in b/src/kernel/platform/amd/acp_7_0/acp_7_0.x.in similarity index 100% rename from src/platform/amd/acp_7_0/acp_7_0.x.in rename to src/kernel/platform/amd/acp_7_0/acp_7_0.x.in diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/core-isa.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/core-isa.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/core-isa.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/core-isa.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/core-matmap.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/core-matmap.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/core-matmap.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/core-matmap.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/defs.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/defs.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/defs.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/defs.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/specreg.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/specreg.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/specreg.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/specreg.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/system.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/system.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/system.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/system.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/tie-asm.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/tie-asm.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/tie-asm.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/tie-asm.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/config/tie.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/tie.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/config/tie.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/config/tie.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/idma.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/idma.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/idma.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/idma.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_datacache.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_datacache.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_datacache.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_datacache.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_externalregisters.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_externalregisters.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_externalregisters.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/tie/xt_externalregisters.h diff --git a/src/platform/amd/acp_7_0/include/arch/xtensa/xtensa-types.h b/src/kernel/platform/amd/acp_7_0/include/arch/xtensa/xtensa-types.h similarity index 100% rename from src/platform/amd/acp_7_0/include/arch/xtensa/xtensa-types.h rename to src/kernel/platform/amd/acp_7_0/include/arch/xtensa/xtensa-types.h diff --git a/src/platform/amd/acp_7_0/include/platform/drivers/interrupt.h b/src/kernel/platform/amd/acp_7_0/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/amd/acp_7_0/include/platform/drivers/interrupt.h rename to src/kernel/platform/amd/acp_7_0/include/platform/drivers/interrupt.h diff --git a/src/platform/amd/acp_7_0/include/platform/fw_scratch_mem.h b/src/kernel/platform/amd/acp_7_0/include/platform/fw_scratch_mem.h similarity index 100% rename from src/platform/amd/acp_7_0/include/platform/fw_scratch_mem.h rename to src/kernel/platform/amd/acp_7_0/include/platform/fw_scratch_mem.h diff --git a/src/platform/amd/acp_7_0/include/platform/lib/memory.h b/src/kernel/platform/amd/acp_7_0/include/platform/lib/memory.h similarity index 100% rename from src/platform/amd/acp_7_0/include/platform/lib/memory.h rename to src/kernel/platform/amd/acp_7_0/include/platform/lib/memory.h diff --git a/src/platform/amd/acp_7_0/include/platform/platform.h b/src/kernel/platform/amd/acp_7_0/include/platform/platform.h similarity index 100% rename from src/platform/amd/acp_7_0/include/platform/platform.h rename to src/kernel/platform/amd/acp_7_0/include/platform/platform.h diff --git a/src/platform/amd/acp_7_0/include/platform/platform_misc.h b/src/kernel/platform/amd/acp_7_0/include/platform/platform_misc.h similarity index 100% rename from src/platform/amd/acp_7_0/include/platform/platform_misc.h rename to src/kernel/platform/amd/acp_7_0/include/platform/platform_misc.h diff --git a/src/platform/amd/acp_7_0/lib/clk.c b/src/kernel/platform/amd/acp_7_0/lib/clk.c similarity index 100% rename from src/platform/amd/acp_7_0/lib/clk.c rename to src/kernel/platform/amd/acp_7_0/lib/clk.c diff --git a/src/platform/amd/acp_7_0/platform.c b/src/kernel/platform/amd/acp_7_0/platform.c similarity index 99% rename from src/platform/amd/acp_7_0/platform.c rename to src/kernel/platform/amd/acp_7_0/platform.c index 72162bfd68f4..6d40612c22d7 100644 --- a/src/platform/amd/acp_7_0/platform.c +++ b/src/kernel/platform/amd/acp_7_0/platform.c @@ -26,7 +26,6 @@ #include #include #include -#include #include #include #include diff --git a/src/platform/amd/acp_7_x/include/arch/xtensa/tie/xt_datacache.h b/src/kernel/platform/amd/acp_7_x/include/arch/xtensa/tie/xt_datacache.h similarity index 100% rename from src/platform/amd/acp_7_x/include/arch/xtensa/tie/xt_datacache.h rename to src/kernel/platform/amd/acp_7_x/include/arch/xtensa/tie/xt_datacache.h diff --git a/src/platform/amd/acp_7_x/include/platform/fw_scratch_mem.h b/src/kernel/platform/amd/acp_7_x/include/platform/fw_scratch_mem.h similarity index 100% rename from src/platform/amd/acp_7_x/include/platform/fw_scratch_mem.h rename to src/kernel/platform/amd/acp_7_x/include/platform/fw_scratch_mem.h diff --git a/src/platform/amd/acp_7_x/include/platform/lib/memory.h b/src/kernel/platform/amd/acp_7_x/include/platform/lib/memory.h similarity index 100% rename from src/platform/amd/acp_7_x/include/platform/lib/memory.h rename to src/kernel/platform/amd/acp_7_x/include/platform/lib/memory.h diff --git a/src/platform/amd/acp_7_x/include/platform/platform.h b/src/kernel/platform/amd/acp_7_x/include/platform/platform.h similarity index 100% rename from src/platform/amd/acp_7_x/include/platform/platform.h rename to src/kernel/platform/amd/acp_7_x/include/platform/platform.h diff --git a/src/platform/amd/acp_7_x/include/platform/platform_misc.h b/src/kernel/platform/amd/acp_7_x/include/platform/platform_misc.h similarity index 100% rename from src/platform/amd/acp_7_x/include/platform/platform_misc.h rename to src/kernel/platform/amd/acp_7_x/include/platform/platform_misc.h diff --git a/src/platform/amd/acp_7_x/lib/clk.c b/src/kernel/platform/amd/acp_7_x/lib/clk.c similarity index 100% rename from src/platform/amd/acp_7_x/lib/clk.c rename to src/kernel/platform/amd/acp_7_x/lib/clk.c diff --git a/src/platform/amd/acp_7_x/platform.c b/src/kernel/platform/amd/acp_7_x/platform.c similarity index 99% rename from src/platform/amd/acp_7_x/platform.c rename to src/kernel/platform/amd/acp_7_x/platform.c index 37d6d3d45ed9..740f2eb687aa 100644 --- a/src/platform/amd/acp_7_x/platform.c +++ b/src/kernel/platform/amd/acp_7_x/platform.c @@ -27,7 +27,6 @@ #include #include #include -#include #include #include #include diff --git a/src/platform/amd/common/include/platform/acp_dma.h b/src/kernel/platform/amd/common/include/platform/acp_dma.h similarity index 100% rename from src/platform/amd/common/include/platform/acp_dma.h rename to src/kernel/platform/amd/common/include/platform/acp_dma.h diff --git a/src/platform/amd/common/include/platform/acp_dmic_dma.h b/src/kernel/platform/amd/common/include/platform/acp_dmic_dma.h similarity index 100% rename from src/platform/amd/common/include/platform/acp_dmic_dma.h rename to src/kernel/platform/amd/common/include/platform/acp_dmic_dma.h diff --git a/src/platform/amd/common/include/platform/acp_sp_dma.h b/src/kernel/platform/amd/common/include/platform/acp_sp_dma.h similarity index 100% rename from src/platform/amd/common/include/platform/acp_sp_dma.h rename to src/kernel/platform/amd/common/include/platform/acp_sp_dma.h diff --git a/src/platform/amd/common/include/platform/drivers/idc.h b/src/kernel/platform/amd/common/include/platform/drivers/idc.h similarity index 100% rename from src/platform/amd/common/include/platform/drivers/idc.h rename to src/kernel/platform/amd/common/include/platform/drivers/idc.h diff --git a/src/platform/amd/common/include/platform/ipc.h b/src/kernel/platform/amd/common/include/platform/ipc.h similarity index 100% rename from src/platform/amd/common/include/platform/ipc.h rename to src/kernel/platform/amd/common/include/platform/ipc.h diff --git a/src/platform/amd/common/include/platform/lib/clk.h b/src/kernel/platform/amd/common/include/platform/lib/clk.h similarity index 100% rename from src/platform/amd/common/include/platform/lib/clk.h rename to src/kernel/platform/amd/common/include/platform/lib/clk.h diff --git a/src/platform/amd/common/include/platform/lib/cpu.h b/src/kernel/platform/amd/common/include/platform/lib/cpu.h similarity index 100% rename from src/platform/amd/common/include/platform/lib/cpu.h rename to src/kernel/platform/amd/common/include/platform/lib/cpu.h diff --git a/src/platform/amd/common/include/platform/lib/dai.h b/src/kernel/platform/amd/common/include/platform/lib/dai.h similarity index 100% rename from src/platform/amd/common/include/platform/lib/dai.h rename to src/kernel/platform/amd/common/include/platform/lib/dai.h diff --git a/src/platform/amd/common/include/platform/lib/dma.h b/src/kernel/platform/amd/common/include/platform/lib/dma.h similarity index 100% rename from src/platform/amd/common/include/platform/lib/dma.h rename to src/kernel/platform/amd/common/include/platform/lib/dma.h diff --git a/src/platform/amd/common/include/platform/lib/mailbox.h b/src/kernel/platform/amd/common/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/amd/common/include/platform/lib/mailbox.h rename to src/kernel/platform/amd/common/include/platform/lib/mailbox.h diff --git a/src/platform/amd/common/include/platform/lib/pm_runtime.h b/src/kernel/platform/amd/common/include/platform/lib/pm_runtime.h similarity index 100% rename from src/platform/amd/common/include/platform/lib/pm_runtime.h rename to src/kernel/platform/amd/common/include/platform/lib/pm_runtime.h diff --git a/src/platform/amd/common/include/platform/trace/trace.h b/src/kernel/platform/amd/common/include/platform/trace/trace.h similarity index 100% rename from src/platform/amd/common/include/platform/trace/trace.h rename to src/kernel/platform/amd/common/include/platform/trace/trace.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/core-isa.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/core-isa.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/core-isa.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/core-isa.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/core-matmap.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/core-matmap.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/core-matmap.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/core-matmap.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/defs.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/defs.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/defs.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/defs.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/specreg.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/specreg.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/specreg.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/specreg.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/system.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/system.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/system.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/system.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/tie-asm.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/tie-asm.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/tie-asm.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/tie-asm.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/config/tie.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/tie.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/config/tie.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/config/tie.h diff --git a/src/platform/amd/rembrandt/include/arch/xtensa/tie/xt_datacache.h b/src/kernel/platform/amd/rembrandt/include/arch/xtensa/tie/xt_datacache.h similarity index 100% rename from src/platform/amd/rembrandt/include/arch/xtensa/tie/xt_datacache.h rename to src/kernel/platform/amd/rembrandt/include/arch/xtensa/tie/xt_datacache.h diff --git a/src/platform/amd/rembrandt/include/platform/chip_offset_byte.h b/src/kernel/platform/amd/rembrandt/include/platform/chip_offset_byte.h similarity index 100% rename from src/platform/amd/rembrandt/include/platform/chip_offset_byte.h rename to src/kernel/platform/amd/rembrandt/include/platform/chip_offset_byte.h diff --git a/src/platform/amd/rembrandt/include/platform/chip_registers.h b/src/kernel/platform/amd/rembrandt/include/platform/chip_registers.h similarity index 100% rename from src/platform/amd/rembrandt/include/platform/chip_registers.h rename to src/kernel/platform/amd/rembrandt/include/platform/chip_registers.h diff --git a/src/platform/amd/rembrandt/include/platform/drivers/interrupt.h b/src/kernel/platform/amd/rembrandt/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/amd/rembrandt/include/platform/drivers/interrupt.h rename to src/kernel/platform/amd/rembrandt/include/platform/drivers/interrupt.h diff --git a/src/platform/amd/rembrandt/include/platform/fw_scratch_mem.h b/src/kernel/platform/amd/rembrandt/include/platform/fw_scratch_mem.h similarity index 100% rename from src/platform/amd/rembrandt/include/platform/fw_scratch_mem.h rename to src/kernel/platform/amd/rembrandt/include/platform/fw_scratch_mem.h diff --git a/src/platform/amd/rembrandt/include/platform/lib/memory.h b/src/kernel/platform/amd/rembrandt/include/platform/lib/memory.h similarity index 100% rename from src/platform/amd/rembrandt/include/platform/lib/memory.h rename to src/kernel/platform/amd/rembrandt/include/platform/lib/memory.h diff --git a/src/platform/amd/rembrandt/include/platform/platform.h b/src/kernel/platform/amd/rembrandt/include/platform/platform.h similarity index 100% rename from src/platform/amd/rembrandt/include/platform/platform.h rename to src/kernel/platform/amd/rembrandt/include/platform/platform.h diff --git a/src/platform/amd/rembrandt/lib/clk.c b/src/kernel/platform/amd/rembrandt/lib/clk.c similarity index 100% rename from src/platform/amd/rembrandt/lib/clk.c rename to src/kernel/platform/amd/rembrandt/lib/clk.c diff --git a/src/platform/amd/rembrandt/lib/dai.c b/src/kernel/platform/amd/rembrandt/lib/dai.c similarity index 100% rename from src/platform/amd/rembrandt/lib/dai.c rename to src/kernel/platform/amd/rembrandt/lib/dai.c diff --git a/src/platform/amd/rembrandt/lib/dma.c b/src/kernel/platform/amd/rembrandt/lib/dma.c similarity index 100% rename from src/platform/amd/rembrandt/lib/dma.c rename to src/kernel/platform/amd/rembrandt/lib/dma.c diff --git a/src/platform/amd/rembrandt/platform.c b/src/kernel/platform/amd/rembrandt/platform.c similarity index 91% rename from src/platform/amd/rembrandt/platform.c rename to src/kernel/platform/amd/rembrandt/platform.c index 1c26e8e8fcfa..b915c4963438 100644 --- a/src/platform/amd/rembrandt/platform.c +++ b/src/kernel/platform/amd/rembrandt/platform.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -158,17 +157,6 @@ int platform_init(struct sof *sof) ret = dai_init(sof); if (ret < 0) return -ENODEV; -#if CONFIG_TRACE - /* Initialize DMA for Trace*/ - trace_point(TRACE_BOOT_PLATFORM_DMA_TRACE); - sof->dmat->config.elem_array.elems = rzalloc(SOF_MEM_FLAG_KERNEL, - sizeof(struct dma_sg_elem) * 1); - sof->dmat->config.elem_array.count = 1; - sof->dmat->config.elem_array.elems->dest = 0x03800000; - sof->dmat->config.elem_array.elems->size = 65536; - sof->dmat->config.scatter = 0; - dma_trace_init_complete(sof->dmat); -#endif /* show heap status */ heap_trace_all(1); return 0; diff --git a/src/platform/amd/rembrandt/rembrandt.x.in b/src/kernel/platform/amd/rembrandt/rembrandt.x.in similarity index 100% rename from src/platform/amd/rembrandt/rembrandt.x.in rename to src/kernel/platform/amd/rembrandt/rembrandt.x.in diff --git a/src/platform/imx8/imx8.x.in b/src/kernel/platform/imx8/imx8.x.in similarity index 100% rename from src/platform/imx8/imx8.x.in rename to src/kernel/platform/imx8/imx8.x.in diff --git a/src/platform/imx8/include/arch/xtensa/config/core-isa.h b/src/kernel/platform/imx8/include/arch/xtensa/config/core-isa.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/core-isa.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/core-isa.h diff --git a/src/platform/imx8/include/arch/xtensa/config/core-matmap.h b/src/kernel/platform/imx8/include/arch/xtensa/config/core-matmap.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/core-matmap.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/core-matmap.h diff --git a/src/platform/imx8/include/arch/xtensa/config/defs.h b/src/kernel/platform/imx8/include/arch/xtensa/config/defs.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/defs.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/defs.h diff --git a/src/platform/imx8/include/arch/xtensa/config/specreg.h b/src/kernel/platform/imx8/include/arch/xtensa/config/specreg.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/specreg.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/specreg.h diff --git a/src/platform/imx8/include/arch/xtensa/config/system.h b/src/kernel/platform/imx8/include/arch/xtensa/config/system.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/system.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/system.h diff --git a/src/platform/imx8/include/arch/xtensa/config/tie-asm.h b/src/kernel/platform/imx8/include/arch/xtensa/config/tie-asm.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/tie-asm.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/tie-asm.h diff --git a/src/platform/imx8/include/arch/xtensa/config/tie.h b/src/kernel/platform/imx8/include/arch/xtensa/config/tie.h similarity index 100% rename from src/platform/imx8/include/arch/xtensa/config/tie.h rename to src/kernel/platform/imx8/include/arch/xtensa/config/tie.h diff --git a/src/platform/imx8/include/platform/drivers/idc.h b/src/kernel/platform/imx8/include/platform/drivers/idc.h similarity index 100% rename from src/platform/imx8/include/platform/drivers/idc.h rename to src/kernel/platform/imx8/include/platform/drivers/idc.h diff --git a/src/platform/imx8/include/platform/drivers/interrupt.h b/src/kernel/platform/imx8/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/imx8/include/platform/drivers/interrupt.h rename to src/kernel/platform/imx8/include/platform/drivers/interrupt.h diff --git a/src/platform/imx8/include/platform/lib/clk.h b/src/kernel/platform/imx8/include/platform/lib/clk.h similarity index 100% rename from src/platform/imx8/include/platform/lib/clk.h rename to src/kernel/platform/imx8/include/platform/lib/clk.h diff --git a/src/platform/imx8/include/platform/lib/cpu.h b/src/kernel/platform/imx8/include/platform/lib/cpu.h similarity index 100% rename from src/platform/imx8/include/platform/lib/cpu.h rename to src/kernel/platform/imx8/include/platform/lib/cpu.h diff --git a/src/platform/imx8/include/platform/lib/dai.h b/src/kernel/platform/imx8/include/platform/lib/dai.h similarity index 100% rename from src/platform/imx8/include/platform/lib/dai.h rename to src/kernel/platform/imx8/include/platform/lib/dai.h diff --git a/src/platform/imx8/include/platform/lib/dma.h b/src/kernel/platform/imx8/include/platform/lib/dma.h similarity index 100% rename from src/platform/imx8/include/platform/lib/dma.h rename to src/kernel/platform/imx8/include/platform/lib/dma.h diff --git a/src/platform/imx8/include/platform/lib/mailbox.h b/src/kernel/platform/imx8/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/imx8/include/platform/lib/mailbox.h rename to src/kernel/platform/imx8/include/platform/lib/mailbox.h diff --git a/src/platform/imx8/include/platform/lib/memory.h b/src/kernel/platform/imx8/include/platform/lib/memory.h similarity index 100% rename from src/platform/imx8/include/platform/lib/memory.h rename to src/kernel/platform/imx8/include/platform/lib/memory.h diff --git a/src/platform/imx8/include/platform/platform.h b/src/kernel/platform/imx8/include/platform/platform.h similarity index 100% rename from src/platform/imx8/include/platform/platform.h rename to src/kernel/platform/imx8/include/platform/platform.h diff --git a/src/platform/imx8/include/platform/trace/trace.h b/src/kernel/platform/imx8/include/platform/trace/trace.h similarity index 100% rename from src/platform/imx8/include/platform/trace/trace.h rename to src/kernel/platform/imx8/include/platform/trace/trace.h diff --git a/src/platform/imx8/lib/clk.c b/src/kernel/platform/imx8/lib/clk.c similarity index 100% rename from src/platform/imx8/lib/clk.c rename to src/kernel/platform/imx8/lib/clk.c diff --git a/src/platform/imx8/platform.c b/src/kernel/platform/imx8/platform.c similarity index 100% rename from src/platform/imx8/platform.c rename to src/kernel/platform/imx8/platform.c diff --git a/src/platform/imx8m/imx8m.x.in b/src/kernel/platform/imx8m/imx8m.x.in similarity index 100% rename from src/platform/imx8m/imx8m.x.in rename to src/kernel/platform/imx8m/imx8m.x.in diff --git a/src/platform/imx8m/include/arch/xtensa/config/core-isa.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/core-isa.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/core-isa.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/core-isa.h diff --git a/src/platform/imx8m/include/arch/xtensa/config/core-matmap.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/core-matmap.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/core-matmap.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/core-matmap.h diff --git a/src/platform/imx8m/include/arch/xtensa/config/defs.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/defs.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/defs.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/defs.h diff --git a/src/platform/imx8m/include/arch/xtensa/config/specreg.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/specreg.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/specreg.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/specreg.h diff --git a/src/platform/imx8m/include/arch/xtensa/config/system.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/system.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/system.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/system.h diff --git a/src/platform/imx8m/include/arch/xtensa/config/tie-asm.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/tie-asm.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/tie-asm.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/tie-asm.h diff --git a/src/platform/imx8m/include/arch/xtensa/config/tie.h b/src/kernel/platform/imx8m/include/arch/xtensa/config/tie.h similarity index 100% rename from src/platform/imx8m/include/arch/xtensa/config/tie.h rename to src/kernel/platform/imx8m/include/arch/xtensa/config/tie.h diff --git a/src/platform/imx8m/include/platform/drivers/idc.h b/src/kernel/platform/imx8m/include/platform/drivers/idc.h similarity index 100% rename from src/platform/imx8m/include/platform/drivers/idc.h rename to src/kernel/platform/imx8m/include/platform/drivers/idc.h diff --git a/src/platform/imx8m/include/platform/drivers/interrupt.h b/src/kernel/platform/imx8m/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/imx8m/include/platform/drivers/interrupt.h rename to src/kernel/platform/imx8m/include/platform/drivers/interrupt.h diff --git a/src/platform/imx8m/include/platform/lib/clk.h b/src/kernel/platform/imx8m/include/platform/lib/clk.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/clk.h rename to src/kernel/platform/imx8m/include/platform/lib/clk.h diff --git a/src/platform/imx8m/include/platform/lib/cpu.h b/src/kernel/platform/imx8m/include/platform/lib/cpu.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/cpu.h rename to src/kernel/platform/imx8m/include/platform/lib/cpu.h diff --git a/src/platform/imx8m/include/platform/lib/dai.h b/src/kernel/platform/imx8m/include/platform/lib/dai.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/dai.h rename to src/kernel/platform/imx8m/include/platform/lib/dai.h diff --git a/src/platform/imx8m/include/platform/lib/dma.h b/src/kernel/platform/imx8m/include/platform/lib/dma.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/dma.h rename to src/kernel/platform/imx8m/include/platform/lib/dma.h diff --git a/src/platform/imx8m/include/platform/lib/mailbox.h b/src/kernel/platform/imx8m/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/mailbox.h rename to src/kernel/platform/imx8m/include/platform/lib/mailbox.h diff --git a/src/platform/imx8m/include/platform/lib/memory.h b/src/kernel/platform/imx8m/include/platform/lib/memory.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/memory.h rename to src/kernel/platform/imx8m/include/platform/lib/memory.h diff --git a/src/platform/imx8m/include/platform/lib/pm_runtime.h b/src/kernel/platform/imx8m/include/platform/lib/pm_runtime.h similarity index 100% rename from src/platform/imx8m/include/platform/lib/pm_runtime.h rename to src/kernel/platform/imx8m/include/platform/lib/pm_runtime.h diff --git a/src/platform/imx8m/include/platform/platform.h b/src/kernel/platform/imx8m/include/platform/platform.h similarity index 100% rename from src/platform/imx8m/include/platform/platform.h rename to src/kernel/platform/imx8m/include/platform/platform.h diff --git a/src/platform/imx8m/include/platform/trace/trace.h b/src/kernel/platform/imx8m/include/platform/trace/trace.h similarity index 100% rename from src/platform/imx8m/include/platform/trace/trace.h rename to src/kernel/platform/imx8m/include/platform/trace/trace.h diff --git a/src/platform/imx8m/lib/clk.c b/src/kernel/platform/imx8m/lib/clk.c similarity index 100% rename from src/platform/imx8m/lib/clk.c rename to src/kernel/platform/imx8m/lib/clk.c diff --git a/src/platform/imx8m/platform.c b/src/kernel/platform/imx8m/platform.c similarity index 96% rename from src/platform/imx8m/platform.c rename to src/kernel/platform/imx8m/platform.c index 46e60a7e8608..8bc719dc52ec 100644 --- a/src/platform/imx8m/platform.c +++ b/src/kernel/platform/imx8m/platform.c @@ -24,7 +24,6 @@ #include #include #include -#include #include #include #include @@ -172,12 +171,6 @@ int platform_init(struct sof *sof) if (ret < 0) return -ENODEV; -#if CONFIG_TRACE - /* Initialize DMA for Trace*/ - trace_point(TRACE_BOOT_PLATFORM_DMA_TRACE); - dma_trace_init_complete(sof->dmat); -#endif - /* show heap status */ heap_trace_all(1); diff --git a/src/platform/imx8m_cm7/include/platform/lib/clk.h b/src/kernel/platform/imx8m_cm7/include/platform/lib/clk.h similarity index 100% rename from src/platform/imx8m_cm7/include/platform/lib/clk.h rename to src/kernel/platform/imx8m_cm7/include/platform/lib/clk.h diff --git a/src/platform/imx8m_cm7/include/platform/lib/dma.h b/src/kernel/platform/imx8m_cm7/include/platform/lib/dma.h similarity index 100% rename from src/platform/imx8m_cm7/include/platform/lib/dma.h rename to src/kernel/platform/imx8m_cm7/include/platform/lib/dma.h diff --git a/src/platform/imx8m_cm7/include/platform/lib/mailbox.h b/src/kernel/platform/imx8m_cm7/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/imx8m_cm7/include/platform/lib/mailbox.h rename to src/kernel/platform/imx8m_cm7/include/platform/lib/mailbox.h diff --git a/src/platform/imx8m_cm7/include/platform/lib/memory.h b/src/kernel/platform/imx8m_cm7/include/platform/lib/memory.h similarity index 100% rename from src/platform/imx8m_cm7/include/platform/lib/memory.h rename to src/kernel/platform/imx8m_cm7/include/platform/lib/memory.h diff --git a/src/platform/imx8m_cm7/include/platform/platform.h b/src/kernel/platform/imx8m_cm7/include/platform/platform.h similarity index 100% rename from src/platform/imx8m_cm7/include/platform/platform.h rename to src/kernel/platform/imx8m_cm7/include/platform/platform.h diff --git a/src/platform/imx8m_cm7/include/platform/trace/trace.h b/src/kernel/platform/imx8m_cm7/include/platform/trace/trace.h similarity index 100% rename from src/platform/imx8m_cm7/include/platform/trace/trace.h rename to src/kernel/platform/imx8m_cm7/include/platform/trace/trace.h diff --git a/src/platform/imx8m_cm7/lib/clk.c b/src/kernel/platform/imx8m_cm7/lib/clk.c similarity index 100% rename from src/platform/imx8m_cm7/lib/clk.c rename to src/kernel/platform/imx8m_cm7/lib/clk.c diff --git a/src/platform/imx8m_cm7/linker/data-sections.ld b/src/kernel/platform/imx8m_cm7/linker/data-sections.ld similarity index 100% rename from src/platform/imx8m_cm7/linker/data-sections.ld rename to src/kernel/platform/imx8m_cm7/linker/data-sections.ld diff --git a/src/platform/imx8m_cm7/platform.c b/src/kernel/platform/imx8m_cm7/platform.c similarity index 100% rename from src/platform/imx8m_cm7/platform.c rename to src/kernel/platform/imx8m_cm7/platform.c diff --git a/src/platform/imx8ulp/imx8ulp.x.in b/src/kernel/platform/imx8ulp/imx8ulp.x.in similarity index 100% rename from src/platform/imx8ulp/imx8ulp.x.in rename to src/kernel/platform/imx8ulp/imx8ulp.x.in diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/core-isa.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/core-isa.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/core-isa.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/core-isa.h diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/core-matmap.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/core-matmap.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/core-matmap.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/core-matmap.h diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/defs.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/defs.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/defs.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/defs.h diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/specreg.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/specreg.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/specreg.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/specreg.h diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/system.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/system.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/system.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/system.h diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/tie-asm.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/tie-asm.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/tie-asm.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/tie-asm.h diff --git a/src/platform/imx8ulp/include/arch/xtensa/config/tie.h b/src/kernel/platform/imx8ulp/include/arch/xtensa/config/tie.h similarity index 100% rename from src/platform/imx8ulp/include/arch/xtensa/config/tie.h rename to src/kernel/platform/imx8ulp/include/arch/xtensa/config/tie.h diff --git a/src/platform/imx8ulp/include/platform/drivers/idc.h b/src/kernel/platform/imx8ulp/include/platform/drivers/idc.h similarity index 100% rename from src/platform/imx8ulp/include/platform/drivers/idc.h rename to src/kernel/platform/imx8ulp/include/platform/drivers/idc.h diff --git a/src/platform/imx8ulp/include/platform/drivers/interrupt.h b/src/kernel/platform/imx8ulp/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/imx8ulp/include/platform/drivers/interrupt.h rename to src/kernel/platform/imx8ulp/include/platform/drivers/interrupt.h diff --git a/src/platform/imx8ulp/include/platform/lib/clk.h b/src/kernel/platform/imx8ulp/include/platform/lib/clk.h similarity index 100% rename from src/platform/imx8ulp/include/platform/lib/clk.h rename to src/kernel/platform/imx8ulp/include/platform/lib/clk.h diff --git a/src/platform/imx8ulp/include/platform/lib/cpu.h b/src/kernel/platform/imx8ulp/include/platform/lib/cpu.h similarity index 100% rename from src/platform/imx8ulp/include/platform/lib/cpu.h rename to src/kernel/platform/imx8ulp/include/platform/lib/cpu.h diff --git a/src/platform/imx8ulp/include/platform/lib/dai.h b/src/kernel/platform/imx8ulp/include/platform/lib/dai.h similarity index 100% rename from src/platform/imx8ulp/include/platform/lib/dai.h rename to src/kernel/platform/imx8ulp/include/platform/lib/dai.h diff --git a/src/platform/imx8ulp/include/platform/lib/dma.h b/src/kernel/platform/imx8ulp/include/platform/lib/dma.h similarity index 100% rename from src/platform/imx8ulp/include/platform/lib/dma.h rename to src/kernel/platform/imx8ulp/include/platform/lib/dma.h diff --git a/src/platform/imx8ulp/include/platform/lib/mailbox.h b/src/kernel/platform/imx8ulp/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/imx8ulp/include/platform/lib/mailbox.h rename to src/kernel/platform/imx8ulp/include/platform/lib/mailbox.h diff --git a/src/platform/imx8ulp/include/platform/lib/memory.h b/src/kernel/platform/imx8ulp/include/platform/lib/memory.h similarity index 100% rename from src/platform/imx8ulp/include/platform/lib/memory.h rename to src/kernel/platform/imx8ulp/include/platform/lib/memory.h diff --git a/src/platform/imx8ulp/include/platform/platform.h b/src/kernel/platform/imx8ulp/include/platform/platform.h similarity index 100% rename from src/platform/imx8ulp/include/platform/platform.h rename to src/kernel/platform/imx8ulp/include/platform/platform.h diff --git a/src/platform/imx8ulp/include/platform/trace/trace.h b/src/kernel/platform/imx8ulp/include/platform/trace/trace.h similarity index 100% rename from src/platform/imx8ulp/include/platform/trace/trace.h rename to src/kernel/platform/imx8ulp/include/platform/trace/trace.h diff --git a/src/platform/imx8ulp/lib/clk.c b/src/kernel/platform/imx8ulp/lib/clk.c similarity index 100% rename from src/platform/imx8ulp/lib/clk.c rename to src/kernel/platform/imx8ulp/lib/clk.c diff --git a/src/platform/imx8ulp/platform.c b/src/kernel/platform/imx8ulp/platform.c similarity index 95% rename from src/platform/imx8ulp/platform.c rename to src/kernel/platform/imx8ulp/platform.c index 88e6dfb8d56d..96706dd9854a 100644 --- a/src/platform/imx8ulp/platform.c +++ b/src/kernel/platform/imx8ulp/platform.c @@ -25,9 +25,6 @@ #include #include #include -#if CONFIG_TRACE -#include -#endif #include #include #include @@ -171,12 +168,6 @@ int platform_init(struct sof *sof) if (ret < 0) return -ENODEV; -#if CONFIG_TRACE - /* Initialize DMA for Trace*/ - trace_point(TRACE_BOOT_PLATFORM_DMA_TRACE); - dma_trace_init_complete(sof->dmat); -#endif - /* show heap status */ heap_trace_all(1); diff --git a/src/platform/imx93_a55/include/platform/drivers/idc.h b/src/kernel/platform/imx93_a55/include/platform/drivers/idc.h similarity index 100% rename from src/platform/imx93_a55/include/platform/drivers/idc.h rename to src/kernel/platform/imx93_a55/include/platform/drivers/idc.h diff --git a/src/platform/imx93_a55/include/platform/lib/clk.h b/src/kernel/platform/imx93_a55/include/platform/lib/clk.h similarity index 100% rename from src/platform/imx93_a55/include/platform/lib/clk.h rename to src/kernel/platform/imx93_a55/include/platform/lib/clk.h diff --git a/src/platform/imx93_a55/include/platform/lib/dai.h b/src/kernel/platform/imx93_a55/include/platform/lib/dai.h similarity index 100% rename from src/platform/imx93_a55/include/platform/lib/dai.h rename to src/kernel/platform/imx93_a55/include/platform/lib/dai.h diff --git a/src/platform/imx93_a55/include/platform/lib/dma.h b/src/kernel/platform/imx93_a55/include/platform/lib/dma.h similarity index 100% rename from src/platform/imx93_a55/include/platform/lib/dma.h rename to src/kernel/platform/imx93_a55/include/platform/lib/dma.h diff --git a/src/platform/imx93_a55/include/platform/lib/mailbox.h b/src/kernel/platform/imx93_a55/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/imx93_a55/include/platform/lib/mailbox.h rename to src/kernel/platform/imx93_a55/include/platform/lib/mailbox.h diff --git a/src/platform/imx93_a55/include/platform/lib/memory.h b/src/kernel/platform/imx93_a55/include/platform/lib/memory.h similarity index 100% rename from src/platform/imx93_a55/include/platform/lib/memory.h rename to src/kernel/platform/imx93_a55/include/platform/lib/memory.h diff --git a/src/platform/imx93_a55/include/platform/platform.h b/src/kernel/platform/imx93_a55/include/platform/platform.h similarity index 100% rename from src/platform/imx93_a55/include/platform/platform.h rename to src/kernel/platform/imx93_a55/include/platform/platform.h diff --git a/src/platform/imx93_a55/include/platform/trace/trace.h b/src/kernel/platform/imx93_a55/include/platform/trace/trace.h similarity index 100% rename from src/platform/imx93_a55/include/platform/trace/trace.h rename to src/kernel/platform/imx93_a55/include/platform/trace/trace.h diff --git a/src/platform/imx93_a55/lib/clk.c b/src/kernel/platform/imx93_a55/lib/clk.c similarity index 100% rename from src/platform/imx93_a55/lib/clk.c rename to src/kernel/platform/imx93_a55/lib/clk.c diff --git a/src/platform/imx93_a55/platform.c b/src/kernel/platform/imx93_a55/platform.c similarity index 100% rename from src/platform/imx93_a55/platform.c rename to src/kernel/platform/imx93_a55/platform.c diff --git a/src/platform/imx95/include/platform/lib/clk.h b/src/kernel/platform/imx95/include/platform/lib/clk.h similarity index 100% rename from src/platform/imx95/include/platform/lib/clk.h rename to src/kernel/platform/imx95/include/platform/lib/clk.h diff --git a/src/platform/imx95/include/platform/lib/dma.h b/src/kernel/platform/imx95/include/platform/lib/dma.h similarity index 100% rename from src/platform/imx95/include/platform/lib/dma.h rename to src/kernel/platform/imx95/include/platform/lib/dma.h diff --git a/src/platform/imx95/include/platform/lib/mailbox.h b/src/kernel/platform/imx95/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/imx95/include/platform/lib/mailbox.h rename to src/kernel/platform/imx95/include/platform/lib/mailbox.h diff --git a/src/platform/imx95/include/platform/lib/memory.h b/src/kernel/platform/imx95/include/platform/lib/memory.h similarity index 100% rename from src/platform/imx95/include/platform/lib/memory.h rename to src/kernel/platform/imx95/include/platform/lib/memory.h diff --git a/src/platform/imx95/include/platform/platform.h b/src/kernel/platform/imx95/include/platform/platform.h similarity index 100% rename from src/platform/imx95/include/platform/platform.h rename to src/kernel/platform/imx95/include/platform/platform.h diff --git a/src/platform/imx95/include/platform/trace/trace.h b/src/kernel/platform/imx95/include/platform/trace/trace.h similarity index 100% rename from src/platform/imx95/include/platform/trace/trace.h rename to src/kernel/platform/imx95/include/platform/trace/trace.h diff --git a/src/platform/imx95/lib/clk.c b/src/kernel/platform/imx95/lib/clk.c similarity index 100% rename from src/platform/imx95/lib/clk.c rename to src/kernel/platform/imx95/lib/clk.c diff --git a/src/platform/imx95/linker/data-sections.ld b/src/kernel/platform/imx95/linker/data-sections.ld similarity index 100% rename from src/platform/imx95/linker/data-sections.ld rename to src/kernel/platform/imx95/linker/data-sections.ld diff --git a/src/platform/imx95/platform.c b/src/kernel/platform/imx95/platform.c similarity index 100% rename from src/platform/imx95/platform.c rename to src/kernel/platform/imx95/platform.c diff --git a/src/platform/intel/ace/include/ace/lib/clk.h b/src/kernel/platform/intel/ace/include/ace/lib/clk.h similarity index 100% rename from src/platform/intel/ace/include/ace/lib/clk.h rename to src/kernel/platform/intel/ace/include/ace/lib/clk.h diff --git a/src/platform/intel/ace/include/ace/lib/mailbox.h b/src/kernel/platform/intel/ace/include/ace/lib/mailbox.h similarity index 100% rename from src/platform/intel/ace/include/ace/lib/mailbox.h rename to src/kernel/platform/intel/ace/include/ace/lib/mailbox.h diff --git a/src/platform/intel/ace/include/ace/lib/memory.h b/src/kernel/platform/intel/ace/include/ace/lib/memory.h similarity index 100% rename from src/platform/intel/ace/include/ace/lib/memory.h rename to src/kernel/platform/intel/ace/include/ace/lib/memory.h diff --git a/src/platform/intel/ace/include/ace/version.h b/src/kernel/platform/intel/ace/include/ace/version.h similarity index 100% rename from src/platform/intel/ace/include/ace/version.h rename to src/kernel/platform/intel/ace/include/ace/version.h diff --git a/src/platform/intel/ace/include/auth/intel_auth_api.h b/src/kernel/platform/intel/ace/include/auth/intel_auth_api.h similarity index 100% rename from src/platform/intel/ace/include/auth/intel_auth_api.h rename to src/kernel/platform/intel/ace/include/auth/intel_auth_api.h diff --git a/src/platform/intel/ace/include/auth/intel_status_logger_api.h b/src/kernel/platform/intel/ace/include/auth/intel_status_logger_api.h similarity index 100% rename from src/platform/intel/ace/include/auth/intel_status_logger_api.h rename to src/kernel/platform/intel/ace/include/auth/intel_status_logger_api.h diff --git a/src/platform/intel/ace/include/auth/intel_status_logger_iface.h b/src/kernel/platform/intel/ace/include/auth/intel_status_logger_iface.h similarity index 100% rename from src/platform/intel/ace/include/auth/intel_status_logger_iface.h rename to src/kernel/platform/intel/ace/include/auth/intel_status_logger_iface.h diff --git a/src/platform/intel/ace/lib/watchdog.c b/src/kernel/platform/intel/ace/lib/watchdog.c similarity index 100% rename from src/platform/intel/ace/lib/watchdog.c rename to src/kernel/platform/intel/ace/lib/watchdog.c diff --git a/src/platform/intel/ace/platform.c b/src/kernel/platform/intel/ace/platform.c similarity index 100% rename from src/platform/intel/ace/platform.c rename to src/kernel/platform/intel/ace/platform.c diff --git a/src/platform/intel/cavs/include/cavs/drivers/mn.h b/src/kernel/platform/intel/cavs/include/cavs/drivers/mn.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/drivers/mn.h rename to src/kernel/platform/intel/cavs/include/cavs/drivers/mn.h diff --git a/src/platform/intel/cavs/include/cavs/drivers/sideband-ipc.h b/src/kernel/platform/intel/cavs/include/cavs/drivers/sideband-ipc.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/drivers/sideband-ipc.h rename to src/kernel/platform/intel/cavs/include/cavs/drivers/sideband-ipc.h diff --git a/src/platform/intel/cavs/include/cavs/drivers/timestamp.h b/src/kernel/platform/intel/cavs/include/cavs/drivers/timestamp.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/drivers/timestamp.h rename to src/kernel/platform/intel/cavs/include/cavs/drivers/timestamp.h diff --git a/src/platform/intel/cavs/include/cavs/ext_manifest.h b/src/kernel/platform/intel/cavs/include/cavs/ext_manifest.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/ext_manifest.h rename to src/kernel/platform/intel/cavs/include/cavs/ext_manifest.h diff --git a/src/platform/intel/cavs/include/cavs/lib/mailbox.h b/src/kernel/platform/intel/cavs/include/cavs/lib/mailbox.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/lib/mailbox.h rename to src/kernel/platform/intel/cavs/include/cavs/lib/mailbox.h diff --git a/src/platform/intel/cavs/include/cavs/lib/memory.h b/src/kernel/platform/intel/cavs/include/cavs/lib/memory.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/lib/memory.h rename to src/kernel/platform/intel/cavs/include/cavs/lib/memory.h diff --git a/src/platform/intel/cavs/include/cavs/lib/power_down.h b/src/kernel/platform/intel/cavs/include/cavs/lib/power_down.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/lib/power_down.h rename to src/kernel/platform/intel/cavs/include/cavs/lib/power_down.h diff --git a/src/platform/intel/cavs/include/cavs/mem_window.h b/src/kernel/platform/intel/cavs/include/cavs/mem_window.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/mem_window.h rename to src/kernel/platform/intel/cavs/include/cavs/mem_window.h diff --git a/src/platform/intel/cavs/include/cavs/trace/trace.h b/src/kernel/platform/intel/cavs/include/cavs/trace/trace.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/trace/trace.h rename to src/kernel/platform/intel/cavs/include/cavs/trace/trace.h diff --git a/src/platform/intel/cavs/include/cavs/version.h b/src/kernel/platform/intel/cavs/include/cavs/version.h similarity index 100% rename from src/platform/intel/cavs/include/cavs/version.h rename to src/kernel/platform/intel/cavs/include/cavs/version.h diff --git a/src/platform/intel/cavs/platform.c b/src/kernel/platform/intel/cavs/platform.c similarity index 100% rename from src/platform/intel/cavs/platform.c rename to src/kernel/platform/intel/cavs/platform.c diff --git a/src/platform/library/CMakeLists.txt b/src/kernel/platform/library/CMakeLists.txt similarity index 100% rename from src/platform/library/CMakeLists.txt rename to src/kernel/platform/library/CMakeLists.txt diff --git a/src/platform/library/include/platform/drivers/idc.h b/src/kernel/platform/library/include/platform/drivers/idc.h similarity index 100% rename from src/platform/library/include/platform/drivers/idc.h rename to src/kernel/platform/library/include/platform/drivers/idc.h diff --git a/src/platform/library/include/platform/drivers/interrupt.h b/src/kernel/platform/library/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/library/include/platform/drivers/interrupt.h rename to src/kernel/platform/library/include/platform/drivers/interrupt.h diff --git a/src/platform/library/include/platform/drivers/pmc.h b/src/kernel/platform/library/include/platform/drivers/pmc.h similarity index 100% rename from src/platform/library/include/platform/drivers/pmc.h rename to src/kernel/platform/library/include/platform/drivers/pmc.h diff --git a/src/platform/library/include/platform/lib/clk.h b/src/kernel/platform/library/include/platform/lib/clk.h similarity index 100% rename from src/platform/library/include/platform/lib/clk.h rename to src/kernel/platform/library/include/platform/lib/clk.h diff --git a/src/platform/library/include/platform/lib/cpu.h b/src/kernel/platform/library/include/platform/lib/cpu.h similarity index 100% rename from src/platform/library/include/platform/lib/cpu.h rename to src/kernel/platform/library/include/platform/lib/cpu.h diff --git a/src/platform/library/include/platform/lib/dai.h b/src/kernel/platform/library/include/platform/lib/dai.h similarity index 100% rename from src/platform/library/include/platform/lib/dai.h rename to src/kernel/platform/library/include/platform/lib/dai.h diff --git a/src/platform/library/include/platform/lib/dma.h b/src/kernel/platform/library/include/platform/lib/dma.h similarity index 100% rename from src/platform/library/include/platform/lib/dma.h rename to src/kernel/platform/library/include/platform/lib/dma.h diff --git a/src/platform/library/include/platform/lib/ll_schedule.h b/src/kernel/platform/library/include/platform/lib/ll_schedule.h similarity index 100% rename from src/platform/library/include/platform/lib/ll_schedule.h rename to src/kernel/platform/library/include/platform/lib/ll_schedule.h diff --git a/src/platform/library/include/platform/lib/mailbox.h b/src/kernel/platform/library/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/library/include/platform/lib/mailbox.h rename to src/kernel/platform/library/include/platform/lib/mailbox.h diff --git a/src/platform/library/include/platform/lib/memory.h b/src/kernel/platform/library/include/platform/lib/memory.h similarity index 100% rename from src/platform/library/include/platform/lib/memory.h rename to src/kernel/platform/library/include/platform/lib/memory.h diff --git a/src/platform/library/include/platform/lib/pm_runtime.h b/src/kernel/platform/library/include/platform/lib/pm_runtime.h similarity index 100% rename from src/platform/library/include/platform/lib/pm_runtime.h rename to src/kernel/platform/library/include/platform/lib/pm_runtime.h diff --git a/src/platform/library/include/platform/platform.h b/src/kernel/platform/library/include/platform/platform.h similarity index 100% rename from src/platform/library/include/platform/platform.h rename to src/kernel/platform/library/include/platform/platform.h diff --git a/src/platform/library/init.c b/src/kernel/platform/library/init.c similarity index 100% rename from src/platform/library/init.c rename to src/kernel/platform/library/init.c diff --git a/src/platform/library/lib/CMakeLists.txt b/src/kernel/platform/library/lib/CMakeLists.txt similarity index 65% rename from src/platform/library/lib/CMakeLists.txt rename to src/kernel/platform/library/lib/CMakeLists.txt index 05965cd482ee..b8d0224bc4aa 100644 --- a/src/platform/library/lib/CMakeLists.txt +++ b/src/kernel/platform/library/lib/CMakeLists.txt @@ -6,8 +6,5 @@ add_local_sources(sof dai.c pm_runtime.c memory.c + trace.c ) - -if (CONFIG_TRACE) - add_local_sources(sof trace.c) -endif() diff --git a/src/platform/library/lib/alloc.c b/src/kernel/platform/library/lib/alloc.c similarity index 100% rename from src/platform/library/lib/alloc.c rename to src/kernel/platform/library/lib/alloc.c diff --git a/src/platform/library/lib/clk.c b/src/kernel/platform/library/lib/clk.c similarity index 100% rename from src/platform/library/lib/clk.c rename to src/kernel/platform/library/lib/clk.c diff --git a/src/platform/library/lib/dai.c b/src/kernel/platform/library/lib/dai.c similarity index 100% rename from src/platform/library/lib/dai.c rename to src/kernel/platform/library/lib/dai.c diff --git a/src/platform/library/lib/memory.c b/src/kernel/platform/library/lib/memory.c similarity index 100% rename from src/platform/library/lib/memory.c rename to src/kernel/platform/library/lib/memory.c diff --git a/src/platform/library/lib/pm_runtime.c b/src/kernel/platform/library/lib/pm_runtime.c similarity index 100% rename from src/platform/library/lib/pm_runtime.c rename to src/kernel/platform/library/lib/pm_runtime.c diff --git a/src/platform/library/lib/trace.c b/src/kernel/platform/library/lib/trace.c similarity index 95% rename from src/platform/library/lib/trace.c rename to src/kernel/platform/library/lib/trace.c index 7ca847afabec..c949a81f5f41 100644 --- a/src/platform/library/lib/trace.c +++ b/src/kernel/platform/library/lib/trace.c @@ -9,7 +9,6 @@ #include #include -#include /* enable trace by default in testbench */ int host_trace_level = LOG_LEVEL_ERROR; diff --git a/src/platform/library/panic.c b/src/kernel/platform/library/panic.c similarity index 100% rename from src/platform/library/panic.c rename to src/kernel/platform/library/panic.c diff --git a/src/platform/library/platform.c b/src/kernel/platform/library/platform.c similarity index 100% rename from src/platform/library/platform.c rename to src/kernel/platform/library/platform.c diff --git a/src/platform/library/schedule/CMakeLists.txt b/src/kernel/platform/library/schedule/CMakeLists.txt similarity index 100% rename from src/platform/library/schedule/CMakeLists.txt rename to src/kernel/platform/library/schedule/CMakeLists.txt diff --git a/src/platform/library/schedule/edf_schedule.c b/src/kernel/platform/library/schedule/edf_schedule.c similarity index 100% rename from src/platform/library/schedule/edf_schedule.c rename to src/kernel/platform/library/schedule/edf_schedule.c diff --git a/src/platform/library/schedule/ll_schedule.c b/src/kernel/platform/library/schedule/ll_schedule.c similarity index 100% rename from src/platform/library/schedule/ll_schedule.c rename to src/kernel/platform/library/schedule/ll_schedule.c diff --git a/src/platform/library/schedule/schedule.c b/src/kernel/platform/library/schedule/schedule.c similarity index 100% rename from src/platform/library/schedule/schedule.c rename to src/kernel/platform/library/schedule/schedule.c diff --git a/src/platform/lunarlake/include/platform/lib/clk.h b/src/kernel/platform/lunarlake/include/platform/lib/clk.h similarity index 100% rename from src/platform/lunarlake/include/platform/lib/clk.h rename to src/kernel/platform/lunarlake/include/platform/lib/clk.h diff --git a/src/platform/lunarlake/include/platform/lib/mailbox.h b/src/kernel/platform/lunarlake/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/lunarlake/include/platform/lib/mailbox.h rename to src/kernel/platform/lunarlake/include/platform/lib/mailbox.h diff --git a/src/platform/lunarlake/include/platform/lib/memory.h b/src/kernel/platform/lunarlake/include/platform/lib/memory.h similarity index 100% rename from src/platform/lunarlake/include/platform/lib/memory.h rename to src/kernel/platform/lunarlake/include/platform/lib/memory.h diff --git a/src/platform/lunarlake/include/platform/platform.h b/src/kernel/platform/lunarlake/include/platform/platform.h similarity index 100% rename from src/platform/lunarlake/include/platform/platform.h rename to src/kernel/platform/lunarlake/include/platform/platform.h diff --git a/src/platform/lunarlake/include/platform/trace/trace.h b/src/kernel/platform/lunarlake/include/platform/trace/trace.h similarity index 100% rename from src/platform/lunarlake/include/platform/trace/trace.h rename to src/kernel/platform/lunarlake/include/platform/trace/trace.h diff --git a/src/platform/lunarlake/lib/clk.c b/src/kernel/platform/lunarlake/lib/clk.c similarity index 100% rename from src/platform/lunarlake/lib/clk.c rename to src/kernel/platform/lunarlake/lib/clk.c diff --git a/src/platform/meteorlake/include/platform/lib/clk.h b/src/kernel/platform/meteorlake/include/platform/lib/clk.h similarity index 100% rename from src/platform/meteorlake/include/platform/lib/clk.h rename to src/kernel/platform/meteorlake/include/platform/lib/clk.h diff --git a/src/platform/meteorlake/include/platform/lib/mailbox.h b/src/kernel/platform/meteorlake/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/meteorlake/include/platform/lib/mailbox.h rename to src/kernel/platform/meteorlake/include/platform/lib/mailbox.h diff --git a/src/platform/meteorlake/include/platform/lib/memory.h b/src/kernel/platform/meteorlake/include/platform/lib/memory.h similarity index 100% rename from src/platform/meteorlake/include/platform/lib/memory.h rename to src/kernel/platform/meteorlake/include/platform/lib/memory.h diff --git a/src/platform/meteorlake/include/platform/platform.h b/src/kernel/platform/meteorlake/include/platform/platform.h similarity index 100% rename from src/platform/meteorlake/include/platform/platform.h rename to src/kernel/platform/meteorlake/include/platform/platform.h diff --git a/src/platform/meteorlake/include/platform/trace/trace.h b/src/kernel/platform/meteorlake/include/platform/trace/trace.h similarity index 100% rename from src/platform/meteorlake/include/platform/trace/trace.h rename to src/kernel/platform/meteorlake/include/platform/trace/trace.h diff --git a/src/platform/meteorlake/lib/clk.c b/src/kernel/platform/meteorlake/lib/clk.c similarity index 100% rename from src/platform/meteorlake/lib/clk.c rename to src/kernel/platform/meteorlake/lib/clk.c diff --git a/src/platform/mtk/dai.c b/src/kernel/platform/mtk/dai.c similarity index 100% rename from src/platform/mtk/dai.c rename to src/kernel/platform/mtk/dai.c diff --git a/src/platform/mtk/include/platform/lib/clk.h b/src/kernel/platform/mtk/include/platform/lib/clk.h similarity index 100% rename from src/platform/mtk/include/platform/lib/clk.h rename to src/kernel/platform/mtk/include/platform/lib/clk.h diff --git a/src/platform/mtk/include/platform/lib/cpu.h b/src/kernel/platform/mtk/include/platform/lib/cpu.h similarity index 100% rename from src/platform/mtk/include/platform/lib/cpu.h rename to src/kernel/platform/mtk/include/platform/lib/cpu.h diff --git a/src/platform/mtk/include/platform/lib/dai.h b/src/kernel/platform/mtk/include/platform/lib/dai.h similarity index 100% rename from src/platform/mtk/include/platform/lib/dai.h rename to src/kernel/platform/mtk/include/platform/lib/dai.h diff --git a/src/platform/mtk/include/platform/lib/dma.h b/src/kernel/platform/mtk/include/platform/lib/dma.h similarity index 100% rename from src/platform/mtk/include/platform/lib/dma.h rename to src/kernel/platform/mtk/include/platform/lib/dma.h diff --git a/src/platform/mtk/include/platform/lib/mailbox.h b/src/kernel/platform/mtk/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/mtk/include/platform/lib/mailbox.h rename to src/kernel/platform/mtk/include/platform/lib/mailbox.h diff --git a/src/platform/mtk/include/platform/lib/memory.h b/src/kernel/platform/mtk/include/platform/lib/memory.h similarity index 100% rename from src/platform/mtk/include/platform/lib/memory.h rename to src/kernel/platform/mtk/include/platform/lib/memory.h diff --git a/src/platform/mtk/include/platform/lib/pm_runtime.h b/src/kernel/platform/mtk/include/platform/lib/pm_runtime.h similarity index 100% rename from src/platform/mtk/include/platform/lib/pm_runtime.h rename to src/kernel/platform/mtk/include/platform/lib/pm_runtime.h diff --git a/src/platform/mtk/include/platform/platform.h b/src/kernel/platform/mtk/include/platform/platform.h similarity index 100% rename from src/platform/mtk/include/platform/platform.h rename to src/kernel/platform/mtk/include/platform/platform.h diff --git a/src/platform/mtk/include/platform/trace/trace.h b/src/kernel/platform/mtk/include/platform/trace/trace.h similarity index 100% rename from src/platform/mtk/include/platform/trace/trace.h rename to src/kernel/platform/mtk/include/platform/trace/trace.h diff --git a/src/platform/mtk/platform.c b/src/kernel/platform/mtk/platform.c similarity index 100% rename from src/platform/mtk/platform.c rename to src/kernel/platform/mtk/platform.c diff --git a/src/platform/mtk/tools/README.txt b/src/kernel/platform/mtk/tools/README.txt similarity index 100% rename from src/platform/mtk/tools/README.txt rename to src/kernel/platform/mtk/tools/README.txt diff --git a/src/platform/mtk/tools/build.sh b/src/kernel/platform/mtk/tools/build.sh similarity index 61% rename from src/platform/mtk/tools/build.sh rename to src/kernel/platform/mtk/tools/build.sh index 1b17c8e38cf8..e38d16b7051b 100755 --- a/src/platform/mtk/tools/build.sh +++ b/src/kernel/platform/mtk/tools/build.sh @@ -13,11 +13,11 @@ SOF=`cd ../../../..; /bin/pwd` for p in $PLATFORMS; do - SRCS="$SOF/src/platform/$p/lib/dai.c $SOF/src/platform/$p/afe-platform.c" + SRCS="$SOF/src/kernel/platform/$p/lib/dai.c $SOF/src/kernel/platform/$p/afe-platform.c" - INCS="-I$SOF/src/include -I$SOF/src/platform/posix/include -I$SOF/posix/include" - INCS="$INCS -I$SOF/src/arch/host/include -I$SOF/src/platform/$p/include/platform" - INCS="$INCS -I$SOF/src/platform/$p/include" + INCS="-I$SOF/src/include -I$SOF/src/kernel/platform/posix/include -I$SOF/posix/include" + INCS="$INCS -I$SOF/src/kernel/arch/host/include -I$SOF/src/kernel/platform/$p/include/platform" + INCS="$INCS -I$SOF/src/kernel/platform/$p/include" DEFS="-DRELATIVE_FILE=\"mt-dai-gen.c\" -DCONFIG_CORE_COUNT=1 -DCONFIG_IPC_MAJOR_3=1" diff --git a/src/platform/mtk/tools/mt-dai-gen.c b/src/kernel/platform/mtk/tools/mt-dai-gen.c similarity index 100% rename from src/platform/mtk/tools/mt-dai-gen.c rename to src/kernel/platform/mtk/tools/mt-dai-gen.c diff --git a/src/platform/novalake/include/platform/lib/clk.h b/src/kernel/platform/novalake/include/platform/lib/clk.h similarity index 100% rename from src/platform/novalake/include/platform/lib/clk.h rename to src/kernel/platform/novalake/include/platform/lib/clk.h diff --git a/src/platform/novalake/include/platform/lib/mailbox.h b/src/kernel/platform/novalake/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/novalake/include/platform/lib/mailbox.h rename to src/kernel/platform/novalake/include/platform/lib/mailbox.h diff --git a/src/platform/novalake/include/platform/lib/memory.h b/src/kernel/platform/novalake/include/platform/lib/memory.h similarity index 100% rename from src/platform/novalake/include/platform/lib/memory.h rename to src/kernel/platform/novalake/include/platform/lib/memory.h diff --git a/src/platform/novalake/include/platform/platform.h b/src/kernel/platform/novalake/include/platform/platform.h similarity index 100% rename from src/platform/novalake/include/platform/platform.h rename to src/kernel/platform/novalake/include/platform/platform.h diff --git a/src/platform/novalake/include/platform/trace/trace.h b/src/kernel/platform/novalake/include/platform/trace/trace.h similarity index 100% rename from src/platform/novalake/include/platform/trace/trace.h rename to src/kernel/platform/novalake/include/platform/trace/trace.h diff --git a/src/platform/novalake/lib/clk.c b/src/kernel/platform/novalake/lib/clk.c similarity index 100% rename from src/platform/novalake/lib/clk.c rename to src/kernel/platform/novalake/lib/clk.c diff --git a/src/platform/posix/dai.c b/src/kernel/platform/posix/dai.c similarity index 100% rename from src/platform/posix/dai.c rename to src/kernel/platform/posix/dai.c diff --git a/src/platform/posix/dma.c b/src/kernel/platform/posix/dma.c similarity index 100% rename from src/platform/posix/dma.c rename to src/kernel/platform/posix/dma.c diff --git a/src/platform/posix/fuzz.c b/src/kernel/platform/posix/fuzz.c similarity index 100% rename from src/platform/posix/fuzz.c rename to src/kernel/platform/posix/fuzz.c diff --git a/src/platform/posix/include/platform/drivers/idc.h b/src/kernel/platform/posix/include/platform/drivers/idc.h similarity index 100% rename from src/platform/posix/include/platform/drivers/idc.h rename to src/kernel/platform/posix/include/platform/drivers/idc.h diff --git a/src/platform/posix/include/platform/drivers/interrupt.h b/src/kernel/platform/posix/include/platform/drivers/interrupt.h similarity index 100% rename from src/platform/posix/include/platform/drivers/interrupt.h rename to src/kernel/platform/posix/include/platform/drivers/interrupt.h diff --git a/src/platform/posix/include/platform/fuzz.h b/src/kernel/platform/posix/include/platform/fuzz.h similarity index 100% rename from src/platform/posix/include/platform/fuzz.h rename to src/kernel/platform/posix/include/platform/fuzz.h diff --git a/src/platform/posix/include/platform/lib/clk.h b/src/kernel/platform/posix/include/platform/lib/clk.h similarity index 100% rename from src/platform/posix/include/platform/lib/clk.h rename to src/kernel/platform/posix/include/platform/lib/clk.h diff --git a/src/platform/posix/include/platform/lib/cpu.h b/src/kernel/platform/posix/include/platform/lib/cpu.h similarity index 100% rename from src/platform/posix/include/platform/lib/cpu.h rename to src/kernel/platform/posix/include/platform/lib/cpu.h diff --git a/src/platform/posix/include/platform/lib/dai.h b/src/kernel/platform/posix/include/platform/lib/dai.h similarity index 100% rename from src/platform/posix/include/platform/lib/dai.h rename to src/kernel/platform/posix/include/platform/lib/dai.h diff --git a/src/platform/posix/include/platform/lib/dma.h b/src/kernel/platform/posix/include/platform/lib/dma.h similarity index 100% rename from src/platform/posix/include/platform/lib/dma.h rename to src/kernel/platform/posix/include/platform/lib/dma.h diff --git a/src/platform/posix/include/platform/lib/mailbox.h b/src/kernel/platform/posix/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/posix/include/platform/lib/mailbox.h rename to src/kernel/platform/posix/include/platform/lib/mailbox.h diff --git a/src/platform/posix/include/platform/lib/memory.h b/src/kernel/platform/posix/include/platform/lib/memory.h similarity index 100% rename from src/platform/posix/include/platform/lib/memory.h rename to src/kernel/platform/posix/include/platform/lib/memory.h diff --git a/src/platform/posix/include/platform/lib/pm_runtime.h b/src/kernel/platform/posix/include/platform/lib/pm_runtime.h similarity index 100% rename from src/platform/posix/include/platform/lib/pm_runtime.h rename to src/kernel/platform/posix/include/platform/lib/pm_runtime.h diff --git a/src/platform/posix/include/platform/platform.h b/src/kernel/platform/posix/include/platform/platform.h similarity index 100% rename from src/platform/posix/include/platform/platform.h rename to src/kernel/platform/posix/include/platform/platform.h diff --git a/src/platform/posix/include/platform/trace/trace.h b/src/kernel/platform/posix/include/platform/trace/trace.h similarity index 100% rename from src/platform/posix/include/platform/trace/trace.h rename to src/kernel/platform/posix/include/platform/trace/trace.h diff --git a/src/platform/posix/ipc.c b/src/kernel/platform/posix/ipc.c similarity index 100% rename from src/platform/posix/ipc.c rename to src/kernel/platform/posix/ipc.c diff --git a/src/platform/posix/posix.c b/src/kernel/platform/posix/posix.c similarity index 100% rename from src/platform/posix/posix.c rename to src/kernel/platform/posix/posix.c diff --git a/src/platform/qemu_xtensa/include/platform/lib/clk.h b/src/kernel/platform/qemu_xtensa/include/platform/lib/clk.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/lib/clk.h rename to src/kernel/platform/qemu_xtensa/include/platform/lib/clk.h diff --git a/src/platform/qemu_xtensa/include/platform/lib/dai.h b/src/kernel/platform/qemu_xtensa/include/platform/lib/dai.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/lib/dai.h rename to src/kernel/platform/qemu_xtensa/include/platform/lib/dai.h diff --git a/src/platform/qemu_xtensa/include/platform/lib/dma.h b/src/kernel/platform/qemu_xtensa/include/platform/lib/dma.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/lib/dma.h rename to src/kernel/platform/qemu_xtensa/include/platform/lib/dma.h diff --git a/src/platform/qemu_xtensa/include/platform/lib/mailbox.h b/src/kernel/platform/qemu_xtensa/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/lib/mailbox.h rename to src/kernel/platform/qemu_xtensa/include/platform/lib/mailbox.h diff --git a/src/platform/qemu_xtensa/include/platform/lib/memory.h b/src/kernel/platform/qemu_xtensa/include/platform/lib/memory.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/lib/memory.h rename to src/kernel/platform/qemu_xtensa/include/platform/lib/memory.h diff --git a/src/platform/qemu_xtensa/include/platform/platform.h b/src/kernel/platform/qemu_xtensa/include/platform/platform.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/platform.h rename to src/kernel/platform/qemu_xtensa/include/platform/platform.h diff --git a/src/platform/qemu_xtensa/include/platform/trace/trace.h b/src/kernel/platform/qemu_xtensa/include/platform/trace/trace.h similarity index 100% rename from src/platform/qemu_xtensa/include/platform/trace/trace.h rename to src/kernel/platform/qemu_xtensa/include/platform/trace/trace.h diff --git a/src/platform/qemu_xtensa/platform.c b/src/kernel/platform/qemu_xtensa/platform.c similarity index 100% rename from src/platform/qemu_xtensa/platform.c rename to src/kernel/platform/qemu_xtensa/platform.c diff --git a/src/platform/tigerlake/include/platform/lib/mailbox.h b/src/kernel/platform/tigerlake/include/platform/lib/mailbox.h similarity index 100% rename from src/platform/tigerlake/include/platform/lib/mailbox.h rename to src/kernel/platform/tigerlake/include/platform/lib/mailbox.h diff --git a/src/platform/tigerlake/include/platform/lib/memory.h b/src/kernel/platform/tigerlake/include/platform/lib/memory.h similarity index 99% rename from src/platform/tigerlake/include/platform/lib/memory.h rename to src/kernel/platform/tigerlake/include/platform/lib/memory.h index 63802920ac95..bf82e2483b8f 100644 --- a/src/platform/tigerlake/include/platform/lib/memory.h +++ b/src/kernel/platform/tigerlake/include/platform/lib/memory.h @@ -208,11 +208,7 @@ /* window 3 */ #define SRAM_TRACE_BASE (SRAM_STREAM_BASE + SRAM_STREAM_SIZE) -#if CONFIG_TRACE -#define SRAM_TRACE_SIZE 0x2000 -#else #define SRAM_TRACE_SIZE 0x0 -#endif #define HP_SRAM_WIN0_BASE SRAM_SW_REG_BASE #define HP_SRAM_WIN0_SIZE (SRAM_SW_REG_SIZE + SRAM_OUTBOX_SIZE) diff --git a/src/platform/tigerlake/include/platform/platform.h b/src/kernel/platform/tigerlake/include/platform/platform.h similarity index 100% rename from src/platform/tigerlake/include/platform/platform.h rename to src/kernel/platform/tigerlake/include/platform/platform.h diff --git a/src/platform/tigerlake/include/platform/trace/trace.h b/src/kernel/platform/tigerlake/include/platform/trace/trace.h similarity index 100% rename from src/platform/tigerlake/include/platform/trace/trace.h rename to src/kernel/platform/tigerlake/include/platform/trace/trace.h diff --git a/src/kernel/schedule/CMakeLists.txt b/src/kernel/schedule/CMakeLists.txt index 277378449165..8711fde99b95 100644 --- a/src/kernel/schedule/CMakeLists.txt +++ b/src/kernel/schedule/CMakeLists.txt @@ -2,7 +2,36 @@ is_zephyr(zephyr) if(zephyr) + zephyr_library_sources( + schedule-kernel.c + zephyr_domain-kernel.c + ) + + # The kernel part of zephyr_ll.c, built with it, see src/schedule/CMakeLists.txt + if(NOT CONFIG_SOC_ACP_6_0 OR CONFIG_DMA_DOMAIN) + zephyr_library_sources( + zephyr_ll-kernel.c + ) + endif() + zephyr_library_sources_ifdef(CONFIG_SOF_USERSPACE_LL zephyr_ll_user.c ) + + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER + zephyr_dp_schedule-kernel.c + ) + if(CONFIG_SOF_USERSPACE_APPLICATION) + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER + zephyr_dp_schedule_application-kernel.c + ) + else() + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_DP_SCHEDULER + zephyr_dp_schedule_thread-kernel.c + ) + endif() + + zephyr_library_sources_ifdef(CONFIG_ZEPHYR_TWB_SCHEDULER + zephyr_twb_schedule.c + ) endif() diff --git a/src/kernel/schedule/schedule-kernel.c b/src/kernel/schedule/schedule-kernel.c new file mode 100644 index 000000000000..bdefa7f35567 --- /dev/null +++ b/src/kernel/schedule/schedule-kernel.c @@ -0,0 +1,81 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2019 Intel Corporation. All rights reserved. +// +// Author: Bartosz Kokoszko + +/* Generic scheduler */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(schedule, CONFIG_SOF_LOG_LEVEL); + +extern struct tr_ctx sch_tr; + +/* Kernel part of the generic scheduler: scheduler registration. The rest is in + * src/schedule/schedule.c + */ + +__cold static void scheduler_register(struct schedule_data *scheduler) +{ + struct schedulers **sch = arch_schedulers_get(); + struct k_heap *heap = NULL; + + assert_can_be_cold(); + + if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(scheduler->type)) { + sch = arch_user_schedulers_get(); + heap = sof_sys_user_heap_get(); + } + + if (!*sch) { + /* init schedulers list */ + *sch = sof_heap_alloc(heap, 0, sizeof(**sch), 0); + if (!*sch) { + tr_err(&sch_tr, "allocation failed"); + return; + } + memset(*sch, 0, sizeof(**sch)); + list_init(&(*sch)->list); + } + + list_item_append(&scheduler->list, &(*sch)->list); +} + +__cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data) +{ + struct schedule_data *sch; + struct k_heap *heap = NULL; + + assert_can_be_cold(); + + if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(type)) + heap = sof_sys_user_heap_get(); + + if (!ops || !ops->schedule_task || !ops->schedule_task_cancel || + !ops->schedule_task_free) + return; + + sch = sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, sizeof(*sch), 0); + if (!sch) { + tr_err(&sch_tr, "allocation failed"); + sof_panic(SOF_IPC_PANIC_IPC); + } + memset(sch, 0, sizeof(*sch)); + list_init(&sch->list); + sch->type = type; + sch->ops = ops; + sch->data = data; + + scheduler_register(sch); +} diff --git a/src/kernel/schedule/zephyr_domain-kernel.c b/src/kernel/schedule/zephyr_domain-kernel.c new file mode 100644 index 000000000000..de320d93c9e4 --- /dev/null +++ b/src/kernel/schedule/zephyr_domain-kernel.c @@ -0,0 +1,452 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2019-2021 Intel Corporation. All rights reserved. +// +// Author: Tomasz Lauda + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include + +LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); + +/* Kernel part of the Zephyr LL scheduler timer domain: the scheduler threads, the + * timer and its interrupt handler. The thread function and the parts that run in the + * LL scheduler threads are in src/schedule/zephyr_domain.c + */ + +/* + * Currently the Zephyr clock rate is part it's Kconfig known at build time. + * SOF on Intel CAVS platforms currently only aligns with Zephyr when both + * use the CAVS 19.2 MHz SSP clock. TODO - needs runtime alignment. + */ +#if CONFIG_XTENSA && CONFIG_CAVS && !CONFIG_INTEL_ADSP_TIMER +#error "Zephyr uses 19.2MHz clock derived from SSP which must be enabled." +#endif + +#define ZEPHYR_LL_STACK_SIZE 8192 + +#if CONFIG_SOF_USERSPACE_LL +K_THREAD_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); +#else +K_KERNEL_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); +#endif + +/* Timer callback: runs in timer IRQ context */ +static void zephyr_domain_timer_fn(struct k_timer *timer) +{ + struct zephyr_domain *zephyr_domain = k_timer_user_data_get(timer); + int core; + + /* + * A race is possible when the Zephyr LL scheduling domain is being + * unregistered while a timer IRQ is processed on a different core. Then + * the timer is removed by the former but then re-added by the latter, + * but this time with no user data and no handler set. This leads to the + * timer continuing to trigger and then leading to a lock up when it is + * registered again next time. + */ + if (!zephyr_domain) { + k_timer_stop(timer); + return; + } + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + if (dt->handler) + k_sem_give(dt->sem); + } +} + +/* The normal kernel-space implementation for register/unregister */ +#ifndef CONFIG_SOF_USERSPACE_LL + +static int zephyr_domain_register(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + char thread_name[] = "ll_thread0"; + k_tid_t thread; + k_spinlock_key_t key; + + tr_dbg(&ll_tr, "entry"); + + /* domain work only needs registered once on each core */ + if (dt->handler) + return 0; + + dt->handler = handler; + dt->arg = arg; + + /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ + k_sem_init(dt->sem, 0, 10); + + thread_name[sizeof(thread_name) - 2] = '0' + core; + + /* not allocated dynamically when LL in kernel space */ + dt->ll_thread = &dt->ll_thread_obj; + + thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, + zephyr_domain_thread_fn, zephyr_domain, INT_TO_POINTER(core), + NULL, CONFIG_LL_THREAD_PRIORITY, 0, K_FOREVER); + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, core); +#endif + k_thread_name_set(thread, thread_name); + + k_thread_start(thread); + + key = k_spin_lock(&domain->lock); + + if (!k_timer_user_data_get(zephyr_domain->timer)) { + k_timeout_t start = {0}; + + k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); + k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); + + k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); + + /* Enable the watchdog */ + watchdog_enable(core); + } + + k_spin_unlock(&domain->lock, key); + + tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", + domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); + + return 0; +} + +static int zephyr_domain_unregister(struct ll_schedule_domain *domain, + struct task *task, uint32_t num_tasks) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + k_spinlock_key_t key; + + tr_dbg(&ll_tr, "entry"); + + /* tasks still registered on this core */ + if (num_tasks) + return 0; + + key = k_spin_lock(&domain->lock); + + if (!atomic_read(&domain->total_num_tasks)) { + /* Disable the watchdog */ + watchdog_disable(core); + + k_timer_stop(zephyr_domain->timer); + k_timer_user_data_set(zephyr_domain->timer, NULL); + } + + zephyr_domain->domain_thread[core].handler = NULL; + + k_spin_unlock(&domain->lock, key); + + tr_info(&ll_tr, "domain->type %d domain->clk %d", + domain->type, domain->clk); + + /* + * If running in the context of the domain thread, k_thread_abort() will + * not return + */ + k_thread_abort(zephyr_domain->domain_thread[core].ll_thread); + + tr_dbg(&ll_tr, "exit"); + + return 0; +} + +#else /* CONFIG_SOF_USERSPACE_LL */ + +/* + * Kernel-owned per-core LL thread table. Populated from the privileged + * domain-thread init path and consulted by z_impl_zephyr_ll_task_sem_alloc() + * so that privileged code never has to traverse the user-accessible + * scheduler/domain objects to find the LL thread of a given core. + */ +static struct k_thread *ll_thread_tid[CONFIG_CORE_COUNT]; + +/* + * Privileged thread initialization for userspace LL scheduling. + * Creates the scheduling thread, sets up timer, grants access to kernel + * objects. Must be called from kernel context before any user-space + * domain_register() calls. + */ +static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, + struct task *task) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + struct zephyr_domain_thread *dt; + char thread_name[] = "ll_thread0"; + k_tid_t thread; + int core = task->core; + + tr_dbg(&ll_tr, "entry"); + + if (core >= CONFIG_CORE_COUNT) + return -EINVAL; + + dt = zephyr_domain->domain_thread + core; + + /* thread only needs to be created once per core */ + if (dt->ll_thread) + return 0; + + dt->handler = NULL; + dt->arg = NULL; + + dt->sem = k_object_alloc(K_OBJ_SEM); + if (!dt->sem) { + tr_err(&ll_tr, "Failed to allocate semaphore for core %d", core); + return -ENOMEM; + } + + /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ + k_sem_init(dt->sem, 0, 10); + + thread_name[sizeof(thread_name) - 2] = '0' + core; + + /* Allocate thread structure dynamically */ + dt->ll_thread = k_object_alloc(K_OBJ_THREAD); + if (!dt->ll_thread) { + tr_err(&ll_tr, "Failed to allocate thread object for core %d", core); + k_object_free(dt->sem); + dt->sem = NULL; + return -ENOMEM; + } + + thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, + zephyr_domain_thread_fn, zephyr_domain, + INT_TO_POINTER(core), NULL, CONFIG_LL_THREAD_PRIORITY, + K_USER, K_FOREVER); + + /* record in the kernel-only table for syscall-context lookups */ + ll_thread_tid[core] = dt->ll_thread; + +#ifdef CONFIG_SCHED_CPU_MASK + k_thread_cpu_pin(thread, core); +#endif + k_thread_name_set(thread, thread_name); + + k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); + k_thread_access_grant(thread, dt->sem, domain->lock, zephyr_domain->timer); + user_grant_dai_access_all(thread); + user_grant_dma_access_all(thread); + user_grant_uaol_access_all(thread); + tr_dbg(&ll_tr, "granted LL access to thread %p (core %d)", thread, core); + + k_thread_start(thread); + + k_mutex_lock(domain->lock, K_FOREVER); + if (!k_timer_user_data_get(zephyr_domain->timer)) { + k_timeout_t start = {0}; + + k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); + k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); + + k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); + + /* Enable the watchdog */ + watchdog_enable(core); + } + + k_mutex_unlock(domain->lock); + + tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", + domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); + + return 0; +} + +/* + * Free resources acquired by zephyr_domain_thread_init(). + * Stops the timer, aborts the scheduling thread and frees the thread object. + * Must be called from kernel context. + */ +static void zephyr_domain_thread_free(struct ll_schedule_domain *domain, + uint32_t num_tasks) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + tr_dbg(&ll_tr, "thread_free entry, core %d, num_tasks %u", core, num_tasks); + + /* Still tasks on other cores, only clean up this core's thread */ + k_mutex_lock(domain->lock, K_FOREVER); + + if (!num_tasks && !atomic_read(&domain->total_num_tasks)) { + /* Last task globally: stop the timer and watchdog */ + watchdog_disable(core); + + k_timer_stop(zephyr_domain->timer); + k_timer_user_data_set(zephyr_domain->timer, NULL); + } + + dt->handler = NULL; + dt->arg = NULL; + + k_mutex_unlock(domain->lock); + + if (dt->ll_thread) { + k_thread_abort(dt->ll_thread); + k_object_free(dt->ll_thread); + dt->ll_thread = NULL; + ll_thread_tid[core] = NULL; + } + + if (dt->sem) { + k_object_free(dt->sem); + dt->sem = NULL; + } + + tr_info(&ll_tr, "thread_free done, core %d", core); +} + +struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain) +{ + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); + int core = cpu_get_id(); + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + tr_dbg(&ll_tr, "entry"); + + return dt->ll_thread; +} + +/* + * Return the LL scheduling thread for an explicitly given core. + * + * Reads a kernel-only table keyed by core, so it is safe to call from a + * privileged syscall context without dereferencing any user-accessible + * scheduler or domain object, and without relying on cpu_get_id(). + */ +struct k_thread *zephyr_domain_thread_tid_for_core(int core) +{ + if (core < 0 || core >= CONFIG_CORE_COUNT) + return NULL; + + return ll_thread_tid[core]; +} + +struct k_thread *zephyr_ll_domain_thread(void) +{ + struct ll_schedule_domain *ll_domain = zephyr_ll_domain(); + + if (!ll_domain) + return NULL; + + struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(ll_domain); + + return zephyr_domain->domain_thread[cpu_get_id()].ll_thread; +} + +#endif /* CONFIG_SOF_USERSPACE_LL */ + +APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { +#ifdef CONFIG_SOF_USERSPACE_LL + .domain_register = zephyr_domain_register_user, + .domain_unregister = zephyr_domain_unregister_user, + .domain_thread_init = zephyr_domain_thread_init, + .domain_thread_free = zephyr_domain_thread_free, +#else + .domain_register = zephyr_domain_register, + .domain_unregister = zephyr_domain_unregister, +#endif +#if CONFIG_CROSS_CORE_STREAM + .domain_block = zephyr_domain_block, + .domain_unblock = zephyr_domain_unblock, +#endif +}; + +/* Core 0 only */ +struct ll_schedule_domain *zephyr_domain_init(int clk) +{ + struct ll_schedule_domain *domain; + struct zephyr_domain *zephyr_domain; + int core; + + domain = domain_init(SOF_SCHEDULE_LL_TIMER, clk, false, + &zephyr_domain_ops); + if (!domain) { + tr_err(&ll_tr, "domain init failed"); + return NULL; + } + +#if CONFIG_SOF_USERSPACE_LL + zephyr_domain = sof_heap_alloc(zephyr_ll_user_heap(), + SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*zephyr_domain), sizeof(void *)); + if (zephyr_domain) + memset(zephyr_domain, 0, sizeof(*zephyr_domain)); +#else + zephyr_domain = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + sizeof(*zephyr_domain)); +#endif + if (!zephyr_domain) { + tr_err(&ll_tr, "domain allocation failed"); + k_panic(); + return NULL; + } + + zephyr_domain->ll_domain = domain; + +#if CONFIG_SOF_USERSPACE_LL + /* Allocate timer dynamically for userspace access */ + zephyr_domain->timer = k_object_alloc(K_OBJ_TIMER); + if (!zephyr_domain->timer) { + tr_err(&ll_tr, "timer allocation failed"); + k_panic(); + return NULL; + } +#else + /* not allocated dynamically when LL in kernel space */ + zephyr_domain->timer = &zephyr_domain->timer_obj; +#endif + +#if CONFIG_CROSS_CORE_STREAM + atomic_set(&zephyr_domain->block, 0); + k_mutex_init(&zephyr_domain->block_mutex); + k_condvar_init(&zephyr_domain->block_condvar); +#endif + + ll_sch_domain_set_pdata(domain, zephyr_domain); + + for (core = 0; core < CONFIG_CORE_COUNT; core++) { +#ifndef CONFIG_SOF_USERSPACE_LL + struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; + + /* not allocated dynamically when LL in kernel space */ + dt->sem = &dt->sem_obj; +#endif + } + + return domain; +} diff --git a/src/kernel/schedule/zephyr_dp_schedule-kernel.c b/src/kernel/schedule/zephyr_dp_schedule-kernel.c new file mode 100644 index 000000000000..2fd035e106ea --- /dev/null +++ b/src/kernel/schedule/zephyr_dp_schedule-kernel.c @@ -0,0 +1,279 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2023 Intel Corporation. All rights reserved. + * + * Author: Marcin Szkudlinski + */ + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include + +#include +#include + +LOG_MODULE_REGISTER(dp_schedule, CONFIG_SOF_LOG_LEVEL); +SOF_DEFINE_REG_UUID(dp_sched); + +/* unused with Zephyr, generates no output */ +DECLARE_TR_CTX(dp_tr, SOF_UUID(dp_sched_uuid), LOG_LEVEL_INFO); + +/* Kernel part of the DP scheduler: the scheduler locks, their access grants, the LL + * tick system call and the scheduler initialization. The rest is in + * src/schedule/zephyr_dp_schedule.c + */ + +#define DP_LOCK_INIT(i, _) Z_SEM_INITIALIZER(dp_lock[i], 1, 1) +#define DP_LOCK_INIT_LIST LISTIFY(CONFIG_MP_MAX_NUM_CPUS, DP_LOCK_INIT, (,)) + +/* User threads don't need access to this array. Access is performed from + * the kernel space via a syscall. Array must be placed in special section + * to be qualified as initialized by the gen_kobject_list.py script. + */ +STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_lock, CONFIG_MP_MAX_NUM_CPUS) = { DP_LOCK_INIT_LIST }; + +void scheduler_dp_grant(k_tid_t thread_id, uint16_t core) +{ + if (thread_id) + k_thread_access_grant(thread_id, &dp_lock[core]); +} + +/* dummy LL task - to start LL on secondary cores */ +static enum task_state scheduler_dp_ll_tick_dummy(void *data) +{ + return SOF_TASK_STATE_RESCHEDULE; +} + +/* + * function called after every LL tick + * + * This function checks if the queued DP tasks are ready to processing (meaning + * the module run by the task has enough data at all sources and enough free space + * on all sinks) + * + * if the task becomes ready, a deadline is set allowing Zephyr to schedule threads + * in right order + * + * TODO: currently there's a limitation - DP module must be surrounded by LL modules. + * it simplifies algorithm - there's no need to browse through DP chains calculating + * deadlines for each module in function of all modules execution status. + * Now is simple - modules deadline is its start + tick time. + * + * example: + * Lets assume we do have a pipeline: + * + * LL1 -> DP1 -> LL2 -> DP2 -> LL3 -> DP3 -> LL4 + * + * all LLs starts in 1ms tick + * + * for simplification lets assume + * - all LLs are on primary core, all DPs on secondary (100% CPU is for DP) + * - context switching requires 0 cycles + * + * DP1 - starts every 1ms, needs 0.5ms to finish processing + * DP2 - starts every 2ms, needs 0.6ms to finish processing + * DP3 - starts every 10ms, needs 0.3ms to finish processing + * + * TICK0 + * only LL1 is ready to run + * LL1 processing (producing data chunk for DP1) + * + * TICK1 + * LL1 is ready to run + * DP1 is ready tu run (has data from LL1) set deadline to TICK2 + * LL1 processing (producing second data chunk for DP1) + * DP1 processing for 0.5ms (consuming first data chunk, producing data chunk for LL2) + * CPU is idle for 0.5ms + * + * TICK2 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK3 + * LL2 is ready to run + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * CPU is idle for 0.5ms + * + * TICK3 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK4 + * LL2 is ready to run + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * CPU is idle for 0.5ms + * + * TICK4 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK5 + * LL2 is ready to run + * DP2 is ready to run set deadline to TICK6 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% of second data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used + * + * !!!!!! Note here - DP1 must do before DP2 as it MUST finish in this tick. DP2 can wait + * >>>>>>> this is what we call EDF - EARIEST DEADLINE FIRST <<<<<< + * + * TICK5 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK6 + * LL2 is ready to run + * DP2 is in progress, deadline is set to TICK6 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of second data chunk for DP2) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.1ms (producing TWO data chunks for LL3) + * CPU is idle for 0.4ms (60% used) + * + * TICK6 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK7 + * LL2 is ready to run + * DP2 is ready to run set deadline to TICK8 + * LL3 is ready to run + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% of second data chunk for DP2) + * LL3 processing (producing 10% of first data chunk for DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used + * + * + * + * (........ 9 more cycles - LL3 procuces 100% of data for DP3......) + * + * + * TICK15 + * LL1 is ready to run + * DP1 is ready tu run set deadline to TICK16 + * LL2 is ready to run + * DP2 is ready to run set deadline to TICK17 + * LL3 is ready to run + * DP3 is ready to run set deadline to TICK25 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing 50% of data chunk for DP2) + * LL3 processing (producing 10% of second data chunk for DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used - + * !!! note that DP3 is ready but has no chance to get CPU in this cycle + * + * TICK16 + * LL1 is ready to run set deadline to TICK17 + * DP1 is ready tu run + * LL2 is ready to run + * DP2 is in progress, deadline is set to TICK17 + * LL3 is ready to run + * DP3 is in progress, deadline is set to TICK25 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of data chunk for DP2) + * LL3 processing (producing 10% of second data chunk for DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.1ms (producing data) + * DP3 processing for 0.2ms (producing 10 data chunks for LL4) + * 90% CPU used + * + * TICK17 + * LL1 is ready to run + * DP1 is ready tu run + * LL2 is ready to run + * DP2 is ready to run + * LL3 is ready to run + * LL4 is ready to run + * !! NOTE that DP3 is not ready - it will be ready again in TICK25 + * LL1 processing (producing data chunk for DP1) + * LL2 processing (producing rest of data chunk for DP2) + * LL3 processing (producing next 10% of second data chunk for DP3) + * LL4 processing (consuming 10% of data prepared by DP3) + * DP1 processing for 0.5ms (producing data chunk for LL2) + * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) + * 100% CPU used + * + * + * Now - pipeline is in stable state, CPU used almost in 100% (it would be 100% if DP3 + * needed 1.2ms for processing - but the example would be too complicated) + */ + +void z_impl_scheduler_dp_ll_tick(void) +{ + unsigned int lock_key; +#if CONFIG_SOF_USERSPACE_LL + struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); +#else + struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); +#endif + + if (!dp_sch) + return; + + /* remember current timestamp as "NOW" */ + dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); + + lock_key = scheduler_dp_lock(cpu_get_id()); + scheduler_dp_recalculate(dp_sch); + scheduler_dp_unlock(lock_key); +} + +#ifdef CONFIG_USERSPACE +#include +void z_vrfy_scheduler_dp_ll_tick(void) +{ + z_impl_scheduler_dp_ll_tick(); +} +#include +#endif + +APP_SYSUSER_DATA static struct scheduler_ops schedule_dp_ops = { + .schedule_task = scheduler_dp_task_shedule, +#if CONFIG_SOF_USERSPACE_APPLICATION + .schedule_task_cancel = scheduler_dp_task_cancel, +#else + .schedule_task_cancel = scheduler_dp_task_stop, +#endif + .schedule_task_free = scheduler_dp_task_free, +}; + +/* Runs on each core */ +__cold int scheduler_dp_init(void) +{ + assert_can_be_cold(); + + struct scheduler_dp_data *dp_sch = sof_heap_alloc(sof_sys_user_heap_get(), + SOF_MEM_FLAG_KERNEL, sizeof(*dp_sch), 0); + if (!dp_sch) + return -ENOMEM; + + memset(dp_sch, 0, sizeof(*dp_sch)); + list_init(&dp_sch->tasks); + + scheduler_init(SOF_SCHEDULE_DP, &schedule_dp_ops, dp_sch); + + /* init src of DP tick */ + return schedule_task_init_ll(&dp_sch->ll_tick_src, + SOF_UUID(dp_sched_uuid), + SOF_SCHEDULE_LL_TIMER, + 0, scheduler_dp_ll_tick_dummy, dp_sch, + cpu_get_id(), 0); +} diff --git a/src/kernel/schedule/zephyr_dp_schedule_application-kernel.c b/src/kernel/schedule/zephyr_dp_schedule_application-kernel.c new file mode 100644 index 000000000000..395f1f5129be --- /dev/null +++ b/src/kernel/schedule/zephyr_dp_schedule_application-kernel.c @@ -0,0 +1,533 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Marcin Szkudlinski + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include +#include +#include +#include + +#include +#include +#include + +#include + +LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); +extern struct tr_ctx dp_tr; + +/* Kernel part of the application variant of the DP scheduler: the DP thread creation, + * its memory domain, the kernel-only record of its kernel objects and the + * scheduler_dp_internal_free() system call. The DP thread function and the IPC + * forwarding to it are in src/schedule/zephyr_dp_schedule_application.c + */ + +static struct objpool_head dp_mdom_head = {.list = LIST_INIT(dp_mdom_head.list)}; + +/* Synchronization semaphore for the scheduler thread to wait for DP startup */ +#define DP_SYNC_INIT(i, _) Z_SEM_INITIALIZER(dp_sync[i], 0, 1) +#define DP_SYNC_INIT_LIST LISTIFY(CONFIG_CORE_COUNT, DP_SYNC_INIT, (,)) +STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_sync, CONFIG_CORE_COUNT) = { DP_SYNC_INIT_LIST }; + +/* + * Kernel-only record of the kernel objects owned by a DP task. + * + * struct scheduler_dp_task_memory lives in the module's heap, which is mapped + * into the DP thread's memory domain, and is also writable by the LL and IPC + * user threads. Freeing must therefore never use the pointers stored there: + * scheduler_dp_internal_free() takes them from this record instead. The task + * pointer is only used as a lookup key. + */ +struct scheduler_dp_task_kobj { + struct list_item list; + struct task *task; + struct processing_module *mod; + struct k_event *event; + struct k_thread *thread; + struct k_mem_domain *mdom; + struct k_mem_partition mpart[SOF_DP_PART_TYPE_COUNT]; + uint32_t comp_id; + uint32_t gen; /* unique per record, see dp_task_kobj_take() */ +}; + +static struct list_item dp_task_kobj_list = LIST_INIT(dp_task_kobj_list); +static struct k_spinlock dp_task_kobj_lock; +/* generation of the last added record, under dp_task_kobj_lock */ +static uint32_t dp_task_kobj_gen; + +static void dp_task_kobj_add(struct scheduler_dp_task_kobj *kobj) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + + kobj->gen = ++dp_task_kobj_gen; + list_item_append(&kobj->list, &dp_task_kobj_list); + k_spin_unlock(&dp_task_kobj_lock, key); +} + +/* Find the record for @task, caller holds dp_task_kobj_lock */ +static struct scheduler_dp_task_kobj *dp_task_kobj_find(const struct task *task) +{ + struct scheduler_dp_task_kobj *kobj; + struct list_item *item; + + list_for_item(item, &dp_task_kobj_list) { + kobj = container_of(item, struct scheduler_dp_task_kobj, list); + if (kobj->task == task) + return kobj; + } + + return NULL; +} + +/* Copy the record for @task to @copy without unlinking it */ +static bool dp_task_kobj_peek(const struct task *task, struct scheduler_dp_task_kobj *copy) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); + + if (kobj) + *copy = *kobj; + + k_spin_unlock(&dp_task_kobj_lock, key); + + return kobj != NULL; +} + +/* + * Find the record for @task and unlink it, so it can only be freed once. + * If @expected is given, only take the record it was copied from, so a + * record replaced since @expected was peeked is left alone. + */ +static struct scheduler_dp_task_kobj * +dp_task_kobj_take(const struct task *task, const struct scheduler_dp_task_kobj *expected) +{ + k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); + struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); + + /* + * Compare the generation, not the object addresses: the task could + * have been freed and recreated at the same addresses meanwhile. + */ + if (kobj && expected && kobj->gen != expected->gen) + kobj = NULL; + + if (kobj) + list_item_del(&kobj->list); + + k_spin_unlock(&dp_task_kobj_lock, key); + + return kobj; +} + +/* + * Safe to call with partial successful initialisation, + * k_mem_domain_remove_partition() then just returns -ENOENT + */ +static void scheduler_dp_domain_free(struct processing_module *pmod, struct k_mem_domain *mdom, + struct k_mem_partition *mpart, uint32_t comp_id) +{ + llext_manager_rm_domain(comp_id, mdom); + + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP); + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP_CACHE); + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG); + k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG_CACHE); + + if (mdom->num_partitions) { + /* Violation: all partitions should be freed by now */ + unsigned int n = mdom->num_partitions; + + LOG_WRN("%u domain partition(s) not freed, force-freeing them all", n); + + for (unsigned int i = 0; i < arch_mem_domain_max_partitions_get() && n; i++) { + struct k_mem_partition *dpart = &mdom->partitions[i]; + + if (dpart->size) { + k_mem_domain_remove_partition(mdom, dpart); + n--; + } + } + } + + /* All partitions removed, the domain can be freed now */ + pmod->mdom = NULL; + k_mem_domain_deinit(mdom); + objpool_free(&dp_mdom_head, mdom); +} + +/* + * Free the kernel objects of @task, using only the kernel-only record. + * + * @return false if @task has no record (unknown task or already freed) + */ +static bool scheduler_dp_internal_free_kobj(struct task *task, + const struct scheduler_dp_task_kobj *expected) +{ + struct scheduler_dp_task_kobj *kobj = dp_task_kobj_take(task, expected); + + if (!kobj) + return false; + + /* normally already terminated by scheduler_dp_task_free(), no-op then */ + k_thread_abort(kobj->thread); + k_object_free(kobj->event); + k_object_free(kobj->thread); + scheduler_dp_domain_free(kobj->mod, kobj->mdom, kobj->mpart, kobj->comp_id); + + mod_free(kobj->mod, container_of(task, struct scheduler_dp_task_memory, task)); + rfree(kobj); + + return true; +} + +void z_impl_scheduler_dp_internal_free(struct task *task) +{ + if (!scheduler_dp_internal_free_kobj(task, NULL)) + tr_err(&dp_tr, "unknown DP task %p", task); +} + +#ifdef CONFIG_THREAD_NAME +static void scheduler_dp_thread_name_set(k_tid_t thread_id, struct processing_module *mod) +{ + char name[CONFIG_THREAD_MAX_NAME_LEN]; + + snprintf(name, sizeof(name), "DP:%#x", mod->dev->ipc_config.id); + + k_thread_name_set(thread_id, name); +} +#else +/* k_thread_name_set() is a no-op so skip constructing a thread name */ +#define scheduler_dp_thread_name_set(x, y) +#endif + +/* Called only in IPC context in kernel mode (this can change) */ +int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, + const struct task_ops *ops, struct processing_module *mod, + uint16_t core, size_t stack_size, uint32_t options) +{ + k_thread_stack_t *p_stack; + struct scheduler_dp_task_memory *task_memory; + uint8_t *ipc_config_data = NULL; + k_tid_t thread_id; + uint32_t comp_id; + + int ret; + + /* must be called on the same core the task will be bound to */ + assert(cpu_get_id() == core); + assert(!k_is_user_context()); + + /* + * allocate memory + * to avoid multiple malloc operations allocate all required memory as a single structure + * and return pointer to task_memory->task + * As the structure contains zephyr kernel specific data, it must be located in + * shared, non cached memory + */ + task_memory = mod_alloc_ext(mod, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*task_memory), 0); + if (!task_memory) { + tr_err(&dp_tr, "memory alloc failed"); + return -ENOMEM; + } + + memset(task_memory, 0, sizeof(*task_memory)); + + const struct comp_driver *drv = mod->dev->drv; + + task_memory->drv = *mod->dev->drv; + mod->dev->drv = &task_memory->drv; + + /* allocate stack - must be aligned and cached so a separate alloc */ + p_stack = user_stack_allocate(stack_size, options); + if (!p_stack) { + tr_err(&dp_tr, "stack alloc failed"); + ret = -ENOMEM; + goto e_tmem; + } + + struct task *ptask = &task_memory->task; + + /* internal SOF task init */ + ret = schedule_task_init(ptask, uid, SOF_SCHEDULE_DP, 0, ops->run, mod, core, options); + if (ret < 0) { + tr_err(&dp_tr, "schedule_task_init failed"); + goto e_stack; + } + + /* + * task_memory, including pdata, is in the module heap, which user + * threads can write while this function runs. Keep everything the + * kernel uses here or at teardown in kobj and locals, and only mirror + * it into pdata for the DP thread. + */ + struct task_dp_pdata *pdata = &task_memory->pdata; + struct scheduler_dp_task_kobj *kobj = rmalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, + sizeof(*kobj)); + + if (!kobj) { + tr_err(&dp_tr, "task record alloc failed"); + ret = -ENOMEM; + goto e_stack; + } + + kobj->task = ptask; + kobj->mod = mod; + comp_id = mod->dev->ipc_config.id; + kobj->comp_id = comp_id; + + if (mod->dev->drv->adapter_ops->get_configuration) { + ipc_config_data = mod_alloc_ext(mod, SOF_MEM_FLAG_USER, SOF_IPC_MSG_MAX_SIZE, 0); + if (!ipc_config_data) { + tr_err(&dp_tr, "Failed to allocate memory for get-config"); + ret = -ENOMEM; + goto e_kobj; + } + } + pdata->ipc_config_data = ipc_config_data; + + pdata->flat = &task_memory->flat; + + kobj->event = k_object_alloc(K_OBJ_EVENT); + if (!kobj->event) { + tr_err(&dp_tr, "Event object allocation failed"); + ret = -ENOMEM; + goto e_config; + } + + pdata->event = kobj->event; + + kobj->thread = k_object_alloc(K_OBJ_THREAD); + if (!kobj->thread) { + tr_err(&dp_tr, "Thread object allocation failed"); + ret = -ENOMEM; + goto e_event; + } + memset(&kobj->thread->arch, 0, sizeof(kobj->thread->arch)); + pdata->thread = kobj->thread; + + /* success, fill the structures */ + pdata->p_stack = p_stack; + pdata->mod = mod; + + /* initialize other task structures */ + ptask->ops.complete = ops->complete; + ptask->ops.get_deadline = ops->get_deadline; + ptask->priv_data = pdata; + list_init(&ptask->list); + + /* create a zephyr thread for the task */ + thread_id = k_thread_create(kobj->thread, p_stack, + stack_size, dp_thread_fn, ptask, NULL, NULL, + CONFIG_DP_THREAD_PRIORITY, ptask->flags, K_FOREVER); + pdata->thread_id = thread_id; + scheduler_dp_thread_name_set(thread_id, mod); + + unsigned int pidx; + size_t size; + uintptr_t start; + bool on_pool = false; + +#ifdef CONFIG_SCHED_CPU_MASK + /* pin the thread to specific core */ + ret = k_thread_cpu_pin(thread_id, core); + if (ret < 0) { + tr_err(&dp_tr, "zephyr task pin to core failed"); + goto e_thread; + } +#endif + + k_thread_access_grant(thread_id, kobj->event, &dp_sync[core]); + scheduler_dp_grant(thread_id, core); +#if CONFIG_SOF_USERSPACE_LL + struct k_thread *thread_ipc = ipc_thread_user(core); + + k_thread_access_grant(thread_ipc, kobj->event, thread_id, p_stack, &dp_sync[core]); + scheduler_dp_grant(thread_ipc, core); + scheduler_dp_grant(zephyr_ll_domain_thread(), core); +#endif + + struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), + SOF_MEM_FLAG_COHERENT); + + if (!mdom) { + tr_err(&dp_tr, "objpool allocation failed"); + ret = -ENOMEM; + goto e_thread; + } + + on_pool = true; + + mod->mdom = mdom; + kobj->mdom = mdom; + + if (!mdom->arch.ptables) { + ret = k_mem_domain_init(mdom, 0, NULL); + if (ret < 0) + goto e_thread; + } + + /* Module heap partition */ + mod_heap_info(mod, &size, &start); + kobj->mpart[SOF_DP_PART_HEAP] = (struct k_mem_partition){ + .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)start), + .size = size, + .attr = K_MEM_PARTITION_P_RW_U_RW, + }; + kobj->mpart[SOF_DP_PART_HEAP_CACHE] = (struct k_mem_partition){ + .start = start, + .size = size, + .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, + }; + /* Host mailbox partition for additional IPC parameters: read-only */ + kobj->mpart[SOF_DP_PART_CFG] = (struct k_mem_partition){ + .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)MAILBOX_HOSTBOX_BASE), + .size = MAILBOX_HOSTBOX_SIZE, + .attr = K_MEM_PARTITION_P_RO_U_RO, + }; + kobj->mpart[SOF_DP_PART_CFG_CACHE] = (struct k_mem_partition){ + .start = (uintptr_t)MAILBOX_HOSTBOX_BASE, + .size = MAILBOX_HOSTBOX_SIZE, + .attr = K_MEM_PARTITION_P_RO_U_RO | XTENSA_MMU_CACHED_WB, + }; + + for (pidx = 0; pidx < SOF_DP_PART_TYPE_COUNT; pidx++) { + ret = k_mem_domain_add_partition(mdom, kobj->mpart + pidx); + if (ret < 0) + goto e_dom; + } + + memcpy(pdata->mpart, kobj->mpart, sizeof(pdata->mpart)); + + ret = llext_manager_add_domain(comp_id, mdom); + if (ret < 0) { + tr_err(&dp_tr, "failed to add LLEXT to domain %d", ret); + goto e_dom; + } + + /* + * Keep this call last, able to fail, otherwise domain will be removed + * before its thread + */ + ret = k_mem_domain_add_thread(mdom, thread_id); + if (ret < 0) { + tr_err(&dp_tr, "failed to add thread to domain %d", ret); + goto e_dom; + } + + /* publish the record before the thread can run */ + dp_task_kobj_add(kobj); + + /* start the thread, it should immediately stop at the semaphore */ + k_event_init(kobj->event); + k_thread_start(thread_id); + + /* ptask points into task_memory, so only publish it once it cannot be freed */ + *task = ptask; + + return 0; + +e_dom: + scheduler_dp_domain_free(mod, mdom, kobj->mpart, comp_id); + on_pool = false; +e_thread: + if (on_pool) + objpool_free(&dp_mdom_head, mdom); + k_thread_abort(thread_id); + k_object_free(kobj->thread); +e_event: + k_object_free(kobj->event); +e_config: + mod_free(mod, ipc_config_data); +e_kobj: + rfree(kobj); +e_stack: + user_stack_free(p_stack); +e_tmem: + /* the copy lives in task_memory, so stop pointing at it before freeing */ + mod->dev->drv = drv; + mod_free(mod, task_memory); + return ret; +} + +#ifdef CONFIG_USERSPACE +#include + +/* + * mod_free() follows the module's resource and allocator pointers in + * supervisor mode. They are user-writable, so at least check that they + * point to memory the caller can access. + */ +static void scheduler_dp_mod_vrfy(struct processing_module *mod) +{ + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); + K_OOPS(K_SYSCALL_MEMORY_WRITE(mod->dev, sizeof(*mod->dev))); + K_OOPS(K_SYSCALL_MEMORY_READ(mod->dev->drv, sizeof(*mod->dev->drv))); + + struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(alloc, sizeof(*alloc))); + if (alloc->heap) { + size_t h_size = 0; + uintptr_t h_start; + + mod_heap_info(mod, &h_size, &h_start); + if (h_size) + K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); + } + if (alloc->vreg) { + size_t vr_size = 0; + uintptr_t vr_start; + + vregion_mem_info(alloc->vreg, &vr_size, &vr_start); + K_OOPS(K_SYSCALL_MEMORY_WRITE(vr_start, vr_size)); + } +} + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + struct scheduler_dp_task_kobj kobj; + + /* + * Do not read task->priv_data: the kernel objects come from the + * kernel-only record, and an unknown task pointer is rejected. + */ + K_OOPS(!dp_task_kobj_peek(task, &kobj)); + + /* + * A DP thread holds its own thread and event objects, so it would pass + * the checks below for its own task. Freeing would then abort the + * caller after the record is unlinked, leaking everything it holds. + */ + K_OOPS(kobj.thread == k_current_get()); + + /* + * The caller must have been granted the task's objects. Check before + * unlinking the record, so an unauthorized call leaves it intact. + */ + K_OOPS(K_SYSCALL_OBJ(kobj.event, K_OBJ_EVENT)); + K_OOPS(K_SYSCALL_OBJ_INIT(kobj.thread, K_OBJ_THREAD)); + scheduler_dp_mod_vrfy(kobj.mod); + + K_OOPS(!scheduler_dp_internal_free_kobj(task, &kobj)); +} +#include +#endif /* CONFIG_USERSPACE */ diff --git a/src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c b/src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c new file mode 100644 index 000000000000..05c85ad98b1e --- /dev/null +++ b/src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c @@ -0,0 +1,208 @@ +// SPDX-License-Identifier: BSD-3-Clause +/* + * Copyright(c) 2025 Intel Corporation. All rights reserved. + * + * Author: Marcin Szkudlinski + * Adrian Warecki + */ + +#include +#include + +#include +#include +#include +#include +#include +#include +#include +#include +#include + +#include + +#include +#include + +#include + +LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); +extern struct tr_ctx dp_tr; + +/* Kernel part of the thread variant of the DP scheduler: the DP thread creation and + * the scheduler_dp_internal_free() system call. The DP thread function is in + * src/schedule/zephyr_dp_schedule_thread.c + */ + +int scheduler_dp_task_init(struct task **task, + const struct sof_uuid_entry *uid, + const struct task_ops *ops, + struct processing_module *mod, + uint16_t core, + size_t stack_size, + uint32_t options) +{ + void __sparse_cache *p_stack = NULL; + struct k_heap *const user_heap = mod->dev->drv->user_heap; + + /* memory allocation helper structure */ + struct { + struct task task; /* keep first, used for freeing below */ + struct task_dp_pdata pdata; + } *task_memory; + + int ret; + + /* must be called on the same core the task will be binded to */ + assert(cpu_get_id() == core); + + /* + * allocate memory + * to avoid multiple malloc operations allocate all required memory as a single structure + * and return pointer to task_memory->task + * As the structure contains zephyr kernel specific data, it must be located in + * shared, non cached memory + */ + task_memory = sof_heap_alloc(user_heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, + sizeof(*task_memory), 0); + if (!task_memory) { + tr_err(&dp_tr, "memory alloc failed"); + return -ENOMEM; + } + + memset(task_memory, 0, sizeof(*task_memory)); + /* allocate stack - must be aligned and cached so a separate alloc */ + p_stack = user_stack_allocate(stack_size, options); + if (!p_stack) { + tr_err(&dp_tr, "stack alloc failed"); + ret = -ENOMEM; + goto err; + } + + /* internal SOF task init */ + ret = schedule_task_init(&task_memory->task, uid, SOF_SCHEDULE_DP, 0, ops->run, + mod, core, options); + if (ret < 0) { + tr_err(&dp_tr, "schedule_task_init failed"); + goto err; + } + + struct task_dp_pdata *pdata = &task_memory->pdata; + + /* Point to event_struct event for kernel threads synchronization */ + /* It will be overwritten for K_USER threads to dynamic ones. */ + pdata->event = &pdata->event_struct; + pdata->thread = &pdata->thread_struct; + +#ifdef CONFIG_USERSPACE + if (options & K_USER) { + pdata->event = k_object_alloc(K_OBJ_EVENT); + if (!pdata->event) { + tr_err(&dp_tr, "Event object allocation failed"); + ret = -ENOMEM; + goto err; + } + + pdata->thread = k_object_alloc(K_OBJ_THREAD); + if (!pdata->thread) { + tr_err(&dp_tr, "Thread object allocation failed"); + ret = -ENOMEM; + goto err; + } + } +#endif /* CONFIG_USERSPACE */ + + /* initialize other task structures */ + task_memory->task.ops.complete = ops->complete; + task_memory->task.ops.get_deadline = ops->get_deadline; + task_memory->task.state = SOF_TASK_STATE_INIT; + task_memory->task.core = core; + task_memory->task.priv_data = pdata; + k_event_init(pdata->event); + + /* success, fill the structures */ + pdata->p_stack = p_stack; + pdata->mod = mod; + + /* create a zephyr thread for the task */ + pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)p_stack, + stack_size, dp_thread_fn, &task_memory->task, NULL, NULL, + CONFIG_DP_THREAD_PRIORITY, task_memory->task.flags, + K_FOREVER); + +#ifdef CONFIG_SCHED_CPU_MASK + /* pin the thread to specific core */ + ret = k_thread_cpu_pin(pdata->thread_id, core); + if (ret < 0) { + tr_err(&dp_tr, "zephyr task pin to core failed"); + goto e_thread; + } +#endif + +#ifdef CONFIG_USERSPACE + if (options & K_USER) { + k_thread_access_grant(pdata->thread_id, pdata->event); + scheduler_dp_grant(pdata->thread_id, core); + + ret = k_mem_domain_add_thread(pdata->mod->user_ctx->comp_dom, pdata->thread_id); + if (ret < 0) { + tr_err(&dp_tr, "k_mem_domain_add_thread() failed %d", ret); + goto e_thread; + } + +#if IS_ENABLED(CONFIG_SOF_USERSPACE_MOD_IPC_BY_DP_THREAD) + pdata->ipc_work_item = userspace_proxy_register_ipc_handler(mod, pdata->event); + assert(pdata->ipc_work_item); +#endif + } +#endif /* CONFIG_USERSPACE */ + + /* start the thread, it should immediately stop at an event */ + k_thread_start(pdata->thread_id); + + /* success, fill output parameter */ + *task = &task_memory->task; + return 0; + +e_thread: + k_thread_abort(pdata->thread_id); +err: + /* cleanup - free all allocated resources */ + if (user_stack_free((__sparse_force void *)p_stack)) + tr_err(&dp_tr, "user_stack_free failed!"); + + /* k_object_free looks for a pointer in the list, any invalid value can be passed */ + k_object_free(task_memory->pdata.event); + k_object_free(task_memory->pdata.thread); + sof_heap_free(user_heap, task_memory); + return ret; +} + +void z_impl_scheduler_dp_internal_free(struct task *task) +{ + struct task_dp_pdata *pdata = task->priv_data; + +#ifdef CONFIG_USERSPACE + if (pdata->event != &pdata->event_struct) + k_object_free(pdata->event); + if (pdata->thread != &pdata->thread_struct) + k_object_free(pdata->thread); +#endif + + /* task is the first member in task_memory above */ + sof_heap_free(pdata->mod->dev->drv->user_heap, task); +} + +#ifdef CONFIG_USERSPACE +#include + +void z_vrfy_scheduler_dp_internal_free(struct task *task) +{ + /* + * With the thread DP scheduler variant scheduler_dp_internal_free() is + * never called from the userspace context + */ + K_OOPS(true); +} +#include +#endif diff --git a/src/kernel/schedule/zephyr_ll-kernel.c b/src/kernel/schedule/zephyr_ll-kernel.c new file mode 100644 index 000000000000..4118733699d4 --- /dev/null +++ b/src/kernel/schedule/zephyr_ll-kernel.c @@ -0,0 +1,274 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2021 Intel Corporation. All rights reserved. +// +// Author: Guennadi Liakhovetski + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); + +/* Kernel part of the Zephyr LL scheduler: the scheduler initialization, the task + * semaphore syscalls and the access grants to the scheduler objects. The rest is in + * src/schedule/zephyr_ll.c + */ + +#if CONFIG_DYNAMIC_OBJECTS +static struct list_item zephyr_ll_task_sem_list = LIST_INIT(zephyr_ll_task_sem_list); + +struct zephyr_ll_task_sem { + struct task *task; + struct k_sem *sem; + struct list_item list; +}; + +/* + * The syscall validators pass in pdata that they read once from user memory + * and validated, so these helpers must not read task->priv_data again. + */ +static int zephyr_ll_task_sem_alloc_pdata(struct task *task, struct zephyr_ll_pdata *pdata) +{ + struct zephyr_ll_task_sem *ts = rmalloc(SOF_MEM_FLAG_COHERENT, sizeof(*ts)); + + if (!ts) + return -ENOMEM; + + ts->sem = k_object_alloc(K_OBJ_SEM); + if (!ts->sem) { + rfree(ts); + return -ENOMEM; + } + + k_sem_init(ts->sem, 0, 1); + +#if CONFIG_SOF_USERSPACE_LL + /* + * The per-task semaphore is signalled from zephyr_ll_task_done(), + * which runs in the (unprivileged) LL scheduler thread when a task is + * freed while it is still running. k_object_alloc() only grants access + * to the calling thread (the IPC handler that creates the task), so the + * LL thread must be granted access explicitly, otherwise its + * k_sem_give() traps with a userspace permission fault. + * + * Resolve the LL thread from kernel-only per-core state keyed by the + * task's target core; never traverse the user-accessible scheduler or + * domain objects from privileged context. + */ + struct k_thread *ll_tid = zephyr_domain_thread_tid_for_core(task->core); + + if (ll_tid) + k_thread_access_grant(ll_tid, ts->sem); +#endif /* CONFIG_SOF_USERSPACE_LL */ + + ts->task = task; + pdata->sem_p = ts->sem; + /* List is protected by IPC serialization */ + list_item_append(&ts->list, &zephyr_ll_task_sem_list); + + return 0; +} + +int z_impl_zephyr_ll_task_sem_alloc(struct task *task) +{ + return zephyr_ll_task_sem_alloc_pdata(task, task->priv_data); +} + +static int zephyr_ll_task_sem_free_pdata(struct task *task, struct zephyr_ll_pdata *pdata) +{ + struct list_item *list; + struct zephyr_ll_task_sem *ts; + bool found = false; + + /* List is protected by IPC serialization */ + list_for_item(list, &zephyr_ll_task_sem_list) { + ts = container_of(list, struct zephyr_ll_task_sem, list); + if (ts->task == task) { + found = true; + break; + } + } + + if (!found) + return -ENOENT; + + if (pdata->sem_p != ts->sem) + return -EINVAL; + + list_item_del(list); + k_object_free(ts->sem); + rfree(ts); + + return 0; +} + +int z_impl_zephyr_ll_task_sem_free(struct task *task) +{ + return zephyr_ll_task_sem_free_pdata(task, task->priv_data); +} + +#ifdef CONFIG_USERSPACE +#include +/* + * Read task->priv_data exactly once. The task lives in user memory and a + * thread on another core can change the pointer after it has been checked. + * + * @return validated pdata pointer, or NULL if the task has none + */ +static struct zephyr_ll_pdata *zephyr_ll_task_pdata_vrfy(struct task *task) +{ + struct zephyr_ll_pdata *pdata; + + K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); + K_OOPS(k_usermode_from_copy(&pdata, &task->priv_data, sizeof(pdata))); + if (pdata) + K_OOPS(K_SYSCALL_MEMORY_WRITE(pdata, sizeof(*pdata))); + + return pdata; +} + +static inline int z_vrfy_zephyr_ll_task_sem_alloc(struct task *task) +{ + struct zephyr_ll_pdata *pdata; + + if (!task) + return -EINVAL; + pdata = zephyr_ll_task_pdata_vrfy(task); + if (!pdata) + return -EINVAL; + + return zephyr_ll_task_sem_alloc_pdata(task, pdata); +} +#include + +static inline int z_vrfy_zephyr_ll_task_sem_free(struct task *task) +{ + struct zephyr_ll_pdata *pdata; + + if (!task) + return -EINVAL; + pdata = zephyr_ll_task_pdata_vrfy(task); + if (!pdata) + return -EINVAL; + + return zephyr_ll_task_sem_free_pdata(task, pdata); +} +#include +#endif /* CONFIG_USERSPACE */ +#endif /* CONFIG_DYNAMIC_OBJECTS */ + +#if CONFIG_SOF_USERSPACE_LL +struct k_thread *zephyr_ll_init_context(void *data, struct task *task) +{ + struct zephyr_ll *sch = data; + struct zephyr_ll_pdata *pdata = task->priv_data; + int ret; + + /* + * Use domain_thread_init() for privileged setup (thread creation, + * timer, access grants). domain_register() is now bookkeeping only + * and will be called later from user context when scheduling tasks. + */ + ret = domain_thread_init(sch->ll_domain, task); + if (ret < 0) { + tr_err(&ll_tr, "cannot init_context %d", ret); + return NULL; + } + + assert(!k_is_user_context()); + k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock, pdata->sem_p); + + tr_dbg(&ll_tr, "granting access to lock %p for thread %p", sch->lock, + zephyr_domain_thread_tid(sch->ll_domain)); + tr_dbg(&ll_tr, "granting access to domain lock %p for thread %p", &sch->ll_domain->lock, + zephyr_domain_thread_tid(sch->ll_domain)); + + return zephyr_domain_thread_tid(sch->ll_domain); +} + +void user_ll_grant_access(struct k_thread *thread, int core) +{ + assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] != NULL); + + k_thread_access_grant(thread, zephyr_ll_locks[core]); +} +#endif /* CONFIG_SOF_USERSPACE_LL */ + +static const struct scheduler_ops zephyr_ll_ops = { + .schedule_task = zephyr_ll_task_schedule, + .schedule_task_before = zephyr_ll_task_schedule_before, + .schedule_task_after = zephyr_ll_task_schedule_after, + .schedule_task_free = zephyr_ll_task_free, + .schedule_task_cancel = zephyr_ll_task_cancel, +#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY + .scheduler_free = zephyr_ll_scheduler_free, +#endif +#if CONFIG_SOF_USERSPACE_LL + .scheduler_init_context = zephyr_ll_init_context, +#endif +}; + +/* TODO: low-power mode clock support */ +/* Runs on each core during initialisation with the same domain argument */ +__cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) +{ + struct zephyr_ll *sch; + int core = cpu_get_id(); + struct k_heap *heap = sof_sys_heap_get(); + int flags = SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT; + + assert_can_be_cold(); + +#if CONFIG_SOF_USERSPACE_LL + heap = zephyr_ll_user_heap(); + flags = SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT; +#endif + tr_dbg(&ll_tr, "init on core %d", core); + + /* initialize per-core scheduler private data */ + sch = sof_heap_alloc(heap, flags, sizeof(*sch), 0); + if (!sch) { + tr_err(&ll_tr, "allocation failed"); + return -ENOMEM; + } + + memset(sch, 0, sizeof(*sch)); + + list_init(&sch->tasks); + sch->ll_domain = domain; + sch->core = core; + sch->n_tasks = 0; + sch->heap = heap; + +#if CONFIG_SOF_USERSPACE_LL + /* Allocate mutex dynamically for userspace access */ + sch->lock = k_object_alloc(K_OBJ_MUTEX); + if (!sch->lock) { + tr_err(&ll_tr, "mutex allocation failed"); + sof_heap_free(sch->heap, sch); + return -ENOMEM; + } + assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] == NULL); + zephyr_ll_locks[core] = sch->lock; + k_mutex_init(sch->lock); + + tr_dbg(&ll_tr, "ll-scheduler init done, sch %p sch->lock %p", sch, sch->lock); +#endif + + scheduler_init(domain->type, &zephyr_ll_ops, sch); + + return 0; +} diff --git a/src/schedule/zephyr_twb_schedule.c b/src/kernel/schedule/zephyr_twb_schedule.c similarity index 100% rename from src/schedule/zephyr_twb_schedule.c rename to src/kernel/schedule/zephyr_twb_schedule.c diff --git a/src/schedule/CMakeLists.txt b/src/schedule/CMakeLists.txt index bc14c6ecff46..ee9ceb32ef9d 100644 --- a/src/schedule/CMakeLists.txt +++ b/src/schedule/CMakeLists.txt @@ -50,8 +50,4 @@ else() ) endif() -zephyr_library_sources_ifdef(CONFIG_ZEPHYR_TWB_SCHEDULER - zephyr_twb_schedule.c -) - endif() diff --git a/src/schedule/README.md b/src/schedule/README.md index 0bef23248837..d2a86f6bf55c 100644 --- a/src/schedule/README.md +++ b/src/schedule/README.md @@ -1,12 +1,15 @@ # SOF Scheduling Architecture -> **Runs in:** Boundary / bridge — this directory owns the LL/DP schedulers and their thread + -> memory-domain setup. `zephyr_ll.c` and `zephyr_domain.c` are kernel-side; the LL user memory -> domain and heap are set up by `src/kernel/schedule/zephyr_ll_user.c`. -> `zephyr_dp_schedule_thread.c` is boundary code that can create and manage both kernel and -> `K_USER` DP threads depending on configuration; `zephyr_dp_schedule_application.c` is the -> DP application-mode path. This directory hosts the `zephyr_ll_task_sem_*` and -> `scheduler_dp_*` syscalls. See [User/kernel split](../include/sof/userspace/README.md). +> **Runs in:** Boundary / bridge — this directory holds the parts of the LL and DP schedulers +> that run in the scheduler and IPC threads, which are user threads with user-space LL. Their +> kernel parts are in `src/kernel/schedule`, one `*-kernel.c` file per source here: scheduler +> registration and initialization, the LL thread and timer setup and the timer interrupt, the +> DP thread creation and memory domains, the access grants, and the kernel halves of the +> `zephyr_ll_task_sem_*` and `scheduler_dp_*` syscalls. The LL user memory domain and heap are +> set up by `src/kernel/schedule/zephyr_ll_user.c`, and the TWB scheduler, which is kernel-only, +> is `src/kernel/schedule/zephyr_twb_schedule.c`. The scheduler data structures shared by both +> parts are in `sof/schedule/zephyr_ll.h` and `sof/schedule/zephyr_dp_schedule.h`. See +> [User/kernel split](../include/sof/userspace/README.md). This directory (`src/schedule`) contains the Sound Open Firmware (SOF) scheduling infrastructure, deeply integrated with the underlying Zephyr RTOS. SOF utilizes a multi-tiered scheduling approach to cater to different real-time constraints, ranging from hard real-time, low-latency requirements to more relaxed, compute-intensive data processing tasks. @@ -139,7 +142,7 @@ stateDiagram-v2 ## 3. TWB (Thread With Budget) Scheduler -The TWB scheduler (`zephyr_twb_schedule.c`) provides execution budget limits for specific tasks to prevent them from starving the CPU. This is useful for intensive workloads that shouldn't disrupt the overall systemic low-latency chain. +The TWB scheduler (`src/kernel/schedule/zephyr_twb_schedule.c`) provides execution budget limits for specific tasks to prevent them from starving the CPU. This is useful for intensive workloads that shouldn't disrupt the overall systemic low-latency chain. ### Architecture diff --git a/src/schedule/schedule.c b/src/schedule/schedule.c index fd3349a14902..62438be7c663 100644 --- a/src/schedule/schedule.c +++ b/src/schedule/schedule.c @@ -25,16 +25,9 @@ SOF_DEFINE_REG_UUID(schedule); /* unused with Zephyr, generates no output */ DECLARE_TR_CTX(sch_tr, SOF_UUID(schedule_uuid), LOG_LEVEL_INFO); -static inline bool scheduler_is_user(int type) -{ - /* - * currently only LL managed in user-space, but longterm - * goal is to move all audio application level scheduling - * to user-space and only keep Zephyr scheduler logic in - * kernel - */ - return type == SOF_SCHEDULE_LL_TIMER || type == SOF_SCHEDULE_DP; -} +/* Scheduler registration, which runs in kernel mode, is in + * src/kernel/schedule/schedule-kernel.c + */ int schedule_task_init(struct task *task, const struct sof_uuid_entry *uid, uint16_t type, @@ -84,60 +77,6 @@ int schedule_task_init(struct task *task, return 0; } -__cold static void scheduler_register(struct schedule_data *scheduler) -{ - struct schedulers **sch = arch_schedulers_get(); - struct k_heap *heap = NULL; - - assert_can_be_cold(); - - if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(scheduler->type)) { - sch = arch_user_schedulers_get(); - heap = sof_sys_user_heap_get(); - } - - if (!*sch) { - /* init schedulers list */ - *sch = sof_heap_alloc(heap, 0, sizeof(**sch), 0); - if (!*sch) { - tr_err(&sch_tr, "allocation failed"); - return; - } - memset(*sch, 0, sizeof(**sch)); - list_init(&(*sch)->list); - } - - list_item_append(&scheduler->list, &(*sch)->list); -} - -__cold void scheduler_init(int type, const struct scheduler_ops *ops, void *data) -{ - struct schedule_data *sch; - struct k_heap *heap = NULL; - - assert_can_be_cold(); - - if (IS_ENABLED(CONFIG_SOF_USERSPACE_LL) && scheduler_is_user(type)) - heap = sof_sys_user_heap_get(); - - if (!ops || !ops->schedule_task || !ops->schedule_task_cancel || - !ops->schedule_task_free) - return; - - sch = sof_heap_alloc(heap, SOF_MEM_FLAG_KERNEL, sizeof(*sch), 0); - if (!sch) { - tr_err(&sch_tr, "allocation failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - memset(sch, 0, sizeof(*sch)); - list_init(&sch->list); - sch->type = type; - sch->ops = ops; - sch->data = data; - - scheduler_register(sch); -} - /* Locks for the list here should be held by the caller, * as different schedulers use different locks */ diff --git a/src/schedule/zephyr_domain.c b/src/schedule/zephyr_domain.c index 9e10212a1cda..a26e1cea854c 100644 --- a/src/schedule/zephyr_domain.c +++ b/src/schedule/zephyr_domain.c @@ -4,17 +4,14 @@ // // Author: Tomasz Lauda -#include #include #include -#include #include -#include #include #include -#include #include #include +#include #include #include #include @@ -28,47 +25,9 @@ LOG_MODULE_DECLARE(ll_schedule, CONFIG_SOF_LOG_LEVEL); -/* - * Currently the Zephyr clock rate is part it's Kconfig known at build time. - * SOF on Intel CAVS platforms currently only aligns with Zephyr when both - * use the CAVS 19.2 MHz SSP clock. TODO - needs runtime alignment. +/* The thread and timer setup, which runs in kernel mode, is in + * src/kernel/schedule/zephyr_domain-kernel.c */ -#if CONFIG_XTENSA && CONFIG_CAVS && !CONFIG_INTEL_ADSP_TIMER -#error "Zephyr uses 19.2MHz clock derived from SSP which must be enabled." -#endif - -#define ZEPHYR_LL_STACK_SIZE 8192 - -#if CONFIG_SOF_USERSPACE_LL -K_THREAD_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); -#else -K_KERNEL_STACK_ARRAY_DEFINE(ll_sched_stack, CONFIG_CORE_COUNT, ZEPHYR_LL_STACK_SIZE); -#endif - -struct zephyr_domain_thread { - struct k_thread *ll_thread; - struct k_sem *sem; -#ifndef CONFIG_SOF_USERSPACE_LL - struct k_thread ll_thread_obj; - struct k_sem sem_obj; -#endif - void (*handler)(void *arg); - void *arg; -}; - -struct zephyr_domain { - struct k_timer *timer; -#ifndef CONFIG_SOF_USERSPACE_LL - struct k_timer timer_obj; -#endif - struct zephyr_domain_thread domain_thread[CONFIG_CORE_COUNT]; - struct ll_schedule_domain *ll_domain; -#if CONFIG_CROSS_CORE_STREAM - atomic_t block; - struct k_mutex block_mutex; - struct k_condvar block_condvar; -#endif -}; /* perf measurement windows size 2^x */ #define CYCLES_WINDOW_SIZE CONFIG_SCHEDULE_LL_STATS_LOG_WINDOW_SIZE @@ -86,7 +45,7 @@ static inline void stats_report(unsigned int runs, int core, unsigned int cycles } #endif /* CONFIG_SCHEDULE_LL_STATS_LOG */ -static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) +void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) { struct zephyr_domain *zephyr_domain = p1; int core = POINTER_TO_INT(p2); @@ -155,247 +114,14 @@ static void zephyr_domain_thread_fn(void *p1, void *p2, void *p3) } } -/* Timer callback: runs in timer IRQ context */ -static void zephyr_domain_timer_fn(struct k_timer *timer) -{ - struct zephyr_domain *zephyr_domain = k_timer_user_data_get(timer); - int core; - - /* - * A race is possible when the Zephyr LL scheduling domain is being - * unregistered while a timer IRQ is processed on a different core. Then - * the timer is removed by the former but then re-added by the latter, - * but this time with no user data and no handler set. This leads to the - * timer continuing to trigger and then leading to a lock up when it is - * registered again next time. - */ - if (!zephyr_domain) { - k_timer_stop(timer); - return; - } - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - if (dt->handler) - k_sem_give(dt->sem); - } -} - -/* The normal kernel-space implementation for register/unregister */ -#ifndef CONFIG_SOF_USERSPACE_LL - -static int zephyr_domain_register(struct ll_schedule_domain *domain, - struct task *task, - void (*handler)(void *arg), void *arg) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - char thread_name[] = "ll_thread0"; - k_tid_t thread; - k_spinlock_key_t key; - - tr_dbg(&ll_tr, "entry"); - - /* domain work only needs registered once on each core */ - if (dt->handler) - return 0; - - dt->handler = handler; - dt->arg = arg; - - /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ - k_sem_init(dt->sem, 0, 10); - - thread_name[sizeof(thread_name) - 2] = '0' + core; - - /* not allocated dynamically when LL in kernel space */ - dt->ll_thread = &dt->ll_thread_obj; - - thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, - zephyr_domain_thread_fn, zephyr_domain, INT_TO_POINTER(core), - NULL, CONFIG_LL_THREAD_PRIORITY, 0, K_FOREVER); - -#ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_pin(thread, core); -#endif - k_thread_name_set(thread, thread_name); - - k_thread_start(thread); - - key = k_spin_lock(&domain->lock); - - if (!k_timer_user_data_get(zephyr_domain->timer)) { - k_timeout_t start = {0}; - - k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); - k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); - - k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); - - /* Enable the watchdog */ - watchdog_enable(core); - } - - k_spin_unlock(&domain->lock, key); - - tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", - domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); - - return 0; -} - -static int zephyr_domain_unregister(struct ll_schedule_domain *domain, - struct task *task, uint32_t num_tasks) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - k_spinlock_key_t key; - - tr_dbg(&ll_tr, "entry"); - - /* tasks still registered on this core */ - if (num_tasks) - return 0; - - key = k_spin_lock(&domain->lock); - - if (!atomic_read(&domain->total_num_tasks)) { - /* Disable the watchdog */ - watchdog_disable(core); - - k_timer_stop(zephyr_domain->timer); - k_timer_user_data_set(zephyr_domain->timer, NULL); - } - - zephyr_domain->domain_thread[core].handler = NULL; - - k_spin_unlock(&domain->lock, key); - - tr_info(&ll_tr, "domain->type %d domain->clk %d", - domain->type, domain->clk); - - /* - * If running in the context of the domain thread, k_thread_abort() will - * not return - */ - k_thread_abort(zephyr_domain->domain_thread[core].ll_thread); - - tr_dbg(&ll_tr, "exit"); - - return 0; -} - -#else /* CONFIG_SOF_USERSPACE_LL */ - -/* - * Kernel-owned per-core LL thread table. Populated from the privileged - * domain-thread init path and consulted by z_impl_zephyr_ll_task_sem_alloc() - * so that privileged code never has to traverse the user-accessible - * scheduler/domain objects to find the LL thread of a given core. - */ -static struct k_thread *ll_thread_tid[CONFIG_CORE_COUNT]; - -/* - * Privileged thread initialization for userspace LL scheduling. - * Creates the scheduling thread, sets up timer, grants access to kernel - * objects. Must be called from kernel context before any user-space - * domain_register() calls. - */ -static int zephyr_domain_thread_init(struct ll_schedule_domain *domain, - struct task *task) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - struct zephyr_domain_thread *dt; - char thread_name[] = "ll_thread0"; - k_tid_t thread; - int core = task->core; - - tr_dbg(&ll_tr, "entry"); - - if (core >= CONFIG_CORE_COUNT) - return -EINVAL; - - dt = zephyr_domain->domain_thread + core; - - /* thread only needs to be created once per core */ - if (dt->ll_thread) - return 0; - - dt->handler = NULL; - dt->arg = NULL; - - dt->sem = k_object_alloc(K_OBJ_SEM); - if (!dt->sem) { - tr_err(&ll_tr, "Failed to allocate semaphore for core %d", core); - return -ENOMEM; - } - - /* 10 is rather random, we better not accumulate 10 missed timer interrupts */ - k_sem_init(dt->sem, 0, 10); - - thread_name[sizeof(thread_name) - 2] = '0' + core; - - /* Allocate thread structure dynamically */ - dt->ll_thread = k_object_alloc(K_OBJ_THREAD); - if (!dt->ll_thread) { - tr_err(&ll_tr, "Failed to allocate thread object for core %d", core); - k_object_free(dt->sem); - dt->sem = NULL; - return -ENOMEM; - } - - thread = k_thread_create(dt->ll_thread, ll_sched_stack[core], ZEPHYR_LL_STACK_SIZE, - zephyr_domain_thread_fn, zephyr_domain, - INT_TO_POINTER(core), NULL, CONFIG_LL_THREAD_PRIORITY, - K_USER, K_FOREVER); - - /* record in the kernel-only table for syscall-context lookups */ - ll_thread_tid[core] = dt->ll_thread; - -#ifdef CONFIG_SCHED_CPU_MASK - k_thread_cpu_pin(thread, core); -#endif - k_thread_name_set(thread, thread_name); - - k_mem_domain_add_thread(zephyr_ll_mem_domain(), thread); - k_thread_access_grant(thread, dt->sem, domain->lock, zephyr_domain->timer); - user_grant_dai_access_all(thread); - user_grant_dma_access_all(thread); - user_grant_uaol_access_all(thread); - tr_dbg(&ll_tr, "granted LL access to thread %p (core %d)", thread, core); - - k_thread_start(thread); - - k_mutex_lock(domain->lock, K_FOREVER); - if (!k_timer_user_data_get(zephyr_domain->timer)) { - k_timeout_t start = {0}; - - k_timer_init(zephyr_domain->timer, zephyr_domain_timer_fn, NULL); - k_timer_user_data_set(zephyr_domain->timer, zephyr_domain); - - k_timer_start(zephyr_domain->timer, start, K_USEC(LL_TIMER_PERIOD_US)); - - /* Enable the watchdog */ - watchdog_enable(core); - } - - k_mutex_unlock(domain->lock); - - tr_info(&ll_tr, "domain->type %d domain->clk %d domain->ticks_per_ms %d period %d", - domain->type, domain->clk, domain->ticks_per_ms, (uint32_t)LL_TIMER_PERIOD_US); - - return 0; -} - +#ifdef CONFIG_SOF_USERSPACE_LL /* * User-space register: bookkeeping only. The privileged thread setup has * already been done by domain_thread_init() called from kernel context. */ -static int zephyr_domain_register_user(struct ll_schedule_domain *domain, - struct task *task, - void (*handler)(void *arg), void *arg) +int zephyr_domain_register_user(struct ll_schedule_domain *domain, + struct task *task, + void (*handler)(void *arg), void *arg) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); struct zephyr_domain_thread *dt; @@ -426,8 +152,8 @@ static int zephyr_domain_register_user(struct ll_schedule_domain *domain, return 0; } -static int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, - struct task *task, uint32_t num_tasks) +int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, + struct task *task, uint32_t num_tasks) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); int core = task->core; @@ -454,94 +180,10 @@ static int zephyr_domain_unregister_user(struct ll_schedule_domain *domain, return 0; } - -/* - * Free resources acquired by zephyr_domain_thread_init(). - * Stops the timer, aborts the scheduling thread and frees the thread object. - * Must be called from kernel context. - */ -static void zephyr_domain_thread_free(struct ll_schedule_domain *domain, - uint32_t num_tasks) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - tr_dbg(&ll_tr, "thread_free entry, core %d, num_tasks %u", core, num_tasks); - - /* Still tasks on other cores, only clean up this core's thread */ - k_mutex_lock(domain->lock, K_FOREVER); - - if (!num_tasks && !atomic_read(&domain->total_num_tasks)) { - /* Last task globally: stop the timer and watchdog */ - watchdog_disable(core); - - k_timer_stop(zephyr_domain->timer); - k_timer_user_data_set(zephyr_domain->timer, NULL); - } - - dt->handler = NULL; - dt->arg = NULL; - - k_mutex_unlock(domain->lock); - - if (dt->ll_thread) { - k_thread_abort(dt->ll_thread); - k_object_free(dt->ll_thread); - dt->ll_thread = NULL; - ll_thread_tid[core] = NULL; - } - - if (dt->sem) { - k_object_free(dt->sem); - dt->sem = NULL; - } - - tr_info(&ll_tr, "thread_free done, core %d", core); -} - -struct k_thread *zephyr_domain_thread_tid(struct ll_schedule_domain *domain) -{ - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); - int core = cpu_get_id(); - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - tr_dbg(&ll_tr, "entry"); - - return dt->ll_thread; -} - -/* - * Return the LL scheduling thread for an explicitly given core. - * - * Reads a kernel-only table keyed by core, so it is safe to call from a - * privileged syscall context without dereferencing any user-accessible - * scheduler or domain object, and without relying on cpu_get_id(). - */ -struct k_thread *zephyr_domain_thread_tid_for_core(int core) -{ - if (core < 0 || core >= CONFIG_CORE_COUNT) - return NULL; - - return ll_thread_tid[core]; -} - -struct k_thread *zephyr_ll_domain_thread(void) -{ - struct ll_schedule_domain *ll_domain = zephyr_ll_domain(); - - if (!ll_domain) - return NULL; - - struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(ll_domain); - - return zephyr_domain->domain_thread[cpu_get_id()].ll_thread; -} - #endif /* CONFIG_SOF_USERSPACE_LL */ #if CONFIG_CROSS_CORE_STREAM -static void zephyr_domain_block(struct ll_schedule_domain *domain) +void zephyr_domain_block(struct ll_schedule_domain *domain) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); @@ -552,7 +194,7 @@ static void zephyr_domain_block(struct ll_schedule_domain *domain) k_mutex_unlock(&zephyr_domain->block_mutex); } -static void zephyr_domain_unblock(struct ll_schedule_domain *domain) +void zephyr_domain_unblock(struct ll_schedule_domain *domain) { struct zephyr_domain *zephyr_domain = ll_sch_domain_get_pdata(domain); @@ -565,87 +207,6 @@ static void zephyr_domain_unblock(struct ll_schedule_domain *domain) } #endif /* CONFIG_CROSS_CORE_STREAM */ -APP_TASK_DATA static const struct ll_schedule_domain_ops zephyr_domain_ops = { -#ifdef CONFIG_SOF_USERSPACE_LL - .domain_register = zephyr_domain_register_user, - .domain_unregister = zephyr_domain_unregister_user, - .domain_thread_init = zephyr_domain_thread_init, - .domain_thread_free = zephyr_domain_thread_free, -#else - .domain_register = zephyr_domain_register, - .domain_unregister = zephyr_domain_unregister, -#endif -#if CONFIG_CROSS_CORE_STREAM - .domain_block = zephyr_domain_block, - .domain_unblock = zephyr_domain_unblock, -#endif -}; - -/* Core 0 only */ -struct ll_schedule_domain *zephyr_domain_init(int clk) -{ - struct ll_schedule_domain *domain; - struct zephyr_domain *zephyr_domain; - int core; - - domain = domain_init(SOF_SCHEDULE_LL_TIMER, clk, false, - &zephyr_domain_ops); - if (!domain) { - tr_err(&ll_tr, "domain init failed"); - return NULL; - } - -#if CONFIG_SOF_USERSPACE_LL - zephyr_domain = sof_heap_alloc(zephyr_ll_user_heap(), - SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*zephyr_domain), sizeof(void *)); - if (zephyr_domain) - memset(zephyr_domain, 0, sizeof(*zephyr_domain)); -#else - zephyr_domain = rzalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, - sizeof(*zephyr_domain)); -#endif - if (!zephyr_domain) { - tr_err(&ll_tr, "domain allocation failed"); - k_panic(); - return NULL; - } - - zephyr_domain->ll_domain = domain; - -#if CONFIG_SOF_USERSPACE_LL - /* Allocate timer dynamically for userspace access */ - zephyr_domain->timer = k_object_alloc(K_OBJ_TIMER); - if (!zephyr_domain->timer) { - tr_err(&ll_tr, "timer allocation failed"); - k_panic(); - return NULL; - } -#else - /* not allocated dynamically when LL in kernel space */ - zephyr_domain->timer = &zephyr_domain->timer_obj; -#endif - -#if CONFIG_CROSS_CORE_STREAM - atomic_set(&zephyr_domain->block, 0); - k_mutex_init(&zephyr_domain->block_mutex); - k_condvar_init(&zephyr_domain->block_condvar); -#endif - - ll_sch_domain_set_pdata(domain, zephyr_domain); - - for (core = 0; core < CONFIG_CORE_COUNT; core++) { -#ifndef CONFIG_SOF_USERSPACE_LL - struct zephyr_domain_thread *dt = zephyr_domain->domain_thread + core; - - /* not allocated dynamically when LL in kernel space */ - dt->sem = &dt->sem_obj; -#endif - } - - return domain; -} - /* Check if currently running in the LL scheduler thread context */ bool ll_sch_is_current(void) { diff --git a/src/schedule/zephyr_dp_schedule.c b/src/schedule/zephyr_dp_schedule.c index 314f8df5bba0..ac50500abd7f 100644 --- a/src/schedule/zephyr_dp_schedule.c +++ b/src/schedule/zephyr_dp_schedule.c @@ -24,28 +24,19 @@ #include #include -#include "zephyr_dp_schedule.h" +#include #include #include #include -LOG_MODULE_REGISTER(dp_schedule, CONFIG_SOF_LOG_LEVEL); -SOF_DEFINE_REG_UUID(dp_sched); +LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); +extern struct tr_ctx dp_tr; -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(dp_tr, SOF_UUID(dp_sched_uuid), LOG_LEVEL_INFO); - -#define DP_LOCK_INIT(i, _) Z_SEM_INITIALIZER(dp_lock[i], 1, 1) -#define DP_LOCK_INIT_LIST LISTIFY(CONFIG_MP_MAX_NUM_CPUS, DP_LOCK_INIT, (,)) - -/* User threads don't need access to this array. Access is performed from - * the kernel space via a syscall. Array must be placed in special section - * to be qualified as initialized by the gen_kobject_list.py script. +/* The scheduler locks, the LL tick system call and the scheduler initialization, which + * run in kernel mode, are in src/kernel/schedule/zephyr_dp_schedule-kernel.c */ -static -STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_lock, CONFIG_MP_MAX_NUM_CPUS) = { DP_LOCK_INIT_LIST }; /* Each per-core instance of DP scheduler has separate structures; hence, locks are per-core. * @@ -62,202 +53,8 @@ void scheduler_dp_unlock(unsigned int key) k_sem_give(&dp_lock[key]); } -void scheduler_dp_grant(k_tid_t thread_id, uint16_t core) -{ - if (thread_id) - k_thread_access_grant(thread_id, &dp_lock[core]); -} - -/* dummy LL task - to start LL on secondary cores */ -static enum task_state scheduler_dp_ll_tick_dummy(void *data) -{ - return SOF_TASK_STATE_RESCHEDULE; -} - -/* - * function called after every LL tick - * - * This function checks if the queued DP tasks are ready to processing (meaning - * the module run by the task has enough data at all sources and enough free space - * on all sinks) - * - * if the task becomes ready, a deadline is set allowing Zephyr to schedule threads - * in right order - * - * TODO: currently there's a limitation - DP module must be surrounded by LL modules. - * it simplifies algorithm - there's no need to browse through DP chains calculating - * deadlines for each module in function of all modules execution status. - * Now is simple - modules deadline is its start + tick time. - * - * example: - * Lets assume we do have a pipeline: - * - * LL1 -> DP1 -> LL2 -> DP2 -> LL3 -> DP3 -> LL4 - * - * all LLs starts in 1ms tick - * - * for simplification lets assume - * - all LLs are on primary core, all DPs on secondary (100% CPU is for DP) - * - context switching requires 0 cycles - * - * DP1 - starts every 1ms, needs 0.5ms to finish processing - * DP2 - starts every 2ms, needs 0.6ms to finish processing - * DP3 - starts every 10ms, needs 0.3ms to finish processing - * - * TICK0 - * only LL1 is ready to run - * LL1 processing (producing data chunk for DP1) - * - * TICK1 - * LL1 is ready to run - * DP1 is ready tu run (has data from LL1) set deadline to TICK2 - * LL1 processing (producing second data chunk for DP1) - * DP1 processing for 0.5ms (consuming first data chunk, producing data chunk for LL2) - * CPU is idle for 0.5ms - * - * TICK2 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK3 - * LL2 is ready to run - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * CPU is idle for 0.5ms - * - * TICK3 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK4 - * LL2 is ready to run - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * CPU is idle for 0.5ms - * - * TICK4 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK5 - * LL2 is ready to run - * DP2 is ready to run set deadline to TICK6 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% of second data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - * - * !!!!!! Note here - DP1 must do before DP2 as it MUST finish in this tick. DP2 can wait - * >>>>>>> this is what we call EDF - EARIEST DEADLINE FIRST <<<<<< - * - * TICK5 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK6 - * LL2 is ready to run - * DP2 is in progress, deadline is set to TICK6 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of second data chunk for DP2) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.1ms (producing TWO data chunks for LL3) - * CPU is idle for 0.4ms (60% used) - * - * TICK6 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK7 - * LL2 is ready to run - * DP2 is ready to run set deadline to TICK8 - * LL3 is ready to run - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% of second data chunk for DP2) - * LL3 processing (producing 10% of first data chunk for DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - * - * - * - * (........ 9 more cycles - LL3 procuces 100% of data for DP3......) - * - * - * TICK15 - * LL1 is ready to run - * DP1 is ready tu run set deadline to TICK16 - * LL2 is ready to run - * DP2 is ready to run set deadline to TICK17 - * LL3 is ready to run - * DP3 is ready to run set deadline to TICK25 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing 50% of data chunk for DP2) - * LL3 processing (producing 10% of second data chunk for DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - - * !!! note that DP3 is ready but has no chance to get CPU in this cycle - * - * TICK16 - * LL1 is ready to run set deadline to TICK17 - * DP1 is ready tu run - * LL2 is ready to run - * DP2 is in progress, deadline is set to TICK17 - * LL3 is ready to run - * DP3 is in progress, deadline is set to TICK25 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of data chunk for DP2) - * LL3 processing (producing 10% of second data chunk for DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.1ms (producing data) - * DP3 processing for 0.2ms (producing 10 data chunks for LL4) - * 90% CPU used - * - * TICK17 - * LL1 is ready to run - * DP1 is ready tu run - * LL2 is ready to run - * DP2 is ready to run - * LL3 is ready to run - * LL4 is ready to run - * !! NOTE that DP3 is not ready - it will be ready again in TICK25 - * LL1 processing (producing data chunk for DP1) - * LL2 processing (producing rest of data chunk for DP2) - * LL3 processing (producing next 10% of second data chunk for DP3) - * LL4 processing (consuming 10% of data prepared by DP3) - * DP1 processing for 0.5ms (producing data chunk for LL2) - * DP2 processing for 0.5ms (no data produced as DP2 has 0.1ms to go) - * 100% CPU used - * - * - * Now - pipeline is in stable state, CPU used almost in 100% (it would be 100% if DP3 - * needed 1.2ms for processing - but the example would be too complicated) - */ - -void z_impl_scheduler_dp_ll_tick(void) -{ - unsigned int lock_key; -#if CONFIG_SOF_USERSPACE_LL - struct scheduler_dp_data *dp_sch = scheduler_get_user_data(SOF_SCHEDULE_DP); -#else - struct scheduler_dp_data *dp_sch = scheduler_get_data(SOF_SCHEDULE_DP); -#endif - - if (!dp_sch) - return; - - /* remember current timestamp as "NOW" */ - dp_sch->last_ll_tick_timestamp = k_cycle_get_32(); - - lock_key = scheduler_dp_lock(cpu_get_id()); - scheduler_dp_recalculate(dp_sch); - scheduler_dp_unlock(lock_key); -} - -#ifdef CONFIG_USERSPACE -#include -void z_vrfy_scheduler_dp_ll_tick(void) -{ - z_impl_scheduler_dp_ll_tick(); -} -#include -#endif - #if CONFIG_SOF_USERSPACE_APPLICATION -static int scheduler_dp_task_cancel(void *data, struct task *task) +int scheduler_dp_task_cancel(void *data, struct task *task) { /* Should never be called */ k_panic(); @@ -265,7 +62,7 @@ static int scheduler_dp_task_cancel(void *data, struct task *task) } #endif -static int scheduler_dp_task_stop(void *data, struct task *task) +int scheduler_dp_task_stop(void *data, struct task *task) { unsigned int lock_key; struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)data; @@ -293,7 +90,7 @@ static int scheduler_dp_task_stop(void *data, struct task *task) return 0; } -static int scheduler_dp_task_free(void *data, struct task *task) +int scheduler_dp_task_free(void *data, struct task *task) { struct task_dp_pdata *pdata = task->priv_data; int ret; @@ -317,8 +114,8 @@ static int scheduler_dp_task_free(void *data, struct task *task) return ret; } -static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, - uint64_t period) +int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t start, + uint64_t period) { struct scheduler_dp_data *dp_sch = (struct scheduler_dp_data *)data; struct task_dp_pdata *pdata = task->priv_data; @@ -354,39 +151,6 @@ static int scheduler_dp_task_shedule(void *data, struct task *task, uint64_t sta return 0; } -APP_SYSUSER_DATA static struct scheduler_ops schedule_dp_ops = { - .schedule_task = scheduler_dp_task_shedule, -#if CONFIG_SOF_USERSPACE_APPLICATION - .schedule_task_cancel = scheduler_dp_task_cancel, -#else - .schedule_task_cancel = scheduler_dp_task_stop, -#endif - .schedule_task_free = scheduler_dp_task_free, -}; - -/* Runs on each core */ -__cold int scheduler_dp_init(void) -{ - assert_can_be_cold(); - - struct scheduler_dp_data *dp_sch = sof_heap_alloc(sof_sys_user_heap_get(), - SOF_MEM_FLAG_KERNEL, sizeof(*dp_sch), 0); - if (!dp_sch) - return -ENOMEM; - - memset(dp_sch, 0, sizeof(*dp_sch)); - list_init(&dp_sch->tasks); - - scheduler_init(SOF_SCHEDULE_DP, &schedule_dp_ops, dp_sch); - - /* init src of DP tick */ - return schedule_task_init_ll(&dp_sch->ll_tick_src, - SOF_UUID(dp_sched_uuid), - SOF_SCHEDULE_LL_TIMER, - 0, scheduler_dp_ll_tick_dummy, dp_sch, - cpu_get_id(), 0); -} - void scheduler_get_task_info_dp(struct scheduler_props *scheduler_props, uint32_t *data_off_size) { unsigned int lock_key; diff --git a/src/schedule/zephyr_dp_schedule_application.c b/src/schedule/zephyr_dp_schedule_application.c index 5e69d0943419..bb9cf29b87a3 100644 --- a/src/schedule/zephyr_dp_schedule_application.c +++ b/src/schedule/zephyr_dp_schedule_application.c @@ -29,57 +29,14 @@ #include #include -#include "zephyr_dp_schedule.h" +#include LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx dp_tr; -static struct objpool_head dp_mdom_head = {.list = LIST_INIT(dp_mdom_head.list)}; - -/* Synchronization semaphore for the scheduler thread to wait for DP startup */ -#define DP_SYNC_INIT(i, _) Z_SEM_INITIALIZER(dp_sync[i], 0, 1) -#define DP_SYNC_INIT_LIST LISTIFY(CONFIG_CORE_COUNT, DP_SYNC_INIT, (,)) -static STRUCT_SECTION_ITERABLE_ARRAY(k_sem, dp_sync, CONFIG_CORE_COUNT) = { DP_SYNC_INIT_LIST }; - -struct ipc4_flat { - unsigned int cmd; - int ret; - union { - struct { - struct ipc4_module_bind_unbind bu; - enum bind_type type; - } bind; - struct { - unsigned int trigger_cmd; - enum ipc4_pipeline_state state; - int n_sources; - int n_sinks; - struct sof_source *source[CONFIG_MODULE_MAX_CONNECTIONS]; - struct sof_sink *sink[CONFIG_MODULE_MAX_CONNECTIONS]; - } pipeline_state; - /* - * SOF_IPC4_MOD_INIT_INSTANCE: a by-value copy of the caller's - * ext_data. The original lives on the calling thread's stack, - * which the DP thread's memory domain doesn't have access to. - * The pointers it contains reference the IPC mailbox, which stays - * valid and is covered by the DP thread's SOF_DP_PART_CFG partition. - */ - struct module_ext_init_data init_instance; - struct { - uint32_t param_id; - enum module_cfg_fragment_position position; - uint32_t data_offset_size; - size_t fragment_size; - const char *data; - } set_config; - struct { - uint32_t param_id; - uint32_t data_offset_size; - size_t fragment_size; - char *data; - } get_config; - }; -}; +/* The DP thread creation and the scheduler_dp_internal_free() system call, which run + * in kernel mode, are in src/kernel/schedule/zephyr_dp_schedule_application-kernel.c + */ /* Pack IPC input data */ static int ipc_thread_flatten(unsigned int cmd, const union scheduler_dp_thread_ipc_param *param, @@ -142,104 +99,6 @@ static int ipc_thread_flatten(unsigned int cmd, const union scheduler_dp_thread_ return 0; } -/* memory allocation helper structure */ -struct scheduler_dp_task_memory { - struct task task; - struct task_dp_pdata pdata; - struct comp_driver drv; - struct ipc4_flat flat; -}; - -/* - * Kernel-only record of the kernel objects owned by a DP task. - * - * struct scheduler_dp_task_memory lives in the module's heap, which is mapped - * into the DP thread's memory domain, and is also writable by the LL and IPC - * user threads. Freeing must therefore never use the pointers stored there: - * scheduler_dp_internal_free() takes them from this record instead. The task - * pointer is only used as a lookup key. - */ -struct scheduler_dp_task_kobj { - struct list_item list; - struct task *task; - struct processing_module *mod; - struct k_event *event; - struct k_thread *thread; - struct k_mem_domain *mdom; - struct k_mem_partition mpart[SOF_DP_PART_TYPE_COUNT]; - uint32_t comp_id; - uint32_t gen; /* unique per record, see dp_task_kobj_take() */ -}; - -static struct list_item dp_task_kobj_list = LIST_INIT(dp_task_kobj_list); -static struct k_spinlock dp_task_kobj_lock; -/* generation of the last added record, under dp_task_kobj_lock */ -static uint32_t dp_task_kobj_gen; - -static void dp_task_kobj_add(struct scheduler_dp_task_kobj *kobj) -{ - k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); - - kobj->gen = ++dp_task_kobj_gen; - list_item_append(&kobj->list, &dp_task_kobj_list); - k_spin_unlock(&dp_task_kobj_lock, key); -} - -/* Find the record for @task, caller holds dp_task_kobj_lock */ -static struct scheduler_dp_task_kobj *dp_task_kobj_find(const struct task *task) -{ - struct scheduler_dp_task_kobj *kobj; - struct list_item *item; - - list_for_item(item, &dp_task_kobj_list) { - kobj = container_of(item, struct scheduler_dp_task_kobj, list); - if (kobj->task == task) - return kobj; - } - - return NULL; -} - -/* Copy the record for @task to @copy without unlinking it */ -static bool dp_task_kobj_peek(const struct task *task, struct scheduler_dp_task_kobj *copy) -{ - k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); - struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); - - if (kobj) - *copy = *kobj; - - k_spin_unlock(&dp_task_kobj_lock, key); - - return kobj != NULL; -} - -/* - * Find the record for @task and unlink it, so it can only be freed once. - * If @expected is given, only take the record it was copied from, so a - * record replaced since @expected was peeked is left alone. - */ -static struct scheduler_dp_task_kobj * -dp_task_kobj_take(const struct task *task, const struct scheduler_dp_task_kobj *expected) -{ - k_spinlock_key_t key = k_spin_lock(&dp_task_kobj_lock); - struct scheduler_dp_task_kobj *kobj = dp_task_kobj_find(task); - - /* - * Compare the generation, not the object addresses: the task could - * have been freed and recreated at the same addresses meanwhile. - */ - if (kobj && expected && kobj->gen != expected->gen) - kobj = NULL; - - if (kobj) - list_item_del(&kobj->list); - - k_spin_unlock(&dp_task_kobj_lock, key); - - return kobj; -} - /* Unpack IPC data and execute a callback */ static void ipc_thread_unflatten_run(struct processing_module *pmod, struct ipc4_flat *flat) { @@ -529,398 +388,3 @@ void dp_thread_fn(void *p1, void *p2, void *p3) if (task->state == SOF_TASK_STATE_COMPLETED) task_complete(task); } - -/* - * Safe to call with partial successful initialisation, - * k_mem_domain_remove_partition() then just returns -ENOENT - */ -static void scheduler_dp_domain_free(struct processing_module *pmod, struct k_mem_domain *mdom, - struct k_mem_partition *mpart, uint32_t comp_id) -{ - llext_manager_rm_domain(comp_id, mdom); - - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP); - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_HEAP_CACHE); - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG); - k_mem_domain_remove_partition(mdom, mpart + SOF_DP_PART_CFG_CACHE); - - if (mdom->num_partitions) { - /* Violation: all partitions should be freed by now */ - unsigned int n = mdom->num_partitions; - - LOG_WRN("%u domain partition(s) not freed, force-freeing them all", n); - - for (unsigned int i = 0; i < arch_mem_domain_max_partitions_get() && n; i++) { - struct k_mem_partition *dpart = &mdom->partitions[i]; - - if (dpart->size) { - k_mem_domain_remove_partition(mdom, dpart); - n--; - } - } - } - - /* All partitions removed, the domain can be freed now */ - pmod->mdom = NULL; - k_mem_domain_deinit(mdom); - objpool_free(&dp_mdom_head, mdom); -} - -/* - * Free the kernel objects of @task, using only the kernel-only record. - * - * @return false if @task has no record (unknown task or already freed) - */ -static bool scheduler_dp_internal_free_kobj(struct task *task, - const struct scheduler_dp_task_kobj *expected) -{ - struct scheduler_dp_task_kobj *kobj = dp_task_kobj_take(task, expected); - - if (!kobj) - return false; - - /* normally already terminated by scheduler_dp_task_free(), no-op then */ - k_thread_abort(kobj->thread); - k_object_free(kobj->event); - k_object_free(kobj->thread); - scheduler_dp_domain_free(kobj->mod, kobj->mdom, kobj->mpart, kobj->comp_id); - - mod_free(kobj->mod, container_of(task, struct scheduler_dp_task_memory, task)); - rfree(kobj); - - return true; -} - -void z_impl_scheduler_dp_internal_free(struct task *task) -{ - if (!scheduler_dp_internal_free_kobj(task, NULL)) - tr_err(&dp_tr, "unknown DP task %p", task); -} - -#ifdef CONFIG_THREAD_NAME -static void scheduler_dp_thread_name_set(k_tid_t thread_id, struct processing_module *mod) -{ - char name[CONFIG_THREAD_MAX_NAME_LEN]; - - snprintf(name, sizeof(name), "DP:%#x", mod->dev->ipc_config.id); - - k_thread_name_set(thread_id, name); -} -#else -/* k_thread_name_set() is a no-op so skip constructing a thread name */ -#define scheduler_dp_thread_name_set(x, y) -#endif - -/* Called only in IPC context in kernel mode (this can change) */ -int scheduler_dp_task_init(struct task **task, const struct sof_uuid_entry *uid, - const struct task_ops *ops, struct processing_module *mod, - uint16_t core, size_t stack_size, uint32_t options) -{ - k_thread_stack_t *p_stack; - struct scheduler_dp_task_memory *task_memory; - uint8_t *ipc_config_data = NULL; - k_tid_t thread_id; - uint32_t comp_id; - - int ret; - - /* must be called on the same core the task will be bound to */ - assert(cpu_get_id() == core); - assert(!k_is_user_context()); - - /* - * allocate memory - * to avoid multiple malloc operations allocate all required memory as a single structure - * and return pointer to task_memory->task - * As the structure contains zephyr kernel specific data, it must be located in - * shared, non cached memory - */ - task_memory = mod_alloc_ext(mod, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*task_memory), 0); - if (!task_memory) { - tr_err(&dp_tr, "memory alloc failed"); - return -ENOMEM; - } - - memset(task_memory, 0, sizeof(*task_memory)); - - const struct comp_driver *drv = mod->dev->drv; - - task_memory->drv = *mod->dev->drv; - mod->dev->drv = &task_memory->drv; - - /* allocate stack - must be aligned and cached so a separate alloc */ - p_stack = user_stack_allocate(stack_size, options); - if (!p_stack) { - tr_err(&dp_tr, "stack alloc failed"); - ret = -ENOMEM; - goto e_tmem; - } - - struct task *ptask = &task_memory->task; - - /* internal SOF task init */ - ret = schedule_task_init(ptask, uid, SOF_SCHEDULE_DP, 0, ops->run, mod, core, options); - if (ret < 0) { - tr_err(&dp_tr, "schedule_task_init failed"); - goto e_stack; - } - - /* - * task_memory, including pdata, is in the module heap, which user - * threads can write while this function runs. Keep everything the - * kernel uses here or at teardown in kobj and locals, and only mirror - * it into pdata for the DP thread. - */ - struct task_dp_pdata *pdata = &task_memory->pdata; - struct scheduler_dp_task_kobj *kobj = rmalloc(SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT, - sizeof(*kobj)); - - if (!kobj) { - tr_err(&dp_tr, "task record alloc failed"); - ret = -ENOMEM; - goto e_stack; - } - - kobj->task = ptask; - kobj->mod = mod; - comp_id = mod->dev->ipc_config.id; - kobj->comp_id = comp_id; - - if (mod->dev->drv->adapter_ops->get_configuration) { - ipc_config_data = mod_alloc_ext(mod, SOF_MEM_FLAG_USER, SOF_IPC_MSG_MAX_SIZE, 0); - if (!ipc_config_data) { - tr_err(&dp_tr, "Failed to allocate memory for get-config"); - ret = -ENOMEM; - goto e_kobj; - } - } - pdata->ipc_config_data = ipc_config_data; - - pdata->flat = &task_memory->flat; - - kobj->event = k_object_alloc(K_OBJ_EVENT); - if (!kobj->event) { - tr_err(&dp_tr, "Event object allocation failed"); - ret = -ENOMEM; - goto e_config; - } - - pdata->event = kobj->event; - - kobj->thread = k_object_alloc(K_OBJ_THREAD); - if (!kobj->thread) { - tr_err(&dp_tr, "Thread object allocation failed"); - ret = -ENOMEM; - goto e_event; - } - memset(&kobj->thread->arch, 0, sizeof(kobj->thread->arch)); - pdata->thread = kobj->thread; - - /* success, fill the structures */ - pdata->p_stack = p_stack; - pdata->mod = mod; - - /* initialize other task structures */ - ptask->ops.complete = ops->complete; - ptask->ops.get_deadline = ops->get_deadline; - ptask->priv_data = pdata; - list_init(&ptask->list); - - /* create a zephyr thread for the task */ - thread_id = k_thread_create(kobj->thread, p_stack, - stack_size, dp_thread_fn, ptask, NULL, NULL, - CONFIG_DP_THREAD_PRIORITY, ptask->flags, K_FOREVER); - pdata->thread_id = thread_id; - scheduler_dp_thread_name_set(thread_id, mod); - - unsigned int pidx; - size_t size; - uintptr_t start; - bool on_pool = false; - -#ifdef CONFIG_SCHED_CPU_MASK - /* pin the thread to specific core */ - ret = k_thread_cpu_pin(thread_id, core); - if (ret < 0) { - tr_err(&dp_tr, "zephyr task pin to core failed"); - goto e_thread; - } -#endif - - k_thread_access_grant(thread_id, kobj->event, &dp_sync[core]); - scheduler_dp_grant(thread_id, core); -#if CONFIG_SOF_USERSPACE_LL - struct k_thread *thread_ipc = ipc_thread_user(core); - - k_thread_access_grant(thread_ipc, kobj->event, thread_id, p_stack, &dp_sync[core]); - scheduler_dp_grant(thread_ipc, core); - scheduler_dp_grant(zephyr_ll_domain_thread(), core); -#endif - - struct k_mem_domain *mdom = objpool_alloc(&dp_mdom_head, sizeof(*mdom), - SOF_MEM_FLAG_COHERENT); - - if (!mdom) { - tr_err(&dp_tr, "objpool allocation failed"); - ret = -ENOMEM; - goto e_thread; - } - - on_pool = true; - - mod->mdom = mdom; - kobj->mdom = mdom; - - if (!mdom->arch.ptables) { - ret = k_mem_domain_init(mdom, 0, NULL); - if (ret < 0) - goto e_thread; - } - - /* Module heap partition */ - mod_heap_info(mod, &size, &start); - kobj->mpart[SOF_DP_PART_HEAP] = (struct k_mem_partition){ - .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)start), - .size = size, - .attr = K_MEM_PARTITION_P_RW_U_RW, - }; - kobj->mpart[SOF_DP_PART_HEAP_CACHE] = (struct k_mem_partition){ - .start = start, - .size = size, - .attr = K_MEM_PARTITION_P_RW_U_RW | XTENSA_MMU_CACHED_WB, - }; - /* Host mailbox partition for additional IPC parameters: read-only */ - kobj->mpart[SOF_DP_PART_CFG] = (struct k_mem_partition){ - .start = (uintptr_t)sys_cache_uncached_ptr_get((void *)MAILBOX_HOSTBOX_BASE), - .size = MAILBOX_HOSTBOX_SIZE, - .attr = K_MEM_PARTITION_P_RO_U_RO, - }; - kobj->mpart[SOF_DP_PART_CFG_CACHE] = (struct k_mem_partition){ - .start = (uintptr_t)MAILBOX_HOSTBOX_BASE, - .size = MAILBOX_HOSTBOX_SIZE, - .attr = K_MEM_PARTITION_P_RO_U_RO | XTENSA_MMU_CACHED_WB, - }; - - for (pidx = 0; pidx < SOF_DP_PART_TYPE_COUNT; pidx++) { - ret = k_mem_domain_add_partition(mdom, kobj->mpart + pidx); - if (ret < 0) - goto e_dom; - } - - memcpy(pdata->mpart, kobj->mpart, sizeof(pdata->mpart)); - - ret = llext_manager_add_domain(comp_id, mdom); - if (ret < 0) { - tr_err(&dp_tr, "failed to add LLEXT to domain %d", ret); - goto e_dom; - } - - /* - * Keep this call last, able to fail, otherwise domain will be removed - * before its thread - */ - ret = k_mem_domain_add_thread(mdom, thread_id); - if (ret < 0) { - tr_err(&dp_tr, "failed to add thread to domain %d", ret); - goto e_dom; - } - - /* publish the record before the thread can run */ - dp_task_kobj_add(kobj); - - /* start the thread, it should immediately stop at the semaphore */ - k_event_init(kobj->event); - k_thread_start(thread_id); - - /* ptask points into task_memory, so only publish it once it cannot be freed */ - *task = ptask; - - return 0; - -e_dom: - scheduler_dp_domain_free(mod, mdom, kobj->mpart, comp_id); - on_pool = false; -e_thread: - if (on_pool) - objpool_free(&dp_mdom_head, mdom); - k_thread_abort(thread_id); - k_object_free(kobj->thread); -e_event: - k_object_free(kobj->event); -e_config: - mod_free(mod, ipc_config_data); -e_kobj: - rfree(kobj); -e_stack: - user_stack_free(p_stack); -e_tmem: - /* the copy lives in task_memory, so stop pointing at it before freeing */ - mod->dev->drv = drv; - mod_free(mod, task_memory); - return ret; -} - -#ifdef CONFIG_USERSPACE -#include - -/* - * mod_free() follows the module's resource and allocator pointers in - * supervisor mode. They are user-writable, so at least check that they - * point to memory the caller can access. - */ -static void scheduler_dp_mod_vrfy(struct processing_module *mod) -{ - K_OOPS(K_SYSCALL_MEMORY_WRITE(mod, sizeof(*mod))); - K_OOPS(K_SYSCALL_MEMORY_WRITE(mod->dev, sizeof(*mod->dev))); - K_OOPS(K_SYSCALL_MEMORY_READ(mod->dev->drv, sizeof(*mod->dev->drv))); - - struct mod_alloc_ctx *alloc = mod->priv.resources.alloc; - - K_OOPS(K_SYSCALL_MEMORY_WRITE(alloc, sizeof(*alloc))); - if (alloc->heap) { - size_t h_size = 0; - uintptr_t h_start; - - mod_heap_info(mod, &h_size, &h_start); - if (h_size) - K_OOPS(K_SYSCALL_MEMORY_WRITE(h_start, h_size)); - } - if (alloc->vreg) { - size_t vr_size = 0; - uintptr_t vr_start; - - vregion_mem_info(alloc->vreg, &vr_size, &vr_start); - K_OOPS(K_SYSCALL_MEMORY_WRITE(vr_start, vr_size)); - } -} - -void z_vrfy_scheduler_dp_internal_free(struct task *task) -{ - struct scheduler_dp_task_kobj kobj; - - /* - * Do not read task->priv_data: the kernel objects come from the - * kernel-only record, and an unknown task pointer is rejected. - */ - K_OOPS(!dp_task_kobj_peek(task, &kobj)); - - /* - * A DP thread holds its own thread and event objects, so it would pass - * the checks below for its own task. Freeing would then abort the - * caller after the record is unlinked, leaking everything it holds. - */ - K_OOPS(kobj.thread == k_current_get()); - - /* - * The caller must have been granted the task's objects. Check before - * unlinking the record, so an unauthorized call leaves it intact. - */ - K_OOPS(K_SYSCALL_OBJ(kobj.event, K_OBJ_EVENT)); - K_OOPS(K_SYSCALL_OBJ_INIT(kobj.thread, K_OBJ_THREAD)); - scheduler_dp_mod_vrfy(kobj.mod); - - K_OOPS(!scheduler_dp_internal_free_kobj(task, &kobj)); -} -#include -#endif /* CONFIG_USERSPACE */ diff --git a/src/schedule/zephyr_dp_schedule_thread.c b/src/schedule/zephyr_dp_schedule_thread.c index fa8583832a03..9c6db72feacd 100644 --- a/src/schedule/zephyr_dp_schedule_thread.c +++ b/src/schedule/zephyr_dp_schedule_thread.c @@ -24,11 +24,15 @@ #include #include -#include "zephyr_dp_schedule.h" +#include LOG_MODULE_DECLARE(dp_schedule, CONFIG_SOF_LOG_LEVEL); extern struct tr_ctx dp_tr; +/* The DP thread creation and the scheduler_dp_internal_free() system call, which run + * in kernel mode, are in src/kernel/schedule/zephyr_dp_schedule_thread-kernel.c + */ + /* Go through all DP tasks and recalculate their readiness and deadlines * NOT REENTRANT, should be called with scheduler_dp_lock() */ @@ -197,176 +201,3 @@ void dp_thread_fn(void *p1, void *p2, void *p3) if (task->state == SOF_TASK_STATE_COMPLETED) task_complete(task); } - -int scheduler_dp_task_init(struct task **task, - const struct sof_uuid_entry *uid, - const struct task_ops *ops, - struct processing_module *mod, - uint16_t core, - size_t stack_size, - uint32_t options) -{ - void __sparse_cache *p_stack = NULL; - struct k_heap *const user_heap = mod->dev->drv->user_heap; - - /* memory allocation helper structure */ - struct { - struct task task; /* keep first, used for freeing below */ - struct task_dp_pdata pdata; - } *task_memory; - - int ret; - - /* must be called on the same core the task will be binded to */ - assert(cpu_get_id() == core); - - /* - * allocate memory - * to avoid multiple malloc operations allocate all required memory as a single structure - * and return pointer to task_memory->task - * As the structure contains zephyr kernel specific data, it must be located in - * shared, non cached memory - */ - task_memory = sof_heap_alloc(user_heap, SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, - sizeof(*task_memory), 0); - if (!task_memory) { - tr_err(&dp_tr, "memory alloc failed"); - return -ENOMEM; - } - - memset(task_memory, 0, sizeof(*task_memory)); - /* allocate stack - must be aligned and cached so a separate alloc */ - p_stack = user_stack_allocate(stack_size, options); - if (!p_stack) { - tr_err(&dp_tr, "stack alloc failed"); - ret = -ENOMEM; - goto err; - } - - /* internal SOF task init */ - ret = schedule_task_init(&task_memory->task, uid, SOF_SCHEDULE_DP, 0, ops->run, - mod, core, options); - if (ret < 0) { - tr_err(&dp_tr, "schedule_task_init failed"); - goto err; - } - - struct task_dp_pdata *pdata = &task_memory->pdata; - - /* Point to event_struct event for kernel threads synchronization */ - /* It will be overwritten for K_USER threads to dynamic ones. */ - pdata->event = &pdata->event_struct; - pdata->thread = &pdata->thread_struct; - -#ifdef CONFIG_USERSPACE - if (options & K_USER) { - pdata->event = k_object_alloc(K_OBJ_EVENT); - if (!pdata->event) { - tr_err(&dp_tr, "Event object allocation failed"); - ret = -ENOMEM; - goto err; - } - - pdata->thread = k_object_alloc(K_OBJ_THREAD); - if (!pdata->thread) { - tr_err(&dp_tr, "Thread object allocation failed"); - ret = -ENOMEM; - goto err; - } - } -#endif /* CONFIG_USERSPACE */ - - /* initialize other task structures */ - task_memory->task.ops.complete = ops->complete; - task_memory->task.ops.get_deadline = ops->get_deadline; - task_memory->task.state = SOF_TASK_STATE_INIT; - task_memory->task.core = core; - task_memory->task.priv_data = pdata; - k_event_init(pdata->event); - - /* success, fill the structures */ - pdata->p_stack = p_stack; - pdata->mod = mod; - - /* create a zephyr thread for the task */ - pdata->thread_id = k_thread_create(pdata->thread, (__sparse_force void *)p_stack, - stack_size, dp_thread_fn, &task_memory->task, NULL, NULL, - CONFIG_DP_THREAD_PRIORITY, task_memory->task.flags, - K_FOREVER); - -#ifdef CONFIG_SCHED_CPU_MASK - /* pin the thread to specific core */ - ret = k_thread_cpu_pin(pdata->thread_id, core); - if (ret < 0) { - tr_err(&dp_tr, "zephyr task pin to core failed"); - goto e_thread; - } -#endif - -#ifdef CONFIG_USERSPACE - if (options & K_USER) { - k_thread_access_grant(pdata->thread_id, pdata->event); - scheduler_dp_grant(pdata->thread_id, core); - - ret = k_mem_domain_add_thread(pdata->mod->user_ctx->comp_dom, pdata->thread_id); - if (ret < 0) { - tr_err(&dp_tr, "k_mem_domain_add_thread() failed %d", ret); - goto e_thread; - } - -#if IS_ENABLED(CONFIG_SOF_USERSPACE_MOD_IPC_BY_DP_THREAD) - pdata->ipc_work_item = userspace_proxy_register_ipc_handler(mod, pdata->event); - assert(pdata->ipc_work_item); -#endif - } -#endif /* CONFIG_USERSPACE */ - - /* start the thread, it should immediately stop at an event */ - k_thread_start(pdata->thread_id); - - /* success, fill output parameter */ - *task = &task_memory->task; - return 0; - -e_thread: - k_thread_abort(pdata->thread_id); -err: - /* cleanup - free all allocated resources */ - if (user_stack_free((__sparse_force void *)p_stack)) - tr_err(&dp_tr, "user_stack_free failed!"); - - /* k_object_free looks for a pointer in the list, any invalid value can be passed */ - k_object_free(task_memory->pdata.event); - k_object_free(task_memory->pdata.thread); - sof_heap_free(user_heap, task_memory); - return ret; -} - -void z_impl_scheduler_dp_internal_free(struct task *task) -{ - struct task_dp_pdata *pdata = task->priv_data; - -#ifdef CONFIG_USERSPACE - if (pdata->event != &pdata->event_struct) - k_object_free(pdata->event); - if (pdata->thread != &pdata->thread_struct) - k_object_free(pdata->thread); -#endif - - /* task is the first member in task_memory above */ - sof_heap_free(pdata->mod->dev->drv->user_heap, task); -} - -#ifdef CONFIG_USERSPACE -#include - -void z_vrfy_scheduler_dp_internal_free(struct task *task) -{ - /* - * With the thread DP scheduler variant scheduler_dp_internal_free() is - * never called from the userspace context - */ - K_OOPS(true); -} -#include -#endif diff --git a/src/schedule/zephyr_ll.c b/src/schedule/zephyr_ll.c index 3edebbca5b20..b8d5906e6bbf 100644 --- a/src/schedule/zephyr_ll.c +++ b/src/schedule/zephyr_ll.c @@ -12,6 +12,7 @@ #include #include #include +#include #include #include #include @@ -26,27 +27,9 @@ SOF_DEFINE_REG_UUID(zll_sched); /* unused with Zephyr, generates no output */ DECLARE_TR_CTX(ll_tr, SOF_UUID(zll_sched_uuid), LOG_LEVEL_INFO); -/* per-scheduler data */ -struct zephyr_ll { - struct list_item tasks; /* list of ll tasks */ - unsigned int n_tasks; /* task counter */ - struct ll_schedule_domain *ll_domain; /* scheduling domain */ - unsigned int core; /* core ID of this instance */ -#if CONFIG_SOF_USERSPACE_LL - struct k_mutex *lock; /* mutex for userspace */ -#endif - struct k_heap *heap; -}; - -/* per-task scheduler data */ -struct zephyr_ll_pdata { - bool run; - bool freeing; - struct k_sem *sem_p; -#if !CONFIG_DYNAMIC_OBJECTS - struct k_sem sem; -#endif -}; +/* The scheduler syscalls and access grants, which run in kernel mode, are in + * src/kernel/schedule/zephyr_ll-kernel.c + */ #if CONFIG_SOF_USERSPACE_LL /* @@ -54,7 +37,7 @@ struct zephyr_ll_pdata { * can read the pointer for syscalls. The actual lock object resides * in kernel memory, there are just handles! */ -static APP_SYSUSER_BSS struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; +APP_SYSUSER_BSS struct k_mutex *zephyr_ll_locks[CONFIG_CORE_COUNT]; /* * Shadow ownership tracking for the per-core LL mutex, used only to @@ -426,178 +409,35 @@ static int zephyr_ll_task_schedule_common(struct zephyr_ll *sch, struct task *ta return 0; } -static int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start, - uint64_t period) +int zephyr_ll_task_schedule(void *data, struct task *task, uint64_t start, + uint64_t period) { struct zephyr_ll *sch = data; return zephyr_ll_task_schedule_common(sch, task, start, period, NULL, false); } -static int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start, - uint64_t period, struct task *before) +int zephyr_ll_task_schedule_before(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *before) { struct zephyr_ll *sch = data; return zephyr_ll_task_schedule_common(sch, task, start, period, before, true); } -static int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start, - uint64_t period, struct task *after) +int zephyr_ll_task_schedule_after(void *data, struct task *task, uint64_t start, + uint64_t period, struct task *after) { struct zephyr_ll *sch = data; return zephyr_ll_task_schedule_common(sch, task, start, period, after, false); } -#if CONFIG_DYNAMIC_OBJECTS -static struct list_item zephyr_ll_task_sem_list = LIST_INIT(zephyr_ll_task_sem_list); - -struct zephyr_ll_task_sem { - struct task *task; - struct k_sem *sem; - struct list_item list; -}; - -/* - * The syscall validators pass in pdata that they read once from user memory - * and validated, so these helpers must not read task->priv_data again. - */ -static int zephyr_ll_task_sem_alloc_pdata(struct task *task, struct zephyr_ll_pdata *pdata) -{ - struct zephyr_ll_task_sem *ts = rmalloc(SOF_MEM_FLAG_COHERENT, sizeof(*ts)); - - if (!ts) - return -ENOMEM; - - ts->sem = k_object_alloc(K_OBJ_SEM); - if (!ts->sem) { - rfree(ts); - return -ENOMEM; - } - - k_sem_init(ts->sem, 0, 1); - -#if CONFIG_SOF_USERSPACE_LL - /* - * The per-task semaphore is signalled from zephyr_ll_task_done(), - * which runs in the (unprivileged) LL scheduler thread when a task is - * freed while it is still running. k_object_alloc() only grants access - * to the calling thread (the IPC handler that creates the task), so the - * LL thread must be granted access explicitly, otherwise its - * k_sem_give() traps with a userspace permission fault. - * - * Resolve the LL thread from kernel-only per-core state keyed by the - * task's target core; never traverse the user-accessible scheduler or - * domain objects from privileged context. - */ - struct k_thread *ll_tid = zephyr_domain_thread_tid_for_core(task->core); - - if (ll_tid) - k_thread_access_grant(ll_tid, ts->sem); -#endif /* CONFIG_SOF_USERSPACE_LL */ - - ts->task = task; - pdata->sem_p = ts->sem; - /* List is protected by IPC serialization */ - list_item_append(&ts->list, &zephyr_ll_task_sem_list); - - return 0; -} - -int z_impl_zephyr_ll_task_sem_alloc(struct task *task) -{ - return zephyr_ll_task_sem_alloc_pdata(task, task->priv_data); -} - -static int zephyr_ll_task_sem_free_pdata(struct task *task, struct zephyr_ll_pdata *pdata) -{ - struct list_item *list; - struct zephyr_ll_task_sem *ts; - bool found = false; - - /* List is protected by IPC serialization */ - list_for_item(list, &zephyr_ll_task_sem_list) { - ts = container_of(list, struct zephyr_ll_task_sem, list); - if (ts->task == task) { - found = true; - break; - } - } - - if (!found) - return -ENOENT; - - if (pdata->sem_p != ts->sem) - return -EINVAL; - - list_item_del(list); - k_object_free(ts->sem); - rfree(ts); - - return 0; -} - -int z_impl_zephyr_ll_task_sem_free(struct task *task) -{ - return zephyr_ll_task_sem_free_pdata(task, task->priv_data); -} - -#ifdef CONFIG_USERSPACE -#include -/* - * Read task->priv_data exactly once. The task lives in user memory and a - * thread on another core can change the pointer after it has been checked. - * - * @return validated pdata pointer, or NULL if the task has none - */ -static struct zephyr_ll_pdata *zephyr_ll_task_pdata_vrfy(struct task *task) -{ - struct zephyr_ll_pdata *pdata; - - K_OOPS(K_SYSCALL_MEMORY_WRITE(task, sizeof(*task))); - K_OOPS(k_usermode_from_copy(&pdata, &task->priv_data, sizeof(pdata))); - if (pdata) - K_OOPS(K_SYSCALL_MEMORY_WRITE(pdata, sizeof(*pdata))); - - return pdata; -} - -static inline int z_vrfy_zephyr_ll_task_sem_alloc(struct task *task) -{ - struct zephyr_ll_pdata *pdata; - - if (!task) - return -EINVAL; - pdata = zephyr_ll_task_pdata_vrfy(task); - if (!pdata) - return -EINVAL; - - return zephyr_ll_task_sem_alloc_pdata(task, pdata); -} -#include - -static inline int z_vrfy_zephyr_ll_task_sem_free(struct task *task) -{ - struct zephyr_ll_pdata *pdata; - - if (!task) - return -EINVAL; - pdata = zephyr_ll_task_pdata_vrfy(task); - if (!pdata) - return -EINVAL; - - return zephyr_ll_task_sem_free_pdata(task, pdata); -} -#include -#endif /* CONFIG_USERSPACE */ -#endif /* CONFIG_DYNAMIC_OBJECTS */ - /* * This is synchronous - after this returns the object can be destroyed! * Assertion: under Zephyr this is always called from a thread context! */ -static int zephyr_ll_task_free(void *data, struct task *task) +int zephyr_ll_task_free(void *data, struct task *task) { struct zephyr_ll *sch = data; uint32_t flags; @@ -665,7 +505,7 @@ static int zephyr_ll_task_free(void *data, struct task *task) } /* This is asynchronous */ -static int zephyr_ll_task_cancel(void *data, struct task *task) +int zephyr_ll_task_cancel(void *data, struct task *task) { struct zephyr_ll *sch = data; uint32_t flags; @@ -695,7 +535,7 @@ static int zephyr_ll_task_cancel(void *data, struct task *task) } #if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY -static void zephyr_ll_scheduler_free(void *data, uint32_t flags) +void zephyr_ll_scheduler_free(void *data, uint32_t flags) { struct zephyr_ll *sch = data; @@ -717,50 +557,6 @@ static void zephyr_ll_scheduler_free(void *data, uint32_t flags) } #endif /* CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY */ -#if CONFIG_SOF_USERSPACE_LL -struct k_thread *zephyr_ll_init_context(void *data, struct task *task) -{ - struct zephyr_ll *sch = data; - struct zephyr_ll_pdata *pdata = task->priv_data; - int ret; - - /* - * Use domain_thread_init() for privileged setup (thread creation, - * timer, access grants). domain_register() is now bookkeeping only - * and will be called later from user context when scheduling tasks. - */ - ret = domain_thread_init(sch->ll_domain, task); - if (ret < 0) { - tr_err(&ll_tr, "cannot init_context %d", ret); - return NULL; - } - - assert(!k_is_user_context()); - k_thread_access_grant(zephyr_domain_thread_tid(sch->ll_domain), sch->lock, pdata->sem_p); - - tr_dbg(&ll_tr, "granting access to lock %p for thread %p", sch->lock, - zephyr_domain_thread_tid(sch->ll_domain)); - tr_dbg(&ll_tr, "granting access to domain lock %p for thread %p", &sch->ll_domain->lock, - zephyr_domain_thread_tid(sch->ll_domain)); - - return zephyr_domain_thread_tid(sch->ll_domain); -} -#endif /* CONFIG_SOF_USERSPACE_LL */ - -static const struct scheduler_ops zephyr_ll_ops = { - .schedule_task = zephyr_ll_task_schedule, - .schedule_task_before = zephyr_ll_task_schedule_before, - .schedule_task_after = zephyr_ll_task_schedule_after, - .schedule_task_free = zephyr_ll_task_free, - .schedule_task_cancel = zephyr_ll_task_cancel, -#if CONFIG_SOF_BOOT_TEST_STANDALONE || CONFIG_LIBRARY - .scheduler_free = zephyr_ll_scheduler_free, -#endif -#if CONFIG_SOF_USERSPACE_LL - .scheduler_init_context = zephyr_ll_init_context, -#endif -}; - #if CONFIG_SOF_USERSPACE_LL struct task *zephyr_ll_task_alloc(void) { @@ -774,13 +570,6 @@ struct task *zephyr_ll_task_alloc(void) return task; } -void user_ll_grant_access(struct k_thread *thread, int core) -{ - assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] != NULL); - - k_thread_access_grant(thread, zephyr_ll_locks[core]); -} - /** * Lock the LL scheduler to prevent it from processing tasks. * @@ -866,58 +655,6 @@ int zephyr_ll_task_init(struct task *task, } EXPORT_SYMBOL(zephyr_ll_task_init); -/* TODO: low-power mode clock support */ -/* Runs on each core during initialisation with the same domain argument */ -__cold int zephyr_ll_scheduler_init(struct ll_schedule_domain *domain) -{ - struct zephyr_ll *sch; - int core = cpu_get_id(); - struct k_heap *heap = sof_sys_heap_get(); - int flags = SOF_MEM_FLAG_KERNEL | SOF_MEM_FLAG_COHERENT; - - assert_can_be_cold(); - -#if CONFIG_SOF_USERSPACE_LL - heap = zephyr_ll_user_heap(); - flags = SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT; -#endif - tr_dbg(&ll_tr, "init on core %d", core); - - /* initialize per-core scheduler private data */ - sch = sof_heap_alloc(heap, flags, sizeof(*sch), 0); - if (!sch) { - tr_err(&ll_tr, "allocation failed"); - return -ENOMEM; - } - - memset(sch, 0, sizeof(*sch)); - - list_init(&sch->tasks); - sch->ll_domain = domain; - sch->core = core; - sch->n_tasks = 0; - sch->heap = heap; - -#if CONFIG_SOF_USERSPACE_LL - /* Allocate mutex dynamically for userspace access */ - sch->lock = k_object_alloc(K_OBJ_MUTEX); - if (!sch->lock) { - tr_err(&ll_tr, "mutex allocation failed"); - sof_heap_free(sch->heap, sch); - return -ENOMEM; - } - assert(core < CONFIG_CORE_COUNT && zephyr_ll_locks[core] == NULL); - zephyr_ll_locks[core] = sch->lock; - k_mutex_init(sch->lock); - - tr_dbg(&ll_tr, "ll-scheduler init done, sch %p sch->lock %p", sch, sch->lock); -#endif - - scheduler_init(domain->type, &zephyr_ll_ops, sch); - - return 0; -} - void scheduler_get_task_info_ll(struct scheduler_props *scheduler_props, uint32_t *data_off_size) { diff --git a/src/trace/CMakeLists.txt b/src/trace/CMakeLists.txt deleted file mode 100644 index 7fab6cf5eaaa..000000000000 --- a/src/trace/CMakeLists.txt +++ /dev/null @@ -1,5 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause - -if(CONFIG_TRACE) - add_local_sources(sof dma-trace.c trace.c) -endif() diff --git a/src/trace/Kconfig b/src/trace/Kconfig deleted file mode 100644 index 92ccb461ed18..000000000000 --- a/src/trace/Kconfig +++ /dev/null @@ -1,139 +0,0 @@ -# SPDX-License-Identifier: BSD-3-Clause - -# Trace configs - -menu "Trace" - -config TRACE - bool "Trace" - default n if ZEPHYR_SOF_MODULE - default y - help - Enable SOF DMA based traces compatible with the sof-logger tool. - With Zephyr, RTOS side events are not seen in the SOF trace, so native - Zephyr logging (CONFIG_ZEPHYR_LOG) is recommended instead. - -config TRACEV - bool "Trace verbose" - depends on TRACE - default n - help - Enabling verbose traces. tr_dbg() statements depend on this at - compile-time and on run-time filtering below too. - -config TRACEE - bool "Trace error" - depends on TRACE - default y - help - Sending error traces by mailbox additionally. - -config TRACEM - bool "Trace mailbox" - depends on TRACE - default n - help - Sending all traces by mailbox additionally. - -config TRACE_FILTERING - bool "Trace filtering" - depends on TRACE - default y - help - Filtering of trace messages based on their verbosity level and/or frequency. - -config TRACE_FILTERING_VERBOSITY - bool "Filter by verbosity" - depends on TRACE_FILTERING - default y - help - Filtering by log verbosity level, where maximum verbosity allowed is specified for each - context and may be adjusted in runtime. Most basic feature found in every logger. - -config TRACE_FILTERING_ADAPTIVE - bool "Adaptive rate limiting" - depends on TRACE_FILTERING - default y - help - Adaptive filtering of repeating trace messages, tracking up to - CONFIG_TRACE_RECENT_ENTRIES_COUNT and suppressing messages repeated in less than - CONFIG_TRACE_RECENT_TIME_THRESHOLD microseconds. This filtering and suppression are - turned off as soon as the user makes any runtime configuration change using the - sof-logger -F flag. Warning: log message parameters (formatted with a conversion - specifier %_) are ignored when comparing repeated messages and they are dropped by - this filtering. - -config TRACE_RECENT_ENTRIES_COUNT - int "How many recent log messages are stored" - depends on TRACE_FILTERING_ADAPTIVE - default 5 - help - The most recent log messages are stored to match the currently processed - message. The size of that window affects effectiveness and performance. More recent - entries allow to better filter repetitive messages out, but also slightly decrease - performance due to the increased number of comparisons necessary. - -config TRACE_RECENT_TIME_THRESHOLD - int "Period of time considered recent (microseconds)" - depends on TRACE_FILTERING_ADAPTIVE - default 1500 - range 1 TRACE_RECENT_MAX_TIME - help - Period of time during which entries are tracked and will be suppressed if reported again. - Entries expire after this time. - -config TRACE_RECENT_MAX_TIME - int "Maximum period of time that message can be suppressed (microseconds)" - depends on TRACE_FILTERING_ADAPTIVE - default 5000000 - help - Maximum amount of time message can be suppressed for, due to repeated suppression. - -config TRACE_BURST_COUNT - int "Allowed amount of rapidly repeated messages, that will not be suppressed by the filter" - depends on TRACE_FILTERING_ADAPTIVE - default 4 - help - Amount of messages that will pass through the filter even if sent in rapid succession. - Allowed message burst size before filter suppresses the message. - -config LOG_BACKEND_SOF_PROBE - bool "Logging backend with SOF probes" - depends on LOG - select LOG_OUTPUT - help - Enable backend for log output via SOF probe interface. - Probe interface allows to transmit logs and PCM data, muxed over - a shared DMA channel. - Logging is enabled by setting up a probe point with - probe purpose value of PROBE_PURPOSE_LOGGING. - -config LOG_BACKEND_SOF_PROBE_OUTPUT_AUTO_ENABLE - bool "Automatically enable probes logging when probe dma is started" - depends on LOG_BACKEND_SOF_PROBE - help - Automatically enable logging as soon as probes DMA - starts. This option simplifies probes logging back - end usage as there is no need to enable logs after - every boot. The probes DMA is still needed before the - log output can be enabled. - -config LOG_BACKEND_SOF_PROBE_OUTPUT_DICTIONARY - bool "Dictionary" - select LOG_DICTIONARY_SUPPORT - help - Set output format of the SOF probe logging backend to - Zephyr log dictionary. - The resulting log file can be parsed with tools in - zephyr/scripts/logging/dictionary - A dictionary for the binary is created during build. - -config LOG_BACKEND_SOF_PROBE_OUTPUT - int "Set logging output format" - default 1 if LOG_MIPI_SYST_ENABLE - default 2 if LOG_BACKEND_SOF_PROBE_OUTPUT_DICTIONARY - default 0 # Text encoding is default - help - Set the logging format for SOF probe output. - -endmenu diff --git a/src/trace/dma-trace.c b/src/trace/dma-trace.c deleted file mode 100644 index 328bb2767f65..000000000000 --- a/src/trace/dma-trace.c +++ /dev/null @@ -1,645 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause -// -// Copyright(c) 2016 Intel Corporation. All rights reserved. -// -// Author: Yan Wang - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -#ifdef __ZEPHYR__ -#include -#endif - -#include -#include -#include - -LOG_MODULE_REGISTER(dma_trace, CONFIG_SOF_LOG_LEVEL); - -SOF_DEFINE_REG_UUID(dma_trace); - -/* unused with Zephyr, generates no output */ -DECLARE_TR_CTX(dt_tr, SOF_UUID(dma_trace_uuid), LOG_LEVEL_INFO); - -SOF_DEFINE_REG_UUID(dma_trace_task); - -static int dma_trace_get_avail_data(struct dma_trace_data *d, - struct dma_trace_buf *buffer, - int avail); - -/** Periodically runs and starts the DMA even when the buffer is not - * full. - */ -static enum task_state trace_work(void *data) -{ - struct dma_trace_data *d = data; - struct dma_trace_buf *buffer = &d->dmatb; - struct dma_sg_config *config = &d->config; - k_spinlock_key_t key; - uint32_t avail = buffer->avail; - int32_t size; - uint32_t overflow; - - /* The host DMA channel is not available */ - if (!d->dc.chan) - return SOF_TASK_STATE_RESCHEDULE; - - if (!ipc_trigger_trace_xfer(avail)) - return SOF_TASK_STATE_RESCHEDULE; - - /* make sure we don't write more than buffer */ - if (avail > DMA_TRACE_LOCAL_SIZE) { - overflow = avail - DMA_TRACE_LOCAL_SIZE; - avail = DMA_TRACE_LOCAL_SIZE; - } else { - overflow = 0; - } - - /* dma gateway supports wrap mode copy, but GPDMA doesn't - * support, so do it differently based on HW features - */ - size = dma_trace_get_avail_data(d, buffer, avail); - - /* any data to copy ? */ - if (size == 0) { - return SOF_TASK_STATE_RESCHEDULE; - } - - d->posn.overflow = overflow; - - /* DMA trace copying is working */ - d->copy_in_progress = 1; - - /* copy this section to host */ - size = dma_copy_to_host(&d->dc, config, d->posn.host_offset, - buffer->r_ptr, size); - if (size < 0) { - tr_err(&dt_tr, "dma_copy_to_host() failed"); - goto out; - } - - /* update host pointer and check for wrap */ - d->posn.host_offset += size; - if (d->posn.host_offset >= d->host_size) - d->posn.host_offset -= d->host_size; - - /* update local pointer and check for wrap */ - buffer->r_ptr = (char *)buffer->r_ptr + size; - if (buffer->r_ptr >= buffer->end_addr) - buffer->r_ptr = (char *)buffer->r_ptr - DMA_TRACE_LOCAL_SIZE; - - ipc_msg_send(d->msg, &d->posn, false); - -out: - key = k_spin_lock(&d->lock); - - /* disregard any old messages and don't resend them if we overflow */ - if (size > 0) { - if (d->posn.overflow) - buffer->avail = DMA_TRACE_LOCAL_SIZE - size; - else - buffer->avail -= size; - } - - /* DMA trace copying is done, allow reschedule */ - d->copy_in_progress = 0; - - k_spin_unlock(&d->lock, key); - - /* reschedule the trace copying work */ - return SOF_TASK_STATE_RESCHEDULE; -} - -/** Do this early so we can log at initialization time even before the - * DMA runs. The rest happens later in dma_trace_init_complete() and - * dma_trace_enable() - */ -int dma_trace_init_early(struct sof *sof) -{ - int ret; - - /* If this assert is wrong then traces have been corrupting - * random parts of memory. Some functions run before _and_ after - * DMA trace initialization and we don't want to ask them to - * never trace. So dma_trace_initialized() must be either - * clearly false/NULL or clearly true, we can't tolerate random - * uninitialized values in sof->dmat etc. - */ - assert(!dma_trace_initialized(sof->dmat)); - - sof->dmat = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*sof->dmat)); - if (!sof->dmat) { - mtrace_printf(LOG_LEVEL_ERROR, - "alloc failed"); - return -ENOMEM; - } - - dma_sg_init(&sof->dmat->config.elem_array); - k_spinlock_init(&sof->dmat->lock); - - ipc_build_trace_posn(&sof->dmat->posn); - sof->dmat->msg = ipc_msg_init(NULL, sof->dmat->posn.rhdr.hdr.cmd, - sof->dmat->posn.rhdr.hdr.size); - if (!sof->dmat->msg) { - ret = -ENOMEM; - goto err; - } - - return 0; - -err: - mtrace_printf(LOG_LEVEL_ERROR, - "failed: %d", ret); - - rfree(sof->dmat); - sof->dmat = NULL; - - return ret; -} - -/** Run after dma_trace_init_early() and before dma_trace_enable() */ -int dma_trace_init_complete(struct dma_trace_data *d) -{ - int ret = 0; - - tr_info(&dt_tr, "entry"); - - if (!d) { - mtrace_printf(LOG_LEVEL_ERROR, - "failed, no dma_trace_data"); - return -ENOMEM; - } - - /* init DMA copy context */ - ret = dma_copy_new(&d->dc); - if (ret < 0) { - mtrace_printf(LOG_LEVEL_ERROR, - "dma_copy_new() failed: %d", ret); - goto out; - } -#if CONFIG_ZEPHYR_NATIVE_DRIVERS - ret = dma_get_attribute(d->dc.dmac->z_dev, DMA_ATTR_COPY_ALIGNMENT, - &d->dma_copy_align); -#else - ret = dma_get_attribute_legacy(d->dc.dmac, DMA_ATTR_COPY_ALIGNMENT, - &d->dma_copy_align); -#endif - if (ret < 0) { - mtrace_printf(LOG_LEVEL_ERROR, - "dma_get_attribute() failed: %d", ret); - - goto out; - } - - schedule_task_init_ll(&d->dmat_work, SOF_UUID(dma_trace_task_uuid), - SOF_SCHEDULE_LL_TIMER, - SOF_TASK_PRI_MED, trace_work, d, 0, 0); - -out: - - return ret; -} - -#if (CONFIG_HOST_PTABLE) -int dma_trace_host_buffer(struct dma_trace_data *d, - struct dma_sg_elem_array *elem_array, - uint32_t host_size) -{ - d->host_size = host_size; - d->config.elem_array = *elem_array; - - return 0; -} -#endif - -static void dma_trace_buffer_free(struct dma_trace_data *d) -{ - struct dma_trace_buf *buffer = &d->dmatb; - k_spinlock_key_t key; - - key = k_spin_lock(&d->lock); - - rfree(buffer->addr); - memset(buffer, 0, sizeof(*buffer)); - - k_spin_unlock(&d->lock, key); -} - -static int dma_trace_buffer_init(struct dma_trace_data *d) -{ - struct dma_trace_buf *buffer = &d->dmatb; - void *buf; - k_spinlock_key_t key; - uint32_t addr_align; - int err; - - /* - * Keep the existing dtrace buffer to avoid memory leak, unlikely to - * happen if host correctly using the dma_trace_disable(). - * - * The buffer can not be freed up here as it is likely in use. - * The (re-)initialization will happen in dma_trace_start() when it is - * safe to do (the DMA is stopped) - */ - if (dma_trace_initialized(d)) - return 0; - - if (!d || !d->dc.dmac) { - mtrace_printf(LOG_LEVEL_ERROR, - "failed, no DMAC! d=%p", d); - return -ENODEV; - } -#if CONFIG_ZEPHYR_NATIVE_DRIVERS - err = dma_get_attribute(d->dc.dmac->z_dev, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, - &addr_align); -#else - err = dma_get_attribute_legacy(d->dc.dmac, DMA_ATTR_BUFFER_ADDRESS_ALIGNMENT, - &addr_align); -#endif - if (err < 0) - return err; - - /* For DMA to work properly the buffer must be correctly aligned */ - buf = rballoc_align(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_DMA, - DMA_TRACE_LOCAL_SIZE, addr_align); - if (!buf) { - mtrace_printf(LOG_LEVEL_ERROR, "alloc failed"); - return -ENOMEM; - } - - bzero(buf, DMA_TRACE_LOCAL_SIZE); - dcache_writeback_region((__sparse_force void __sparse_cache *)buf, DMA_TRACE_LOCAL_SIZE); - - /* initialise the DMA buffer, whole sequence in section */ - key = k_spin_lock(&d->lock); - - buffer->addr = buf; - buffer->size = DMA_TRACE_LOCAL_SIZE; - buffer->w_ptr = buffer->addr; - buffer->r_ptr = buffer->addr; - buffer->end_addr = (char *)buffer->addr + buffer->size; - buffer->avail = 0; - - k_spin_unlock(&d->lock, key); - -#ifdef __ZEPHYR__ -#define ZEPHYR_VER_OPT " zephyr:" STRINGIFY(BUILD_VERSION) -#else -#define ZEPHYR_VER_OPT -#endif - - /* STRINGIFY(SOF_SRC_HASH) is part of the format string so it - * goes to the .ldc file and does not go to the firmware - * binary. It will be different from SOF_SRC_HASH in case of - * mismatch. - */ -#define SOF_BANNER_COMMON \ - "FW ABI 0x%x DBG ABI 0x%x tags SOF:" SOF_GIT_TAG ZEPHYR_VER_OPT \ - " src hash 0x%08x (ldc hash " STRINGIFY(SOF_SRC_HASH) ")" - - /* It should be the very first sent log for easy identification. */ - mtrace_printf(LOG_LEVEL_INFO, - "SHM: " SOF_BANNER_COMMON, - SOF_ABI_VERSION, SOF_ABI_DBG_VERSION, SOF_SRC_HASH); - - /* Use a different, DMA: prefix to ease identification of log files */ - tr_info(&dt_tr, - "DMA: " SOF_BANNER_COMMON, - SOF_ABI_VERSION, SOF_ABI_DBG_VERSION, SOF_SRC_HASH); - - return 0; -} - -static int dma_trace_get_avail_data(struct dma_trace_data *d, - struct dma_trace_buf *buffer, - int avail) -{ - uint32_t hsize; - uint32_t lsize; - int32_t size; - - /* copy to host in sections if we wrap */ - lsize = avail; - hsize = avail; - - if (avail == 0) - return 0; - - /* host buffer wrap ? */ - if (d->posn.host_offset + avail > d->host_size) - hsize = d->host_size - d->posn.host_offset; - - /* local buffer wrap ? */ - if ((char *)buffer->r_ptr + avail > (char *)buffer->end_addr) - lsize = (char *)buffer->end_addr - (char *)buffer->r_ptr; - - /* get smallest size */ - if (hsize < lsize) - size = hsize; - else - size = lsize; - - return size; -} - -/** Invoked remotely by SOF_IPC_TRACE_DMA_PARAMS* Depends on - * dma_trace_init_complete() - */ -int dma_trace_enable(struct dma_trace_data *d) -{ - int err; - - /* Allocate and initialize the dma trace buffer if needed */ - err = dma_trace_buffer_init(d); - if (err < 0) - return err; - - /* validate DMA context */ - if (!d->dc.dmac || !d->dc.chan) { - tr_err_atomic(&dt_tr, "not valid"); - err = -ENODEV; - goto out; - } - - d->enabled = 1; - schedule_task(&d->dmat_work, DMA_TRACE_PERIOD, DMA_TRACE_PERIOD); - -out: - if (err < 0) - dma_trace_buffer_free(d); - - return err; -} - -void dma_trace_disable(struct dma_trace_data *d) -{ - /* cancel trace work */ - schedule_task_cancel(&d->dmat_work); - - if (d->dc.chan) { - dma_stop_legacy(d->dc.chan); - dma_channel_put_legacy(d->dc.chan); - d->dc.chan = NULL; - } - -#if (CONFIG_HOST_PTABLE) - /* Free up the host SG if it is set */ - if (d->host_size) { - dma_sg_free(NULL, &d->config.elem_array); - d->host_size = 0; - } -#endif -} - -/** Sends all pending DMA messages to mailbox (for emergencies) */ -void dma_trace_flush(void *t) -{ - struct dma_trace_data *trace_data = dma_trace_data_get(); - struct dma_trace_buf *buffer = NULL; - uint32_t avail; - int32_t size; - int32_t wrap_count; - int ret; - - if (!dma_trace_initialized(trace_data)) - return; - - buffer = &trace_data->dmatb; - avail = buffer->avail; - - /* number of bytes to flush */ - if (avail > DMA_FLUSH_TRACE_SIZE) { - size = DMA_FLUSH_TRACE_SIZE; - } else { - /* check for buffer wrap */ - if (buffer->w_ptr > buffer->r_ptr) - size = (char *)buffer->w_ptr - (char *)buffer->r_ptr; - else - size = (char *)buffer->end_addr - - (char *)buffer->r_ptr + - (char *)buffer->w_ptr - - (char *)buffer->addr; - } - - size = MIN(size, MAILBOX_TRACE_SIZE); - - /* invalidate trace data */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)t, size); - - /* check for buffer wrap */ - if ((char *)buffer->w_ptr - size < (char *)buffer->addr) { - wrap_count = (char *)buffer->w_ptr - (char *)buffer->addr; - ret = memcpy_s(t, size - wrap_count, - (char *)buffer->end_addr - - (size - wrap_count), size - wrap_count); - assert(!ret); - ret = memcpy_s((char *)t + (size - wrap_count), wrap_count, - buffer->addr, wrap_count); - assert(!ret); - } else { - ret = memcpy_s(t, size, (char *)buffer->w_ptr - size, size); - assert(!ret); - } - - /* writeback trace data */ - dcache_writeback_region((__sparse_force void __sparse_cache *)t, size); - -} - -void dma_trace_on(void) -{ - if (!dma_trace_initialized(sof_get()->dmat)) - return; - - struct dma_trace_data *trace_data = dma_trace_data_get(); - - if (trace_data->enabled) { - return; - } - - trace_data->enabled = 1; - schedule_task(&trace_data->dmat_work, DMA_TRACE_PERIOD, - DMA_TRACE_PERIOD); - -} - -void dma_trace_off(void) -{ - struct dma_trace_data *trace_data = dma_trace_data_get(); - - if (!trace_data->enabled) { - return; - } - - schedule_task_cancel(&trace_data->dmat_work); - trace_data->enabled = 0; - -} - -static int dtrace_calc_buf_overflow(struct dma_trace_buf *buffer, - uint32_t length) -{ - uint32_t margin; - uint32_t overflow_margin; - uint32_t overflow = 0; - - margin = dtrace_calc_buf_margin(buffer); - - /* overflow calculating */ - if (buffer->w_ptr < buffer->r_ptr) - overflow_margin = (char *)buffer->r_ptr - - (char *)buffer->w_ptr - 1; - else - overflow_margin = margin + (char *)buffer->r_ptr - - (char *)buffer->addr - 1; - - if (overflow_margin < length) - overflow = length - overflow_margin; - - return overflow; -} - -/** Ring buffer implementation, drops on overflow. */ -static void dtrace_add_event(const char *e, uint32_t length) -{ - struct dma_trace_data *trace_data = dma_trace_data_get(); - struct dma_trace_buf *buffer = &trace_data->dmatb; - uint32_t margin; - uint32_t overflow; - int ret; - - margin = dtrace_calc_buf_margin(buffer); - overflow = dtrace_calc_buf_overflow(buffer, length); - - /* tracing dropped entries */ - if (trace_data->dropped_entries) { - if (!overflow) { - /* - * if any dropped entries have appeared and there - * is not any overflow, their amount will be logged - */ - uint32_t tmp_dropped_entries = - trace_data->dropped_entries; - trace_data->dropped_entries = 0; - mtrace_printf(LOG_LEVEL_WARNING, - "number of dropped logs = %u", - tmp_dropped_entries); - } - } - - /* checking overflow */ - if (!overflow) { - /* check for buffer wrap */ - if (margin > length) { - /* no wrap */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)buffer->w_ptr, - length); - ret = memcpy_s(buffer->w_ptr, length, e, length); - assert(!ret); - dcache_writeback_region((__sparse_force void __sparse_cache *)buffer->w_ptr, - length); - buffer->w_ptr = (char *)buffer->w_ptr + length; - } else { - /* data is bigger than remaining margin so we wrap */ - dcache_invalidate_region((__sparse_force void __sparse_cache *)buffer->w_ptr, - margin); - ret = memcpy_s(buffer->w_ptr, margin, e, margin); - assert(!ret); - dcache_writeback_region((__sparse_force void __sparse_cache *)buffer->w_ptr, - margin); - buffer->w_ptr = buffer->addr; - - dcache_invalidate_region((__sparse_force void __sparse_cache *)buffer->w_ptr, - length - margin); - ret = memcpy_s(buffer->w_ptr, length - margin, - e + margin, length - margin); - assert(!ret); - dcache_writeback_region((__sparse_force void __sparse_cache *)buffer->w_ptr, - length - margin); - buffer->w_ptr = (char *)buffer->w_ptr + length - margin; - } - - buffer->avail += length; - trace_data->posn.messages++; - } else { - /* if there is not enough memory for new log, we drop it */ - trace_data->dropped_entries++; - } - -} - -/** Main dma-trace entry point */ -void dtrace_event(const char *e, uint32_t length) -{ - struct dma_trace_data *trace_data = dma_trace_data_get(); - struct dma_trace_buf *buffer = NULL; - k_spinlock_key_t key; - - if (!dma_trace_initialized(trace_data) || - length > DMA_TRACE_LOCAL_SIZE / 8 || length == 0) { - return; - } - - buffer = &trace_data->dmatb; - - key = k_spin_lock(&trace_data->lock); - dtrace_add_event(e, length); - - /* if DMA trace copying is working or secondary core - * don't check if local buffer is half full - */ - if (trace_data->copy_in_progress || - cpu_get_id() != PLATFORM_PRIMARY_CORE_ID) { - k_spin_unlock(&trace_data->lock, key); - return; - } - - k_spin_unlock(&trace_data->lock, key); - - /* schedule copy now if buffer > 50% full */ - if (trace_data->enabled && - buffer->avail >= (DMA_TRACE_LOCAL_SIZE / 2)) { - reschedule_task(&trace_data->dmat_work, - DMA_TRACE_RESCHEDULE_TIME); - /* reschedule should not be interrupted - * just like we are in copy progress - */ - trace_data->copy_in_progress = 1; - } - -} - -void dtrace_event_atomic(const char *e, uint32_t length) -{ - struct dma_trace_data *trace_data = dma_trace_data_get(); - - if (!dma_trace_initialized(trace_data) || - length > DMA_TRACE_LOCAL_SIZE / 8 || length == 0) { - return; - } - - dtrace_add_event(e, length); -} diff --git a/src/trace/trace.c b/src/trace/trace.c deleted file mode 100644 index bc17c848a902..000000000000 --- a/src/trace/trace.c +++ /dev/null @@ -1,579 +0,0 @@ -// SPDX-License-Identifier: BSD-3-Clause -// -// Copyright(c) 2016 Intel Corporation. All rights reserved. -// -// Author: Liam Girdwood -// Artur Kloniecki -// Karol Trzcinski - -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include -#include - -/* every trace calls uses IPC context in this file */ -LOG_MODULE_DECLARE(ipc, CONFIG_SOF_LOG_LEVEL); - -extern struct tr_ctx dt_tr; - -#if CONFIG_TRACE_FILTERING_ADAPTIVE -struct recent_log_entry { - uint32_t entry_id; - uint64_t message_ts; - uint64_t first_suppression_ts; - uint32_t trigger_count; -}; - -struct recent_trace_context { - struct recent_log_entry recent_entries[CONFIG_TRACE_RECENT_ENTRIES_COUNT]; -}; -#endif /* CONFIG_TRACE_FILTERING_ADAPTIVE */ - -/** MAILBOX_TRACE_BASE ring buffer */ -struct trace { - uintptr_t pos ; /**< offset of the next byte to write */ - uint32_t enable; -#if CONFIG_TRACE_FILTERING_ADAPTIVE - bool user_filter_override; /* whether filtering was overridden by user or not */ -#endif /* CONFIG_TRACE_FILTERING_ADAPTIVE */ - struct k_spinlock lock; /* locking mechanism */ - -#if CONFIG_TRACE_FILTERING_ADAPTIVE - struct recent_trace_context trace_core_context[CONFIG_CORE_COUNT]; -#endif -}; - -/* calculates total message size, both header and payload in bytes */ -#define MESSAGE_SIZE(args_num) \ - (sizeof(struct log_entry_header) + args_num * sizeof(uint32_t)) - -/* calculated total message size in dwords */ -#define MESSAGE_SIZE_DWORDS(args_num) \ - (MESSAGE_SIZE(args_num) / sizeof(uint32_t)) - -/* calculates offset to param_idx of payload */ -#define PAYLOAD_OFFSET(param_idx) \ - (MESSAGE_SIZE_DWORDS(0) + param_idx) - -#define TRACE_ID_MASK ((1 << TRACE_ID_LENGTH) - 1) - -static void put_header(void *dst, const struct sof_uuid_entry *uid, - uint32_t id_1, uint32_t id_2, - uint32_t entry, uint64_t timestamp) -{ - struct dma_trace_data *trace_data = dma_trace_data_get(); - /* Support very early tracing */ - uint64_t delta = dma_trace_initialized(trace_data) ? trace_data->time_delta : 0; - struct log_entry_header header; - int ret; - - header.uid = (uintptr_t)uid; - header.id_0 = id_1 & TRACE_ID_MASK; - header.id_1 = id_2 & TRACE_ID_MASK; - header.core_id = cpu_get_id(); - header.timestamp = timestamp + delta; - header.log_entry_address = entry; - - ret = memcpy_s(dst, sizeof(header), &header, sizeof(header)); - assert(!ret); -} - -#ifndef __ZEPHYR__ -/** Ring buffer for the mailbox trace */ -void mtrace_event(const char *data, uint32_t length) -{ - struct trace *trace = trace_get(); - char *t = (char *)MAILBOX_TRACE_BASE; - const int32_t available = MAILBOX_TRACE_SIZE - trace->pos; - - if (available < length) { /* wrap */ - memset(t + trace->pos, 0xff, available); - dcache_writeback_region((__sparse_force char __sparse_cache *)t + trace->pos, - available); - trace->pos = 0; - } - - memcpy_s(t + trace->pos, MAILBOX_TRACE_SIZE - trace->pos, - data, length); - dcache_writeback_region((__sparse_force char __sparse_cache *)t + trace->pos, length); - trace->pos += length; -} -#endif /* __ZEPHYR__ */ - -#if CONFIG_TRACE_FILTERING_VERBOSITY -/** - * \brief Runtime trace filtering based on verbosity level - * \param lvl log level (LOG_LEVEL_ ERROR, INFO, DEBUG ...) - * \param uuid uuid address - * \return false when trace is filtered out, otherwise true - */ -static inline bool trace_filter_verbosity(uint32_t lvl, const struct tr_ctx *ctx) -{ - STATIC_ASSERT(LOG_LEVEL_CRITICAL < LOG_LEVEL_VERBOSE, - LOG_LEVEL_CRITICAL_MUST_HAVE_LOWEST_VALUE); - return lvl <= ctx->level; -} -#endif /* CONFIG_TRACE_FILTERING_VERBOSITY */ - -#if CONFIG_TRACE_FILTERING_ADAPTIVE -/** Report how many times an entry was suppressed and clear it. */ -static void emit_suppressed_entry(struct recent_log_entry *entry) -{ - _log_message(trace_log_unfiltered, false, LOG_LEVEL_INFO, _TRACE_INV_CLASS, &dt_tr, - _TRACE_INV_ID, _TRACE_INV_ID, "Suppressed %u similar messages: %pQ", - entry->trigger_count - CONFIG_TRACE_BURST_COUNT, - (void *)entry->entry_id); - - memset(entry, 0, sizeof(*entry)); -} - -/** Flush entries that have not been seen again in the last - * CONFIG_TRACE_RECENT_TIME_THRESHOLD microseconds before current_ts. - */ -static void emit_recent_entries(uint64_t current_ts) -{ - struct trace *trace = trace_get(); - struct recent_log_entry *recent_entries = - trace->trace_core_context[cpu_get_id()].recent_entries; - int i; - - /* Check if any tracked entries were dormant long enough to unsuppress them */ - for (i = 0; i < CONFIG_TRACE_RECENT_ENTRIES_COUNT; i++) { - if (recent_entries[i].entry_id) { - if (current_ts - recent_entries[i].message_ts > - CONFIG_TRACE_RECENT_TIME_THRESHOLD) { - if (recent_entries[i].trigger_count > CONFIG_TRACE_BURST_COUNT) - emit_suppressed_entry(&recent_entries[i]); - else - memset(&recent_entries[i], 0, sizeof(recent_entries[i])); - } - } - } - -} - -/** - * \brief Runtime trace flood suppression - * \param log_level log verbosity level - * \param entry ID of log entry - * \param message_ts message timestamp - * \return true when the message must be printed by the caller because it was not filtered - */ -static bool trace_filter_flood(uint32_t log_level, uint32_t entry, uint64_t message_ts) -{ - struct trace *trace = trace_get(); - struct recent_log_entry *recent_entries = - trace->trace_core_context[cpu_get_id()].recent_entries; - struct recent_log_entry *oldest_entry = recent_entries; - int i; - - /* don't attempt to suppress debug messages using this method, it would be uneffective */ - if (log_level >= LOG_LEVEL_DEBUG) - return true; - - /* check if same log entry was sent recently */ - for (i = 0; i < CONFIG_TRACE_RECENT_ENTRIES_COUNT; i++) { - /* Identify the oldest (or empty) entry to evict if we have no match */ - if (oldest_entry->first_suppression_ts > recent_entries[i].first_suppression_ts) - oldest_entry = &recent_entries[i]; - - if (recent_entries[i].entry_id == entry) { - /* We have a match but include this message in this burst only if the - * burst: - - 1. hasn't lasted for too long; - - 2. hasn't been quiet for too long. - */ - if (message_ts - recent_entries[i].first_suppression_ts < - CONFIG_TRACE_RECENT_MAX_TIME && - message_ts - recent_entries[i].message_ts < - CONFIG_TRACE_RECENT_TIME_THRESHOLD) { - recent_entries[i].trigger_count++; - /* Refresh last seen time */ - recent_entries[i].message_ts = message_ts; - - /* Allow the start of a burst to be printed normally */ - return recent_entries[i].trigger_count <= CONFIG_TRACE_BURST_COUNT; - } - - /* Emit and clear this burst */ - if (recent_entries[i].trigger_count > CONFIG_TRACE_BURST_COUNT) - emit_suppressed_entry(&recent_entries[i]); - else - memset(&recent_entries[i], 0, sizeof(recent_entries[i])); - - return true; - } - } - - /* Make room for tracking new entry, by emitting the oldest one in the filter */ - if (oldest_entry->entry_id && oldest_entry->trigger_count > CONFIG_TRACE_BURST_COUNT) - emit_suppressed_entry(oldest_entry); - - /* Start a new burst */ - oldest_entry->entry_id = entry; - oldest_entry->message_ts = message_ts; - oldest_entry->first_suppression_ts = message_ts; - oldest_entry->trigger_count = 1; - - return true; -} -#endif /* CONFIG_TRACE_FILTERING_ADAPTIVE */ - -/** Implementation shared and invoked by both adaptive filtering and - * not. Serializes events into trace messages and passes them to - * dtrace_event() - */ -static void dma_trace_log(bool send_atomic, uint32_t log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list vargs) -{ - uint32_t data[MESSAGE_SIZE_DWORDS(_TRACE_EVENT_MAX_ARGUMENT_COUNT)]; - const int message_size = MESSAGE_SIZE(arg_count); - int i; - - /* fill log content. arg_count is in the dictionary. */ - put_header(data, ctx->uuid_p, id_1, id_2, log_entry, sof_cycle_get_64_safe()); - - for (i = 0; i < arg_count; ++i) - data[PAYLOAD_OFFSET(i)] = va_arg(vargs, uint32_t); - - /* send event by */ - if (send_atomic) - dtrace_event_atomic((const char *)data, message_size); - else - dtrace_event((const char *)data, message_size); - -} - -void trace_log_unfiltered(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list vl) -{ - struct trace *trace = trace_get(); - - if (!trace->enable) { - return; - } - - dma_trace_log(send_atomic, (uint32_t)log_entry, ctx, lvl, id_1, id_2, arg_count, vl); -} - -void trace_log_filtered(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, va_list vl) -{ - struct trace *trace = trace_get(); - - if (!trace->enable) { - return; - } - -#if CONFIG_TRACE_FILTERING_VERBOSITY - if (!trace_filter_verbosity(lvl, ctx)) - return; -#endif /* CONFIG_TRACE_FILTERING_VERBOSITY */ - -#if CONFIG_TRACE_FILTERING_ADAPTIVE - if (!trace->user_filter_override) { - const uint64_t current_ts = sof_cycle_get_64_safe(); - - emit_recent_entries(current_ts); - - if (!trace_filter_flood(lvl, (uint32_t)log_entry, current_ts)) - return; - } -#endif /* CONFIG_TRACE_FILTERING_ADAPTIVE */ - - dma_trace_log(send_atomic, (uint32_t)log_entry, ctx, lvl, id_1, id_2, arg_count, vl); -} - -struct sof_ipc_trace_filter_elem *trace_filter_fill(struct sof_ipc_trace_filter_elem *elem, - struct sof_ipc_trace_filter_elem *end, - struct trace_filter *filter) -{ - filter->log_level = -1; - filter->uuid_id = 0; - filter->comp_id = -1; - filter->pipe_id = -1; - - while (elem <= end) { - switch (elem->key & SOF_IPC_TRACE_FILTER_ELEM_TYPE_MASK) { - case SOF_IPC_TRACE_FILTER_ELEM_SET_LEVEL: - filter->log_level = elem->value; - break; - case SOF_IPC_TRACE_FILTER_ELEM_BY_UUID: - filter->uuid_id = elem->value; - break; - case SOF_IPC_TRACE_FILTER_ELEM_BY_COMP: - filter->comp_id = elem->value; - break; - case SOF_IPC_TRACE_FILTER_ELEM_BY_PIPE: - filter->pipe_id = elem->value; - break; - default: - tr_err(&ipc_tr, "Invalid SOF_IPC_TRACE_FILTER_ELEM 0x%x", - elem->key); - return NULL; - } - - /* each filter set must be terminated with FIN flag and have new log level */ - if (elem->key & SOF_IPC_TRACE_FILTER_ELEM_FIN) { - if (filter->log_level < 0) { - tr_err(&ipc_tr, "Each trace filter set must specify new log level"); - return NULL; - } else { - return elem + 1; - } - } - - ++elem; - } - - tr_err(&ipc_tr, "Trace filter elements set is not properly terminated"); - return NULL; -} - -/* update global components, which tr_ctx is stored inside special section */ -static int trace_filter_update_global(int32_t log_level, uint32_t uuid_id) -{ - int cnt = 0; -#if !defined(CONFIG_LIBRARY) - extern void *_trace_ctx_start; - extern void *_trace_ctx_end; - struct tr_ctx *ptr = (struct tr_ctx *)&_trace_ctx_start; - struct tr_ctx *end = (struct tr_ctx *)&_trace_ctx_end; - - /* iterate over global `tr_ctx` entries located in their own section */ - /* cppcheck-suppress comparePointers */ - while (ptr < end) { - /* - * when looking for specific uuid element, - * then find, update and stop searching - */ - if ((uintptr_t)ptr->uuid_p == uuid_id) { - ptr->level = log_level; - return 1; - } - /* otherwise each element should be updated */ - if (!ptr->uuid_p) { - ptr->level = log_level; - ++cnt; - } - ++ptr; - } -#endif - - return cnt; -} - -/* return trace context from any ipc component type */ -static struct tr_ctx *trace_filter_ipc_comp_context(struct ipc_comp_dev *icd) -{ - switch (icd->type) { - case COMP_TYPE_COMPONENT: - return &icd->cd->tctx; - case COMP_TYPE_BUFFER: - return &icd->cb->tctx; - case COMP_TYPE_PIPELINE: - return &icd->pipeline->tctx; - /* each COMP_TYPE must be specified */ - default: - tr_err(&ipc_tr, "Unknown trace context for ipc component type 0x%X", - icd->type); - return NULL; - } -} - -/* update ipc components, wchich tr_ctx may be read from ipc_comp_dev structure */ -static int trace_filter_update_instances(int32_t log_level, uint32_t uuid_id, - int32_t pipe_id, int32_t comp_id) -{ - struct ipc *ipc = ipc_get(); - struct ipc_comp_dev *icd; - struct list_item *clist; - struct tr_ctx *ctx; - bool correct_comp; - int cnt = 0; - - /* compare each ipc component with filter settings and update log level after pass */ - list_for_item(clist, &ipc->comp_list) { - icd = container_of(clist, struct ipc_comp_dev, list); - ctx = trace_filter_ipc_comp_context(icd); - if (!ctx) - return -EINVAL; - correct_comp = comp_id == -1 || icd->id == comp_id; /* todo: icd->topo_id */ - correct_comp &= uuid_id == 0 || (uintptr_t)ctx->uuid_p == uuid_id; - correct_comp &= pipe_id == -1 || ipc_comp_pipe_id(icd) == pipe_id; - if (correct_comp) { - ctx->level = log_level; - ++cnt; - } - - } - return cnt; -} - -int trace_filter_update(const struct trace_filter *filter) -{ - int ret = 0; -#if CONFIG_TRACE_FILTERING_ADAPTIVE - struct trace *trace = trace_get(); - - if (!trace->user_filter_override) { - trace->user_filter_override = true; - tr_info(&ipc_tr, "Adaptive filtering disabled by user"); - } -#endif /* CONFIG_TRACE_FILTERING_ADAPTIVE */ - - /* validate log level, LOG_LEVEL_CRITICAL has low value, LOG_LEVEL_VERBOSE high */ - if (filter->log_level < LOG_LEVEL_CRITICAL || - filter->log_level > LOG_LEVEL_VERBOSE) - return -EINVAL; - - /* update `*`, `name*` or global `name` */ - if (filter->pipe_id == -1 && filter->comp_id == -1) - ret = trace_filter_update_global(filter->log_level, filter->uuid_id); - - /* update `*`, `name*`, `nameX.*` or `nameX.Y`, `name` may be '*' */ - ret += trace_filter_update_instances(filter->log_level, filter->uuid_id, - filter->pipe_id, filter->comp_id); - return ret > 0 ? ret : -EINVAL; -} - -/** Sends all pending DMA messages to mailbox (for emergencies) */ -void trace_flush_dma_to_mbox(void) -{ - struct trace *trace = trace_get(); - volatile uint64_t *t; - k_spinlock_key_t key; - - key = k_spin_lock(&trace->lock); - - /* get mailbox position */ - t = (volatile uint64_t *)(MAILBOX_TRACE_BASE + trace->pos); - - /* flush dma trace messages */ - dma_trace_flush((void *)t); - - k_spin_unlock(&trace->lock, key); -} - -void trace_on(void) -{ - struct trace *trace = trace_get(); - k_spinlock_key_t key; - - key = k_spin_lock(&trace->lock); - - trace->enable = 1; - dma_trace_on(); - - k_spin_unlock(&trace->lock, key); -} - -void trace_off(void) -{ - struct trace *trace = trace_get(); - k_spinlock_key_t key; - - key = k_spin_lock(&trace->lock); - - trace->enable = 0; - dma_trace_off(); - - k_spin_unlock(&trace->lock, key); -} - -void trace_init(struct sof *sof) -{ - sof->trace = rzalloc(SOF_MEM_FLAG_USER | SOF_MEM_FLAG_COHERENT, sizeof(*sof->trace)); - if (!sof->trace) { - mtrace_printf(LOG_LEVEL_ERROR, "allocation failed"); - sof_panic(SOF_IPC_PANIC_IPC); - } - - sof->trace->enable = 1; - sof->trace->pos = 0; -#if CONFIG_TRACE_FILTERING_ADAPTIVE - sof->trace->user_filter_override = false; -#endif /* CONFIG_TRACE_FILTERING_ADAPTIVE */ - k_spinlock_init(&sof->trace->lock); - -#ifndef __ZEPHYR__ - /* Zephyr owns and has already initialized this buffer (and - * likely has already logged to it by the time we get here). - * Don't touch. - */ - bzero((void *)MAILBOX_TRACE_BASE, MAILBOX_TRACE_SIZE); - dcache_writeback_invalidate_region((__sparse_force void __sparse_cache *)MAILBOX_TRACE_BASE, - MAILBOX_TRACE_SIZE); -#endif - - dma_trace_init_early(sof); -} - -static void mtrace_dict_entry_vl(bool atomic_context, uint32_t dict_entry_address, - int n_args, va_list ap) -{ - int i; - char packet[MESSAGE_SIZE(_TRACE_EVENT_MAX_ARGUMENT_COUNT)]; - uint32_t *args = (uint32_t *)&packet[MESSAGE_SIZE(0)]; - const uint64_t tstamp = sof_cycle_get_64_safe(); - - put_header(packet, dt_tr.uuid_p, _TRACE_INV_ID, _TRACE_INV_ID, - dict_entry_address, tstamp); - - for (i = 0; i < MIN(n_args, _TRACE_EVENT_MAX_ARGUMENT_COUNT); i++) - args[i] = va_arg(ap, uint32_t); - - if (atomic_context) { - mtrace_event(packet, MESSAGE_SIZE(n_args)); - } else { - struct trace * const trace = trace_get(); - k_spinlock_key_t key; - - key = k_spin_lock(&trace->lock); - mtrace_event(packet, MESSAGE_SIZE(n_args)); - k_spin_unlock(&trace->lock, key); - } -} - -void mtrace_dict_entry(bool atomic_context, uint32_t dict_entry_address, int n_args, ...) -{ - va_list ap; - - va_start(ap, n_args); - mtrace_dict_entry_vl(atomic_context, dict_entry_address, n_args, ap); - va_end(ap); -} - -void _log_sofdict(log_func_t sofdict_logf, bool atomic, const void *log_entry, - const struct tr_ctx *ctx, const uint32_t lvl, - uint32_t id_1, uint32_t id_2, int arg_count, ...) -{ - va_list ap; - -#ifndef __ZEPHYR__ /* for Zephyr see _log_nodict() in trace.h */ - if (lvl <= MTRACE_DUPLICATION_LEVEL) { - va_start(ap, arg_count); - mtrace_dict_entry_vl(atomic, (uint32_t)log_entry, arg_count, ap); - va_end(ap); - } -#endif - - va_start(ap, arg_count); - sofdict_logf(atomic, log_entry, ctx, lvl, id_1, id_2, arg_count, ap); - va_end(ap); -} diff --git a/test/cmocka/CMakeLists.txt b/test/cmocka/CMakeLists.txt index 288282d11457..98a64af7d584 100644 --- a/test/cmocka/CMakeLists.txt +++ b/test/cmocka/CMakeLists.txt @@ -42,9 +42,9 @@ else() endif() if(CONFIG_CAVS) - target_include_directories(sof_options INTERFACE ${PROJECT_SOURCE_DIR}/src/platform/intel/cavs/include) + target_include_directories(sof_options INTERFACE ${PROJECT_SOURCE_DIR}/src/kernel/platform/intel/cavs/include) elseif(CONFIG_ACE) - target_include_directories(sof_options INTERFACE ${PROJECT_SOURCE_DIR}/src/platform/intel/ace/include) + target_include_directories(sof_options INTERFACE ${PROJECT_SOURCE_DIR}/src/kernel/platform/intel/ace/include) endif() # linker script, just for log entries diff --git a/test/cmocka/configs/acp_7_0_defconfig b/test/cmocka/configs/acp_7_0_defconfig index c5e3d4952294..0f4a75aadf9c 100644 --- a/test/cmocka/configs/acp_7_0_defconfig +++ b/test/cmocka/configs/acp_7_0_defconfig @@ -1,6 +1,5 @@ CONFIG_ACP_7_0=y CONFIG_RIMAGE_SIGNING_SCHEMA="acp_7_0" -CONFIG_TRACE_CHANNEL=7 CONFIG_WRAP_ACTUAL_POSITION=y CONFIG_AMD_BT=n CONFIG_AMD_SP=n diff --git a/test/cmocka/configs/acp_7_x_defconfig b/test/cmocka/configs/acp_7_x_defconfig index d978611fadde..8f321d541d62 100644 --- a/test/cmocka/configs/acp_7_x_defconfig +++ b/test/cmocka/configs/acp_7_x_defconfig @@ -1,6 +1,5 @@ CONFIG_ACP_7_X=y CONFIG_RIMAGE_SIGNING_SCHEMA="acp_7_3" -CONFIG_TRACE_CHANNEL=7 CONFIG_WRAP_ACTUAL_POSITION=y CONFIG_CORE_COUNT=1 CONFIG_FORMAT_CONVERT_HIFI3=n diff --git a/test/cmocka/configs/rembrandt_defconfig b/test/cmocka/configs/rembrandt_defconfig index 6d5a00745050..a7a66e29f918 100644 --- a/test/cmocka/configs/rembrandt_defconfig +++ b/test/cmocka/configs/rembrandt_defconfig @@ -1,6 +1,5 @@ CONFIG_REMBRANDT=y CONFIG_RIMAGE_SIGNING_SCHEMA="rmb" -CONFIG_TRACE_CHANNEL=7 CONFIG_WRAP_ACTUAL_POSITION=y CONFIG_AMD_BT=n CONFIG_AMD_SP=n diff --git a/test/cmocka/src/audio/buffer/CMakeLists.txt b/test/cmocka/src/audio/buffer/CMakeLists.txt index cb60f07aa22e..805aaa2da784 100644 --- a/test/cmocka/src/audio/buffer/CMakeLists.txt +++ b/test/cmocka/src/audio/buffer/CMakeLists.txt @@ -14,6 +14,7 @@ cmocka_test(buffer_copy ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c @@ -38,6 +39,7 @@ cmocka_test(buffer_new ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c @@ -62,6 +64,7 @@ cmocka_test(buffer_wrap ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c @@ -86,6 +89,7 @@ cmocka_test(buffer_write ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c diff --git a/test/cmocka/src/audio/component/CMakeLists.txt b/test/cmocka/src/audio/component/CMakeLists.txt index b82756073b25..50389cfe4535 100644 --- a/test/cmocka/src/audio/component/CMakeLists.txt +++ b/test/cmocka/src/audio/component/CMakeLists.txt @@ -8,6 +8,7 @@ cmocka_test(comp_set_state ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c diff --git a/test/cmocka/src/audio/dcblock/CMakeLists.txt b/test/cmocka/src/audio/dcblock/CMakeLists.txt index 9c7f58e8feed..0e5754b022d5 100644 --- a/test/cmocka/src/audio/dcblock/CMakeLists.txt +++ b/test/cmocka/src/audio/dcblock/CMakeLists.txt @@ -41,6 +41,7 @@ add_library(audio_for_dcblock STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/audio/eq_fir/CMakeLists.txt b/test/cmocka/src/audio/eq_fir/CMakeLists.txt index 0bb81cc57b0a..485d1996dd8d 100644 --- a/test/cmocka/src/audio/eq_fir/CMakeLists.txt +++ b/test/cmocka/src/audio/eq_fir/CMakeLists.txt @@ -37,6 +37,7 @@ add_library(audio_for_eq_fir STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/audio/eq_iir/CMakeLists.txt b/test/cmocka/src/audio/eq_iir/CMakeLists.txt index 5087e77b4b83..53a550b6b6cf 100644 --- a/test/cmocka/src/audio/eq_iir/CMakeLists.txt +++ b/test/cmocka/src/audio/eq_iir/CMakeLists.txt @@ -40,6 +40,7 @@ add_library(audio_for_eq_iir STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/audio/mixer/CMakeLists.txt b/test/cmocka/src/audio/mixer/CMakeLists.txt index a6c1cdb1a78f..b0b086816e3b 100644 --- a/test/cmocka/src/audio/mixer/CMakeLists.txt +++ b/test/cmocka/src/audio/mixer/CMakeLists.txt @@ -9,6 +9,7 @@ cmocka_test(mixer ${PROJECT_SOURCE_DIR}/src/audio/mixer/mixer_hifi3.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/src/audio/module_adapter/module_adapter.c diff --git a/test/cmocka/src/audio/pcm_converter/CMakeLists.txt b/test/cmocka/src/audio/pcm_converter/CMakeLists.txt index 4dca80136e68..ee72311e004b 100644 --- a/test/cmocka/src/audio/pcm_converter/CMakeLists.txt +++ b/test/cmocka/src/audio/pcm_converter/CMakeLists.txt @@ -19,6 +19,7 @@ if(CONFIG_FORMAT_FLOAT) ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c diff --git a/test/cmocka/src/audio/pipeline/CMakeLists.txt b/test/cmocka/src/audio/pipeline/CMakeLists.txt index 628398576a43..27d27757d6ae 100644 --- a/test/cmocka/src/audio/pipeline/CMakeLists.txt +++ b/test/cmocka/src/audio/pipeline/CMakeLists.txt @@ -6,6 +6,7 @@ cmocka_test(pipeline_new ${PROJECT_SOURCE_DIR}/src/audio/component.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c @@ -31,6 +32,7 @@ cmocka_test(pipeline_connect_upstream ${PROJECT_SOURCE_DIR}/src/audio/component.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c @@ -56,6 +58,7 @@ cmocka_test(pipeline_free ${PROJECT_SOURCE_DIR}/src/audio/component.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/comp_buffer.c ${PROJECT_SOURCE_DIR}/src/audio/buffers/audio_buffer.c diff --git a/test/cmocka/src/audio/selector/CMakeLists.txt b/test/cmocka/src/audio/selector/CMakeLists.txt index 8ea8d969ae7c..eb8f742cc5f5 100644 --- a/test/cmocka/src/audio/selector/CMakeLists.txt +++ b/test/cmocka/src/audio/selector/CMakeLists.txt @@ -26,6 +26,7 @@ add_library(audio_for_selector STATIC ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c diff --git a/test/cmocka/src/audio/volume/CMakeLists.txt b/test/cmocka/src/audio/volume/CMakeLists.txt index d5e1a3ecb757..7a6dc7b2c854 100644 --- a/test/cmocka/src/audio/volume/CMakeLists.txt +++ b/test/cmocka/src/audio/volume/CMakeLists.txt @@ -33,6 +33,7 @@ add_library(audio_for_volume STATIC ${PROJECT_SOURCE_DIR}/src/module/audio/sink_api.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c diff --git a/test/cmocka/src/common_mocks.c b/test/cmocka/src/common_mocks.c index 633108a0730e..53a5bc667f8e 100644 --- a/test/cmocka/src/common_mocks.c +++ b/test/cmocka/src/common_mocks.c @@ -181,31 +181,6 @@ void WEAK __panic(uint32_t p, const char *filename, uint32_t linenum) exit(EXIT_FAILURE); } -#if CONFIG_TRACE -void WEAK trace_log_filtered(bool send_atomic, const void *log_entry, const struct tr_ctx *ctx, - uint32_t lvl, uint32_t id_1, uint32_t id_2, int arg_count, - va_list args) -{ - (void) send_atomic; - (void) log_entry; - (void) ctx; - (void) lvl; - (void) id_1; - (void) id_2; - (void) arg_count; - (void) args; -} - -void WEAK _log_sofdict(log_func_t sofdict_logf, bool atomic, const void *log_entry, - const struct tr_ctx *ctx, const uint32_t lvl, - uint32_t id_1, uint32_t id_2, int arg_count, ...) -{ -} - -void WEAK trace_flush_dma_to_mbox(void) -{ -} -#endif #if CONFIG_LIBRARY volatile void *task_context_get(void); @@ -425,18 +400,6 @@ int WEAK arch_cpu_is_core_enabled(int id) int WEAK test_bench_trace = 1; int WEAK debug; -/* ... but not always in unit tests */ -#if CONFIG_TRACE -/* look up subsystem class name from table */ -char * WEAK get_trace_class(uint32_t trace_class) -{ - (void)trace_class; - /* todo: trace class is deprecated, - * uuid should be used only - */ - return "unknown"; -} -#endif uint8_t * WEAK get_library_mailbox(void) { diff --git a/test/cmocka/src/lib/alloc/CMakeLists.txt b/test/cmocka/src/lib/alloc/CMakeLists.txt index 2ec7c8f606db..10bef8205b4b 100644 --- a/test/cmocka/src/lib/alloc/CMakeLists.txt +++ b/test/cmocka/src/lib/alloc/CMakeLists.txt @@ -3,7 +3,7 @@ cmocka_test(alloc alloc.c ${PROJECT_SOURCE_DIR}/src/lib/alloc.c - ${PROJECT_SOURCE_DIR}/src/platform/library/lib/memory.c + ${PROJECT_SOURCE_DIR}/src/kernel/platform/library/lib/memory.c ) -target_include_directories(sof_options INTERFACE ${PROJECT_SOURCE_DIR}/src/platform/intel/cavs/include) +target_include_directories(sof_options INTERFACE ${PROJECT_SOURCE_DIR}/src/kernel/platform/intel/cavs/include) diff --git a/test/cmocka/src/lib/fast-get/CMakeLists.txt b/test/cmocka/src/lib/fast-get/CMakeLists.txt index 8449320f4e6b..cf542366bcce 100644 --- a/test/cmocka/src/lib/fast-get/CMakeLists.txt +++ b/test/cmocka/src/lib/fast-get/CMakeLists.txt @@ -5,7 +5,7 @@ cmocka_test(fast-get-tests ${PROJECT_SOURCE_DIR}/src/lib/objpool.c ${PROJECT_SOURCE_DIR}/zephyr/lib/fast-get.c ${PROJECT_SOURCE_DIR}/src/lib/alloc.c - ${PROJECT_SOURCE_DIR}/src/platform/library/lib/memory.c + ${PROJECT_SOURCE_DIR}/src/kernel/platform/library/lib/memory.c ) target_link_libraries(fast-get-tests PRIVATE "-Wl,--wrap=rzalloc,--wrap=rmalloc,--wrap=rfree") diff --git a/test/cmocka/src/math/fft/CMakeLists.txt b/test/cmocka/src/math/fft/CMakeLists.txt index f8a0510f7027..79234b7b12f9 100644 --- a/test/cmocka/src/math/fft/CMakeLists.txt +++ b/test/cmocka/src/math/fft/CMakeLists.txt @@ -18,6 +18,7 @@ cmocka_test(fft ${PROJECT_SOURCE_DIR}/test/cmocka/src/notifier_mocks.c ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc3/helper.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-common.c + ${PROJECT_SOURCE_DIR}/src/kernel/ipc/ipc-common-kernel.c ${PROJECT_SOURCE_DIR}/src/ipc/ipc-helper.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-graph.c ${PROJECT_SOURCE_DIR}/src/audio/pipeline/pipeline-params.c diff --git a/test/ztest/unit/audio/crossover/CMakeLists.txt b/test/ztest/unit/audio/crossover/CMakeLists.txt index 39fe53fccf8c..cc599bc4e512 100644 --- a/test/ztest/unit/audio/crossover/CMakeLists.txt +++ b/test/ztest/unit/audio/crossover/CMakeLists.txt @@ -12,7 +12,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/src/audio/crossover ${PROJECT_BINARY_DIR}/include/generated ) diff --git a/test/ztest/unit/audio/drc/CMakeLists.txt b/test/ztest/unit/audio/drc/CMakeLists.txt index 4c83db706257..39691eefa371 100644 --- a/test/ztest/unit/audio/drc/CMakeLists.txt +++ b/test/ztest/unit/audio/drc/CMakeLists.txt @@ -17,7 +17,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/src/audio ${SOF_ROOT}/test/cmocka/include ${PROJECT_BINARY_DIR}/include/generated @@ -61,6 +61,7 @@ target_sources(app PRIVATE ${SOF_ROOT}/src/module/audio/sink_api.c ${SOF_ROOT}/src/kernel/ipc/ipc3/helper.c ${SOF_ROOT}/src/ipc/ipc-common.c + ${SOF_ROOT}/src/kernel/ipc/ipc-common-kernel.c ${SOF_ROOT}/src/ipc/ipc-helper.c ${SOF_ROOT}/src/lib/objpool.c ${SOF_ROOT}/src/audio/pipeline/pipeline-graph.c diff --git a/test/ztest/unit/dsrc/CMakeLists.txt b/test/ztest/unit/dsrc/CMakeLists.txt index a198ace9f231..db31f8a536d8 100644 --- a/test/ztest/unit/dsrc/CMakeLists.txt +++ b/test/ztest/unit/dsrc/CMakeLists.txt @@ -11,7 +11,7 @@ include(${SOF_ROOT}/scripts/cmake/misc.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ) # Define SOF-specific configurations for unit testing diff --git a/test/ztest/unit/fast-get/CMakeLists.txt b/test/ztest/unit/fast-get/CMakeLists.txt index 3fe6a87680f4..b58587e15794 100644 --- a/test/ztest/unit/fast-get/CMakeLists.txt +++ b/test/ztest/unit/fast-get/CMakeLists.txt @@ -15,7 +15,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${PROJECT_BINARY_DIR}/include/generated # For uuid-registry.h ) diff --git a/test/ztest/unit/math/advanced/functions/CMakeLists.txt b/test/ztest/unit/math/advanced/functions/CMakeLists.txt index 615fc86a7db6..0afee53408b7 100644 --- a/test/ztest/unit/math/advanced/functions/CMakeLists.txt +++ b/test/ztest/unit/math/advanced/functions/CMakeLists.txt @@ -15,7 +15,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/test/cmocka/include ${PROJECT_BINARY_DIR}/include/generated # For uuid-registry.h ) diff --git a/test/ztest/unit/math/auditory/CMakeLists.txt b/test/ztest/unit/math/auditory/CMakeLists.txt index 6126b05d3e3d..d09c724d80af 100644 --- a/test/ztest/unit/math/auditory/CMakeLists.txt +++ b/test/ztest/unit/math/auditory/CMakeLists.txt @@ -16,7 +16,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/test/ztest/unit/common # Reference test vectors are #included in place from the CMocka tree # rather than copied, so both frameworks share identical data. diff --git a/test/ztest/unit/math/dct/CMakeLists.txt b/test/ztest/unit/math/dct/CMakeLists.txt index 1bdf36a69247..3bddbf06e6ea 100644 --- a/test/ztest/unit/math/dct/CMakeLists.txt +++ b/test/ztest/unit/math/dct/CMakeLists.txt @@ -16,7 +16,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/test/ztest/unit/common # Reference test vectors are #included in place from the CMocka tree # rather than copied, so both frameworks share identical data. diff --git a/test/ztest/unit/math/fft/CMakeLists.txt b/test/ztest/unit/math/fft/CMakeLists.txt index 6c6432bc3efe..0c7a8f13783f 100644 --- a/test/ztest/unit/math/fft/CMakeLists.txt +++ b/test/ztest/unit/math/fft/CMakeLists.txt @@ -16,7 +16,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/src/audio ${SOF_ROOT}/test/ztest/unit/common ${SOF_ROOT}/test/cmocka/include @@ -65,6 +65,7 @@ target_sources(app PRIVATE ${SOF_ROOT}/src/module/audio/sink_api.c ${SOF_ROOT}/src/kernel/ipc/ipc3/helper.c ${SOF_ROOT}/src/ipc/ipc-common.c + ${SOF_ROOT}/src/kernel/ipc/ipc-common-kernel.c ${SOF_ROOT}/src/ipc/ipc-helper.c ${SOF_ROOT}/src/audio/pipeline/pipeline-graph.c ${SOF_ROOT}/src/audio/pipeline/pipeline-params.c diff --git a/test/ztest/unit/math/fft_multi/CMakeLists.txt b/test/ztest/unit/math/fft_multi/CMakeLists.txt index b08ee40c0884..3287559f3050 100644 --- a/test/ztest/unit/math/fft_multi/CMakeLists.txt +++ b/test/ztest/unit/math/fft_multi/CMakeLists.txt @@ -16,7 +16,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/test/ztest/unit/common ${SOF_ROOT}/test/cmocka/include # Reference test vectors are #included in place from the CMocka tree diff --git a/test/ztest/unit/math/matrix/CMakeLists.txt b/test/ztest/unit/math/matrix/CMakeLists.txt index 6399f3c8ce1f..1dbfea88a08a 100644 --- a/test/ztest/unit/math/matrix/CMakeLists.txt +++ b/test/ztest/unit/math/matrix/CMakeLists.txt @@ -16,7 +16,7 @@ include(${SOF_ROOT}/scripts/cmake/uuid-registry.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ${SOF_ROOT}/test/ztest/unit/common # Reference test vectors are #included in place from the CMocka tree # rather than copied, so both frameworks share identical data. diff --git a/test/ztest/unit/objpool/CMakeLists.txt b/test/ztest/unit/objpool/CMakeLists.txt index 2ce499607463..f19b8a3e926c 100644 --- a/test/ztest/unit/objpool/CMakeLists.txt +++ b/test/ztest/unit/objpool/CMakeLists.txt @@ -11,7 +11,7 @@ include(${SOF_ROOT}/scripts/cmake/misc.cmake) target_include_directories(app PRIVATE ${SOF_ROOT}/zephyr/include ${SOF_ROOT}/src/include - ${SOF_ROOT}/src/platform/posix/include + ${SOF_ROOT}/src/kernel/platform/posix/include ) # Define SOF-specific configurations for unit testing diff --git a/tools/testbench/CMakeLists.txt b/tools/testbench/CMakeLists.txt index d73d20ed2c5b..ec949fc6ed14 100644 --- a/tools/testbench/CMakeLists.txt +++ b/tools/testbench/CMakeLists.txt @@ -9,7 +9,7 @@ include(CheckCCompilerFlag) set(default_asoc_h "/usr/include/alsa/sound/uapi/asoc.h") -file(READ "../../src/arch/host/configs/library_defconfig" library_defconfig) +file(READ "../../src/kernel/arch/host/configs/library_defconfig" library_defconfig) if(library_defconfig MATCHES "CONFIG_IPC_MAJOR_4=y") message("-- Selected IPC4 build for testbench") @@ -39,7 +39,7 @@ set(sof_binary_directory "${PROJECT_BINARY_DIR}/sof_ep/build") set(config_h ${sof_binary_directory}/library_autoconfig.h) -target_include_directories(${testbench} PRIVATE "${sof_source_directory}/src/platform/library/include") +target_include_directories(${testbench} PRIVATE "${sof_source_directory}/src/kernel/platform/library/include") target_include_directories(${testbench} PRIVATE "${sof_source_directory}/src/audio") # Configuration time, make copy diff --git a/tools/testbench/README.md b/tools/testbench/README.md index 82e1630dc771..8fb56e906db4 100644 --- a/tools/testbench/README.md +++ b/tools/testbench/README.md @@ -253,10 +253,10 @@ The testbench build is by default IPC4. The IPC version can be changed by applyi patch: ``` -diff --git a/src/arch/host/configs/library_defconfig b/src/arch/host/configs/library_defconfig +diff --git a/src/kernel/arch/host/configs/library_defconfig b/src/kernel/arch/host/configs/library_defconfig index 1b9483ffe..5c944d292 100644 ---- a/src/arch/host/configs/library_defconfig -+++ b/src/arch/host/configs/library_defconfig +--- a/src/kernel/arch/host/configs/library_defconfig ++++ b/src/kernel/arch/host/configs/library_defconfig @@ -22,8 +22,8 @@ CONFIG_COMP_VOLUME=y CONFIG_COMP_VOLUME_LINEAR_RAMP=y CONFIG_COMP_VOLUME_WINDOWS_FADE=y diff --git a/tools/tplg_parser/CMakeLists.txt b/tools/tplg_parser/CMakeLists.txt index f7a447a6e6c2..353808a3ef15 100644 --- a/tools/tplg_parser/CMakeLists.txt +++ b/tools/tplg_parser/CMakeLists.txt @@ -42,8 +42,8 @@ sof_append_relative_path_definitions(sof_tplg_parser) target_include_directories(sof_tplg_parser PRIVATE ${CMAKE_CURRENT_SOURCE_DIR}/include) target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/include) target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/audio) -target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/arch/host/include) -target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/platform/library/include) +target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/kernel/arch/host/include) +target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/src/kernel/platform/library/include) target_include_directories(sof_tplg_parser PRIVATE ${sof_source_directory}/posix/include) # Configuration time, make copy diff --git a/zephyr/CMakeLists.txt b/zephyr/CMakeLists.txt index 8da2dc47fdc5..b6e3ab4d7792 100644 --- a/zephyr/CMakeLists.txt +++ b/zephyr/CMakeLists.txt @@ -156,9 +156,8 @@ zephyr_interface_library_named(SOF) cmake_path(SET sof_top_dir NORMALIZE "${CMAKE_CURRENT_SOURCE_DIR}/..") set(SOF_SRC_PATH "${sof_top_dir}/src") -set(SOF_PLATFORM_PATH "${SOF_SRC_PATH}/platform") +set(SOF_PLATFORM_PATH "${SOF_SRC_PATH}/kernel/platform") set(SOF_DRIVERS_PATH "${SOF_SRC_PATH}/kernel/drivers") -set(SOF_TRACE_PATH "${SOF_SRC_PATH}/trace") set(RIMAGE_TOP ${sof_top_dir}/tools/rimage) @@ -170,7 +169,7 @@ set(RIMAGE_CONFIG_PATH ${RIMAGE_TOP}/config CACHE PATH # default SOF includes target_include_directories(SOF INTERFACE ${RIMAGE_TOP}/src/include) target_include_directories(SOF INTERFACE ${SOF_SRC_PATH}/include) -target_include_directories(SOF INTERFACE ${SOF_SRC_PATH}/arch/${ARCH}/include) +target_include_directories(SOF INTERFACE ${SOF_SRC_PATH}/kernel/arch/${ARCH}/include) target_include_directories(SOF INTERFACE ${sof_top_dir}/third_party/include) target_include_directories(SOF INTERFACE ${sof_top_dir}/zephyr/include) @@ -242,7 +241,6 @@ add_subdirectory(../src/kernel/ kernel_unused_install/) add_subdirectory(../src/lib/ lib_unused_install/) add_subdirectory(../src/samples/ samples_unused_install/) add_subdirectory(../src/schedule/ schedule_unused_install/) -add_subdirectory(../src/trace/ trace_unused_install/) add_subdirectory(test/) # Old way below: all .c files added by this giant CMake file. @@ -364,7 +362,7 @@ if (CONFIG_SOC_MIMX8ML8_M7) ) # SOF-specific linker script additions - zephyr_linker_sources(DATA_SECTIONS ${sof_top_dir}/src/platform/imx8m_cm7/linker/data-sections.ld) + zephyr_linker_sources(DATA_SECTIONS ${sof_top_dir}/src/kernel/platform/imx8m_cm7/linker/data-sections.ld) set(PLATFORM "imx8m_cm7") @@ -428,7 +426,7 @@ if (CONFIG_SOC_MIMX9596_M7) ) # SOF-specific linker script additions - zephyr_linker_sources(DATA_SECTIONS ${sof_top_dir}/src/platform/imx95/linker/data-sections.ld) + zephyr_linker_sources(DATA_SECTIONS ${sof_top_dir}/src/kernel/platform/imx95/linker/data-sections.ld) set(PLATFORM "imx95") @@ -577,7 +575,7 @@ endif() zephyr_library_sources( # SOF core infrastructure - runs on top of Zephyr - ${SOF_SRC_PATH}/arch/xtensa/drivers/cache_attr.c + ${SOF_SRC_PATH}/kernel/arch/xtensa/drivers/cache_attr.c # Bridge wrapper between SOF and Zephyr APIs - Will shrink over time. wrapper.c @@ -666,6 +664,25 @@ include(../scripts/cmake/version.cmake) include(../scripts/cmake/uuid-registry.cmake) +# Everything under src/kernel belongs in sof_kernel, also the platform, driver +# and architecture sources that this file lists itself rather than adding +# them from src/kernel/CMakeLists.txt. Move them out of modules_sof. +get_target_property(modules_sof_sources modules_sof SOURCES) +cmake_path(SET sof_kernel_src_dir NORMALIZE "${SOF_SRC_PATH}/kernel/") +set(sof_kernel_listed_sources) +foreach(src ${modules_sof_sources}) + cmake_path(ABSOLUTE_PATH src NORMALIZE OUTPUT_VARIABLE abs_src) + string(FIND "${abs_src}" "${sof_kernel_src_dir}" pos) + if(pos EQUAL 0) + list(APPEND sof_kernel_listed_sources ${src}) + endif() +endforeach() +if(sof_kernel_listed_sources) + list(REMOVE_ITEM modules_sof_sources ${sof_kernel_listed_sources}) + set_property(TARGET modules_sof PROPERTY SOURCES ${modules_sof_sources}) + target_sources(sof_kernel PRIVATE ${sof_kernel_listed_sources}) +endif() + # Create Trace realtive file paths sof_append_relative_path_definitions(modules_sof) sof_append_relative_path_definitions(sof_kernel) diff --git a/zephyr/include/rtos/sof.h b/zephyr/include/rtos/sof.h index 05726a55fd5f..72b05f7bc8ed 100644 --- a/zephyr/include/rtos/sof.h +++ b/zephyr/include/rtos/sof.h @@ -25,7 +25,6 @@ struct clock_info; struct comp_driver_list; struct dai_info; struct dma_info; -struct dma_trace_data; struct ipc; struct ll_schedule_domain; struct mm; @@ -36,7 +35,6 @@ struct ams_shared_context; struct notify_data; struct pm_runtime_data; struct sa; -struct trace; struct pipeline_posn; struct probe_pdata; @@ -60,11 +58,7 @@ struct sof { /* system agent */ struct sa *sa; - /* DMA for Trace*/ - struct dma_trace_data *dmat; - /* generic trace structure */ - struct trace *trace; /* platform clock information */ struct clock_info *clocks; diff --git a/zephyr/include/sof/trace/preproc.h b/zephyr/include/sof/trace/preproc.h index 83776e6fea4e..d0333c7751c7 100644 --- a/zephyr/include/sof/trace/preproc.h +++ b/zephyr/include/sof/trace/preproc.h @@ -6,11 +6,6 @@ #ifndef __SOF_TRACE_PREPROC_H__ #define __SOF_TRACE_PREPROC_H__ -#ifdef CONFIG_TRACE -#error "SOF dma-trace (CONFIG_TRACE) not supported in Zephyr builds. \ - Please use CONFIG_ZEPHYR_LOG instead." -#endif - #ifndef CONFIG_ZEPHYR_LOG /* diff --git a/zephyr/include/sof/trace/trace.h b/zephyr/include/sof/trace/trace.h index 3032a7147783..2a71baaf5c70 100644 --- a/zephyr/include/sof/trace/trace.h +++ b/zephyr/include/sof/trace/trace.h @@ -31,8 +31,7 @@ uint64_t platform_timer_get(struct timer *timer); /* * Override SOF mtrace_printf() macro for now to support Zephyr's shared - * memory logger. Note the DMA trace can be copied to the shared memory - * too thanks to CONFIG_TRACEM. + * memory logger. */ #undef mtrace_printf