From a0ec8436568edf196fe5f46263a0e7b11856b8cd Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 30 Sep 2019 14:02:06 +0200 Subject: [PATCH 01/27] platform: cavs: include missing headers Signed-off-by: Tomasz Lauda --- src/platform/intel/cavs/platform.c | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/platform/intel/cavs/platform.c b/src/platform/intel/cavs/platform.c index e7d7a1e4739f..34cb3943b658 100644 --- a/src/platform/intel/cavs/platform.c +++ b/src/platform/intel/cavs/platform.c @@ -26,6 +26,8 @@ #include #include #include +#include +#include #include #include #include From a5b2bfb86864f794f7b33228083d518e52da66bf Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Wed, 11 Sep 2019 14:57:46 +0200 Subject: [PATCH 02/27] dma: add interrupt operation Adds interrupt operation and operation type to DMA API. This is the first step in excluding DMA interrupt handling to schedule layer. Signed-off-by: Tomasz Lauda --- src/include/sof/lib/dma.h | 16 ++++++++++++++++ 1 file changed, 16 insertions(+) diff --git a/src/include/sof/lib/dma.h b/src/include/sof/lib/dma.h index f4957585880e..40b7d3015428 100644 --- a/src/include/sof/lib/dma.h +++ b/src/include/sof/lib/dma.h @@ -79,6 +79,14 @@ enum dma_cb_status { DMA_CB_STATUS_END, }; +/* DMA interrupt commands */ +enum dma_irq_cmd { + DMA_IRQ_STATUS_GET = 0, + DMA_IRQ_CLEAR, + DMA_IRQ_MASK, + DMA_IRQ_UNMASK +}; + #define DMA_CHAN_INVALID 0xFFFFFFFF /* DMA attributes */ @@ -166,6 +174,8 @@ struct dma_ops { uint32_t *free); int (*get_attribute)(struct dma *dma, uint32_t type, uint32_t *value); + + int (*interrupt)(struct dma_chan_data *channel, enum dma_irq_cmd cmd); }; /* DMA platform data */ @@ -356,6 +366,12 @@ static inline int dma_get_attribute(struct dma *dma, uint32_t type, return dma->ops->get_attribute(dma, type, value); } +static inline int dma_interrupt(struct dma_chan_data *channel, + enum dma_irq_cmd cmd) +{ + return channel->dma->ops->interrupt(channel, cmd); +} + /* DMA hardware register operations */ static inline uint32_t dma_reg_read(struct dma *dma, uint32_t reg) { From 38c97c518fd60540ba40cb97ad781971c4af7a7e Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Wed, 11 Sep 2019 15:21:54 +0200 Subject: [PATCH 03/27] dma: add interrupt ops implementations Adds interrupt operation implementations for all existing DMA drivers. Signed-off-by: Tomasz Lauda --- src/drivers/dw/dma.c | 39 ++++++++++++++++++++++++++++++++ src/drivers/imx/dummy-dma.c | 7 ++++++ src/drivers/imx/edma.c | 6 +++++ src/drivers/intel/cavs/hda-dma.c | 9 ++++++++ 4 files changed, 61 insertions(+) diff --git a/src/drivers/dw/dma.c b/src/drivers/dw/dma.c index 5ba32737fe1e..8497e3006a70 100644 --- a/src/drivers/dw/dma.c +++ b/src/drivers/dw/dma.c @@ -161,6 +161,19 @@ static void dw_dma_interrupt_clear(struct dma_chan_data *channel) #endif } +static uint32_t dw_dma_interrupt_status(struct dma_chan_data *channel) +{ + uint32_t status; + +#if CONFIG_HW_LLI + status = dma_reg_read(channel->dma, DW_STATUS_BLOCK); +#else + status = dma_reg_read(channel->dma, DW_STATUS_TFR); +#endif + + return status & DW_CHAN(channel->index); +} + static void dw_dma_increment_pointer(struct dw_dma_chan_data *chan, int bytes) { chan->ptr_data.current_ptr += bytes; @@ -1353,6 +1366,31 @@ static int dw_dma_get_attribute(struct dma *dma, uint32_t type, return ret; } +static int dw_dma_interrupt(struct dma_chan_data *channel, enum dma_irq_cmd cmd) +{ + int ret = 0; + + switch (cmd) { + case DMA_IRQ_STATUS_GET: + ret = dw_dma_interrupt_status(channel); + break; + case DMA_IRQ_CLEAR: + dw_dma_interrupt_clear(channel); + break; + case DMA_IRQ_MASK: + dw_dma_interrupt_mask(channel); + break; + case DMA_IRQ_UNMASK: + dw_dma_interrupt_unmask(channel); + break; + default: + ret = -EINVAL; + break; + } + + return ret; +} + const struct dma_ops dw_dma_ops = { .channel_get = dw_dma_channel_get, .channel_put = dw_dma_channel_put, @@ -1370,4 +1408,5 @@ const struct dma_ops dw_dma_ops = { .remove = dw_dma_remove, .get_data_size = dw_dma_get_data_size, .get_attribute = dw_dma_get_attribute, + .interrupt = dw_dma_interrupt, }; diff --git a/src/drivers/imx/dummy-dma.c b/src/drivers/imx/dummy-dma.c index 28e7347dbb5c..03121bf51da0 100644 --- a/src/drivers/imx/dummy-dma.c +++ b/src/drivers/imx/dummy-dma.c @@ -96,6 +96,12 @@ static int dummy_dma_get_data_size(struct dma_chan_data *channel, return 0; } +static int dummy_dma_interrupt(struct dma_chan_data *channel, + enum dma_irq_cmd cmd) +{ + return 0; +} + const struct dma_ops dummy_dma_ops = { .channel_get = dummy_dma_channel_get, .channel_put = dummy_dma_channel_put, @@ -112,4 +118,5 @@ const struct dma_ops dummy_dma_ops = { .probe = dummy_dma_probe, .remove = dummy_dma_remove, .get_data_size = dummy_dma_get_data_size, + .interrupt = dummy_dma_interrupt, }; diff --git a/src/drivers/imx/edma.c b/src/drivers/imx/edma.c index 9ff65aca7e63..e095b698aa54 100644 --- a/src/drivers/imx/edma.c +++ b/src/drivers/imx/edma.c @@ -110,6 +110,11 @@ static int edma_remove(struct dma *dma) return 0; } +static int edma_interrupt(struct dma_chan_data *channel, enum dma_irq_cmd cmd) +{ + return 0; +} + const struct dma_ops edma_ops = { .channel_get = edma_channel_get, .channel_put = edma_channel_put, @@ -124,4 +129,5 @@ const struct dma_ops edma_ops = { .pm_context_store = edma_pm_context_store, .probe = edma_probe, .remove = edma_remove, + .interrupt = edma_interrupt, }; diff --git a/src/drivers/intel/cavs/hda-dma.c b/src/drivers/intel/cavs/hda-dma.c index c3cf5fb46631..0d4441908b6a 100644 --- a/src/drivers/intel/cavs/hda-dma.c +++ b/src/drivers/intel/cavs/hda-dma.c @@ -880,6 +880,13 @@ static int hda_dma_get_attribute(struct dma *dma, uint32_t type, return ret; } +static int hda_dma_interrupt(struct dma_chan_data *channel, + enum dma_irq_cmd cmd) +{ + /* HDA-DMA doesn't support interrupts */ + return -EINVAL; +} + const struct dma_ops hda_host_dma_ops = { .channel_get = hda_dma_channel_get, .channel_put = hda_dma_channel_put, @@ -897,6 +904,7 @@ const struct dma_ops hda_host_dma_ops = { .remove = hda_dma_remove, .get_data_size = hda_dma_data_size, .get_attribute = hda_dma_get_attribute, + .interrupt = hda_dma_interrupt, }; const struct dma_ops hda_link_dma_ops = { @@ -916,4 +924,5 @@ const struct dma_ops hda_link_dma_ops = { .remove = hda_dma_remove, .get_data_size = hda_dma_data_size, .get_attribute = hda_dma_get_attribute, + .interrupt = hda_dma_interrupt, }; From b9d40e2ea41afde51e466b77e0c7b8b4279a591d Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Thu, 12 Sep 2019 10:20:13 +0200 Subject: [PATCH 04/27] host: use blocking copy for one shot mode Changes copy method for host working in one shot mode. Blocking copy replaces interrupt based copy. In an effort of extracting DMA interrupt based scheduling from DMA to scheduler we need to minimize number of irq sources in our flows. This patch also changes the interrupt based implementation of traces copying through DMA. Signed-off-by: Tomasz Lauda --- src/audio/host.c | 99 ++++++++++++++------------------------- src/drivers/dw/dma.c | 19 ++++++-- src/include/sof/lib/dma.h | 2 - src/ipc/dma-copy.c | 24 ++-------- src/ipc/ipc-host-ptable.c | 23 +-------- 5 files changed, 54 insertions(+), 113 deletions(-) diff --git a/src/audio/host.c b/src/audio/host.c index 16795fa9c7f0..aad307a51ccb 100644 --- a/src/audio/host.c +++ b/src/audio/host.c @@ -95,9 +95,8 @@ struct host_data { struct hc_buf *source; struct hc_buf *sink; - /* helpers used in split transactions */ + /* helper used in split transactions */ uint32_t split_value; - uint32_t last_split_value; uint32_t dma_copy_align; /**< Minimal chunk of data possible to be * copied by dma connected to host @@ -176,21 +175,12 @@ static void host_update_position(struct comp_dev *dev, uint32_t bytes) * This means we must check we do not overflow host period/buffer/page * boundaries on each transfer and split the DMA transfer if we do overflow. */ -static void host_dma_cb_irq(struct comp_dev *dev, struct dma_cb_data *next) +static void host_one_shot_cb(struct comp_dev *dev, uint32_t bytes) { struct host_data *hd = comp_get_drvdata(dev); struct dma_sg_elem *local_elem = hd->config.elem_array.elems; struct dma_sg_elem *source_elem; struct dma_sg_elem *sink_elem; - uint32_t curr_split_value = 0; - uint32_t next_bytes = 0; - uint32_t bytes = hd->last_split_value ? hd->last_split_value : - local_elem->size; - - tracev_host("host_dma_cb_irq()"); - - /* update position */ - host_update_position(dev, bytes); /* update src and dest positions and check for overflow */ local_elem->src += bytes; @@ -215,31 +205,6 @@ static void host_dma_cb_irq(struct comp_dev *dev, struct dma_cb_data *next) local_elem->dest = sink_elem->dest; } } - - /* we need to perform split transfer */ - if (hd->split_value) { - /* check for possible double split */ - curr_split_value = host_dma_get_split(hd, hd->split_value); - if (curr_split_value) { - next_bytes = hd->split_value - curr_split_value; - hd->split_value = curr_split_value; - } else { - next_bytes = hd->split_value; - hd->split_value = 0; - } - - hd->last_split_value = next_bytes; - - next->elem.src = local_elem->src; - next->elem.dest = local_elem->dest; - next->elem.size = next_bytes; - next->status = DMA_CB_STATUS_SPLIT; - return; - } - - hd->last_split_value = 0; - - next->status = DMA_CB_STATUS_END; } /* This is called by DMA driver every time when DMA completes its current @@ -248,21 +213,19 @@ static void host_dma_cb_irq(struct comp_dev *dev, struct dma_cb_data *next) static void host_dma_cb(void *data, uint32_t type, struct dma_cb_data *next) { struct comp_dev *dev = (struct comp_dev *)data; + struct host_data *hd = comp_get_drvdata(dev); + uint32_t bytes = next->elem.size; tracev_host("host_dma_cb()"); - switch (type) { - case DMA_CB_TYPE_IRQ: - host_dma_cb_irq(dev, next); - break; - case DMA_CB_TYPE_COPY: - host_update_position(dev, next->elem.size); - break; - default: - trace_host_error("host_dma_cb() error: Wrong callback type " - "= %u", type); - break; - } + /* update position */ + host_update_position(dev, bytes); + + /* callback for one shot copy */ + if (hd->copy_type == COMP_COPY_ONE_SHOT) + host_one_shot_cb(dev, bytes); + + next->status = DMA_CB_STATUS_END; } static int create_local_elems(struct comp_dev *dev, uint32_t buffer_count, @@ -522,7 +485,7 @@ static void host_buffer_cb(void *data, uint32_t bytes) { struct comp_dev *dev = (struct comp_dev *)data; struct host_data *hd = comp_get_drvdata(dev); - uint32_t copy_bytes = host_buffer_get_copy_bytes(dev); + uint32_t copy_bytes = 0; uint32_t flags = 0; int ret; @@ -533,18 +496,28 @@ static void host_buffer_cb(void *data, uint32_t bytes) else if (hd->copy_type == COMP_COPY_ONE_SHOT) flags |= DMA_COPY_ONE_SHOT; - /* reconfigure transfer */ - ret = dma_set_config(hd->chan, &hd->config); - if (ret < 0) { - trace_host_error("host_buffer_cb() error: dma_set_config() " - "failed, ret = %u", ret); - return; - } + /* copy including all split transfers */ + do { + copy_bytes = host_buffer_get_copy_bytes(dev); - ret = dma_copy(hd->chan, copy_bytes, flags); - if (ret < 0) - trace_host_error("host_buffer_cb() error: dma_copy() failed, " - "ret = %u", ret); + /* reconfigure transfer */ + ret = dma_set_config(hd->chan, &hd->config); + if (ret < 0) { + trace_host_error("host_buffer_cb() error: " + "dma_set_config() failed, ret = %u", + ret); + hd->split_value = 0; + return; + } + + ret = dma_copy(hd->chan, copy_bytes, flags); + if (ret < 0) { + trace_host_error("host_buffer_cb() error: dma_copy() " + "failed, ret = %u", ret); + hd->split_value = 0; + return; + } + } while (hd->split_value); } /* configure the DMA params and descriptors for host buffer IO */ @@ -688,8 +661,7 @@ static int host_params(struct comp_dev *dev) } /* set up callback */ - dma_set_cb(hd->chan, DMA_CB_TYPE_IRQ | - DMA_CB_TYPE_COPY, host_dma_cb, dev); + dma_set_cb(hd->chan, DMA_CB_TYPE_COPY, host_dma_cb, dev); return 0; } @@ -711,7 +683,6 @@ static int host_prepare(struct comp_dev *dev) hd->local_pos = 0; hd->report_pos = 0; hd->split_value = 0; - hd->last_split_value = 0; dev->position = 0; return 0; diff --git a/src/drivers/dw/dma.c b/src/drivers/dw/dma.c index 8497e3006a70..3afdd7e9c47c 100644 --- a/src/drivers/dw/dma.c +++ b/src/drivers/dw/dma.c @@ -965,14 +965,25 @@ static int dw_dma_copy(struct dma_chan_data *channel, int bytes, uint32_t flags) { struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); + int ret = 0; struct dma_cb_data next = { .elem = { .size = bytes }, .status = DMA_CB_STATUS_RELOAD }; - /* for preload or one shot copy just start DMA */ - if (flags & (DMA_COPY_PRELOAD | DMA_COPY_ONE_SHOT)) - return dw_dma_start(channel); + /* for preload and one shot copy just start the DMA and wait */ + if (flags & (DMA_COPY_PRELOAD | DMA_COPY_ONE_SHOT)) { + ret = dw_dma_start(channel); + if (ret < 0) + return ret; + + ret = poll_for_register_delay(dma_base(channel->dma) + + DW_DMA_CHAN_EN, + DW_CHAN(channel->index), 0, + PLATFORM_DMA_TIMEOUT); + if (ret < 0) + return ret; + } tracev_dwdma("dw_dma_copy(): dma %d channel %d copy", channel->dma->plat_data.id, channel->index); @@ -982,7 +993,7 @@ static int dw_dma_copy(struct dma_chan_data *channel, int bytes, /* increment current pointer */ dw_dma_increment_pointer(dw_chan, bytes); - return 0; + return ret; } /* interrupt handler for DMA */ diff --git a/src/include/sof/lib/dma.h b/src/include/sof/lib/dma.h index 40b7d3015428..f020799cf532 100644 --- a/src/include/sof/lib/dma.h +++ b/src/include/sof/lib/dma.h @@ -21,7 +21,6 @@ #include #include #include -#include #include #include #include @@ -485,7 +484,6 @@ static inline uint32_t dma_sg_get_size(struct dma_sg_elem_array *ea) struct dma_copy { struct dma_chan_data *chan; struct dma *dmac; - completion_t complete; }; /* init dma copy context */ diff --git a/src/ipc/dma-copy.c b/src/ipc/dma-copy.c index 97af7ec46096..c6f058c0c2a0 100644 --- a/src/ipc/dma-copy.c +++ b/src/ipc/dma-copy.c @@ -7,7 +7,6 @@ #include #include -#include #include #include #include @@ -50,22 +49,6 @@ static struct dma_sg_elem *sg_get_elem_at(struct dma_sg_config *host_sg, } #endif -#if !CONFIG_DMA_GW - -static void dma_complete(void *data, uint32_t type, struct dma_cb_data *next) -{ - completion_t *comp = (completion_t *)data; - - if (type == DMA_CB_TYPE_IRQ) - wait_completed(comp); - - ipc_dma_trace_send_position(); - - next->status = DMA_CB_STATUS_END; -} - -#endif - /* Copy DSP memory to host memory. * Copies DSP memory to host in a single PAGE_SIZE or smaller block. Does not * waits/sleeps and can be used in IRQ context. @@ -129,10 +112,12 @@ int dma_copy_to_host_nowait(struct dma_copy *dc, struct dma_sg_config *host_sg, if (err < 0) return err; - err = dma_start(dc->chan); + err = dma_copy(dc->chan, local_sg_elem.size, DMA_COPY_ONE_SHOT); if (err < 0) return err; + ipc_dma_trace_send_position(); + /* bytes copied */ return local_sg_elem.size; } @@ -160,9 +145,6 @@ int dma_copy_new(struct dma_copy *dc) trace_dma_error("dma_copy_new() error: dc->chan is NULL"); return -ENODEV; } - - dc->complete.timeout = 100; /* wait 100 usecs for DMA to finish */ - dma_set_cb(dc->chan, DMA_CB_TYPE_IRQ, dma_complete, &dc->complete); #endif return 0; diff --git a/src/ipc/ipc-host-ptable.c b/src/ipc/ipc-host-ptable.c index 2e9f368dffb6..7b30cff8bca6 100644 --- a/src/ipc/ipc-host-ptable.c +++ b/src/ipc/ipc-host-ptable.c @@ -8,7 +8,6 @@ #include #include #include -#include #include #include #include @@ -76,14 +75,6 @@ static int ipc_parse_page_descriptors(uint8_t *page_table, return 0; } -static void dma_complete(void *data, uint32_t type, struct dma_cb_data *next) -{ - completion_t *complete = data; - - if (type == DMA_CB_TYPE_IRQ) - wait_completed(complete); -} - /* * Copy the audio buffer page tables from the host to the DSP max of 4K. */ @@ -92,7 +83,6 @@ static int ipc_get_page_descriptors(struct dma *dmac, uint8_t *page_table, { struct dma_sg_config config; struct dma_sg_elem elem; - completion_t complete; struct dma_chan_data *chan; int ret = 0; @@ -129,25 +119,14 @@ static int ipc_get_page_descriptors(struct dma *dmac, uint8_t *page_table, goto out; } - /* set up callback */ - dma_set_cb(chan, DMA_CB_TYPE_IRQ, dma_complete, &complete); - - wait_init(&complete); - /* start the copy of page table to DSP */ - ret = dma_start(chan); + ret = dma_copy(chan, elem.size, DMA_COPY_ONE_SHOT); if (ret < 0) { trace_ipc_error("ipc_get_page_descriptors() error: " "dma_start() failed"); goto out; } - /* wait for DMA to complete */ - ret = poll_for_completion_delay(&complete, PLATFORM_DMA_TIMEOUT); - if (ret < 0) - trace_ipc_error("ipc_get_page_descriptors() error: " - "poll_for_completion_delay() failed"); - /* compressed page tables now in buffer at _ipc->page_table */ out: dma_channel_put(chan); From 0b7fcbe9a67045eab6174e555e808b08f71d592f Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Fri, 13 Sep 2019 10:27:25 +0200 Subject: [PATCH 05/27] pipeline: allocate pipe_task only if needed Changes implementation of pipe_task allocation and initialization to happen in prepare call instead of during pipeline creations. It might be that the pipeline will never be used. Signed-off-by: Tomasz Lauda --- src/audio/pipeline.c | 52 ++++++++++++++----- src/drivers/intel/cavs/idc.c | 10 ++-- src/include/sof/audio/pipeline.h | 2 +- .../pipeline/pipeline_connection_mocks.c | 4 +- .../cmocka/src/audio/pipeline/pipeline_free.c | 6 +-- 5 files changed, 53 insertions(+), 21 deletions(-) diff --git a/src/audio/pipeline.c b/src/audio/pipeline.c index 1942294d1e6e..72cd9999c243 100644 --- a/src/audio/pipeline.c +++ b/src/audio/pipeline.c @@ -44,7 +44,6 @@ struct pipeline *pipeline_new(struct sof_ipc_pipe_new *pipe_desc, struct comp_dev *cd) { struct pipeline *p; - uint32_t type; trace_pipe("pipeline_new()"); @@ -62,12 +61,6 @@ struct pipeline *pipeline_new(struct sof_ipc_pipe_new *pipe_desc, assert(!memcpy_s(&p->ipc_pipe, sizeof(p->ipc_pipe), pipe_desc, sizeof(*pipe_desc))); - /* get pipeline task type */ - type = pipeline_is_timer_driven(p) ? SOF_SCHEDULE_LL : - SOF_SCHEDULE_EDF; - schedule_task_init(&p->pipe_task, type, pipe_desc->priority, - pipeline_task, NULL, p, pipe_desc->core, 0); - return p; } @@ -236,7 +229,8 @@ int pipeline_free(struct pipeline *p) } /* remove from any scheduling */ - schedule_task_free(&p->pipe_task); + if (p->pipe_task) + schedule_task_free(p->pipe_task); /* now free the pipeline */ rfree(p); @@ -390,10 +384,25 @@ static int pipeline_comp_prepare(struct comp_dev *current, void *data, int dir) &buffer_reset_pos, dir); } +static struct task *pipeline_task_init(struct pipeline *p, uint32_t type, + enum task_state (*func)(void *data)) +{ + struct task *task = NULL; + + task = rzalloc(RZONE_RUNTIME, SOF_MEM_CAPS_RAM, sizeof(*task)); + + if (task) + schedule_task_init(task, type, p->ipc_pipe.priority, func, NULL, + p, p->ipc_pipe.core, 0); + + return task; +} + /* prepare the pipeline for usage - preload host buffers here */ int pipeline_prepare(struct pipeline *p, struct comp_dev *dev) { struct pipeline_data ppl_data; + uint32_t type; int ret = 0; trace_pipe_with_ids(p, "pipeline_prepare()"); @@ -412,6 +421,20 @@ int pipeline_prepare(struct pipeline *p, struct comp_dev *dev) */ p->preload = dev->params.direction == SOF_IPC_STREAM_PLAYBACK && p->sink_comp->state != COMP_STATE_ACTIVE; + + /* initialize task if necessary */ + if (!p->pipe_task) { + type = pipeline_is_timer_driven(p) ? SOF_SCHEDULE_LL : + SOF_SCHEDULE_EDF; + + p->pipe_task = pipeline_task_init(p, type, pipeline_task); + if (!p->pipe_task) { + trace_pipe_error("pipeline_prepare() error: task init " + "failed"); + return -ENOMEM; + } + } + p->status = COMP_STATE_PREPARE; return ret; @@ -444,8 +467,10 @@ void pipeline_cache(struct pipeline *p, struct comp_dev *dev, int cmd) uint32_t flags; /* pipeline needs to be invalidated before usage */ - if (cmd == CACHE_INVALIDATE) + if (cmd == CACHE_INVALIDATE) { dcache_invalidate_region(p, sizeof(*p)); + dcache_invalidate_region(p->pipe_task, sizeof(*p->pipe_task)); + } trace_pipe_with_ids(p, "pipeline_cache()"); @@ -458,8 +483,11 @@ void pipeline_cache(struct pipeline *p, struct comp_dev *dev, int cmd) pipeline_comp_cache(dev, &data, dev->params.direction); /* pipeline needs to be flushed after usage */ - if (cmd == CACHE_WRITEBACK_INV) + if (cmd == CACHE_WRITEBACK_INV) { + dcache_writeback_invalidate_region(p->pipe_task, + sizeof(*p->pipe_task)); dcache_writeback_invalidate_region(p, sizeof(*p)); + } irq_local_enable(flags); } @@ -943,12 +971,12 @@ static int pipeline_xrun_recover(struct pipeline *p) void pipeline_schedule_copy(struct pipeline *p, uint64_t start) { if (p->sched_comp->state == COMP_STATE_ACTIVE) - schedule_task(&p->pipe_task, start, p->ipc_pipe.period); + schedule_task(p->pipe_task, start, p->ipc_pipe.period); } void pipeline_schedule_cancel(struct pipeline *p) { - schedule_task_cancel(&p->pipe_task); + schedule_task_cancel(p->pipe_task); } static enum task_state pipeline_task(void *arg) diff --git a/src/drivers/intel/cavs/idc.c b/src/drivers/intel/cavs/idc.c index 12c9435162b2..241f449a5742 100644 --- a/src/drivers/intel/cavs/idc.c +++ b/src/drivers/intel/cavs/idc.c @@ -192,8 +192,9 @@ static int idc_pipeline_trigger(uint32_t cmd) * Should be removed after changing memory management for * slave cores. */ - if (pcm_dev->cd->pipeline->pipe_task.type == SOF_SCHEDULE_EDF) { - task = &pcm_dev->cd->pipeline->pipe_task; + if (pcm_dev->cd->pipeline->pipe_task->type == + SOF_SCHEDULE_EDF) { + task = pcm_dev->cd->pipeline->pipe_task; edf_pdata = edf_sch_get_pdata(task); dcache_invalidate_region(edf_pdata, sizeof(*edf_pdata)); task_context_cache(edf_pdata->ctx, CACHE_INVALIDATE); @@ -213,8 +214,9 @@ static int idc_pipeline_trigger(uint32_t cmd) * Should be removed after changing memory management for * slave cores. */ - if (pcm_dev->cd->pipeline->pipe_task.type == SOF_SCHEDULE_EDF) { - task = &pcm_dev->cd->pipeline->pipe_task; + if (pcm_dev->cd->pipeline->pipe_task->type == + SOF_SCHEDULE_EDF) { + task = pcm_dev->cd->pipeline->pipe_task; edf_pdata = edf_sch_get_pdata(task); task_context_cache(edf_pdata->ctx, CACHE_WRITEBACK_INV); dcache_writeback_invalidate_region(edf_pdata, diff --git a/src/include/sof/audio/pipeline.h b/src/include/sof/audio/pipeline.h index 717345c5c56d..a2c32fe7f78c 100644 --- a/src/include/sof/audio/pipeline.h +++ b/src/include/sof/audio/pipeline.h @@ -78,7 +78,7 @@ struct pipeline { bool preload; /* is pipeline preload needed */ /* scheduling */ - struct task pipe_task; /* pipeline processing task */ + struct task *pipe_task; /* pipeline processing task */ /* component that drives scheduling in this pipe */ struct comp_dev *sched_comp; diff --git a/test/cmocka/src/audio/pipeline/pipeline_connection_mocks.c b/test/cmocka/src/audio/pipeline/pipeline_connection_mocks.c index c1aaf4743a68..07805615cda9 100644 --- a/test/cmocka/src/audio/pipeline/pipeline_connection_mocks.c +++ b/test/cmocka/src/audio/pipeline/pipeline_connection_mocks.c @@ -37,7 +37,9 @@ struct pipeline_connect_data *get_standard_connect_objects(void) pipe->ipc_pipe = pipe_desc; pipe->status = COMP_STATE_INIT; - pipe->pipe_task.type = SOF_SCHEDULE_EDF; + + pipe->pipe_task = calloc(sizeof(struct task), 1); + pipe->pipe_task->type = SOF_SCHEDULE_EDF; schedulers = calloc(sizeof(struct schedulers), 1); list_init(&schedulers->list); diff --git a/test/cmocka/src/audio/pipeline/pipeline_free.c b/test/cmocka/src/audio/pipeline/pipeline_free.c index 3409fc8f4105..5c616413129c 100644 --- a/test/cmocka/src/audio/pipeline/pipeline_free.c +++ b/test/cmocka/src/audio/pipeline/pipeline_free.c @@ -73,9 +73,9 @@ static void test_audio_pipeline_free_sheduler_task_free(void **state) /*Testing component*/ pipeline_free(&result); - assert_int_equal(result.pipe_task.state, SOF_TASK_STATE_FREE); - assert_ptr_equal(NULL, result.pipe_task.data); - assert_ptr_equal(NULL, result.pipe_task.run); + assert_int_equal(result.pipe_task->state, SOF_TASK_STATE_FREE); + assert_ptr_equal(NULL, result.pipe_task->data); + assert_ptr_equal(NULL, result.pipe_task->run); } static void test_audio_pipeline_free_disconnect_full(void **state) From 9f039462303b73423da5e6bbdc9fc7b7e11f3b39 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Fri, 13 Sep 2019 15:02:09 +0200 Subject: [PATCH 06/27] pipeline: add preload_task Adds preload_task to pipeline. This task is only allocated and initialized, when pipeline needs to be preloaded with data e.g. render pipe scheduled on DMA irq. It's always an EDF task, so we make sure it's get scheduled no matter if interrupt was triggered or not. Signed-off-by: Tomasz Lauda --- src/audio/pipeline.c | 34 ++++++++++++++++++++++++++++++++ src/include/sof/audio/pipeline.h | 1 + 2 files changed, 35 insertions(+) diff --git a/src/audio/pipeline.c b/src/audio/pipeline.c index 72cd9999c243..2b5483fa02ca 100644 --- a/src/audio/pipeline.c +++ b/src/audio/pipeline.c @@ -38,6 +38,7 @@ struct pipeline_data { }; static enum task_state pipeline_task(void *arg); +static enum task_state pipeline_preload_task(void *arg); /* create new pipeline - returns pipeline id or negative error */ struct pipeline *pipeline_new(struct sof_ipc_pipe_new *pipe_desc, @@ -232,6 +233,9 @@ int pipeline_free(struct pipeline *p) if (p->pipe_task) schedule_task_free(p->pipe_task); + if (p->preload_task) + schedule_task_free(p->preload_task); + /* now free the pipeline */ rfree(p); @@ -435,6 +439,18 @@ int pipeline_prepare(struct pipeline *p, struct comp_dev *dev) } } + /* initialize preload task if necessary */ + if (p->preload && !p->preload_task) { + /* preload task is always EDF as it guarantees scheduling */ + p->preload_task = pipeline_task_init(p, SOF_SCHEDULE_EDF, + pipeline_preload_task); + if (!p->preload_task) { + trace_pipe_error("pipeline_prepare() error: preload " + "task init failed"); + return -ENOMEM; + } + } + p->status = COMP_STATE_PREPARE; return ret; @@ -470,6 +486,8 @@ void pipeline_cache(struct pipeline *p, struct comp_dev *dev, int cmd) if (cmd == CACHE_INVALIDATE) { dcache_invalidate_region(p, sizeof(*p)); dcache_invalidate_region(p->pipe_task, sizeof(*p->pipe_task)); + dcache_invalidate_region(p->preload_task, + sizeof(*p->preload_task)); } trace_pipe_with_ids(p, "pipeline_cache()"); @@ -486,6 +504,8 @@ void pipeline_cache(struct pipeline *p, struct comp_dev *dev, int cmd) if (cmd == CACHE_WRITEBACK_INV) { dcache_writeback_invalidate_region(p->pipe_task, sizeof(*p->pipe_task)); + dcache_writeback_invalidate_region(p->preload_task, + sizeof(*p->preload_task)); dcache_writeback_invalidate_region(p, sizeof(*p)); } @@ -1014,3 +1034,17 @@ static enum task_state pipeline_task(void *arg) return (pipeline_is_timer_driven(p) || p->preload) ? SOF_TASK_STATE_RESCHEDULE : SOF_TASK_STATE_COMPLETED; } + +static enum task_state pipeline_preload_task(void *arg) +{ + struct pipeline *p = arg; + + if (pipeline_copy(p) < 0) { + trace_pipe_error_with_ids(p, "pipeline_preload_task(): preload " + "has failed"); + return SOF_TASK_STATE_COMPLETED; + } + + return p->preload ? SOF_TASK_STATE_RESCHEDULE : + SOF_TASK_STATE_COMPLETED; +} diff --git a/src/include/sof/audio/pipeline.h b/src/include/sof/audio/pipeline.h index a2c32fe7f78c..82e03115cc51 100644 --- a/src/include/sof/audio/pipeline.h +++ b/src/include/sof/audio/pipeline.h @@ -79,6 +79,7 @@ struct pipeline { /* scheduling */ struct task *pipe_task; /* pipeline processing task */ + struct task *preload_task; /* pipeline preload task */ /* component that drives scheduling in this pipe */ struct comp_dev *sched_comp; From bf48d13967ab95e46dab35b3295545fe10652947 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Fri, 13 Sep 2019 15:18:33 +0200 Subject: [PATCH 07/27] dma: add core to dma_chan_data structure Adds core to dma_chan_data structure. This way we can easily identify which core currently owns the DMA channel. Signed-off-by: Tomasz Lauda --- src/drivers/dw/dma.c | 4 ++++ src/drivers/intel/cavs/hda-dma.c | 4 ++++ src/include/sof/lib/dma.h | 2 ++ 3 files changed, 10 insertions(+) diff --git a/src/drivers/dw/dma.c b/src/drivers/dw/dma.c index 3afdd7e9c47c..db9a370755c0 100644 --- a/src/drivers/dw/dma.c +++ b/src/drivers/dw/dma.c @@ -328,6 +328,9 @@ static int dw_dma_start(struct dma_chan_data *channel) ret = dw_dma_interrupt_register(channel); if (!ret) { + /* assign core */ + channel->core = cpu_get_id(); + /* enable the channel */ channel->status = COMP_STATE_ACTIVE; dma_reg_write(dma, DW_DMA_CHAN_EN, @@ -1240,6 +1243,7 @@ static int dw_dma_probe(struct dma *dma) chan->index = i; chan->dma = dma; chan->index = i; + chan->core = DMA_CORE_INVALID; dw_chan = rzalloc(RZONE_SYS_RUNTIME | RZONE_FLAG_UNCACHED, SOF_MEM_CAPS_RAM, sizeof(*dw_chan)); diff --git a/src/drivers/intel/cavs/hda-dma.c b/src/drivers/intel/cavs/hda-dma.c index 0d4441908b6a..6406fcd7d159 100644 --- a/src/drivers/intel/cavs/hda-dma.c +++ b/src/drivers/intel/cavs/hda-dma.c @@ -13,6 +13,7 @@ #include #include #include +#include #include #include #include @@ -477,6 +478,8 @@ static int hda_dma_start(struct dma_chan_data *channel) hda_dma_enable_unlock(channel); channel->status = COMP_STATE_ACTIVE; + channel->core = cpu_get_id(); + out: irq_local_enable(flags); return ret; @@ -743,6 +746,7 @@ static int hda_dma_probe(struct dma *dma) chan->dma = dma; chan->index = i; chan->status = COMP_STATE_INIT; + chan->core = DMA_CORE_INVALID; hda_chan = rzalloc(RZONE_SYS_RUNTIME | RZONE_FLAG_UNCACHED, SOF_MEM_CAPS_RAM, diff --git a/src/include/sof/lib/dma.h b/src/include/sof/lib/dma.h index f020799cf532..1121e185c43b 100644 --- a/src/include/sof/lib/dma.h +++ b/src/include/sof/lib/dma.h @@ -87,6 +87,7 @@ enum dma_irq_cmd { }; #define DMA_CHAN_INVALID 0xFFFFFFFF +#define DMA_CORE_INVALID 0xFFFFFFFF /* DMA attributes */ #define DMA_ATTR_BUFFER_ALIGNMENT 0 @@ -208,6 +209,7 @@ struct dma_chan_data { uint32_t direction; uint32_t desc_count; uint32_t index; + uint32_t core; /* client callback function */ void (*cb)(void *data, uint32_t type, struct dma_cb_data *next); From 2eaa86234e76547ce60c606139f1b8f220d43c21 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Fri, 13 Sep 2019 15:31:33 +0200 Subject: [PATCH 08/27] component: add comp_is_scheduling_source function Adds comp_is_scheduling_source function, which checks whether component is the one used as a scheduling source for the pipeline. Signed-off-by: Tomasz Lauda --- src/include/sof/audio/component.h | 11 +++++++++++ 1 file changed, 11 insertions(+) diff --git a/src/include/sof/audio/component.h b/src/include/sof/audio/component.h index 91c894714480..47d5265e108f 100644 --- a/src/include/sof/audio/component.h +++ b/src/include/sof/audio/component.h @@ -29,6 +29,7 @@ #include #include #include +#include #include #include @@ -760,6 +761,16 @@ static inline void comp_overrun(struct comp_dev *dev, struct comp_buffer *sink, pipeline_xrun(dev->pipeline, dev, (int32_t)copy_bytes - sink->free); } +/** + * Called to check whether component schedules its pipeline. + * @param dev Component device. + * @return True if this is scheduling component, false otherwise. + */ +static inline bool comp_is_scheduling_source(struct comp_dev *dev) +{ + return dev == dev->pipeline->sched_comp; +} + /** * Called by component in copy. * @param dev Component device. From ea6c8039c127ae5ec221e44856b3e67722dd56ef Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Fri, 13 Sep 2019 15:33:08 +0200 Subject: [PATCH 09/27] dma: add scheduling_source Adds scheduling_source field and dma_is_scheduling_source function. It allows for easy identification whether particular DMA channel is the source of pipeline scheduling. Signed-off-by: Tomasz Lauda --- src/audio/dai.c | 2 ++ src/audio/host.c | 1 + src/drivers/dw/dma.c | 1 + src/drivers/intel/cavs/hda-dma.c | 1 + src/include/sof/lib/dma.h | 9 +++++++++ 5 files changed, 14 insertions(+) diff --git a/src/audio/dai.c b/src/audio/dai.c index be23cc858e15..97ebd835c535 100644 --- a/src/audio/dai.c +++ b/src/audio/dai.c @@ -248,6 +248,7 @@ static int dai_playback_params(struct comp_dev *dev, uint32_t period_bytes) config->irq_disabled = pipeline_is_timer_driven(dev->pipeline); config->dest_dev = dai_get_handshake(dd->dai, dev->params.direction, dd->stream_id); + config->is_scheduling_source = comp_is_scheduling_source(dev); trace_dai_with_ids(dev, "dai_playback_params() " "dest_dev = %d stream_id = %d " @@ -313,6 +314,7 @@ static int dai_capture_params(struct comp_dev *dev, uint32_t period_bytes) config->irq_disabled = pipeline_is_timer_driven(dev->pipeline); config->src_dev = dai_get_handshake(dd->dai, dev->params.direction, dd->stream_id); + config->is_scheduling_source = comp_is_scheduling_source(dev); /* TODO: Make this code platform-specific or move it driver callback */ if (dai_get_info(dd->dai, DAI_INFO_TYPE) == SOF_DAI_INTEL_DMIC) { diff --git a/src/audio/host.c b/src/audio/host.c index aad307a51ccb..06f6fc4801b7 100644 --- a/src/audio/host.c +++ b/src/audio/host.c @@ -627,6 +627,7 @@ static int host_params(struct comp_dev *dev) config->dest_width = comp_sample_bytes(dev); config->cyclic = 0; config->irq_disabled = pipeline_is_timer_driven(dev->pipeline); + config->is_scheduling_source = comp_is_scheduling_source(dev); host_elements_reset(dev); diff --git a/src/drivers/dw/dma.c b/src/drivers/dw/dma.c index db9a370755c0..befdb881a21e 100644 --- a/src/drivers/dw/dma.c +++ b/src/drivers/dw/dma.c @@ -539,6 +539,7 @@ static int dw_dma_set_config(struct dma_chan_data *channel, /* default channel config */ channel->direction = config->direction; + channel->is_scheduling_source = config->is_scheduling_source; dw_chan->irq_disabled = config->irq_disabled; dw_chan->cfg_lo = DW_CFG_LOW_DEF; dw_chan->cfg_hi = DW_CFG_HIGH_DEF; diff --git a/src/drivers/intel/cavs/hda-dma.c b/src/drivers/intel/cavs/hda-dma.c index 6406fcd7d159..6a636de2197d 100644 --- a/src/drivers/intel/cavs/hda-dma.c +++ b/src/drivers/intel/cavs/hda-dma.c @@ -607,6 +607,7 @@ static int hda_dma_set_config(struct dma_chan_data *channel, /* default channel config */ channel->direction = config->direction; channel->desc_count = config->elem_array.count; + channel->is_scheduling_source = config->is_scheduling_source; /* validate - HDA only supports continuous elems of same size */ for (i = 0; i < config->elem_array.count; i++) { diff --git a/src/include/sof/lib/dma.h b/src/include/sof/lib/dma.h index 1121e185c43b..289e1b2fb56e 100644 --- a/src/include/sof/lib/dma.h +++ b/src/include/sof/lib/dma.h @@ -132,6 +132,8 @@ struct dma_sg_config { struct dma_sg_elem_array elem_array; /* array of dma_sg elems */ bool scatter; bool irq_disabled; + /* true if configured DMA channel is the scheduling source */ + bool is_scheduling_source; }; struct dma_chan_status { @@ -210,6 +212,8 @@ struct dma_chan_data { uint32_t desc_count; uint32_t index; uint32_t core; + /* true if this DMA channel is the scheduling source */ + bool is_scheduling_source; /* client callback function */ void (*cb)(void *data, uint32_t type, struct dma_cb_data *next); @@ -434,6 +438,11 @@ static inline void dma_chan_reg_update_bits(struct dma_chan_data *channel, mask, value); } +static inline bool dma_is_scheduling_source(struct dma_chan_data *channel) +{ + return channel->is_scheduling_source; +} + static inline void dma_sg_init(struct dma_sg_elem_array *ea) { ea->count = 0; From 59d0ac927321891e0d1218835c9dd48c389686fa Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Fri, 13 Sep 2019 15:38:05 +0200 Subject: [PATCH 10/27] schedule: rename SOF_SCHEDULE_LL to SOF_SCHEDULE_LL_TIMER There can be many potential low latency scheduling domains, so let be more specific here. Signed-off-by: Tomasz Lauda --- src/audio/pipeline.c | 2 +- src/audio/volume/volume.c | 4 ++-- src/drivers/intel/cavs/dmic.c | 2 +- src/include/sof/lib/wait.h | 2 +- src/include/sof/schedule/schedule.h | 2 +- src/lib/agent.c | 2 +- src/schedule/timer_domain.c | 2 +- src/trace/dma-trace.c | 2 +- 8 files changed, 9 insertions(+), 9 deletions(-) diff --git a/src/audio/pipeline.c b/src/audio/pipeline.c index 2b5483fa02ca..0f57e111a583 100644 --- a/src/audio/pipeline.c +++ b/src/audio/pipeline.c @@ -428,7 +428,7 @@ int pipeline_prepare(struct pipeline *p, struct comp_dev *dev) /* initialize task if necessary */ if (!p->pipe_task) { - type = pipeline_is_timer_driven(p) ? SOF_SCHEDULE_LL : + type = pipeline_is_timer_driven(p) ? SOF_SCHEDULE_LL_TIMER : SOF_SCHEDULE_EDF; p->pipe_task = pipeline_task_init(p, type, pipeline_task); diff --git a/src/audio/volume/volume.c b/src/audio/volume/volume.c index cb80545a4c15..5ed4368a0eb3 100644 --- a/src/audio/volume/volume.c +++ b/src/audio/volume/volume.c @@ -163,8 +163,8 @@ static struct comp_dev *volume_new(struct sof_ipc_comp *comp) } comp_set_drvdata(dev, cd); - schedule_task_init(&cd->volwork, SOF_SCHEDULE_LL, SOF_TASK_PRI_MED, - vol_work, NULL, dev, 0, 0); + schedule_task_init(&cd->volwork, SOF_SCHEDULE_LL_TIMER, + SOF_TASK_PRI_MED, vol_work, NULL, dev, 0, 0); /* Set the default volumes. If IPC sets min_value or max_value to * not-zero, use them. Otherwise set to internal limits and notify diff --git a/src/drivers/intel/cavs/dmic.c b/src/drivers/intel/cavs/dmic.c index a8be891d142d..822dd33731e0 100644 --- a/src/drivers/intel/cavs/dmic.c +++ b/src/drivers/intel/cavs/dmic.c @@ -1067,7 +1067,7 @@ static int dmic_set_config(struct dai *dai, struct sof_ipc_dai_config *config) trace_dmic("dmic_set_config(), dai->index = %d", di); /* Initialize start sequence handler */ - schedule_task_init(&dmic->dmicwork, SOF_SCHEDULE_LL, + schedule_task_init(&dmic->dmicwork, SOF_SCHEDULE_LL_TIMER, SOF_TASK_PRI_MED, dmic_work, NULL, dai, 0, SOF_SCHEDULE_FLAG_ASYNC); diff --git a/src/include/sof/lib/wait.h b/src/include/sof/lib/wait.h index 05a1620c5038..1fc0fafe2a85 100644 --- a/src/include/sof/lib/wait.h +++ b/src/include/sof/lib/wait.h @@ -69,7 +69,7 @@ static inline void wait_init(completion_t *comp) c->complete = 0; - schedule_task_init(&comp->work, SOF_SCHEDULE_LL, SOF_TASK_PRI_MED, + schedule_task_init(&comp->work, SOF_SCHEDULE_LL_TIMER, SOF_TASK_PRI_MED, _wait_cb, NULL, comp, 0, 0); } diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index e8f6eccf1eaf..77f831fb4a39 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -29,7 +29,7 @@ /* SOF_SCHEDULE_ type comes from topology */ enum { SOF_SCHEDULE_EDF = 0, /* EDF scheduler */ - SOF_SCHEDULE_LL, /* Low latency scheduler */ + SOF_SCHEDULE_LL_TIMER, /* Low latency timer scheduler */ SOF_SCHEDULE_COUNT }; diff --git a/src/lib/agent.c b/src/lib/agent.c index 12af04f464b8..7e2d9b337f80 100644 --- a/src/lib/agent.c +++ b/src/lib/agent.c @@ -83,7 +83,7 @@ void sa_init(struct sof *sof) sa->last_idle = platform_timer_get(platform_timer) + sa->ticks; sa->is_active = true; - schedule_task_init(&sa->work, SOF_SCHEDULE_LL, SOF_TASK_PRI_HIGH, + schedule_task_init(&sa->work, SOF_SCHEDULE_LL_TIMER, SOF_TASK_PRI_HIGH, validate, NULL, sa, 0, 0); schedule_task(&sa->work, PLATFORM_IDLE_TIME, PLATFORM_IDLE_TIME); diff --git a/src/schedule/timer_domain.c b/src/schedule/timer_domain.c index ae818c4c7f19..214fb86e808b 100644 --- a/src/schedule/timer_domain.c +++ b/src/schedule/timer_domain.c @@ -117,7 +117,7 @@ struct ll_schedule_domain *timer_domain_init(struct timer *timer, int clk, struct ll_schedule_domain *domain; struct timer_domain *timer_domain; - domain = domain_init(SOF_SCHEDULE_LL, clk, &timer_domain_ops); + domain = domain_init(SOF_SCHEDULE_LL_TIMER, clk, &timer_domain_ops); timer_domain = rzalloc(RZONE_SYS | RZONE_FLAG_UNCACHED, SOF_MEM_CAPS_RAM, sizeof(*timer_domain)); diff --git a/src/trace/dma-trace.c b/src/trace/dma-trace.c index 07044d1b9994..63aaa0b76f72 100644 --- a/src/trace/dma-trace.c +++ b/src/trace/dma-trace.c @@ -138,7 +138,7 @@ int dma_trace_init_complete(struct dma_trace_data *d) return ret; } - schedule_task_init(&d->dmat_work, SOF_SCHEDULE_LL, + schedule_task_init(&d->dmat_work, SOF_SCHEDULE_LL_TIMER, SOF_TASK_PRI_MED, trace_work, NULL, d, 0, 0); return 0; From 4626d3f1b0f12a2cab570975f8e411d4430afa85 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Fri, 13 Sep 2019 15:41:59 +0200 Subject: [PATCH 11/27] schedule: add SOF_SCHEDULE_LL_DMA Adds SOF_SCHEDULE_LL_DMA as the new low latency scheduling domain type. Signed-off-by: Tomasz Lauda --- src/include/sof/schedule/schedule.h | 14 ++++++++++++-- 1 file changed, 12 insertions(+), 2 deletions(-) diff --git a/src/include/sof/schedule/schedule.h b/src/include/sof/schedule/schedule.h index 77f831fb4a39..8fe2309c8afd 100644 --- a/src/include/sof/schedule/schedule.h +++ b/src/include/sof/schedule/schedule.h @@ -28,8 +28,18 @@ /* SOF_SCHEDULE_ type comes from topology */ enum { - SOF_SCHEDULE_EDF = 0, /* EDF scheduler */ - SOF_SCHEDULE_LL_TIMER, /* Low latency timer scheduler */ + /* EDF scheduler - schedules based on task's deadline */ + SOF_SCHEDULE_EDF = 0, + + /* Low latency timer scheduler - schedules immediately on selected + * timer's tick + */ + SOF_SCHEDULE_LL_TIMER, + + /* Low latency DMA scheduler - schedules immediately on scheduling + * component's DMA interrupt + */ + SOF_SCHEDULE_LL_DMA, SOF_SCHEDULE_COUNT }; From 25726047c7ed81eb6bdae42153cc8b60859330ba Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Fri, 13 Sep 2019 15:55:47 +0200 Subject: [PATCH 12/27] dma: move CAVS_PLATFORM_NUM_DMACS to specific platforms Moves CAVS_PLATFORM_NUM_DMACS definition to specific platforms and renames it to PLATFORM_NUM_DMACS. This way we can access to the number of platform DMAs from generic code. Signed-off-by: Tomasz Lauda --- src/platform/apollolake/include/platform/lib/dma.h | 3 +++ src/platform/cannonlake/include/platform/lib/dma.h | 3 +++ src/platform/icelake/include/platform/lib/dma.h | 3 +++ src/platform/intel/cavs/lib/dma.c | 7 ++----- src/platform/suecreek/include/platform/lib/dma.h | 3 +++ src/platform/tigerlake/include/platform/lib/dma.h | 3 +++ 6 files changed, 17 insertions(+), 5 deletions(-) diff --git a/src/platform/apollolake/include/platform/lib/dma.h b/src/platform/apollolake/include/platform/lib/dma.h index 2af38d9ec1ad..2b812ca83486 100644 --- a/src/platform/apollolake/include/platform/lib/dma.h +++ b/src/platform/apollolake/include/platform/lib/dma.h @@ -11,6 +11,9 @@ #ifndef __PLATFORM_LIB_DMA_H__ #define __PLATFORM_LIB_DMA_H__ +/* number of supported DMACs */ +#define PLATFORM_NUM_DMACS 6 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/cannonlake/include/platform/lib/dma.h b/src/platform/cannonlake/include/platform/lib/dma.h index 3768296b9b4b..8db65afe57d9 100644 --- a/src/platform/cannonlake/include/platform/lib/dma.h +++ b/src/platform/cannonlake/include/platform/lib/dma.h @@ -12,6 +12,9 @@ #ifndef __PLATFORM_LIB_DMA_H__ #define __PLATFORM_LIB_DMA_H__ +/* number of supported DMACs */ +#define PLATFORM_NUM_DMACS 6 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/icelake/include/platform/lib/dma.h b/src/platform/icelake/include/platform/lib/dma.h index 3768296b9b4b..8db65afe57d9 100644 --- a/src/platform/icelake/include/platform/lib/dma.h +++ b/src/platform/icelake/include/platform/lib/dma.h @@ -12,6 +12,9 @@ #ifndef __PLATFORM_LIB_DMA_H__ #define __PLATFORM_LIB_DMA_H__ +/* number of supported DMACs */ +#define PLATFORM_NUM_DMACS 6 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/intel/cavs/lib/dma.c b/src/platform/intel/cavs/lib/dma.c index 2fc25cc904f4..95363ebc7d13 100644 --- a/src/platform/intel/cavs/lib/dma.c +++ b/src/platform/intel/cavs/lib/dma.c @@ -22,18 +22,15 @@ #define DMAC_HOST_OUT_CHANNELS_COUNT 6 #define DMAC_LINK_IN_CHANNELS_COUNT 8 #define DMAC_LINK_OUT_CHANNELS_COUNT 8 -#define CAVS_PLATFORM_NUM_DMACS 6 #elif CONFIG_CANNONLAKE || CONFIG_ICELAKE || CONFIG_TIGERLAKE #define DMAC0_CLASS 6 #define DMAC1_CLASS 7 #define DMAC_HOST_OUT_CHANNELS_COUNT 9 #define DMAC_LINK_IN_CHANNELS_COUNT 9 #define DMAC_LINK_OUT_CHANNELS_COUNT 7 -#define CAVS_PLATFORM_NUM_DMACS 6 #elif CONFIG_SUECREEK #define DMAC0_CLASS 6 #define DMAC1_CLASS 7 -#define CAVS_PLATFORM_NUM_DMACS 3 #endif static struct dw_drv_plat_data dmac0 = { @@ -107,7 +104,7 @@ static struct dw_drv_plat_data dmac1 = { }; #if CONFIG_SUECREEK -static struct dma dma[CAVS_PLATFORM_NUM_DMACS] = { +static struct dma dma[PLATFORM_NUM_DMACS] = { { /* LP GP DMAC 0 */ .plat_data = { .id = DMA_GP_LP_DMAC0, @@ -156,7 +153,7 @@ static struct dma dma[CAVS_PLATFORM_NUM_DMACS] = { }; #else -static struct dma dma[CAVS_PLATFORM_NUM_DMACS] = { +static struct dma dma[PLATFORM_NUM_DMACS] = { { /* Low Power GP DMAC 0 */ .plat_data = { .id = DMA_GP_LP_DMAC0, diff --git a/src/platform/suecreek/include/platform/lib/dma.h b/src/platform/suecreek/include/platform/lib/dma.h index b14ab58e6fde..ad4bd8486cde 100644 --- a/src/platform/suecreek/include/platform/lib/dma.h +++ b/src/platform/suecreek/include/platform/lib/dma.h @@ -12,6 +12,9 @@ #ifndef __PLATFORM_LIB_DMA_H__ #define __PLATFORM_LIB_DMA_H__ +/* number of supported DMACs */ +#define PLATFORM_NUM_DMACS 3 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/tigerlake/include/platform/lib/dma.h b/src/platform/tigerlake/include/platform/lib/dma.h index 3768296b9b4b..8db65afe57d9 100644 --- a/src/platform/tigerlake/include/platform/lib/dma.h +++ b/src/platform/tigerlake/include/platform/lib/dma.h @@ -12,6 +12,9 @@ #ifndef __PLATFORM_LIB_DMA_H__ #define __PLATFORM_LIB_DMA_H__ +/* number of supported DMACs */ +#define PLATFORM_NUM_DMACS 6 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 From 2bd632a0d71c062ed9369c1671b7709405ff732f Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Fri, 13 Sep 2019 16:02:13 +0200 Subject: [PATCH 13/27] hda-dma: remove HDA_DMA_MAX_CHANS definition Removes HDA_DMA_MAX_CHANS definition as it's not needed. We can directly use the number of allocated channels. Signed-off-by: Tomasz Lauda --- src/drivers/intel/cavs/hda-dma.c | 4 +--- 1 file changed, 1 insertion(+), 3 deletions(-) diff --git a/src/drivers/intel/cavs/hda-dma.c b/src/drivers/intel/cavs/hda-dma.c index 6a636de2197d..6e4eb015e4f6 100644 --- a/src/drivers/intel/cavs/hda-dma.c +++ b/src/drivers/intel/cavs/hda-dma.c @@ -60,8 +60,6 @@ /* DGBS */ #define DGBS_MASK 0xfffff0 -#define HDA_DMA_MAX_CHANS 9 - #define HDA_STATE_RELEASE BIT(0) /* DGMBS align value */ @@ -783,7 +781,7 @@ static int hda_dma_remove(struct dma *dma) trace_hddma("hda-dmac :%d -> remove", dma->plat_data.id); - for (i = 0; i < HDA_DMA_MAX_CHANS; i++) + for (i = 0; i < dma->plat_data.channels; i++) rfree(dma_chan_get_data(&dma->chan[i])); rfree(dma->chan); From b6c4c03f447f12442561f2231b48dcd69c3794d6 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 16 Sep 2019 11:54:20 +0200 Subject: [PATCH 14/27] dma: add PLATFORM_MAX_DMA_CHAN definition Adds PLATFORM_MAX_DMA_CHAN definition to every platform. This way generic API can retrieve this information. Signed-off-by: Tomasz Lauda --- src/platform/apollolake/include/platform/lib/dma.h | 3 +++ src/platform/baytrail/include/platform/lib/dma.h | 3 +++ src/platform/cannonlake/include/platform/lib/dma.h | 3 +++ src/platform/haswell/include/platform/lib/dma.h | 3 +++ src/platform/icelake/include/platform/lib/dma.h | 3 +++ src/platform/imx8/include/platform/lib/dma.h | 3 +++ src/platform/suecreek/include/platform/lib/dma.h | 3 +++ src/platform/tigerlake/include/platform/lib/dma.h | 3 +++ 8 files changed, 24 insertions(+) diff --git a/src/platform/apollolake/include/platform/lib/dma.h b/src/platform/apollolake/include/platform/lib/dma.h index 2b812ca83486..dad10e016cdd 100644 --- a/src/platform/apollolake/include/platform/lib/dma.h +++ b/src/platform/apollolake/include/platform/lib/dma.h @@ -14,6 +14,9 @@ /* number of supported DMACs */ #define PLATFORM_NUM_DMACS 6 +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 8 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/baytrail/include/platform/lib/dma.h b/src/platform/baytrail/include/platform/lib/dma.h index ff9b6780db3f..161321dbf5d1 100644 --- a/src/platform/baytrail/include/platform/lib/dma.h +++ b/src/platform/baytrail/include/platform/lib/dma.h @@ -19,6 +19,9 @@ #define PLATFORM_NUM_DMACS 2 #endif +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 8 + #define DMA_ID_DMAC0 0 #define DMA_ID_DMAC1 1 #define DMA_ID_DMAC2 2 diff --git a/src/platform/cannonlake/include/platform/lib/dma.h b/src/platform/cannonlake/include/platform/lib/dma.h index 8db65afe57d9..881d870e5b64 100644 --- a/src/platform/cannonlake/include/platform/lib/dma.h +++ b/src/platform/cannonlake/include/platform/lib/dma.h @@ -15,6 +15,9 @@ /* number of supported DMACs */ #define PLATFORM_NUM_DMACS 6 +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 9 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/haswell/include/platform/lib/dma.h b/src/platform/haswell/include/platform/lib/dma.h index 526eb9c9bb15..ef214979bbbb 100644 --- a/src/platform/haswell/include/platform/lib/dma.h +++ b/src/platform/haswell/include/platform/lib/dma.h @@ -12,6 +12,9 @@ #define PLATFORM_NUM_DMACS 2 +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 8 + #define DMA_ID_DMAC0 0 #define DMA_ID_DMAC1 1 diff --git a/src/platform/icelake/include/platform/lib/dma.h b/src/platform/icelake/include/platform/lib/dma.h index 8db65afe57d9..881d870e5b64 100644 --- a/src/platform/icelake/include/platform/lib/dma.h +++ b/src/platform/icelake/include/platform/lib/dma.h @@ -15,6 +15,9 @@ /* number of supported DMACs */ #define PLATFORM_NUM_DMACS 6 +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 9 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/imx8/include/platform/lib/dma.h b/src/platform/imx8/include/platform/lib/dma.h index 7b7c26302478..434ea3cac0cb 100644 --- a/src/platform/imx8/include/platform/lib/dma.h +++ b/src/platform/imx8/include/platform/lib/dma.h @@ -12,6 +12,9 @@ #define PLATFORM_NUM_DMACS 2 +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 32 + #define DMA_ID_EDMA0 0 #define DMA_ID_HOST 1 diff --git a/src/platform/suecreek/include/platform/lib/dma.h b/src/platform/suecreek/include/platform/lib/dma.h index ad4bd8486cde..b6dfcd35b0cc 100644 --- a/src/platform/suecreek/include/platform/lib/dma.h +++ b/src/platform/suecreek/include/platform/lib/dma.h @@ -15,6 +15,9 @@ /* number of supported DMACs */ #define PLATFORM_NUM_DMACS 3 +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 8 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 diff --git a/src/platform/tigerlake/include/platform/lib/dma.h b/src/platform/tigerlake/include/platform/lib/dma.h index 8db65afe57d9..881d870e5b64 100644 --- a/src/platform/tigerlake/include/platform/lib/dma.h +++ b/src/platform/tigerlake/include/platform/lib/dma.h @@ -15,6 +15,9 @@ /* number of supported DMACs */ #define PLATFORM_NUM_DMACS 6 +/* max number of supported DMA channels */ +#define PLATFORM_MAX_DMA_CHAN 9 + /* available DMACs */ #define DMA_GP_LP_DMAC0 0 #define DMA_GP_LP_DMAC1 1 From ac068731c8074d4789bc046dc194d60cc5b4f58d Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Fri, 20 Sep 2019 12:21:34 +0200 Subject: [PATCH 15/27] dma: imx8: add missing dma_chan_irq macro Signed-off-by: Tomasz Lauda --- src/platform/imx8/include/platform/lib/dma.h | 2 ++ 1 file changed, 2 insertions(+) diff --git a/src/platform/imx8/include/platform/lib/dma.h b/src/platform/imx8/include/platform/lib/dma.h index 434ea3cac0cb..af2e9e31b811 100644 --- a/src/platform/imx8/include/platform/lib/dma.h +++ b/src/platform/imx8/include/platform/lib/dma.h @@ -18,6 +18,8 @@ #define DMA_ID_EDMA0 0 #define DMA_ID_HOST 1 +#define dma_chan_irq(dma, chan) (dma_irq(dma) + chan) + int dmac_init(void); #endif /* __PLATFORM_LIB_DMA_H__ */ From a7af43fc654f16279fb681582664dcde59cbab92 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 16 Sep 2019 11:55:33 +0200 Subject: [PATCH 16/27] schedule: implement DMA scheduling domain Implements DMA scheduling domain. It allows to initialize another low latency scheduler and schedule on any DMA channel. The functionality stays the same as the one already done in DMA drivers e.g. DW-DMA. Signed-off-by: Tomasz Lauda --- src/include/sof/schedule/ll_schedule_domain.h | 5 + src/schedule/CMakeLists.txt | 1 + src/schedule/dma_multi_chan_domain.c | 304 ++++++++++++++++++ 3 files changed, 310 insertions(+) create mode 100644 src/schedule/dma_multi_chan_domain.c diff --git a/src/include/sof/schedule/ll_schedule_domain.h b/src/include/sof/schedule/ll_schedule_domain.h index 6ab4f9f44e8e..dc5e0a43c02a 100644 --- a/src/include/sof/schedule/ll_schedule_domain.h +++ b/src/include/sof/schedule/ll_schedule_domain.h @@ -20,6 +20,7 @@ #include #include +struct dma; struct ll_schedule_domain; struct task; struct timer; @@ -126,4 +127,8 @@ static inline bool domain_is_pending(struct ll_schedule_domain *domain, struct ll_schedule_domain *timer_domain_init(struct timer *timer, int clk, uint64_t timeout); +struct ll_schedule_domain *dma_multi_chan_domain_init(struct dma *dma_array, + uint32_t num_dma, int clk, + bool aggregated_irq); + #endif /* __SOF_SCHEDULE_LL_SCHEDULE_DOMAIN_H__ */ diff --git a/src/schedule/CMakeLists.txt b/src/schedule/CMakeLists.txt index 2817fbe1f259..4ffb0ef2ac56 100644 --- a/src/schedule/CMakeLists.txt +++ b/src/schedule/CMakeLists.txt @@ -1,6 +1,7 @@ # SPDX-License-Identifier: BSD-3-Clause add_local_sources(sof + dma_multi_chan_domain.c edf_schedule.c ll_schedule.c schedule.c diff --git a/src/schedule/dma_multi_chan_domain.c b/src/schedule/dma_multi_chan_domain.c new file mode 100644 index 000000000000..11141e70aef8 --- /dev/null +++ b/src/schedule/dma_multi_chan_domain.c @@ -0,0 +1,304 @@ +// SPDX-License-Identifier: BSD-3-Clause +// +// Copyright(c) 2019 Intel Corporation. All rights reserved. +// +// Author: Tomasz Lauda + +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include +#include + +struct dma_domain_data { + int irq; + struct task *task; + void (*handler)(void *arg); + void *arg; +}; + +struct dma_domain { + struct dma *dma_array; /* pointer to scheduling DMAs */ + uint32_t num_dma; /* number of scheduling DMAs */ + bool aggregated_irq; /* true if aggregated interrupts */ + + /* mask of currently running channels */ + uint32_t channel_mask[PLATFORM_NUM_DMACS][PLATFORM_CORE_COUNT]; + /* array of arguments for aggregated mode */ + struct dma_domain_data *arg[PLATFORM_NUM_DMACS][PLATFORM_CORE_COUNT]; + /* array of registered channels data */ + struct dma_domain_data data[PLATFORM_NUM_DMACS][PLATFORM_MAX_DMA_CHAN]; +}; + +struct ll_schedule_domain_ops dma_multi_chan_domain_ops; + +/** + * \brief Generic DMA interrupt handler. + * \param[in,out] data Pointer to DMA domain data. + */ +static void dma_multi_chan_domain_irq_handler(void *data) +{ + struct dma_domain_data *domain_data = data; + + /* just call registered handler */ + domain_data->handler(domain_data->arg); +} + +/** + * \brief Registers and enables selected DMA interrupt. + * \param[in,out] data Pointer to DMA domain data. + * \param[in,out] handler Pointer to DMA interrupt handler. + * \return Error code. + */ +static int dma_multi_chan_domain_irq_register(struct dma_domain_data *data, + void (*handler)(void *arg)) +{ + int ret; + + /* always go through dma_multi_chan_domain_irq_handler, + * so we have different arg registered for every channel + */ + ret = interrupt_register(data->irq, IRQ_AUTO_UNMASK, + dma_multi_chan_domain_irq_handler, data); + if (ret < 0) + return ret; + + interrupt_enable(data->irq, data); + + return 0; +} + +/** + * \brief Registers task to DMA domain. + * \param[in,out] domain Pointer to schedule domain. + * \param[in] period Period of the scheduled task. + * \param[in,out] task Task to be registered. + * \param[in,out] handler Pointer to DMA interrupt handler. + * \param[in,out] arg Pointer to DMA interrupt handler's argument. + * \return Error code. + * + * Keeps track of potential double registrations and handles non aggregated + * DMA interrupts (different irq number per DMA channel). + */ +static int dma_multi_chan_domain_register(struct ll_schedule_domain *domain, + uint64_t period, struct task *task, + void (*handler)(void *arg), void *arg) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + struct dma *dmas = dma_domain->dma_array; + int core = cpu_get_id(); + int ret; + int i; + int j; + + for (i = 0; i < dma_domain->num_dma; ++i) { + for (j = 0; j < dmas[i].plat_data.channels; ++j) { + /* channel not set as scheduling source */ + if (!dma_is_scheduling_source(&dmas[i].chan[j])) + continue; + + /* channel not running */ + if (dmas[i].chan[j].status != COMP_STATE_ACTIVE) + continue; + + /* channel owned by different core */ + if (core != dmas[i].chan[j].core) + continue; + + /* channel has been already running */ + if (dma_domain->channel_mask[i][core] & BIT(j)) + continue; + + dma_interrupt(&dmas[i].chan[j], DMA_IRQ_CLEAR); + + /* register only if not aggregated or not registered */ + if (!dma_domain->aggregated_irq || + !dma_domain->channel_mask[i][core]) { + ret = dma_multi_chan_domain_irq_register( + &dma_domain->data[i][j], + handler); + if (ret < 0) + return ret; + + dma_domain->data[i][j].handler = handler; + dma_domain->data[i][j].arg = arg; + + /* needed to unregister aggregated interrupts */ + dma_domain->arg[i][core] = + &dma_domain->data[i][j]; + } + + interrupt_clear_mask(dma_domain->data[i][j].irq, + BIT(j)); + + dma_interrupt(&dmas[i].chan[j], DMA_IRQ_UNMASK); + + dma_domain->data[i][j].task = task; + dma_domain->channel_mask[i][core] |= BIT(j); + + return 0; + } + } + + return 0; +} + +/** + * \brief Unregisters and disables selected DMA interrupt. + * \param[in,out] data data Pointer to DMA domain data. + */ +static void dma_multi_chan_domain_irq_unregister(struct dma_domain_data *data) +{ + interrupt_disable(data->irq, data); + + interrupt_unregister(data->irq, data); +} + +/** + * \brief Unregisters task from DMA domain. + * \param[in,out] domain Pointer to schedule domain. + * \param[in] num_tasks Number of currently scheduled tasks. + */ +static void dma_multi_chan_domain_unregister(struct ll_schedule_domain *domain, + uint32_t num_tasks) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + struct dma *dmas = dma_domain->dma_array; + int core = cpu_get_id(); + int i; + int j; + + for (i = 0; i < dma_domain->num_dma; ++i) { + for (j = 0; j < dmas[i].plat_data.channels; ++j) { + /* channel not set as scheduling source */ + if (!dma_is_scheduling_source(&dmas[i].chan[j])) + continue; + + /* channel still running */ + if (dmas[i].chan[j].status == COMP_STATE_ACTIVE) + continue; + + /* channel owned by different core */ + if (core != dmas[i].chan[j].core) + continue; + + /* channel hasn't been running */ + if (!(dma_domain->channel_mask[i][core] & BIT(j))) + continue; + + dma_interrupt(&dmas[i].chan[j], DMA_IRQ_MASK); + dma_interrupt(&dmas[i].chan[j], DMA_IRQ_CLEAR); + interrupt_clear_mask(dma_domain->data[i][j].irq, + BIT(j)); + + dma_domain->data[i][j].task = NULL; + dma_domain->channel_mask[i][core] &= ~BIT(j); + + /* unregister interrupt */ + if (!dma_domain->aggregated_irq) + dma_multi_chan_domain_irq_unregister( + &dma_domain->data[i][j]); + else if (!dma_domain->channel_mask[i][core]) + dma_multi_chan_domain_irq_unregister( + dma_domain->arg[i][core]); + + return; + } + } +} + +/** + * \brief Checks if given task should be executed. + * \param[in,out] domain Pointer to schedule domain. + * \param[in,out] task Task to be checked. + * \return True is task should be executed, false otherwise. + */ +static bool dma_multi_chan_domain_is_pending(struct ll_schedule_domain *domain, + struct task *task) +{ + struct dma_domain *dma_domain = ll_sch_domain_get_pdata(domain); + struct dma *dmas = dma_domain->dma_array; + uint32_t status; + int i; + int j; + + for (i = 0; i < dma_domain->num_dma; ++i) { + for (j = 0; j < dmas[i].plat_data.channels; ++j) { + status = dma_interrupt(&dmas[i].chan[j], + DMA_IRQ_STATUS_GET); + if (!status) + continue; + + /* not this task */ + if (dma_domain->data[i][j].task != task) + continue; + + /* clear interrupt */ + dma_interrupt(&dmas[i].chan[j], DMA_IRQ_CLEAR); + interrupt_clear_mask(dma_domain->data[i][j].irq, + BIT(j)); + + return true; + } + } + + return false; +} + +/** + * \brief Initializes DMA multichannel scheduling domain. + * \param[in,out] dma_array Array of DMAs to be scheduled on. + * \param[in] num_dma Number of DMAs passed as dma_array. + * \param[in] clk Platform clock to base calculations on. + * \param[in] aggregated_irq True if all DMAs share the same interrupt line. + * \return Pointer to initialized scheduling domain object. + */ +struct ll_schedule_domain *dma_multi_chan_domain_init(struct dma *dma_array, + uint32_t num_dma, int clk, + bool aggregated_irq) +{ + struct ll_schedule_domain *domain; + struct dma_domain *dma_domain; + struct dma *dma; + int i; + int j; + + domain = domain_init(SOF_SCHEDULE_LL_DMA, clk, + &dma_multi_chan_domain_ops); + + dma_domain = rzalloc(RZONE_SYS | RZONE_FLAG_UNCACHED, SOF_MEM_CAPS_RAM, + sizeof(*dma_domain)); + dma_domain->dma_array = dma_array; + dma_domain->num_dma = num_dma; + dma_domain->aggregated_irq = aggregated_irq; + + /* retrieve IRQ numbers for each DMA channel */ + for (i = 0; i < num_dma; ++i) { + dma = &dma_array[i]; + for (j = 0; j < dma->plat_data.channels; ++j) + dma_domain->data[i][j].irq = interrupt_get_irq( + dma_chan_irq(dma, j), dma_irq_name(dma)); + } + + ll_sch_domain_set_pdata(domain, dma_domain); + + return domain; +} + +struct ll_schedule_domain_ops dma_multi_chan_domain_ops = { + .domain_register = dma_multi_chan_domain_register, + .domain_unregister = dma_multi_chan_domain_unregister, + .domain_is_pending = dma_multi_chan_domain_is_pending, + .domain_set = NULL, + .domain_enable = NULL, + .domain_disable = NULL, + .domain_clear = NULL, +}; From bae6f2c60915b35f15b8a5bfad88caf696bfbef5 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 16 Sep 2019 12:30:43 +0200 Subject: [PATCH 17/27] dw-dma: add PLATFORM_NUM_DW_DMACS definition Adds PLATFORM_NUM_DW_DMACS definition. This way generic API can access this information. Signed-off-by: Tomasz Lauda --- src/platform/baytrail/include/platform/drivers/dw-dma.h | 8 ++++++++ src/platform/haswell/include/platform/drivers/dw-dma.h | 3 +++ src/platform/intel/cavs/include/cavs/drivers/dw-dma.h | 8 ++++++++ 3 files changed, 19 insertions(+) diff --git a/src/platform/baytrail/include/platform/drivers/dw-dma.h b/src/platform/baytrail/include/platform/drivers/dw-dma.h index c18fe75c5e2f..cc635e78d8d9 100644 --- a/src/platform/baytrail/include/platform/drivers/dw-dma.h +++ b/src/platform/baytrail/include/platform/drivers/dw-dma.h @@ -11,6 +11,14 @@ #define __PLATFORM_DRIVERS_DW_DMA_H__ #include +#include + +/* number of supported DW-DMACs */ +#if CONFIG_CHERRYTRAIL_EXTRA_DW_DMA +#define PLATFORM_NUM_DW_DMACS 3 +#else +#define PLATFORM_NUM_DW_DMACS 2 +#endif /* CTL_HI */ #define DW_CTLH_CLASS(x) SET_BITS(31, 29, x) diff --git a/src/platform/haswell/include/platform/drivers/dw-dma.h b/src/platform/haswell/include/platform/drivers/dw-dma.h index 16e82f5d496f..f0110e828537 100644 --- a/src/platform/haswell/include/platform/drivers/dw-dma.h +++ b/src/platform/haswell/include/platform/drivers/dw-dma.h @@ -12,6 +12,9 @@ #include +/* number of supported DW-DMACs */ +#define PLATFORM_NUM_DW_DMACS 2 + /* CTL_HI */ #define DW_CTLH_DONE(x) SET_BIT(12, x) #define DW_CTLH_BLOCK_TS_MASK MASK(11, 0) diff --git a/src/platform/intel/cavs/include/cavs/drivers/dw-dma.h b/src/platform/intel/cavs/include/cavs/drivers/dw-dma.h index 75f0ded6b999..62c048a756ee 100644 --- a/src/platform/intel/cavs/include/cavs/drivers/dw-dma.h +++ b/src/platform/intel/cavs/include/cavs/drivers/dw-dma.h @@ -11,6 +11,14 @@ #define __CAVS_LIB_DW_DMA_H__ #include +#include + +/* number of supported DW-DMACs */ +#if CONFIG_SUECREEK +#define PLATFORM_NUM_DW_DMACS 3 +#else +#define PLATFORM_NUM_DW_DMACS 2 +#endif /* CTL_HI */ #define DW_CTLH_CLASS(x) SET_BITS(31, 29, x) From c7d4d9b3c0549bbedd94c181d0ff156d6bfd9633 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 16 Sep 2019 13:23:59 +0200 Subject: [PATCH 18/27] dma: extract dma array Extracts dma array to be accessible from the generic API. Signed-off-by: Tomasz Lauda --- src/include/sof/lib/dma.h | 2 ++ src/platform/intel/cavs/lib/dma.c | 4 ++-- 2 files changed, 4 insertions(+), 2 deletions(-) diff --git a/src/include/sof/lib/dma.h b/src/include/sof/lib/dma.h index 289e1b2fb56e..36cce8e0fb9b 100644 --- a/src/include/sof/lib/dma.h +++ b/src/include/sof/lib/dma.h @@ -225,6 +225,8 @@ struct dma_chan_data { void *private; }; +extern struct dma dma[]; + /** * \brief Plugs platform specific DMA array once initialized into the lib. * diff --git a/src/platform/intel/cavs/lib/dma.c b/src/platform/intel/cavs/lib/dma.c index 95363ebc7d13..82dbd7060d52 100644 --- a/src/platform/intel/cavs/lib/dma.c +++ b/src/platform/intel/cavs/lib/dma.c @@ -104,7 +104,7 @@ static struct dw_drv_plat_data dmac1 = { }; #if CONFIG_SUECREEK -static struct dma dma[PLATFORM_NUM_DMACS] = { +struct dma dma[PLATFORM_NUM_DMACS] = { { /* LP GP DMAC 0 */ .plat_data = { .id = DMA_GP_LP_DMAC0, @@ -153,7 +153,7 @@ static struct dma dma[PLATFORM_NUM_DMACS] = { }; #else -static struct dma dma[PLATFORM_NUM_DMACS] = { +struct dma dma[PLATFORM_NUM_DMACS] = { { /* Low Power GP DMAC 0 */ .plat_data = { .id = DMA_GP_LP_DMAC0, From f24e9a7e1d56538666ee2efc35d6722877574780 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 16 Sep 2019 13:38:41 +0200 Subject: [PATCH 19/27] dw-dma: rename DMA_AGGREGATED_IRQ definition Renames DMA_AGGREGATED_IRQ to DW_DMA_AGGREGATED_IRQ definition. This is more appropriate as it's not generic DMA setting. Signed-off-by: Tomasz Lauda --- Kconfig | 6 +++--- src/drivers/dw/dma.c | 36 ++++++++++++++++++------------------ src/platform/Kconfig | 16 ++++++++-------- 3 files changed, 29 insertions(+), 29 deletions(-) diff --git a/Kconfig b/Kconfig index 8cff91401937..0ec0e0dbff91 100644 --- a/Kconfig +++ b/Kconfig @@ -85,16 +85,16 @@ config HW_LLI Any platforms with hardware linked list support should set this. -config DMA_AGGREGATED_IRQ +config DW_DMA_AGGREGATED_IRQ bool default n help - Some platforms cannot register interrupt per DMA channel + Some platforms cannot register interrupt per DW-DMA channel and have the possibility only to register interrupts per DMA controller, which require manual handling of aggregated irq. - Any platforms with DMA aggregated interrupts support + Any platforms with DW-DMA aggregated interrupts support should set this. config DMA_SUSPEND_DRAIN diff --git a/src/drivers/dw/dma.c b/src/drivers/dw/dma.c index befdb881a21e..58e157a8d6f5 100644 --- a/src/drivers/dw/dma.c +++ b/src/drivers/dw/dma.c @@ -58,7 +58,7 @@ struct dw_dma_chan_data { uint32_t cfg_lo; uint32_t cfg_hi; bool irq_disabled; -#if !CONFIG_DMA_AGGREGATED_IRQ +#if !CONFIG_DW_DMA_AGGREGATED_IRQ int irq; #endif @@ -66,7 +66,7 @@ struct dw_dma_chan_data { struct dw_dma_ptr_data ptr_data; }; -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ /* private data for DW DMA engine */ struct dma_pdata { uint32_t mask_irq_channels[PLATFORM_CORE_COUNT]; @@ -134,7 +134,7 @@ static void dw_dma_interrupt_unmask(struct dma_chan_data *channel) static void dw_dma_interrupt_clear(struct dma_chan_data *channel) { -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ const struct dma_pdata *p = dma_get_drvdata(channel->dma); #endif const struct dw_dma_chan_data *chan = dma_chan_get_data(channel); @@ -154,7 +154,7 @@ static void dw_dma_interrupt_clear(struct dma_chan_data *channel) dma_reg_write(channel->dma, DW_CLEAR_ERR, DW_CHAN(channel->index)); /* clear platform interrupt */ -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ interrupt_clear_mask(p->irq, DW_CHAN(channel->index)); #else interrupt_clear_mask(chan->irq, DW_CHAN(channel->index)); @@ -1005,7 +1005,7 @@ static void dw_dma_irq_handler(void *data) { struct dma_chan_data *chan = data; struct dma *dma = chan->dma; -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ struct dma_pdata *p = dma_get_drvdata(dma); #endif uint32_t status_intr; @@ -1024,7 +1024,7 @@ static void dw_dma_irq_handler(void *data) /* get the source of our IRQ and clear it */ #if CONFIG_HW_LLI -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ /* skip if channel is not registered on this core */ mask = BIT(chan - dma->chan); #else @@ -1046,7 +1046,7 @@ static void dw_dma_irq_handler(void *data) dma_reg_write(dma, DW_CLEAR_ERR, status_err); } -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ /* clear platform and DSP interrupt */ interrupt_clear_mask(p->irq, status_src | status_err); #endif @@ -1069,7 +1069,7 @@ static inline int dw_dma_interrupt_register(struct dma_chan_data *channel) struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); uint32_t irq = dma_chan_irq(channel->dma, channel->index); int logical_irq = interrupt_get_irq(irq, dma_irq_name(channel->dma)); -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ struct dma_pdata *p = dma_get_drvdata(channel->dma); int cpu = cpu_get_id(); #endif @@ -1084,7 +1084,7 @@ static inline int dw_dma_interrupt_register(struct dma_chan_data *channel) if (logical_irq < 0) return logical_irq; -#if !CONFIG_DMA_AGGREGATED_IRQ +#if !CONFIG_DW_DMA_AGGREGATED_IRQ dw_chan->irq = logical_irq; #else p->irq = logical_irq; @@ -1103,7 +1103,7 @@ static inline int dw_dma_interrupt_register(struct dma_chan_data *channel) } interrupt_enable(logical_irq, channel); -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ } p->mask_irq_channels[cpu] |= BIT(channel->index); @@ -1116,7 +1116,7 @@ static inline void dw_dma_interrupt_unregister(struct dma_chan_data *channel) { struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); int logical_irq; -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ struct dma_pdata *p = dma_get_drvdata(channel->dma); int cpu = cpu_get_id(); #endif @@ -1128,7 +1128,7 @@ static inline void dw_dma_interrupt_unregister(struct dma_chan_data *channel) return; } -#if !CONFIG_DMA_AGGREGATED_IRQ +#if !CONFIG_DW_DMA_AGGREGATED_IRQ logical_irq = dw_chan->irq; #else logical_irq = p->irq; @@ -1139,7 +1139,7 @@ static inline void dw_dma_interrupt_unregister(struct dma_chan_data *channel) #endif interrupt_disable(logical_irq, channel); interrupt_unregister(logical_irq, channel); -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ } #endif } @@ -1194,7 +1194,7 @@ static int dw_dma_setup(struct dma *dma) static int dw_dma_probe(struct dma *dma) { -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ struct dma_pdata *dw_pdata; #endif struct dma_chan_data *chan; @@ -1208,7 +1208,7 @@ static int dw_dma_probe(struct dma *dma) /* disable dynamic clock gating */ pm_runtime_get_sync(DW_DMAC_CLK, dma->plat_data.id); -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ /* allocate private data */ dw_pdata = rzalloc(RZONE_SYS_RUNTIME | RZONE_FLAG_UNCACHED, SOF_MEM_CAPS_RAM, sizeof(*dw_pdata)); @@ -1272,7 +1272,7 @@ static int dw_dma_probe(struct dma *dma) dma->chan = NULL; } -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ rfree(dw_pdata); dma_set_drvdata(dma, NULL); #endif @@ -1282,7 +1282,7 @@ static int dw_dma_probe(struct dma *dma) static int dw_dma_remove(struct dma *dma) { -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ struct dma_pdata *p = dma_get_drvdata(dma); #endif int i; @@ -1296,7 +1296,7 @@ static int dw_dma_remove(struct dma *dma) rfree(dma->chan); dma->chan = NULL; -#if CONFIG_DMA_AGGREGATED_IRQ +#if CONFIG_DW_DMA_AGGREGATED_IRQ rfree(p); dma_set_drvdata(dma, NULL); #endif diff --git a/src/platform/Kconfig b/src/platform/Kconfig index 1d62e59b1cfc..69f2f1d1875c 100644 --- a/src/platform/Kconfig +++ b/src/platform/Kconfig @@ -9,7 +9,7 @@ choice config BAYTRAIL bool "Build for Baytrail" select HOST_PTABLE - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DMA_SUSPEND_DRAIN select DMA_FIFO_PARTITION select DW @@ -22,7 +22,7 @@ config BAYTRAIL config CHERRYTRAIL bool "Build for Cherrytrail" select HOST_PTABLE - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DMA_SUSPEND_DRAIN select DMA_FIFO_PARTITION select DW @@ -35,7 +35,7 @@ config CHERRYTRAIL config HASWELL bool "Build for Haswell" select HOST_PTABLE - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DW select DW_DMA select HAVE_RESET_VECTOR_ROM @@ -46,7 +46,7 @@ config HASWELL config BROADWELL bool "Build for Broadwell" select HOST_PTABLE - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DW select DW_DMA select HAVE_RESET_VECTOR_ROM @@ -78,7 +78,7 @@ config CANNONLAKE select DW_DMA select MEM_WND select HW_LLI - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DMA_FIFO_PARTITION select CAVS select CAVS_VERSION_1_8 @@ -96,7 +96,7 @@ config SUECREEK select INTEL_IOMUX select DW_GPIO select HW_LLI - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DMA_FIFO_PARTITION select CAVS select CAVS_VERSION_2_0 @@ -113,7 +113,7 @@ config ICELAKE select DW_DMA select MEM_WND select HW_LLI - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DMA_FIFO_PARTITION select CAVS select CAVS_VERSION_2_0 @@ -132,7 +132,7 @@ config TIGERLAKE select TASK_HAVE_PRIORITY_LOW select TASK_HAVE_PRIORITY_MEDIUM select HW_LLI - select DMA_AGGREGATED_IRQ + select DW_DMA_AGGREGATED_IRQ select DMA_FIFO_PARTITION select CAVS select CAVS_VERSION_2_5 From 6c7ea6be0f6ebe4ab42d65ed6137789c239ec5aa Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 16 Sep 2019 13:48:07 +0200 Subject: [PATCH 20/27] dw-dma: add PLATFORM_DW_DMA_INDEX definition Adds PLATFORM_DW_DMA_INDEX definition. This allows to access this value from the generic API. Signed-off-by: Tomasz Lauda --- src/platform/baytrail/include/platform/drivers/dw-dma.h | 3 +++ src/platform/haswell/include/platform/drivers/dw-dma.h | 3 +++ src/platform/intel/cavs/include/cavs/drivers/dw-dma.h | 3 +++ 3 files changed, 9 insertions(+) diff --git a/src/platform/baytrail/include/platform/drivers/dw-dma.h b/src/platform/baytrail/include/platform/drivers/dw-dma.h index cc635e78d8d9..4d9dcec17c7d 100644 --- a/src/platform/baytrail/include/platform/drivers/dw-dma.h +++ b/src/platform/baytrail/include/platform/drivers/dw-dma.h @@ -20,6 +20,9 @@ #define PLATFORM_NUM_DW_DMACS 2 #endif +/* index of the first DW-DMAC in the array */ +#define PLATFORM_DW_DMA_INDEX 0 + /* CTL_HI */ #define DW_CTLH_CLASS(x) SET_BITS(31, 29, x) #define DW_CTLH_WEIGHT(x) SET_BITS(28, 18, x) diff --git a/src/platform/haswell/include/platform/drivers/dw-dma.h b/src/platform/haswell/include/platform/drivers/dw-dma.h index f0110e828537..dbf32d713e89 100644 --- a/src/platform/haswell/include/platform/drivers/dw-dma.h +++ b/src/platform/haswell/include/platform/drivers/dw-dma.h @@ -15,6 +15,9 @@ /* number of supported DW-DMACs */ #define PLATFORM_NUM_DW_DMACS 2 +/* index of the first DW-DMAC in the array */ +#define PLATFORM_DW_DMA_INDEX 0 + /* CTL_HI */ #define DW_CTLH_DONE(x) SET_BIT(12, x) #define DW_CTLH_BLOCK_TS_MASK MASK(11, 0) diff --git a/src/platform/intel/cavs/include/cavs/drivers/dw-dma.h b/src/platform/intel/cavs/include/cavs/drivers/dw-dma.h index 62c048a756ee..4f2bcaa4e393 100644 --- a/src/platform/intel/cavs/include/cavs/drivers/dw-dma.h +++ b/src/platform/intel/cavs/include/cavs/drivers/dw-dma.h @@ -20,6 +20,9 @@ #define PLATFORM_NUM_DW_DMACS 2 #endif +/* index of the first DW-DMAC in the array */ +#define PLATFORM_DW_DMA_INDEX 0 + /* CTL_HI */ #define DW_CTLH_CLASS(x) SET_BITS(31, 29, x) #define DW_CTLH_WEIGHT(x) SET_BITS(28, 18, x) From 726e5762ba8ac8a572b797a98a5d30523df7f8a8 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Tue, 1 Oct 2019 09:39:45 +0200 Subject: [PATCH 21/27] platform: hsw: increase system heap size Signed-off-by: Tomasz Lauda --- src/platform/haswell/include/platform/lib/memory.h | 2 +- 1 file changed, 1 insertion(+), 1 deletion(-) diff --git a/src/platform/haswell/include/platform/lib/memory.h b/src/platform/haswell/include/platform/lib/memory.h index 325e23b8eb58..9b1c36f1b0b4 100644 --- a/src/platform/haswell/include/platform/lib/memory.h +++ b/src/platform/haswell/include/platform/lib/memory.h @@ -100,7 +100,7 @@ void platform_init_memmap(void); #define SOF_DATA_SIZE 0xC000 #define HEAP_SYSTEM_BASE (DRAM0_BASE + SOF_DATA_SIZE) -#define HEAP_SYSTEM_SIZE 0x3000 +#define HEAP_SYSTEM_SIZE 0x4000 #define HEAP_SYSTEM_0_BASE HEAP_SYSTEM_BASE From 93c56c79a80b022f80f1178b8391714adcea5377 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 16 Sep 2019 16:33:31 +0200 Subject: [PATCH 22/27] schedule: use DMA multi channel scheduling domain Registers low latency scheduler with DMA multi channel domain as scheduling source. This way we can finally get rid of scheduling code from DW-DMA driver and DAI component. Functionally it works the same as the previous implementation, but transfers scheduling layer to the right places. Signed-off-by: Tomasz Lauda --- src/arch/xtensa/init.c | 1 + src/audio/dai.c | 35 +++----------- src/audio/pipeline.c | 33 ++++++++----- src/drivers/dw/dma.c | 47 ++++++------------- src/drivers/intel/cavs/idc.c | 10 ++-- .../apollolake/include/platform/platform.h | 1 + .../baytrail/include/platform/platform.h | 1 + src/platform/baytrail/platform.c | 13 ++++- .../cannonlake/include/platform/platform.h | 1 + .../haswell/include/platform/platform.h | 1 + src/platform/haswell/platform.c | 12 ++++- .../icelake/include/platform/platform.h | 1 + src/platform/intel/cavs/platform.c | 13 ++++- .../suecreek/include/platform/platform.h | 1 + .../tigerlake/include/platform/platform.h | 1 + 15 files changed, 88 insertions(+), 83 deletions(-) diff --git a/src/arch/xtensa/init.c b/src/arch/xtensa/init.c index 3447cd2846ba..c1016f5a7148 100644 --- a/src/arch/xtensa/init.c +++ b/src/arch/xtensa/init.c @@ -132,6 +132,7 @@ int slave_core_init(struct sof *sof) trace_point(TRACE_BOOT_PLATFORM_SCHED); scheduler_init_edf(sof); scheduler_init_ll(platform_timer_domain); + scheduler_init_ll(platform_dma_domain); /* initialize IDC mechanism */ trace_point(TRACE_BOOT_PLATFORM_IDC); diff --git a/src/audio/dai.c b/src/audio/dai.c index 97ebd835c535..592c5f4e28b3 100644 --- a/src/audio/dai.c +++ b/src/audio/dai.c @@ -73,18 +73,17 @@ struct dai_data { uint64_t wallclock; /* wall clock at stream start */ }; -static void dai_buffer_process(struct comp_dev *dev, struct dma_cb_data *next) +/* this is called by DMA driver every time descriptor has completed */ +static void dai_dma_cb(void *data, uint32_t type, struct dma_cb_data *next) { + struct comp_dev *dev = (struct comp_dev *)data; struct dai_data *dd = comp_get_drvdata(dev); - struct dma_sg_config *config = &dd->config; uint32_t bytes = next->elem.size; void *buffer_ptr; - tracev_dai_with_ids(dev, "dai_buffer_process()"); + tracev_dai_with_ids(dev, "dai_dma_cb()"); - /* lli needs to be reloaded if irq is disabled */ - next->status = config->irq_disabled ? DMA_CB_STATUS_RELOAD : - DMA_CB_STATUS_IGNORE; + next->status = DMA_CB_STATUS_RELOAD; /* stop dma copy for pause/stop/xrun */ if (dev->state != COMP_STATE_ACTIVE || dd->xrun) { @@ -126,27 +125,6 @@ static void dai_buffer_process(struct comp_dev *dev, struct dma_cb_data *next) } } -/* this is called by DMA driver every time descriptor has completed */ -static void dai_dma_cb(void *data, uint32_t type, struct dma_cb_data *next) -{ - struct comp_dev *dev = (struct comp_dev *)data; - - tracev_dai_with_ids(dev, "dai_dma_cb()"); - - switch (type) { - case DMA_CB_TYPE_IRQ: - pipeline_schedule_copy(dev->pipeline, 0); - break; - case DMA_CB_TYPE_COPY: - dai_buffer_process(dev, next); - break; - default: - trace_dai_error_with_ids(dev, "dai_dma_cb() error: Wrong " - "callback type = %u", type); - break; - } -} - static struct comp_dev *dai_new(struct sof_ipc_comp *comp) { struct comp_dev *dev; @@ -822,8 +800,7 @@ static int dai_config(struct comp_dev *dev, struct sof_ipc_dai_config *config) } /* set up callback */ - dma_set_cb(dd->chan, DMA_CB_TYPE_IRQ | DMA_CB_TYPE_COPY, - dai_dma_cb, dev); + dma_set_cb(dd->chan, DMA_CB_TYPE_COPY, dai_dma_cb, dev); dev->is_dma_connected = 1; } diff --git a/src/audio/pipeline.c b/src/audio/pipeline.c index 0f57e111a583..062e2a176eff 100644 --- a/src/audio/pipeline.c +++ b/src/audio/pipeline.c @@ -39,6 +39,7 @@ struct pipeline_data { static enum task_state pipeline_task(void *arg); static enum task_state pipeline_preload_task(void *arg); +static void pipeline_schedule_preload(struct pipeline *p); /* create new pipeline - returns pipeline id or negative error */ struct pipeline *pipeline_new(struct sof_ipc_pipe_new *pipe_desc, @@ -428,8 +429,11 @@ int pipeline_prepare(struct pipeline *p, struct comp_dev *dev) /* initialize task if necessary */ if (!p->pipe_task) { + /* right now we always consider pipeline as a low latency + * component, but it may change in the future + */ type = pipeline_is_timer_driven(p) ? SOF_SCHEDULE_LL_TIMER : - SOF_SCHEDULE_EDF; + SOF_SCHEDULE_LL_DMA; p->pipe_task = pipeline_task_init(p, type, pipeline_task); if (!p->pipe_task) { @@ -530,15 +534,13 @@ static void pipeline_comp_trigger_sched_comp(struct pipeline *p, case COMP_TRIGGER_START: p->xrun_bytes = 0; - /* playback pipelines need to be scheduled now, - * capture pipelines are scheduled only for - * timer driven scheduling - */ - if (comp->params.direction == SOF_IPC_STREAM_PLAYBACK || - pipeline_is_timer_driven(p)) { - /* schedule initial pipeline fill */ + /* schedule preload if pipeline is not timer driven */ + if (p->preload && !pipeline_is_timer_driven(p)) + /* schedule pipeline preload */ + pipeline_schedule_preload(p); + else pipeline_schedule_copy(p, 0); - } + p->status = COMP_STATE_ACTIVE; break; case COMP_TRIGGER_SUSPEND: @@ -849,8 +851,10 @@ static int pipeline_copy(struct pipeline *p) start->comp.id, dir); /* stop preload only after full walkthrough */ - if (ret != PPL_STATUS_PATH_STOP) + if (ret != PPL_STATUS_PATH_STOP && p->preload) { p->preload = false; + pipeline_schedule_copy(p, 0); + } return ret; } @@ -999,6 +1003,11 @@ void pipeline_schedule_cancel(struct pipeline *p) schedule_task_cancel(p->pipe_task); } +static void pipeline_schedule_preload(struct pipeline *p) +{ + schedule_task(p->preload_task, 0, p->ipc_pipe.period); +} + static enum task_state pipeline_task(void *arg) { struct pipeline *p = arg; @@ -1030,9 +1039,7 @@ static enum task_state pipeline_task(void *arg) tracev_pipe("pipeline_task() sched"); - /* automatically reschedule for timer or not finished preload */ - return (pipeline_is_timer_driven(p) || p->preload) ? - SOF_TASK_STATE_RESCHEDULE : SOF_TASK_STATE_COMPLETED; + return SOF_TASK_STATE_RESCHEDULE; } static enum task_state pipeline_preload_task(void *arg) diff --git a/src/drivers/dw/dma.c b/src/drivers/dw/dma.c index 58e157a8d6f5..540ab3785b88 100644 --- a/src/drivers/dw/dma.c +++ b/src/drivers/dw/dma.c @@ -134,9 +134,6 @@ static void dw_dma_interrupt_unmask(struct dma_chan_data *channel) static void dw_dma_interrupt_clear(struct dma_chan_data *channel) { -#if CONFIG_DW_DMA_AGGREGATED_IRQ - const struct dma_pdata *p = dma_get_drvdata(channel->dma); -#endif const struct dw_dma_chan_data *chan = dma_chan_get_data(channel); if (chan->irq_disabled) { @@ -152,13 +149,6 @@ static void dw_dma_interrupt_clear(struct dma_chan_data *channel) dma_reg_write(channel->dma, DW_CLEAR_SRC_TRAN, DW_CHAN(channel->index)); dma_reg_write(channel->dma, DW_CLEAR_DST_TRAN, DW_CHAN(channel->index)); dma_reg_write(channel->dma, DW_CLEAR_ERR, DW_CHAN(channel->index)); - - /* clear platform interrupt */ -#if CONFIG_DW_DMA_AGGREGATED_IRQ - interrupt_clear_mask(p->irq, DW_CHAN(channel->index)); -#else - interrupt_clear_mask(chan->irq, DW_CHAN(channel->index)); -#endif } static uint32_t dw_dma_interrupt_status(struct dma_chan_data *channel) @@ -292,8 +282,6 @@ static int dw_dma_start(struct dma_chan_data *channel) goto out; } - dw_dma_interrupt_clear(channel); - #if CONFIG_HW_LLI /* LLP mode - write LLP pointer unless in scatter mode */ dma_reg_write(dma, DW_LLP(channel->index), lli->ctrl_lo & @@ -323,19 +311,12 @@ static int dw_dma_start(struct dma_chan_data *channel) } #endif - /* enable interrupt only for the first start */ - if (channel->status == COMP_STATE_PREPARE) - ret = dw_dma_interrupt_register(channel); - - if (!ret) { - /* assign core */ - channel->core = cpu_get_id(); + /* assign core */ + channel->core = cpu_get_id(); - /* enable the channel */ - channel->status = COMP_STATE_ACTIVE; - dma_reg_write(dma, DW_DMA_CHAN_EN, - DW_CHAN_UNMASK(channel->index)); - } + /* enable the channel */ + channel->status = COMP_STATE_ACTIVE; + dma_reg_write(dma, DW_DMA_CHAN_EN, DW_CHAN_UNMASK(channel->index)); out: irq_local_enable(flags); @@ -449,9 +430,6 @@ static int dw_dma_stop(struct dma_chan_data *channel) dma_reg_write(dma, DW_DMA_CHAN_EN, DW_CHAN_MASK(channel->index)); - /* disable interrupt */ - dw_dma_interrupt_unregister(channel); - #if CONFIG_HW_LLI /* clear block interrupt */ dma_reg_write(dma, DW_CLEAR_BLOCK, DW_CHAN(channel->index)); @@ -600,8 +578,6 @@ static int dw_dma_set_config(struct dma_chan_data *channel, } } - dw_dma_interrupt_unmask(channel); - dw_chan->ptr_data.buffer_bytes = 0; /* fill in lli for the elems in the list */ @@ -961,8 +937,7 @@ static void dw_dma_irq_callback(struct dma_chan_data *channel, if (channel->cb && channel->cb_type & type) channel->cb(channel->cb_data, type, next); - if (next->status != DMA_CB_STATUS_IGNORE) - dw_dma_verify_transfer(channel, next); + dw_dma_verify_transfer(channel, next); } static int dw_dma_copy(struct dma_chan_data *channel, int bytes, @@ -972,7 +947,7 @@ static int dw_dma_copy(struct dma_chan_data *channel, int bytes, int ret = 0; struct dma_cb_data next = { .elem = { .size = bytes }, - .status = DMA_CB_STATUS_RELOAD + .status = DMA_CB_STATUS_END }; /* for preload and one shot copy just start the DMA and wait */ @@ -1307,7 +1282,11 @@ static int dw_dma_avail_data_size(struct dma_chan_data *channel) { struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); int32_t read_ptr = dw_chan->ptr_data.current_ptr; +#if CONFIG_HW_LLI int32_t write_ptr = dma_reg_read(channel->dma, DW_DAR(channel->index)); +#else + int32_t write_ptr = ((struct dw_lli *)dw_chan->lli_current->llp)->dar; +#endif int size; size = write_ptr - read_ptr; @@ -1325,7 +1304,11 @@ static int dw_dma_avail_data_size(struct dma_chan_data *channel) static int dw_dma_free_data_size(struct dma_chan_data *channel) { struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); +#if CONFIG_HW_LLI int32_t read_ptr = dma_reg_read(channel->dma, DW_SAR(channel->index)); +#else + int32_t read_ptr = ((struct dw_lli *)dw_chan->lli_current->llp)->sar; +#endif int32_t write_ptr = dw_chan->ptr_data.current_ptr; int size; diff --git a/src/drivers/intel/cavs/idc.c b/src/drivers/intel/cavs/idc.c index 241f449a5742..a4c1407f2a16 100644 --- a/src/drivers/intel/cavs/idc.c +++ b/src/drivers/intel/cavs/idc.c @@ -192,9 +192,8 @@ static int idc_pipeline_trigger(uint32_t cmd) * Should be removed after changing memory management for * slave cores. */ - if (pcm_dev->cd->pipeline->pipe_task->type == - SOF_SCHEDULE_EDF) { - task = pcm_dev->cd->pipeline->pipe_task; + if (pcm_dev->cd->pipeline->preload_task) { + task = pcm_dev->cd->pipeline->preload_task; edf_pdata = edf_sch_get_pdata(task); dcache_invalidate_region(edf_pdata, sizeof(*edf_pdata)); task_context_cache(edf_pdata->ctx, CACHE_INVALIDATE); @@ -214,9 +213,8 @@ static int idc_pipeline_trigger(uint32_t cmd) * Should be removed after changing memory management for * slave cores. */ - if (pcm_dev->cd->pipeline->pipe_task->type == - SOF_SCHEDULE_EDF) { - task = pcm_dev->cd->pipeline->pipe_task; + if (pcm_dev->cd->pipeline->preload_task) { + task = pcm_dev->cd->pipeline->preload_task; edf_pdata = edf_sch_get_pdata(task); task_context_cache(edf_pdata->ctx, CACHE_WRITEBACK_INV); dcache_writeback_invalidate_region(edf_pdata, diff --git a/src/platform/apollolake/include/platform/platform.h b/src/platform/apollolake/include/platform/platform.h index dfbeef351b56..f0d0ae0e4fe7 100644 --- a/src/platform/apollolake/include/platform/platform.h +++ b/src/platform/apollolake/include/platform/platform.h @@ -143,6 +143,7 @@ void platform_wait_for_interrupt(int level); extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/platform/baytrail/include/platform/platform.h b/src/platform/baytrail/include/platform/platform.h index 16613f9ffa42..3ff76791b198 100644 --- a/src/platform/baytrail/include/platform/platform.h +++ b/src/platform/baytrail/include/platform/platform.h @@ -128,6 +128,7 @@ static inline void platform_wait_for_interrupt(int level) extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/platform/baytrail/platform.c b/src/platform/baytrail/platform.c index 17d9e50b3979..ee6ab0642e46 100644 --- a/src/platform/baytrail/platform.c +++ b/src/platform/baytrail/platform.c @@ -6,6 +6,7 @@ // Keyon Jie #include +#include #include #include #include @@ -33,6 +34,7 @@ #include #include #include +#include #include #include @@ -123,6 +125,7 @@ struct timer timer = { struct timer *platform_timer = &timer; struct ll_schedule_domain *platform_timer_domain; +struct ll_schedule_domain *platform_dma_domain; int platform_boot_complete(uint32_t boot_message) { @@ -189,12 +192,20 @@ int platform_init(struct sof *sof) trace_point(TRACE_BOOT_PLATFORM_SCHED); scheduler_init_edf(sof); - /* init low latency domains and schedulers */ + /* init low latency timer domain and scheduler */ platform_timer_domain = timer_domain_init(platform_timer, PLATFORM_DEFAULT_CLOCK, PLATFORM_LL_DEFAULT_TIMEOUT); scheduler_init_ll(platform_timer_domain); + /* init low latency multi channel DW-DMA domain and scheduler */ + platform_dma_domain = + dma_multi_chan_domain_init( + &dma[PLATFORM_DW_DMA_INDEX], + PLATFORM_NUM_DW_DMACS, + PLATFORM_DEFAULT_CLOCK, true); + scheduler_init_ll(platform_dma_domain); + /* init the system agent */ trace_point(TRACE_BOOT_PLATFORM_AGENT); sa_init(sof); diff --git a/src/platform/cannonlake/include/platform/platform.h b/src/platform/cannonlake/include/platform/platform.h index 318e03af7363..eb0dce32a4ac 100644 --- a/src/platform/cannonlake/include/platform/platform.h +++ b/src/platform/cannonlake/include/platform/platform.h @@ -140,6 +140,7 @@ void platform_wait_for_interrupt(int level); extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/platform/haswell/include/platform/platform.h b/src/platform/haswell/include/platform/platform.h index e68c56b74ef5..e8cfc4c36225 100644 --- a/src/platform/haswell/include/platform/platform.h +++ b/src/platform/haswell/include/platform/platform.h @@ -117,6 +117,7 @@ static inline void platform_wait_for_interrupt(int level) extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/platform/haswell/platform.c b/src/platform/haswell/platform.c index 46cc6ed67b7d..0aeae47284f0 100644 --- a/src/platform/haswell/platform.c +++ b/src/platform/haswell/platform.c @@ -5,6 +5,7 @@ // Author: Liam Girdwood #include +#include #include #include #include @@ -120,6 +121,7 @@ struct timer timer = { struct timer *platform_timer = &timer; struct ll_schedule_domain *platform_timer_domain; +struct ll_schedule_domain *platform_dma_domain; int platform_boot_complete(uint32_t boot_message) { @@ -178,12 +180,20 @@ int platform_init(struct sof *sof) trace_point(TRACE_BOOT_PLATFORM_SCHED); scheduler_init_edf(sof); - /* init low latency domains and schedulers */ + /* init low latency timer domain and scheduler */ platform_timer_domain = timer_domain_init(platform_timer, PLATFORM_DEFAULT_CLOCK, PLATFORM_LL_DEFAULT_TIMEOUT); scheduler_init_ll(platform_timer_domain); + /* init low latency multi channel DW-DMA domain and scheduler */ + platform_dma_domain = + dma_multi_chan_domain_init( + &dma[PLATFORM_DW_DMA_INDEX], + PLATFORM_NUM_DW_DMACS, + PLATFORM_DEFAULT_CLOCK, true); + scheduler_init_ll(platform_dma_domain); + /* init the system agent */ trace_point(TRACE_BOOT_PLATFORM_AGENT); sa_init(sof); diff --git a/src/platform/icelake/include/platform/platform.h b/src/platform/icelake/include/platform/platform.h index f81681950118..fb2d51b80207 100644 --- a/src/platform/icelake/include/platform/platform.h +++ b/src/platform/icelake/include/platform/platform.h @@ -140,6 +140,7 @@ void platform_wait_for_interrupt(int level); extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/platform/intel/cavs/platform.c b/src/platform/intel/cavs/platform.c index 34cb3943b658..d4b6a76f55b9 100644 --- a/src/platform/intel/cavs/platform.c +++ b/src/platform/intel/cavs/platform.c @@ -10,6 +10,7 @@ #include #include #include +#include #include #include #include @@ -244,6 +245,7 @@ struct timer timer = { struct timer *platform_timer = &timer; struct ll_schedule_domain *platform_timer_domain; +struct ll_schedule_domain *platform_dma_domain; #if CONFIG_DW_SPI @@ -378,12 +380,21 @@ int platform_init(struct sof *sof) trace_point(TRACE_BOOT_PLATFORM_SCHED); scheduler_init_edf(sof); - /* init low latency domains and schedulers */ + /* init low latency timer domain and scheduler */ platform_timer_domain = timer_domain_init(platform_timer, PLATFORM_DEFAULT_CLOCK, PLATFORM_LL_DEFAULT_TIMEOUT); scheduler_init_ll(platform_timer_domain); + /* init low latency multi channel DW-DMA domain and scheduler */ + platform_dma_domain = + dma_multi_chan_domain_init( + &dma[PLATFORM_DW_DMA_INDEX], + PLATFORM_NUM_DW_DMACS, + PLATFORM_DEFAULT_CLOCK, + IS_ENABLED(CONFIG_DW_DMA_AGGREGATED_IRQ)); + scheduler_init_ll(platform_dma_domain); + /* init the system agent */ trace_point(TRACE_BOOT_PLATFORM_AGENT); sa_init(sof); diff --git a/src/platform/suecreek/include/platform/platform.h b/src/platform/suecreek/include/platform/platform.h index ab59378da92c..61a03ea7562c 100644 --- a/src/platform/suecreek/include/platform/platform.h +++ b/src/platform/suecreek/include/platform/platform.h @@ -150,6 +150,7 @@ void platform_wait_for_interrupt(int level); extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; diff --git a/src/platform/tigerlake/include/platform/platform.h b/src/platform/tigerlake/include/platform/platform.h index 9f483b446b8f..bf1c0783febd 100644 --- a/src/platform/tigerlake/include/platform/platform.h +++ b/src/platform/tigerlake/include/platform/platform.h @@ -140,6 +140,7 @@ void platform_wait_for_interrupt(int level); extern struct timer *platform_timer; extern struct ll_schedule_domain *platform_timer_domain; +extern struct ll_schedule_domain *platform_dma_domain; extern intptr_t _module_init_start; extern intptr_t _module_init_end; From 579715634a250c3ac0354517d5780722bb35a19e Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 16 Sep 2019 16:43:33 +0200 Subject: [PATCH 23/27] dw-dma: remove double channel index assignment Removes double channel index assignment in dw_dma_probe. Signed-off-by: Tomasz Lauda --- src/drivers/dw/dma.c | 1 - 1 file changed, 1 deletion(-) diff --git a/src/drivers/dw/dma.c b/src/drivers/dw/dma.c index 540ab3785b88..e0eae8a43f5e 100644 --- a/src/drivers/dw/dma.c +++ b/src/drivers/dw/dma.c @@ -1216,7 +1216,6 @@ static int dw_dma_probe(struct dma *dma) for (i = 0, chan = dma->chan; i < dma->plat_data.channels; i++, chan++) { chan->status = COMP_STATE_INIT; - chan->index = i; chan->dma = dma; chan->index = i; chan->core = DMA_CORE_INVALID; From 3c3f1066b9c479b3ea18297ae0df251073acab25 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 16 Sep 2019 16:55:05 +0200 Subject: [PATCH 24/27] dw-dma: cleanup and remove irq related code Cleanups DW-DMA driver and removes any code related to interrupt handling. Signed-off-by: Tomasz Lauda --- src/drivers/dw/dma.c | 244 +------------------------------------------ 1 file changed, 5 insertions(+), 239 deletions(-) diff --git a/src/drivers/dw/dma.c b/src/drivers/dw/dma.c index e0eae8a43f5e..af8c1375be3f 100644 --- a/src/drivers/dw/dma.c +++ b/src/drivers/dw/dma.c @@ -25,7 +25,6 @@ #include #include #include -#include #include #include #include @@ -57,51 +56,17 @@ struct dw_dma_chan_data { struct dw_lli *lli_current; uint32_t cfg_lo; uint32_t cfg_hi; - bool irq_disabled; -#if !CONFIG_DW_DMA_AGGREGATED_IRQ - int irq; -#endif - - /* pointer data */ - struct dw_dma_ptr_data ptr_data; + struct dw_dma_ptr_data ptr_data; /* pointer data */ }; -#if CONFIG_DW_DMA_AGGREGATED_IRQ -/* private data for DW DMA engine */ -struct dma_pdata { - uint32_t mask_irq_channels[PLATFORM_CORE_COUNT]; - int irq; -}; -#endif - /* use array to get burst_elems for specific slot number setting. * the relation between msize and burst_elems should be * 2 ^ msize = burst_elems */ static const uint32_t burst_elems[] = {1, 2, 4, 8}; -#if !CONFIG_HW_LLI -static inline void dw_dma_chan_reload_lli(struct dma_chan_data *channel); -static inline void dw_dma_chan_reload_next(struct dma_chan_data *channel, - struct dma_sg_elem *next, - int direction); -#endif -static inline int dw_dma_interrupt_register(struct dma_chan_data *channel); -static inline void dw_dma_interrupt_unregister(struct dma_chan_data *channel); -static int dw_dma_copy(struct dma_chan_data *channel, int bytes, - uint32_t flags); - static void dw_dma_interrupt_mask(struct dma_chan_data *channel) { - struct dw_dma_chan_data *dw_chan_data = dma_chan_get_data(channel); - - if (dw_chan_data->irq_disabled) { - tracev_dwdma("dw_dma_interrupt_mask(): dma %d channel %d " - "not working in irq mode", - channel->dma->plat_data.id, channel->index); - return; - } - /* mask block, transfer and error interrupts for channel */ dma_reg_write(channel->dma, DW_MASK_TFR, DW_CHAN_MASK(channel->index)); dma_reg_write(channel->dma, DW_MASK_BLOCK, @@ -111,15 +76,6 @@ static void dw_dma_interrupt_mask(struct dma_chan_data *channel) static void dw_dma_interrupt_unmask(struct dma_chan_data *channel) { - struct dw_dma_chan_data *dw_chan_data = dma_chan_get_data(channel); - - if (dw_chan_data->irq_disabled) { - tracev_dwdma("dw_dma_interrupt_mask(): dma %d channel %d " - "not working in irq mode", - channel->dma->plat_data.id, channel->index); - return; - } - /* unmask block, transfer and error interrupts for channel */ #if CONFIG_HW_LLI dma_reg_write(channel->dma, DW_MASK_BLOCK, @@ -134,15 +90,6 @@ static void dw_dma_interrupt_unmask(struct dma_chan_data *channel) static void dw_dma_interrupt_clear(struct dma_chan_data *channel) { - const struct dw_dma_chan_data *chan = dma_chan_get_data(channel); - - if (chan->irq_disabled) { - tracev_dwdma("dw_dma_interrupt_mask(): dma %d channel %d " - "not working in irq mode", - channel->dma->plat_data.id, channel->index); - return; - } - /* clear transfer, block, src, dst and error interrupts for channel */ dma_reg_write(channel->dma, DW_CLEAR_TFR, DW_CHAN(channel->index)); dma_reg_write(channel->dma, DW_CLEAR_BLOCK, DW_CHAN(channel->index)); @@ -431,9 +378,6 @@ static int dw_dma_stop(struct dma_chan_data *channel) dma_reg_write(dma, DW_DMA_CHAN_EN, DW_CHAN_MASK(channel->index)); #if CONFIG_HW_LLI - /* clear block interrupt */ - dma_reg_write(dma, DW_CLEAR_BLOCK, DW_CHAN(channel->index)); - for (i = 0; i < channel->desc_count; i++) { lli->ctrl_hi &= ~DW_CTLH_DONE(1); lli++; @@ -518,7 +462,6 @@ static int dw_dma_set_config(struct dma_chan_data *channel, /* default channel config */ channel->direction = config->direction; channel->is_scheduling_source = config->is_scheduling_source; - dw_chan->irq_disabled = config->irq_disabled; dw_chan->cfg_lo = DW_CFG_LOW_DEF; dw_chan->cfg_hi = DW_CFG_HIGH_DEF; @@ -931,15 +874,6 @@ static void dw_dma_verify_transfer(struct dma_chan_data *channel, #endif } -static void dw_dma_irq_callback(struct dma_chan_data *channel, - struct dma_cb_data *next, uint32_t type) -{ - if (channel->cb && channel->cb_type & type) - channel->cb(channel->cb_data, type, next); - - dw_dma_verify_transfer(channel, next); -} - static int dw_dma_copy(struct dma_chan_data *channel, int bytes, uint32_t flags) { @@ -967,7 +901,10 @@ static int dw_dma_copy(struct dma_chan_data *channel, int bytes, tracev_dwdma("dw_dma_copy(): dma %d channel %d copy", channel->dma->plat_data.id, channel->index); - dw_dma_irq_callback(channel, &next, DMA_CB_TYPE_COPY); + if (channel->cb && channel->cb_type & DMA_CB_TYPE_COPY) + channel->cb(channel->cb_data, DMA_CB_TYPE_COPY, &next); + + dw_dma_verify_transfer(channel, &next); /* increment current pointer */ dw_dma_increment_pointer(dw_chan, bytes); @@ -975,150 +912,6 @@ static int dw_dma_copy(struct dma_chan_data *channel, int bytes, return ret; } -/* interrupt handler for DMA */ -static void dw_dma_irq_handler(void *data) -{ - struct dma_chan_data *chan = data; - struct dma *dma = chan->dma; -#if CONFIG_DW_DMA_AGGREGATED_IRQ - struct dma_pdata *p = dma_get_drvdata(dma); -#endif - uint32_t status_intr; - uint32_t status_err; - uint32_t status_src; - uint32_t mask; - int i; - struct dma_cb_data next = { .status = DMA_CB_STATUS_RELOAD }; - - status_intr = dma_reg_read(dma, DW_INTR_STATUS); - if (!status_intr) - return; - - tracev_dwdma("dw_dma_irq_handler(): dma %d IRQ status 0x%x", - dma->plat_data.id, status_intr); - - /* get the source of our IRQ and clear it */ -#if CONFIG_HW_LLI -#if CONFIG_DW_DMA_AGGREGATED_IRQ - /* skip if channel is not registered on this core */ - mask = BIT(chan - dma->chan); -#else - mask = ~0; -#endif - - status_src = dma_reg_read(dma, DW_STATUS_BLOCK) & mask; - dma_reg_write(dma, DW_CLEAR_BLOCK, status_src); -#else - status_src = dma_reg_read(dma, DW_STATUS_TFR); - dma_reg_write(dma, DW_CLEAR_TFR, status_src); -#endif - - /* TODO: handle errors, just clear them atm */ - status_err = dma_reg_read(dma, DW_STATUS_ERR); - if (status_err) { - trace_dwdma_error("dw_dma_irq_handler() error: dma %d status " - "error 0x%x", dma->plat_data.id, status_err); - dma_reg_write(dma, DW_CLEAR_ERR, status_err); - } - -#if CONFIG_DW_DMA_AGGREGATED_IRQ - /* clear platform and DSP interrupt */ - interrupt_clear_mask(p->irq, status_src | status_err); -#endif - - for (i = 0; i < dma->plat_data.channels; i++) { - /* skip if channel is not running */ - if (dma->chan[i].status != COMP_STATE_ACTIVE) - continue; - - mask = 0x1 << i; - - if (status_src & mask) - dw_dma_irq_callback(&dma->chan[i], &next, - DMA_CB_TYPE_IRQ); - } -} - -static inline int dw_dma_interrupt_register(struct dma_chan_data *channel) -{ - struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); - uint32_t irq = dma_chan_irq(channel->dma, channel->index); - int logical_irq = interrupt_get_irq(irq, dma_irq_name(channel->dma)); -#if CONFIG_DW_DMA_AGGREGATED_IRQ - struct dma_pdata *p = dma_get_drvdata(channel->dma); - int cpu = cpu_get_id(); -#endif - int ret; - - if (dw_chan->irq_disabled) { - tracev_dwdma("dw_dma_interrupt_register(): dma %d channel %d " - "not working in irq mode", - channel->dma->plat_data.id, channel->index); - return 0; - } - - if (logical_irq < 0) - return logical_irq; -#if !CONFIG_DW_DMA_AGGREGATED_IRQ - dw_chan->irq = logical_irq; -#else - p->irq = logical_irq; - - if (!p->mask_irq_channels[cpu]) { -#endif - ret = interrupt_register(logical_irq, IRQ_AUTO_UNMASK, - dw_dma_irq_handler, channel); - if (ret < 0) { - trace_dwdma_error("dw_dma_interrupt_register() error: " - "dma %d channel %d failed to " - "allocate IRQ", - channel->dma->plat_data.id, - channel->index); - return ret; - } - - interrupt_enable(logical_irq, channel); -#if CONFIG_DW_DMA_AGGREGATED_IRQ - } - - p->mask_irq_channels[cpu] |= BIT(channel->index); -#endif - - return 0; -} - -static inline void dw_dma_interrupt_unregister(struct dma_chan_data *channel) -{ - struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); - int logical_irq; -#if CONFIG_DW_DMA_AGGREGATED_IRQ - struct dma_pdata *p = dma_get_drvdata(channel->dma); - int cpu = cpu_get_id(); -#endif - - if (dw_chan->irq_disabled) { - tracev_dwdma("dw_dma_interrupt_unregister(): dma %d channel %d" - " not working in irq mode", - channel->dma->plat_data.id, channel->index); - return; - } - -#if !CONFIG_DW_DMA_AGGREGATED_IRQ - logical_irq = dw_chan->irq; -#else - logical_irq = p->irq; - - p->mask_irq_channels[cpu] &= ~BIT(channel->index); - - if (!p->mask_irq_channels[cpu]) { -#endif - interrupt_disable(logical_irq, channel); - interrupt_unregister(logical_irq, channel); -#if CONFIG_DW_DMA_AGGREGATED_IRQ - } -#endif -} - static int dw_dma_setup(struct dma *dma) { int i; @@ -1169,9 +962,6 @@ static int dw_dma_setup(struct dma *dma) static int dw_dma_probe(struct dma *dma) { -#if CONFIG_DW_DMA_AGGREGATED_IRQ - struct dma_pdata *dw_pdata; -#endif struct dma_chan_data *chan; struct dw_dma_chan_data *dw_chan; int ret; @@ -1183,18 +973,6 @@ static int dw_dma_probe(struct dma *dma) /* disable dynamic clock gating */ pm_runtime_get_sync(DW_DMAC_CLK, dma->plat_data.id); -#if CONFIG_DW_DMA_AGGREGATED_IRQ - /* allocate private data */ - dw_pdata = rzalloc(RZONE_SYS_RUNTIME | RZONE_FLAG_UNCACHED, - SOF_MEM_CAPS_RAM, sizeof(*dw_pdata)); - if (!dw_pdata) { - trace_dwdma_error("dw_dma_probe() error: dma %d alloc failed", - dma->plat_data.id); - return -ENOMEM; - } - dma_set_drvdata(dma, dw_pdata); -#endif - /* allocate dma channels */ dma->chan = rzalloc(RZONE_SYS_RUNTIME | RZONE_FLAG_UNCACHED, SOF_MEM_CAPS_RAM, sizeof(struct dma_chan_data) * @@ -1246,19 +1024,11 @@ static int dw_dma_probe(struct dma *dma) dma->chan = NULL; } -#if CONFIG_DW_DMA_AGGREGATED_IRQ - rfree(dw_pdata); - dma_set_drvdata(dma, NULL); -#endif - return -ENOMEM; } static int dw_dma_remove(struct dma *dma) { -#if CONFIG_DW_DMA_AGGREGATED_IRQ - struct dma_pdata *p = dma_get_drvdata(dma); -#endif int i; tracev_dwdma("dw_dma_remove(): dma %d remove", dma->plat_data.id); @@ -1270,10 +1040,6 @@ static int dw_dma_remove(struct dma *dma) rfree(dma->chan); dma->chan = NULL; -#if CONFIG_DW_DMA_AGGREGATED_IRQ - rfree(p); - dma_set_drvdata(dma, NULL); -#endif return 0; } From 98f4007be3d8893c4e755dbc5fd094d11d07211f Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 16 Sep 2019 17:03:19 +0200 Subject: [PATCH 25/27] dma: remove unused DMA_CB_STATUS_IGNORE Removes DMA_CB_STATUS_IGNORE, which is no longer used. Signed-off-by: Tomasz Lauda --- src/include/sof/lib/dma.h | 1 - 1 file changed, 1 deletion(-) diff --git a/src/include/sof/lib/dma.h b/src/include/sof/lib/dma.h index 36cce8e0fb9b..710326a9e5c8 100644 --- a/src/include/sof/lib/dma.h +++ b/src/include/sof/lib/dma.h @@ -74,7 +74,6 @@ enum dma_cb_status { DMA_CB_STATUS_RELOAD = 0, DMA_CB_STATUS_SPLIT, - DMA_CB_STATUS_IGNORE, DMA_CB_STATUS_END, }; From 4062fc05c45793dc67f67782c658aac48a5cb56b Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 16 Sep 2019 17:06:44 +0200 Subject: [PATCH 26/27] dma: remove unused DMA_CB_STATUS_SPLIT Removes unused DMA_CB_STATUS_SPLIT and related code. Signed-off-by: Tomasz Lauda --- src/drivers/dw/dma.c | 44 --------------------------------------- src/include/sof/lib/dma.h | 1 - 2 files changed, 45 deletions(-) diff --git a/src/drivers/dw/dma.c b/src/drivers/dw/dma.c index af8c1375be3f..2d085ac1fe9a 100644 --- a/src/drivers/dw/dma.c +++ b/src/drivers/dw/dma.c @@ -788,45 +788,6 @@ static inline void dw_dma_chan_reload_lli(struct dma_chan_data *channel) /* enable the channel */ dma_reg_write(dma, DW_DMA_CHAN_EN, DW_CHAN_UNMASK(channel->index)); } - -/* reload using callback data */ -static inline void dw_dma_chan_reload_next(struct dma_chan_data *channel, - struct dma_sg_elem *next, - int direction) -{ - struct dma *dma = channel->dma; - struct dw_drv_plat_data *dp = dma->plat_data.drv_plat_data; - struct dw_dma_chan_data *dw_chan = dma_chan_get_data(channel); - uint16_t class = dp->chan[channel->index].class; - struct dw_lli *lli = dw_chan->lli_current; - uint32_t sar; - uint32_t dar; - - dw_dma_mask_address(next, &sar, &dar, direction); - - /* channel needs to start from scratch, so write SAR and DAR */ - dma_reg_write(dma, DW_SAR(channel->index), sar); - dma_reg_write(dma, DW_DAR(channel->index), dar); - - lli->ctrl_hi = 0; - - /* set channel class */ - platform_dw_dma_set_class(dw_chan, lli, class); - - /* set transfer size of element */ - platform_dw_dma_set_transfer_size(dw_chan, lli, next->size); - - /* program CTL_LO and CTL_HI */ - dma_reg_write(dma, DW_CTRL_LOW(channel->index), lli->ctrl_lo); - dma_reg_write(dma, DW_CTRL_HIGH(channel->index), lli->ctrl_hi); - - /* program CFG_LO and CFG_HI */ - dma_reg_write(dma, DW_CFG_LOW(channel->index), dw_chan->cfg_lo); - dma_reg_write(dma, DW_CFG_HIGH(channel->index), dw_chan->cfg_hi); - - /* enable the channel */ - dma_reg_write(dma, DW_DMA_CHAN_EN, DW_CHAN_UNMASK(channel->index)); -} #endif static void dw_dma_verify_transfer(struct dma_chan_data *channel, @@ -854,7 +815,6 @@ static void dw_dma_verify_transfer(struct dma_chan_data *channel, #else /* check for reload channel: * next->status is DMA_CB_STATUS_END, stop this dma copy; - * next->status is DMA_CB_STATUS_SPLIT, use next element for next copy; * otherwise, reload lli */ switch (next->status) { @@ -863,10 +823,6 @@ static void dw_dma_verify_transfer(struct dma_chan_data *channel, dw_chan->lli_current = (struct dw_lli *)dw_chan->lli_current->llp; break; - case DMA_CB_STATUS_SPLIT: - dw_dma_chan_reload_next(channel, &next->elem, - channel->direction); - break; default: dw_dma_chan_reload_lli(channel); break; diff --git a/src/include/sof/lib/dma.h b/src/include/sof/lib/dma.h index 710326a9e5c8..c36eb257d17b 100644 --- a/src/include/sof/lib/dma.h +++ b/src/include/sof/lib/dma.h @@ -73,7 +73,6 @@ */ enum dma_cb_status { DMA_CB_STATUS_RELOAD = 0, - DMA_CB_STATUS_SPLIT, DMA_CB_STATUS_END, }; From ebe47492ec3c0ef5d0949fe0c3924876aa4f2bb5 Mon Sep 17 00:00:00 2001 From: Tomasz Lauda Date: Mon, 16 Sep 2019 17:29:48 +0200 Subject: [PATCH 27/27] host: remove not needed set of DMA_CB_STATUS_END Next has this status set by default, so there is no need to set it explicitly. Signed-off-by: Tomasz Lauda --- src/audio/host.c | 2 -- 1 file changed, 2 deletions(-) diff --git a/src/audio/host.c b/src/audio/host.c index 06f6fc4801b7..0e2e46c0a3a4 100644 --- a/src/audio/host.c +++ b/src/audio/host.c @@ -224,8 +224,6 @@ static void host_dma_cb(void *data, uint32_t type, struct dma_cb_data *next) /* callback for one shot copy */ if (hd->copy_type == COMP_COPY_ONE_SHOT) host_one_shot_cb(dev, bytes); - - next->status = DMA_CB_STATUS_END; } static int create_local_elems(struct comp_dev *dev, uint32_t buffer_count,