[RFC] Implement APIs for coherent access of shared objects

src/arch/host/include/arch/lib/cache.h

@@ -18,6 +18,8 @@ static inline void icache_invalidate_region(void *addr, size_t size) {}
 static inline void dcache_writeback_invalidate_region(void *addr,
 	size_t size) {}
 
+#define DCACHE_LINE_SIZE 64
+
 #endif /* __ARCH_LIB_CACHE_H__ */
 
 #else

src/arch/xtensa/Kconfig

@@ -17,3 +17,12 @@ config WAKEUP_HOOK
 	  This config should be selected by other platform-level configs.
 	  Platforms that use it, have to implement hook function
 	  platform_interrupt_on_wakeup.
+
+config INCOHERENT
+	bool
+	default n
+	help
+	  The architecture is cache incoherent. i.e FW has to manually manage
+	  cache coherency amongst objects that are used on multiple cores.
+	  This setting should only be disabled for cache incoherent
+	  architectures if they are being used in UP mode.

src/arch/xtensa/configs/tgph_defconfig

@@ -9,3 +9,4 @@ CONFIG_HP_MEMORY_BANKS=30
 CONFIG_L1_DRAM=y
 CONFIG_L1_DRAM_MEMORY_BANKS=8
 CONFIG_L1_DRAM_MEMORY_BANK_SIZE=8192
+CONFIG_INCOHERENT=y

src/arch/xtensa/configs/tgplp_defconfig

@@ -4,3 +4,4 @@ CONFIG_INTEL_SSP=y
 CONFIG_INTEL_ALH=y
 CONFIG_LP_MEMORY_BANKS=1
 CONFIG_HP_MEMORY_BANKS=46
+CONFIG_INCOHERENT=y

src/audio/asrc/asrc.c

@@ -806,11 +806,10 @@ static void asrc_process(struct comp_dev *dev, struct comp_buffer *source,
 	int produced = 0;
 
 	/* consumed bytes are not known at this point */
-	buffer_invalidate(source, source->stream.size);
+	buffer_stream_invalidate(source, source->stream.size);
 	cd->asrc_func(dev, &source->stream, &sink->stream, &consumed,
 		      &produced);
-	buffer_writeback(sink, produced *
-			 audio_stream_frame_bytes(&sink->stream));
+	buffer_stream_writeback(sink, produced * audio_stream_frame_bytes(&sink->stream));
 
 	comp_dbg(dev, "asrc_copy(), consumed = %u,  produced = %u",
 		 consumed, produced);
@@ -830,7 +829,6 @@ static int asrc_copy(struct comp_dev *dev)
 	int frames_src;
 	int frames_snk;
 	int ret;
-	uint32_t flags = 0;
 
 	comp_dbg(dev, "asrc_copy()");
 
@@ -844,14 +842,14 @@ static int asrc_copy(struct comp_dev *dev)
 	sink = list_first_item(&dev->bsink_list, struct comp_buffer,
 			       source_list);
 
-	buffer_lock(source, &flags);
-	buffer_lock(sink, &flags);
+	source = buffer_acquire_irq(source);
+	sink = buffer_acquire_irq(sink);
 
 	frames_src = audio_stream_get_avail_frames(&source->stream);
 	frames_snk = audio_stream_get_free_frames(&sink->stream);
 
-	buffer_unlock(sink, flags);
-	buffer_unlock(source, flags);
+	buffer_release_irq(sink);
+	buffer_release_irq(source);
 
 	if (cd->mode == ASRC_OM_PULL) {
 		/* Let ASRC access max number of source frames in pull mode.

src/audio/buffer.c

@@ -45,28 +45,18 @@ struct comp_buffer *buffer_alloc(uint32_t size, uint32_t caps, uint32_t align)
 		return NULL;
 	}
 
-	buffer->lock = rzalloc(SOF_MEM_ZONE_RUNTIME_SHARED, 0, SOF_MEM_CAPS_RAM,
-			       sizeof(*buffer->lock));
-	if (!buffer->lock) {
-		rfree(buffer);
-		tr_err(&buffer_tr, "buffer_alloc(): could not alloc lock");
-		return NULL;
-	}
-
 	buffer->stream.addr = rballoc_align(0, caps, size, align);
 	if (!buffer->stream.addr) {
-		rfree(buffer->lock);
 		rfree(buffer);
 		tr_err(&buffer_tr, "buffer_alloc(): could not alloc size = %u bytes of type = %u",
 		       size, caps);
 		return NULL;
 	}
 
 	buffer_init(buffer, size, caps);
-
+	coherent_init(buffer, c);
 	list_init(&buffer->source_list);
 	list_init(&buffer->sink_list);
-	spinlock_init(buffer->lock);
 
 	return buffer;
 }
@@ -145,7 +135,7 @@ int buffer_set_params(struct comp_buffer *buffer, struct sof_ipc_stream_params *
 bool buffer_params_match(struct comp_buffer *buffer, struct sof_ipc_stream_params *params,
 			 uint32_t flag)
 {
-	assert(params && buffer);
+	assert(params);
 
 	if ((flag & BUFF_PARAMS_FRAME_FMT) &&
 	    buffer->stream.frame_fmt != params->frame_fmt)
@@ -177,14 +167,13 @@ void buffer_free(struct comp_buffer *buffer)
 	/* In case some listeners didn't unregister from buffer's callbacks */
 	notifier_unregister_all(NULL, buffer);
 
+	coherent_free(buffer, c);
 	rfree(buffer->stream.addr);
-	rfree(buffer->lock);
 	rfree(buffer);
 }
 
 void comp_update_buffer_produce(struct comp_buffer *buffer, uint32_t bytes)
 {
-	uint32_t flags = 0;
 	struct buffer_cb_transact cb_data = {
 		.buffer = buffer,
 		.transaction_amount = bytes,
@@ -202,15 +191,13 @@ void comp_update_buffer_produce(struct comp_buffer *buffer, uint32_t bytes)
 		return;
 	}
 
-	buffer_lock(buffer, &flags);
+	buffer = buffer_acquire_irq(buffer);
 
 	audio_stream_produce(&buffer->stream, bytes);
 
 	notifier_event(buffer, NOTIFIER_ID_BUFFER_PRODUCE,
 		       NOTIFIER_TARGET_CORE_LOCAL, &cb_data, sizeof(cb_data));
 
-	buffer_unlock(buffer, flags);
-
 	addr = buffer->stream.addr;
 
 	buf_dbg(buffer, "comp_update_buffer_produce(), ((buffer->avail << 16) | buffer->free) = %08x, ((buffer->id << 16) | buffer->size) = %08x",
@@ -220,11 +207,12 @@ void comp_update_buffer_produce(struct comp_buffer *buffer, uint32_t bytes)
 	buf_dbg(buffer, "comp_update_buffer_produce(), ((buffer->r_ptr - buffer->addr) << 16 | (buffer->w_ptr - buffer->addr)) = %08x",
 		((char *)buffer->stream.r_ptr - addr) << 16 |
 		((char *)buffer->stream.w_ptr - addr));
+
+	buffer = buffer_release_irq(buffer);
 }
 
 void comp_update_buffer_consume(struct comp_buffer *buffer, uint32_t bytes)
 {
-	uint32_t flags = 0;
 	struct buffer_cb_transact cb_data = {
 		.buffer = buffer,
 		.transaction_amount = bytes,
@@ -242,15 +230,13 @@ void comp_update_buffer_consume(struct comp_buffer *buffer, uint32_t bytes)
 		return;
 	}
 
-	buffer_lock(buffer, &flags);
+	buffer = buffer_acquire_irq(buffer);
 
 	audio_stream_consume(&buffer->stream, bytes);
 
 	notifier_event(buffer, NOTIFIER_ID_BUFFER_CONSUME,
 		       NOTIFIER_TARGET_CORE_LOCAL, &cb_data, sizeof(cb_data));
 
-	buffer_unlock(buffer, flags);
-
 	addr = buffer->stream.addr;
 
 	buf_dbg(buffer, "comp_update_buffer_consume(), (buffer->avail << 16) | buffer->free = %08x, (buffer->id << 16) | buffer->size = %08x, (buffer->r_ptr - buffer->addr) << 16 | (buffer->w_ptr - buffer->addr)) = %08x",
@@ -259,4 +245,6 @@ void comp_update_buffer_consume(struct comp_buffer *buffer, uint32_t bytes)
 		(buffer->id << 16) | buffer->stream.size,
 		((char *)buffer->stream.r_ptr - addr) << 16 |
 		((char *)buffer->stream.w_ptr - addr));
+
+	buffer = buffer_release_irq(buffer);
 }

src/audio/codec_adapter/codec_adapter.c

@@ -382,7 +382,7 @@ int codec_adapter_copy(struct comp_dev *dev)
 		if (bytes_to_process < codec_buff_size)
 			goto db_verify;
 
-		buffer_invalidate(source, codec_buff_size);
+		buffer_stream_invalidate(source, codec_buff_size);
 		codec_adapter_copy_from_source_to_lib(&source->stream, &codec->cpd,
 						      codec_buff_size);
 		codec->cpd.avail = codec_buff_size;
@@ -411,7 +411,7 @@ int codec_adapter_copy(struct comp_dev *dev)
 	if (local_buff->stream.free < get_output_bytes(dev))
 		goto db_verify;
 
-	buffer_invalidate(source, codec_buff_size);
+	buffer_stream_invalidate(source, codec_buff_size);
 	codec_adapter_copy_from_source_to_lib(&source->stream, &codec->cpd,
 					      codec_buff_size);
 	codec->cpd.avail = codec_buff_size;
@@ -465,7 +465,7 @@ int codec_adapter_copy(struct comp_dev *dev)
 	audio_stream_copy(&local_buff->stream, 0,
 			  &sink->stream, 0,
 			  copy_bytes / cd->stream_params.sample_container_bytes);
-	buffer_writeback(sink, copy_bytes);
+	buffer_stream_writeback(sink, copy_bytes);
 
 	comp_update_buffer_produce(sink, copy_bytes);
 	comp_update_buffer_consume(local_buff, copy_bytes);

src/audio/copier.c

@@ -471,7 +471,7 @@ static int copier_copy(struct comp_dev *dev)
 			sink_bytes = c.frames * c.sink_frame_bytes;
 
 			i = IPC4_SINK_QUEUE_ID(sink->id);
-			buffer_invalidate(src, src_bytes);
+			buffer_stream_invalidate(src, src_bytes);
 			cd->converter[i](&src->stream, 0, &sink->stream, 0,
 					 c.frames * sink->stream.channels);
 			if (cd->attenuation > 0) {
@@ -480,7 +480,7 @@ static int copier_copy(struct comp_dev *dev)
 					return ret;
 			}
 
-			buffer_writeback(sink, sink_bytes);
+			buffer_stream_writeback(sink, sink_bytes);
 
 			comp_update_buffer_produce(sink, sink_bytes);
 		}

src/audio/crossover/crossover.c

@@ -633,7 +633,6 @@ static int crossover_copy(struct comp_dev *dev)
 	uint32_t num_assigned_sinks = 0;
 	uint32_t frames = UINT_MAX;
 	uint32_t source_bytes, avail;
-	uint32_t flags = 0;
 	uint32_t sinks_bytes[SOF_CROSSOVER_MAX_STREAMS] = { 0 };
 
 	comp_dbg(dev, "crossover_copy()");
@@ -672,26 +671,26 @@ static int crossover_copy(struct comp_dev *dev)
 	else
 		num_sinks = num_assigned_sinks;
 
-	buffer_lock(source, &flags);
+	source = buffer_acquire_irq(source);
 
 	/* Check if source is active */
 	if (source->source->state != dev->state) {
-		buffer_unlock(source, flags);
+		source = buffer_release_irq(source);
 		return -EINVAL;
 	}
 
 	/* Find the number of frames to copy over */
 	for (i = 0; i < num_sinks; i++) {
 		if (!sinks[i])
 			continue;
-		buffer_lock(sinks[i], &flags);
+		sinks[i] = buffer_acquire_irq(sinks[i]);
 		avail = audio_stream_avail_frames(&source->stream,
 						  &sinks[i]->stream);
 		frames = MIN(frames, avail);
-		buffer_unlock(sinks[i], flags);
+		buffer_release_irq(sinks[i]);
 	}
 
-	buffer_unlock(source, flags);
+	source = buffer_release_irq(source);
 
 	source_bytes = frames * audio_stream_frame_bytes(&source->stream);
 
@@ -703,13 +702,13 @@ static int crossover_copy(struct comp_dev *dev)
 	}
 
 	/* Process crossover */
-	buffer_invalidate(source, source_bytes);
+	buffer_stream_invalidate(source, source_bytes);
 	cd->crossover_process(dev, source, sinks, num_sinks, frames);
 
 	for (i = 0; i < num_sinks; i++) {
 		if (!sinks[i])
 			continue;
-		buffer_writeback(sinks[i], sinks_bytes[i]);
+		buffer_stream_writeback(sinks[i], sinks_bytes[i]);
 		comp_update_buffer_produce(sinks[i], sinks_bytes[i]);
 	}
 	comp_update_buffer_consume(source, source_bytes);

src/audio/dai.c

@@ -898,7 +898,6 @@ static int dai_copy(struct comp_dev *dev)
 	uint32_t sink_samples;
 	uint32_t samples;
 	int ret = 0;
-	uint32_t flags = 0;
 
 	comp_dbg(dev, "dai_copy()");
 
@@ -909,7 +908,7 @@ static int dai_copy(struct comp_dev *dev)
 		return ret;
 	}
 
-	buffer_lock(buf, &flags);
+	buf = buffer_acquire_irq(buf);
 
 	/* calculate minimum size to copy */
 	if (dev->direction == SOF_IPC_STREAM_PLAYBACK) {
@@ -929,7 +928,7 @@ static int dai_copy(struct comp_dev *dev)
 
 	copy_bytes = samples * sampling;
 
-	buffer_unlock(buf, flags);
+	buffer_release_irq(buf);
 
 	comp_dbg(dev, "dai_copy(), dir: %d copy_bytes= 0x%x, frames= %d",
 		 dev->direction, copy_bytes,

src/audio/dcblock/dcblock.c

@@ -270,11 +270,11 @@ static void dcblock_process(struct comp_dev *dev, struct comp_buffer *source,
 {
 	struct comp_data *cd = comp_get_drvdata(dev);
 
-	buffer_invalidate(source, source_bytes);
+	buffer_stream_invalidate(source, source_bytes);
 
 	cd->dcblock_func(dev, &source->stream, &sink->stream, frames);
 
-	buffer_writeback(sink, sink_bytes);
+	buffer_stream_writeback(sink, sink_bytes);
 
 	/* calc new free and available */
 	comp_update_buffer_consume(source, source_bytes);

src/audio/drc/drc.c

@@ -309,11 +309,11 @@ static void drc_process(struct comp_dev *dev, struct comp_buffer *source,
 {
 	struct drc_comp_data *cd = comp_get_drvdata(dev);
 
-	buffer_invalidate(source, source_bytes);
+	buffer_stream_invalidate(source, source_bytes);
 
 	cd->drc_func(dev, &source->stream, &sink->stream, frames);
 
-	buffer_writeback(sink, sink_bytes);
+	buffer_stream_writeback(sink, sink_bytes);
 
 	/* calc new free and available */
 	comp_update_buffer_consume(source, source_bytes);

src/audio/eq_fir/eq_fir.c

@@ -474,12 +474,12 @@ static void eq_fir_process(struct comp_dev *dev, struct comp_buffer *source,
 {
 	struct comp_data *cd = comp_get_drvdata(dev);
 
-	buffer_invalidate(source, source_bytes);
+	buffer_stream_invalidate(source, source_bytes);
 
 	cd->eq_fir_func(cd->fir, &source->stream, &sink->stream, frames,
 			source->stream.channels);
 
-	buffer_writeback(sink, sink_bytes);
+	buffer_stream_writeback(sink, sink_bytes);
 
 	/* calc new free and available */
 	comp_update_buffer_consume(source, source_bytes);

src/audio/eq_iir/eq_iir.c

@@ -700,11 +700,11 @@ static void eq_iir_process(struct comp_dev *dev, struct comp_buffer *source,
 {
 	struct comp_data *cd = comp_get_drvdata(dev);
 
-	buffer_invalidate(source, source_bytes);
+	buffer_stream_invalidate(source, source_bytes);
 
 	cd->eq_iir_func(dev, &source->stream, &sink->stream, frames);
 
-	buffer_writeback(sink, sink_bytes);
+	buffer_stream_writeback(sink, sink_bytes);
 
 	/* calc new free and available */
 	comp_update_buffer_consume(source, source_bytes);

src/audio/google_hotword_detect.c

@@ -383,7 +383,7 @@ static int ghd_copy(struct comp_dev *dev)
 	struct comp_data *cd = comp_get_drvdata(dev);
 	struct comp_buffer *source;
 	struct audio_stream *stream;
-	uint32_t bytes, flags, tail_bytes, head_bytes = 0;
+	uint32_t bytes, tail_bytes, head_bytes = 0;
 	int ret;
 
 	/* Check for new model */
@@ -399,9 +399,9 @@ static int ghd_copy(struct comp_dev *dev)
 				 struct comp_buffer, sink_list);
 	stream = &source->stream;
 
-	buffer_lock(source, &flags);
+	source = buffer_acquire_irq(source);
 	bytes = audio_stream_get_avail_bytes(&source->stream);
-	buffer_unlock(source, flags);
+	source = buffer_release_irq(source);
 
 	comp_dbg(dev, "ghd_copy() avail_bytes %u", bytes);
 	comp_dbg(dev, "buffer begin/r_ptr/end [0x%x 0x%x 0x%x]",
@@ -410,7 +410,7 @@ static int ghd_copy(struct comp_dev *dev)
 		 (uint32_t)stream->end_addr);
 
 	/* copy and perform detection */
-	buffer_invalidate(source, bytes);
+	buffer_stream_invalidate(source, bytes);
 
 	tail_bytes = (char *)stream->end_addr - (char *)stream->r_ptr;
 	if (bytes <= tail_bytes)