stm32: Support DMA-based UART output

This reduces the number of UART interrupts by a factor of 12, and
reduces the overall interrupt rate on STM32 by a factor of 2.

BUG=chrome-os-partner:20485
BRANCH=none (not required for pit branch)
TEST=Boot pit.  Ctrl+Q pauses debug output; Ctrl+S resumes it.
     'crash divzero' still prints a full crash dump.
     And util/makeall.sh passes builds all platforms and passes tests.

Change-Id: I86993e14b436150298dcb2c6d29086cc3c9db418
Signed-off-by: Randall Spangler <rspangler@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/168814

6 files changed, 127 insertions, 8 deletions

diff --git a/chip/stm32/config_chip.h b/chip/stm32/config_chip.h
index fa839287e0..8c0845f90e 100644
--- a/chip/stm32/config_chip.h
+++ b/chip/stm32/config_chip.h
@@ -50,6 +50,9 @@
 /* Use DMA */
 #define CONFIG_DMA
 
+/* Use DMA for UART output */
+#define CONFIG_UART_TX_DMA
+
 /* Flash protection applies to the next boot, not the current one */
 #define CONFIG_FLASH_PROTECT_NEXT_BOOT
 
diff --git a/chip/stm32/registers.h b/chip/stm32/registers.h
index bb8dcb21e2..71c254c992 100644
--- a/chip/stm32/registers.h
+++ b/chip/stm32/registers.h
@@ -103,6 +103,7 @@
 
 #define STM32_USART_SR(n)          STM32_USART_REG(n, 0x00)
 #define STM32_USART_SR_RXNE		(1 << 5)
+#define STM32_USART_SR_TC		(1 << 6)
 #define STM32_USART_SR_TXE		(1 << 7)
 #define STM32_USART_DR(n)          STM32_USART_REG(n, 0x04)
 #define STM32_USART_BRR(n)         STM32_USART_REG(n, 0x08)
@@ -110,11 +111,14 @@
 #define STM32_USART_CR1_RE		(1 << 2)
 #define STM32_USART_CR1_TE		(1 << 3)
 #define STM32_USART_CR1_RXNEIE		(1 << 5)
+#define STM32_USART_CR1_TCIE		(1 << 6)
 #define STM32_USART_CR1_TXEIE		(1 << 7)
 #define STM32_USART_CR1_UE		(1 << 13)
 #define STM32_USART_CR1_OVER8		(1 << 15) /* STM32L only */
 #define STM32_USART_CR2(n)         STM32_USART_REG(n, 0x10)
 #define STM32_USART_CR3(n)         STM32_USART_REG(n, 0x14)
+#define STM32_USART_CR3_DMAR		(1 << 6)
+#define STM32_USART_CR3_DMAT		(1 << 7)
 #define STM32_USART_CR3_ONEBIT		(1 << 11) /* STM32L only */
 #define STM32_USART_GTPR(n)        STM32_USART_REG(n, 0x18)
 
diff --git a/chip/stm32/uart.c b/chip/stm32/uart.c
index 42461b2a90..c848f0b20c 100644
--- a/chip/stm32/uart.c
+++ b/chip/stm32/uart.c
@@ -7,6 +7,7 @@
 
 #include "common.h"
 #include "clock.h"
+#include "dma.h"
 #include "gpio.h"
 #include "hooks.h"
 #include "registers.h"
@@ -17,6 +18,19 @@
 /* Console USART index */
 #define UARTN CONFIG_UART_CONSOLE
 
+#ifdef CONFIG_UART_TX_DMA
+#define UART_TX_INT_ENABLE STM32_USART_CR1_TCIE
+
+/* DMA channel options; assumes UART1 */
+static const struct dma_option dma_tx_option = {
+	STM32_DMAC_USART1_TX, (void *)&STM32_USART_DR(UARTN),
+	STM32_DMA_CCR_MSIZE_8_BIT | STM32_DMA_CCR_PSIZE_8_BIT
+};
+
+#else
+#define UART_TX_INT_ENABLE STM32_USART_CR1_TXEIE
+#endif
+
 static int init_done;    /* Initialization done? */
 static int should_stop;  /* Last TX control action */
 
@@ -28,21 +42,21 @@ int uart_init_done(void)
 void uart_tx_start(void)
 {
 	disable_sleep(SLEEP_MASK_UART);
-	STM32_USART_CR1(UARTN) |= STM32_USART_CR1_TXEIE;
 	should_stop = 0;
+	STM32_USART_CR1(UARTN) |= UART_TX_INT_ENABLE;
 	task_trigger_irq(STM32_IRQ_USART(UARTN));
 }
 
 void uart_tx_stop(void)
 {
-	STM32_USART_CR1(UARTN) &= ~STM32_USART_CR1_TXEIE;
+	STM32_USART_CR1(UARTN) &= ~UART_TX_INT_ENABLE;
 	should_stop = 1;
 	enable_sleep(SLEEP_MASK_UART);
 }
 
 int uart_tx_stopped(void)
 {
-	return !(STM32_USART_CR1(UARTN) & STM32_USART_CR1_TXEIE);
+	return !(STM32_USART_CR1(UARTN) & UART_TX_INT_ENABLE);
 }
 
 void uart_tx_flush(void)
@@ -56,6 +70,27 @@ int uart_tx_ready(void)
 	return STM32_USART_SR(UARTN) & STM32_USART_SR_TXE;
 }
 
+#ifdef CONFIG_UART_TX_DMA
+
+int uart_tx_dma_ready(void)
+{
+	return STM32_USART_SR(UARTN) & STM32_USART_SR_TC;
+}
+
+void uart_tx_dma_start(const char *src, int len)
+{
+	/* Prepare DMA */
+	dma_prepare_tx(&dma_tx_option, len, src);
+
+	/* Force clear TC so we don't re-interrupt */
+	STM32_USART_SR(UARTN) &= ~STM32_USART_SR_TC;
+
+	/* Start DMA */
+	dma_go(dma_get_channel(dma_tx_option.channel));
+}
+
+#endif /* CONFIG_UART_TX_DMA */
+
 int uart_rx_available(void)
 {
 	return STM32_USART_SR(UARTN) & STM32_USART_SR_RXNE;
@@ -63,11 +98,10 @@ int uart_rx_available(void)
 
 void uart_write_char(char c)
 {
-	/* we normally never wait here since uart_write_char is normally called
-	 * when the buffer is ready, excepted when we insert a carriage return
-	 * before a line feed in the interrupt routine.
-	 */
-	while (!uart_tx_ready()) ;
+	/* Wait for space */
+	while (!uart_tx_ready())
+		;
+
 	STM32_USART_DR(UARTN) = c;
 }
 
@@ -89,22 +123,30 @@ void uart_enable_interrupt(void)
 /* Interrupt handler for console USART */
 static void uart_interrupt(void)
 {
+#ifdef CONFIG_UART_TX_DMA
+	/* Disable transmission complete interrupt if DMA done */
+	if (STM32_USART_SR(UARTN) & STM32_USART_SR_TC)
+		STM32_USART_CR1(UARTN) &= ~STM32_USART_CR1_TCIE;
+#else
 	/*
 	 * Disable the TX empty interrupt before filling the TX buffer since it
 	 * needs an actual write to DR to be cleared.
 	 */
 	STM32_USART_CR1(UARTN) &= ~STM32_USART_CR1_TXEIE;
+#endif
 
 	/* Read input FIFO until empty, then fill output FIFO */
 	uart_process_input();
 	uart_process_output();
 
+#ifndef CONFIG_UART_TX_DMA
 	/*
 	 * Re-enable TX empty interrupt only if it was not disabled by
 	 * uart_process.
 	 */
 	if (!should_stop)
 		STM32_USART_CR1(UARTN) |= STM32_USART_CR1_TXEIE;
+#endif
 }
 DECLARE_IRQ(STM32_IRQ_USART(UARTN), uart_interrupt, 2);
 
@@ -162,8 +204,13 @@ void uart_init(void)
 	/* 1 stop bit, no fancy stuff */
 	STM32_USART_CR2(UARTN) = 0x0000;
 
+#ifdef CONFIG_UART_TX_DMA
+	/* Enable DMA transmitter */
+	STM32_USART_CR3(UARTN) |= STM32_USART_CR3_DMAT;
+#else
 	/* DMA disabled, special modes disabled, error interrupt disabled */
 	STM32_USART_CR3(UARTN) = 0x0000;
+#endif
 
 #ifdef CHIP_FAMILY_stm32l
 	/* Use single-bit sampling */
diff --git a/common/uart_buffering.c b/common/uart_buffering.c
index 1394b3d457..3a55d10631 100644
--- a/common/uart_buffering.c
+++ b/common/uart_buffering.c
@@ -65,11 +65,59 @@ static int __tx_char(void *context, int c)
 	return 0;
 }
 
+#ifdef CONFIG_UART_TX_DMA
+
+/**
+ * Process UART output via DMA
+ */
+static void uart_process_output_dma(void)
+{
+	/* Size of current DMA transfer */
+	static int tx_dma_in_progress;
+
+	/*
+	 * Get head pointer now, to avoid math problems if some other task
+	 * or interrupt adds output during this call.
+	 */
+	int head = tx_buf_head;
+
+	/* If DMA is still busy, nothing to do. */
+	if(!uart_tx_dma_ready())
+		return;
+
+	/* If a previous DMA transfer completed, free up the buffer it used */
+	if (tx_dma_in_progress) {
+		tx_buf_tail = (tx_buf_tail + tx_dma_in_progress) &
+			(CONFIG_UART_TX_BUF_SIZE - 1);
+		tx_dma_in_progress = 0;
+	}
+
+	/* Disable DMA-done interrupt if nothing to send */
+	if(head == tx_buf_tail) {
+		uart_tx_stop();
+		return;
+	}
+
+	/*
+	 * Get the largest contiguous block of output.  If the transmit buffer
+	 * wraps, only use the part before the wrap.
+	 */
+	tx_dma_in_progress = (head > tx_buf_tail ? head :
+			      CONFIG_UART_TX_BUF_SIZE) - tx_buf_tail;
+
+	uart_tx_dma_start((char *)(tx_buf + tx_buf_tail), tx_dma_in_progress);
+}
+
+#endif /* CONFIG_UART_TX_DMA */
+
 void uart_process_output(void)
 {
 	if (uart_suspended)
 		return;
 
+#ifdef CONFIG_UART_TX_DMA
+	uart_process_output_dma();
+#else
 	/* Copy output from buffer until TX fifo full or output buffer empty */
 	while (uart_tx_ready() && (tx_buf_head != tx_buf_tail)) {
 		uart_write_char(tx_buf[tx_buf_tail]);
@@ -79,6 +127,7 @@ void uart_process_output(void)
 	/* If output buffer is empty, disable transmit interrupt */
 	if (tx_buf_tail == tx_buf_head)
 		uart_tx_stop();
+#endif
 }
 
 void uart_process_input(void)
diff --git a/include/config.h b/include/config.h
index a5263c1985..026b434654 100644
--- a/include/config.h
+++ b/include/config.h
@@ -613,6 +613,9 @@
  */
 #define CONFIG_UART_TX_BUF_SIZE 512
 
+/* Use DMA for UART output */
+#undef CONFIG_UART_TX_DMA
+
 /*
  * UART receive buffer size in bytes.  Must be a power of 2 for macros in
  * common/uart_buffering.c to work properly.  Must be larger than
diff --git a/include/uart.h b/include/uart.h
index fa50645270..fec3fd47ec 100644
--- a/include/uart.h
+++ b/include/uart.h
@@ -127,6 +127,19 @@ void uart_tx_flush(void);
 int uart_tx_ready(void);
 
 /**
+ * Return non-zero if UART is ready to start a DMA transfer.
+ */
+int uart_tx_dma_ready(void);
+
+/**
+ * Start a UART transmit DMA transfer
+ *
+ * @param src		Pointer to data to send
+ * @param len		Length of transfer in bytes
+ */
+void uart_tx_dma_start(const char *src, int len);
+
+/**
  * Return non-zero if the UART has a character available to read.
  */
 int uart_rx_available(void);