board/hyperdebug: Implement flash extensions

Allow dual and quad-lane SPI operations.

BUG=b:273601311
TEST=HyperDebug able to flash 16MB in 31 seconds

Change-Id: I0243154085d018e123053f22c4a3a9d3b5a853d9
Reviewed-on: https://chromium-review.googlesource.com/c/chromiumos/platform/ec/+/4346953
Reviewed-by: Brian Nemec <bnemec@google.com>
Reviewed-by: Jett Rink <jettrink@chromium.org>
Commit-Queue: Jes Klinke <jbk@chromium.org>
Tested-by: Jes Klinke <jbk@chromium.org>

3 files changed, 238 insertions, 185 deletions

diff --git a/board/hyperdebug/board.c b/board/hyperdebug/board.c
index 1c08e5f57d..d685c37964 100644
--- a/board/hyperdebug/board.c
+++ b/board/hyperdebug/board.c
@@ -10,6 +10,7 @@
 #include "queue_policies.h"
 #include "registers.h"
 #include "spi.h"
+#include "stm32-dma.h"
 #include "timer.h"
 #include "usart-stm32l5.h"
 #include "usb-stream.h"
@@ -266,8 +267,9 @@ static void board_init(void)
 			     STM32_OCTOSPI_DCR1_DEVSIZE_MSK;
 	/* Clock prescaler (max value 255) */
 	STM32_OCTOSPI_DCR2 = spi_devices[1].div;
-	/* Zero dummy cycles */
-	STM32_OCTOSPI_TCR = 0;
+
+	/* Select DMA channel */
+	dma_select_channel(STM32_DMAC_CH13, DMAMUX_REQ_OCTOSPI1);
 }
 DECLARE_HOOK(HOOK_INIT, board_init, HOOK_PRIO_DEFAULT);
 
diff --git a/board/hyperdebug/board.h b/board/hyperdebug/board.h
index 8271f81e63..d341a6e570 100644
--- a/board/hyperdebug/board.h
+++ b/board/hyperdebug/board.h
@@ -144,6 +144,7 @@
 /* Enable control of SPI over USB */
 #define CONFIG_USB_SPI
 #define CONFIG_USB_SPI_BUFFER_SIZE 2048
+#define CONFIG_USB_SPI_FLASH_EXTENSIONS
 #define CONFIG_SPI_CONTROLLER
 #define CONFIG_STM32_SPI1_CONTROLLER
 #define CONFIG_SPI_MUTABLE_DEVICE_LIST
diff --git a/board/hyperdebug/spi.c b/board/hyperdebug/spi.c
index 4b549120a5..eee1626281 100644
--- a/board/hyperdebug/spi.c
+++ b/board/hyperdebug/spi.c
@@ -6,9 +6,11 @@
 
 #include "common.h"
 #include "console.h"
+#include "dma.h"
 #include "gpio.h"
 #include "registers.h"
 #include "spi.h"
+#include "stm32-dma.h"
 #include "timer.h"
 #include "usb_spi.h"
 #include "util.h"
@@ -28,8 +30,8 @@ struct spi_device_t spi_devices[] = {
 	  .div = 255,
 	  .gpio_cs = GPIO_CN10_6,
 	  .usb_flags = USB_SPI_ENABLED | USB_SPI_CUSTOM_SPI_DEVICE |
-		       USB_SPI_FLASH_DUAL_SUPPORT |
-		       USB_SPI_FLASH_QUAD_SUPPORT },
+		       USB_SPI_FLASH_DUAL_SUPPORT | USB_SPI_FLASH_QUAD_SUPPORT |
+		       USB_SPI_FLASH_DTR_SUPPORT },
 };
 const unsigned int spi_devices_used = ARRAY_SIZE(spi_devices);
 
@@ -196,234 +198,282 @@ static int octospi_wait_for(uint32_t flags, timestamp_t deadline)
 }
 
 /*
- * Write transaction: Write a number of bytes on the OCTOSPI bus.
+ * Board-specific SPI driver entry point, called by usb_spi.c.
  */
-static int octospi_indirect_write(const uint8_t *txdata, int txlen)
+void usb_spi_board_enable(void)
 {
-	timestamp_t deadline;
-	/* Deadline on the entire SPI transaction. */
-	deadline.val = get_time().val + OCTOSPI_TRANSACTION_TIMEOUT_US;
-
-	/* Enable OCTOSPI, indirect write mode. */
-	STM32_OCTOSPI_CR = STM32_OCTOSPI_CR_FMODE_IND_WRITE |
-			   STM32_OCTOSPI_CR_EN;
-	/* Clear completion flag from last transaction. */
-	STM32_OCTOSPI_FCR = STM32_OCTOSPI_FCR_CTCF;
+	/* All initialization already done in board_init(). */
+}
 
-	/* Data length. */
-	STM32_OCTOSPI_DLR = txlen - 1;
-	/* No instruction or address, only data. */
-	STM32_OCTOSPI_CCR =
-		STM32_OCTOSPI_CCR_IMODE_NONE | STM32_OCTOSPI_CCR_ADMODE_NONE |
-		STM32_OCTOSPI_CCR_ABMODE_NONE | STM32_OCTOSPI_CCR_DMODE_1WIRE;
-
-	/* Transmit data, four bytes at a time. */
-	for (int i = 0; i < txlen; i += 4) {
-		uint32_t value = 0;
-		int rv;
-		for (int j = 0; j < 4; j++) {
-			if (i + j < txlen)
-				value |= txdata[i + j] << (j * 8);
-		}
-		/* Wait for room in the FIFO. */
-		if ((rv = octospi_wait_for(STM32_OCTOSPI_SR_FTF, deadline)))
-			return rv;
-		STM32_OCTOSPI_DR = value;
-	}
-	/* Wait for transaction completion flag. */
-	return octospi_wait_for(STM32_OCTOSPI_SR_TCF, deadline);
+void usb_spi_board_disable(void)
+{
 }
 
+static const struct dma_option dma_octospi_option = {
+	.channel = STM32_DMAC_CH13,
+	.periph = (void *)&STM32_OCTOSPI_DR,
+	.flags = STM32_DMA_CCR_MSIZE_8_BIT | STM32_DMA_CCR_PSIZE_8_BIT,
+};
+
+static bool previous_cs;
+static timestamp_t deadline;
+
 /*
- * Read transaction: Optionally write a few bytes, before reading a number of
- * bytes on the OCTOSPI bus.
+ * Board-specific SPI driver entry point, called by usb_spi.c.  On this board,
+ * the only spi device declared as requiring board specific driver is OCTOSPI.
+ *
+ * Actually, there is nothing board-specific in the code below, as it merely
+ * implements the serial flash USB protocol extensions on the OctoSPI hardware
+ * found on STM32L5 chips (and probably others).  For now, however, HyperDebug
+ * is the only board that uses a chip with such an OctoSPI controller, so until
+ * we have seen this code being generally useful, it will live in
+ * board/hyperdebug.
  */
-static int octospi_indirect_read(const uint8_t *control_data, int control_len,
-				 uint8_t *rxdata, int rxlen)
+int usb_spi_board_transaction_async(const struct spi_device_t *spi_device,
+				    uint32_t flash_flags, const uint8_t *txdata,
+				    int txlen, uint8_t *rxdata, int rxlen)
 {
-	uint32_t instruction = 0, address = 0;
-	timestamp_t deadline;
+	uint32_t opcode = 0, address = 0;
+	const uint32_t mode = flash_flags & FLASH_FLAG_MODE_MSK;
+	const uint8_t width = (flash_flags & FLASH_FLAG_WIDTH_MSK) >>
+			      FLASH_FLAG_WIDTH_POS;
+	uint8_t opcode_len = (flash_flags & FLASH_FLAG_OPCODE_LEN_MSK) >>
+			     FLASH_FLAG_OPCODE_LEN_POS;
+	uint8_t addr_len = (flash_flags & FLASH_FLAG_ADDR_LEN_MSK) >>
+			   FLASH_FLAG_ADDR_LEN_POS;
+	const uint8_t dummy_cycles =
+		(flash_flags & FLASH_FLAG_DUMMY_CYCLES_MSK) >>
+		FLASH_FLAG_DUMMY_CYCLES_POS;
+	uint32_t data_len;
+	uint32_t control_value = 0;
+
+	if (!flash_flags) {
+		/*
+		 * This is a request does not use the extended format with SPI
+		 * flash flags, but is for doing plain write-then-read of single
+		 * lane (COPI/CIPO) SPI data.
+		 */
+		if (rxlen == SPI_READBACK_ALL) {
+			cprints(CC_SPI,
+				"Full duplex not supported by OctoSPI hardware");
+			return EC_ERROR_UNIMPLEMENTED;
+		} else if (!rxlen && !txlen) {
+			/* No operation requested, done. */
+			return EC_SUCCESS;
+		} else if (!rxlen) {
+			/*
+			 * Transmit-only transaction.  This is implemented by
+			 * not using any of the up to 12 bytes of instructions,
+			 * but as all "data".
+			 */
+			flash_flags |= FLASH_FLAG_READ_WRITE_WRITE;
+		} else if (txlen <= 12) {
+			/*
+			 * Sending of up to 12 bytes, followed by reading a
+			 * possibly large number of bytes.  This is implemented
+			 * by a "read" transaction using the instruction and
+			 * address feature of OctoSPI.
+			 */
+			if (txlen <= 4) {
+				opcode_len = txlen;
+			} else {
+				opcode_len = 4;
+				addr_len = txlen - 4;
+			}
+		} else {
+			/*
+			 * Sending many bytes, followed by reading.  This would
+			 * have to be implemented as two separate OctoSPI
+			 * transactions.
+			 */
+			cprints(CC_SPI,
+				"General write-then-read not supported by OctoSPI hardware");
+			return EC_ERROR_UNIMPLEMENTED;
+		}
+	}
+
+	previous_cs = gpio_get_level(spi_device->gpio_cs);
+
+	/* Drive chip select low */
+	gpio_set_level(spi_device->gpio_cs, 0);
 
 	/* Deadline on the entire SPI transaction. */
 	deadline.val = get_time().val + OCTOSPI_TRANSACTION_TIMEOUT_US;
 
-	/* Enable OCTOSPI, indirect read mode. */
-	STM32_OCTOSPI_CR = STM32_OCTOSPI_CR_FMODE_IND_READ |
-			   STM32_OCTOSPI_CR_EN;
+	if ((flash_flags & FLASH_FLAG_READ_WRITE_MSK) ==
+	    FLASH_FLAG_READ_WRITE_WRITE) {
+		data_len = txlen - opcode_len - addr_len;
+		/* Enable OCTOSPI, indirect write mode. */
+		STM32_OCTOSPI_CR = STM32_OCTOSPI_CR_FMODE_IND_WRITE |
+				   STM32_OCTOSPI_CR_DMAEN | STM32_OCTOSPI_CR_EN;
+	} else {
+		data_len = rxlen;
+		/* Enable OCTOSPI, indirect read mode. */
+		STM32_OCTOSPI_CR = STM32_OCTOSPI_CR_FMODE_IND_READ |
+				   STM32_OCTOSPI_CR_DMAEN | STM32_OCTOSPI_CR_EN;
+	}
+
 	/* Clear completion flag from last transaction. */
 	STM32_OCTOSPI_FCR = STM32_OCTOSPI_FCR_CTCF;
 
-	/* Data length (receive). */
-	STM32_OCTOSPI_DLR = rxlen - 1;
-	if (control_len == 0) {
-		/*
-		 * Set up OCTOSPI for: No instruction, no address, then read
-		 * data.
-		 */
-		STM32_OCTOSPI_CCR = STM32_OCTOSPI_CCR_IMODE_NONE |
-				    STM32_OCTOSPI_CCR_ADMODE_NONE |
-				    STM32_OCTOSPI_CCR_ABMODE_NONE |
-				    STM32_OCTOSPI_CCR_DMODE_1WIRE;
-	} else if (control_len <= 4) {
-		/*
-		 * Set up OCTOSPI for: One to four bytes of instruction, no
-		 * address, then read data.
-		 */
-		STM32_OCTOSPI_CCR = STM32_OCTOSPI_CCR_IMODE_1WIRE |
-				    (control_len - 1)
-					    << STM32_OCTOSPI_CCR_ISIZE_POS |
-				    STM32_OCTOSPI_CCR_ADMODE_NONE |
-				    STM32_OCTOSPI_CCR_ABMODE_NONE |
-				    STM32_OCTOSPI_CCR_DMODE_1WIRE;
-		for (int i = 0; i < control_len; i++) {
-			instruction <<= 8;
-			instruction |= control_data[i];
+	/* Data length. */
+	STM32_OCTOSPI_DLR = data_len - 1;
+
+	/*
+	 * Set up the number of bytes and data width of each of the opcode,
+	 * address, and "alternate" stages of the initial command bytes.
+	 */
+	if (opcode_len == 0) {
+		control_value |= STM32_OCTOSPI_CCR_IMODE_NONE;
+	} else {
+		control_value |= (opcode_len - 1)
+				 << STM32_OCTOSPI_CCR_ISIZE_POS;
+		if (mode < FLASH_FLAG_MODE_NNN) {
+			// Opcode phase is single-lane
+			control_value |= 1 << STM32_OCTOSPI_CCR_IMODE_POS;
+		} else {
+			// Opcode phase is multi-lane
+			control_value |= (width + 1)
+					 << STM32_OCTOSPI_CCR_IMODE_POS;
+			if (flash_flags & FLASH_FLAG_DTR)
+				control_value |= STM32_OCTOSPI_CCR_IDTR;
 		}
-	} else if (control_len <= 8) {
-		/*
-		 * Set up OCTOSPI for: One to four bytes of instruction, four
-		 * bytes of address, then read data.
-		 */
-		STM32_OCTOSPI_CCR = STM32_OCTOSPI_CCR_IMODE_1WIRE |
-				    (control_len - 1)
-					    << STM32_OCTOSPI_CCR_ISIZE_POS |
-				    STM32_OCTOSPI_CCR_ADMODE_1WIRE |
-				    STM32_OCTOSPI_CCR_ADSIZE_4BYTES |
-				    STM32_OCTOSPI_CCR_ABMODE_NONE |
-				    STM32_OCTOSPI_CCR_DMODE_1WIRE;
-		for (int i = 0; i < control_len - 4; i++) {
-			instruction <<= 8;
-			instruction |= control_data[i];
+		for (int i = 0; i < opcode_len; i++) {
+			opcode <<= 8;
+			opcode |= *txdata++;
+			txlen--;
 		}
-		for (int i = 0; i < 4; i++) {
+	}
+	if (addr_len == 0) {
+		control_value |= STM32_OCTOSPI_CCR_ADMODE_NONE;
+		control_value |= STM32_OCTOSPI_CCR_ABMODE_NONE;
+	} else if (addr_len <= 4) {
+		control_value |= (addr_len - 1) << STM32_OCTOSPI_CCR_ADSIZE_POS;
+		if (mode < FLASH_FLAG_MODE_1NN) {
+			// Address phase is single-lane
+			control_value |= 1 << STM32_OCTOSPI_CCR_ADMODE_POS;
+		} else {
+			// Address phase is multi-lane
+			control_value |= (width + 1)
+					 << STM32_OCTOSPI_CCR_ADMODE_POS;
+			if (flash_flags & FLASH_FLAG_DTR)
+				control_value |= STM32_OCTOSPI_CCR_ADDTR;
+		}
+		for (int i = 0; i < addr_len; i++) {
 			address <<= 8;
-			address |= control_data[control_len - 4 + i];
+			address |= *txdata++;
+			txlen--;
 		}
-	} else if (control_len <= 12) {
+		control_value |= STM32_OCTOSPI_CCR_ABMODE_NONE;
+	} else {
 		uint32_t alternate = 0;
-		/*
-		 * Set up OCTOSPI for: One to four bytes of instruction, four
-		 * bytes of address, four "alternate" bytes, then read data.
-		 */
-		STM32_OCTOSPI_CCR = STM32_OCTOSPI_CCR_IMODE_1WIRE |
-				    (control_len - 1)
-					    << STM32_OCTOSPI_CCR_ISIZE_POS |
-				    STM32_OCTOSPI_CCR_ADMODE_1WIRE |
-				    STM32_OCTOSPI_CCR_ADSIZE_4BYTES |
-				    STM32_OCTOSPI_CCR_ABMODE_1WIRE |
-				    STM32_OCTOSPI_CCR_ABSIZE_4BYTES |
-				    STM32_OCTOSPI_CCR_DMODE_1WIRE;
-		for (int i = 0; i < control_len - 8; i++) {
-			instruction <<= 8;
-			instruction |= control_data[i];
+		control_value |= 3 << STM32_OCTOSPI_CCR_ADSIZE_POS;
+		control_value |= (addr_len - 5) << STM32_OCTOSPI_CCR_ABSIZE_POS;
+		if (mode < FLASH_FLAG_MODE_1NN) {
+			// Address phase is single-lane
+			control_value |= 1 << STM32_OCTOSPI_CCR_ADMODE_POS;
+			control_value |= 1 << STM32_OCTOSPI_CCR_ABMODE_POS;
+		} else {
+			// Address phase is multi-lane
+			control_value |= (width + 1)
+					 << STM32_OCTOSPI_CCR_ADMODE_POS;
+			control_value |= (width + 1)
+					 << STM32_OCTOSPI_CCR_ABMODE_POS;
+			if (flash_flags & FLASH_FLAG_DTR)
+				control_value |= STM32_OCTOSPI_CCR_ADDTR |
+						 STM32_OCTOSPI_CCR_ABDTR;
 		}
 		for (int i = 0; i < 4; i++) {
 			address <<= 8;
-			address |= control_data[control_len - 8 + i];
+			address |= *txdata++;
+			txlen--;
 		}
-		for (int i = 0; i < 4; i++) {
+		for (int i = 0; i < addr_len - 4; i++) {
 			alternate <<= 8;
-			alternate |= control_data[control_len - 4 + i];
+			alternate |= *txdata++;
+			txlen--;
 		}
 		STM32_OCTOSPI_ABR = alternate;
+	}
+	/* Set up how many bytes to read/write after the initial command. */
+	if (data_len == 0) {
+		control_value |= STM32_OCTOSPI_CCR_DMODE_NONE;
 	} else {
-		return EC_ERROR_UNIMPLEMENTED;
+		if (mode < FLASH_FLAG_MODE_11N) {
+			// Data phase is single-lane
+			control_value |= 1 << STM32_OCTOSPI_CCR_DMODE_POS;
+		} else {
+			// Data phase is multi-lane
+			control_value |= (width + 1)
+					 << STM32_OCTOSPI_CCR_DMODE_POS;
+			if (flash_flags & FLASH_FLAG_DTR)
+				control_value |= STM32_OCTOSPI_CCR_DDTR;
+		}
 	}
+
+	/* Dummy cycles. */
+	STM32_OCTOSPI_TCR = dummy_cycles << STM32_OCTOSPI_TCR_DCYC_POS;
+
+	STM32_OCTOSPI_CCR = control_value;
+
 	/* Set instruction and address registers, triggering the start of the
 	 * write+read transaction. */
-	STM32_OCTOSPI_IR = instruction;
+	STM32_OCTOSPI_IR = opcode;
 	STM32_OCTOSPI_AR = address;
 
-	/* Receive data, four bytes at a time. */
-	for (int i = 0; i < rxlen; i += 4) {
-		int rv;
-		uint32_t value;
-		/* Wait for data available in the FIFO. */
-		if ((rv = octospi_wait_for(STM32_OCTOSPI_SR_FTF, deadline)))
-			return rv;
-		value = STM32_OCTOSPI_DR;
-		for (int j = 0; j < 4; j++) {
-			if (i + j < rxlen)
-				rxdata[i + j] = value >> (j * 8);
+	if ((flash_flags & FLASH_FLAG_READ_WRITE_MSK) ==
+	    FLASH_FLAG_READ_WRITE_WRITE) {
+		if (txlen > 0) {
+			dma_chan_t *txdma = dma_get_channel(STM32_DMAC_CH13);
+			dma_prepare_tx(&dma_octospi_option, txlen, txdata);
+			dma_go(txdma);
+		}
+	} else {
+		if (rxlen > 0) {
+			dma_start_rx(&dma_octospi_option, rxlen, rxdata);
 		}
 	}
-	/* Wait for transaction completion flag. */
-	return octospi_wait_for(STM32_OCTOSPI_SR_TCF, deadline);
-}
 
-/*
- * Board-specific SPI driver entry point, called by usb_spi.c.
- */
-void usb_spi_board_enable(void)
-{
-	/* All initialization already done in board_init(). */
+	return EC_SUCCESS;
 }
 
-void usb_spi_board_disable(void)
+int usb_spi_board_transaction_is_complete(const struct spi_device_t *spi_device)
 {
+	/* Query the "transaction complete flag" of the status register. */
+	return STM32_OCTOSPI_SR & STM32_OCTOSPI_SR_TCF;
 }
 
-/*
- * Board-specific SPI driver entry point, called by usb_spi.c.  On this board,
- * the only spi device declared as requiring board specific driver is OCTOSPI.
- */
-int usb_spi_board_transaction(const struct spi_device_t *spi_device,
-			      uint32_t flash_flags, const uint8_t *txdata,
-			      int txlen, uint8_t *rxdata, int rxlen)
+int usb_spi_board_transaction_flush(const struct spi_device_t *spi_device)
 {
-	int rv = EC_SUCCESS;
-	bool previous_cs;
-
-	if (flash_flags & FLASH_FLAGS_REQUIRING_SUPPORT)
-		return EC_ERROR_UNIMPLEMENTED;
-
-	previous_cs = gpio_get_level(spi_device->gpio_cs);
-
-	/* Drive chip select low */
-	gpio_set_level(spi_device->gpio_cs, 0);
-
 	/*
-	 * STM32L5 OctoSPI in "indirect mode" supports two types of SPI
-	 * operations, "read" and "write", in addition to the main data, each
-	 * type of operation can be preceded by up to 12 bytes of
-	 * "instructions", (which are always written from HyperDebug to the SPI
-	 * device).  We can use the above features to support some combination
-	 * of write-followed-by-read in a single OctoSPI transaction.
+	 * Wait until DMA transfer is complete (no-op if DMA not started because
+	 * of zero-length transfer).
 	 */
+	int rv = dma_wait(STM32_DMAC_CH13);
+	dma_disable(STM32_DMAC_CH13);
+	if (rv != EC_SUCCESS)
+		return rv;
 
-	if (rxlen == SPI_READBACK_ALL) {
-		cprints(CC_SPI,
-			"Full duplex not supported by OctoSPI hardware");
-		rv = EC_ERROR_UNIMPLEMENTED;
-	} else if (!rxlen && !txlen) {
-		/* No operation requested, done. */
-	} else if (!rxlen) {
-		/*
-		 * Transmit-only transaction.  This is implemented by not using
-		 * any of the up to 12 bytes of instructions, but as all "data".
-		 */
-		rv = octospi_indirect_write(txdata, txlen);
-	} else if (txlen <= 12) {
-		/*
-		 * Sending of up to 12 bytes, followed by reading a possibly
-		 * large number of bytes.  This is implemented by a "read"
-		 * transaction using the instruction and address feature of
-		 * OctoSPI.
-		 */
-		rv = octospi_indirect_read(txdata, txlen, rxdata, rxlen);
-	} else {
-		/*
-		 * Sending many bytes, followed by reading.  This is implemented
-		 * as two separate OctoSPI transactions.  (Chip select is kept
-		 * asserted across both transactions, outside the control of the
-		 * OctoSPI hardware.)
-		 */
-		rv = octospi_indirect_write(txdata, txlen);
-		if (rv == EC_SUCCESS)
-			rv = octospi_indirect_read(NULL, 0, rxdata, rxlen);
-	}
+	/*
+	 * Ensure that all bits of the last byte has been shifted onto the SPI
+	 * bus.
+	 */
+	rv = octospi_wait_for(STM32_OCTOSPI_SR_TCF, deadline);
 
 	/* Return chip select to previous level. */
 	gpio_set_level(spi_device->gpio_cs, previous_cs);
+
+	return rv;
+}
+
+int usb_spi_board_transaction(const struct spi_device_t *spi_device,
+			      uint32_t flash_flags, const uint8_t *txdata,
+			      int txlen, uint8_t *rxdata, int rxlen)
+{
+	int rv = usb_spi_board_transaction_async(spi_device, flash_flags,
+						 txdata, txlen, rxdata, rxlen);
+	if (rv == EC_SUCCESS) {
+		rv = usb_spi_board_transaction_flush(spi_device);
+	}
 	return rv;
 }