Cr50: Implement USB according to Programmer's Guide

This is a rewrite of the Cr50 USB Device Control Endpoint
implementation, using the instructions in the DWC USB 2.0 OTG
Programmer's Guide (such as they are). Some of the major
differences:

* Not every USB interrupt indicates the receipt of an incoming
  packet. Many merely provide updates on packet transfer status,
  transaction stages, or other activity. We handle those
  cases correctly.

* We may need to start a new Control transaction at any point,
  even in the middle of an existing transaction.

* Large IN data transfers can be handled with one interrupt by
  chaining multiple together.

* Logical separation of the phases of each transaction (Setup,
  Data, Status).

That said, while this CL matches the Programmer's Guide fairly
closely, that Guide is pretty crappy and this is just the first
commit. There is still a fair amount to do (marked with comments
and bug reports). However, it works at least as well as the
previous version and is much closer to what the supplier claims
is the correct implementation.

BUG=chrome-os-partner:34893
BRANCH=none
TEST=make buildall, manual

Connect the Cr50 to my workstation via USB:
* /bin/dmesg reports no errors
* verify EP0 with lsusb -v -d 18d1:5014
* verify EP1 with './extra/usb_console -e 1 -p 5014' (reverses
  case of input text)
* verify EP2 with the 'hid' command on the EC console (types a 'g')

We also connected this device to Windows, and Mac laptops (and a
Chromebook) and used a USB bus analyzer to monitor the behavior.
It works on machines those, too.

Change-Id: Ic515ea83e217a8d0552d61ac5eb19693661fcd15
Signed-off-by: Bill Richardson <wfrichar@chromium.org>
Reviewed-on: https://chromium-review.googlesource.com/318864
Reviewed-by: Dominic Rizzo <domrizzo@google.com>

1 files changed, 649 insertions, 213 deletions

diff --git a/chip/g/usb.c b/chip/g/usb.c
index 76991f9da6..a23c299990 100644
--- a/chip/g/usb.c
+++ b/chip/g/usb.c
@@ -232,44 +232,149 @@ const uint8_t usb_string_desc[] = {
 /****************************************************************************/
 /* Packet-handling stuff, specific to this SoC */
 
-/* Descriptors for USB controller S/G DMA */
-static struct g_usb_desc ep0_out_desc;
-static struct g_usb_desc ep0_in_desc;
+/* Some internal state to keep track of what's going on */
+static enum {
+	WAITING_FOR_SETUP_PACKET,
+	DATA_STAGE_IN,
+	NO_DATA_STAGE,
+} what_am_i_doing;
+
+/* Programmer's Guide, Table 10-7 */
+enum table_case {
+	BAD_0,
+	TABLE_CASE_A,
+	TABLE_CASE_B,
+	TABLE_CASE_C,
+	TABLE_CASE_D,
+	TABLE_CASE_E,
+	BAD_6,
+	BAD_7,
+};
 
-/* Control endpoint (EP0) buffers */
-static uint8_t ep0_buf_tx[USB_MAX_PACKET_SIZE];
-static uint8_t ep0_buf_rx[USB_MAX_PACKET_SIZE];
+/*
+ * Table 10-7 in the Programmer's Guide decodes OUT endpoint interrupts:
+ *
+ *  Case  StatusPhseRecvd  SetUp  XferCompl   Description
+ *
+ *   A          0            0       1        Out descriptor is updated. Check
+ *                                            its SR bit to see if we got
+ *                                            a SETUP packet or an OUT packet.
+ *   B          0            1       0        SIE has seen an IN or OUT packet
+ *                                            following the SETUP packet.
+ *   C          0            1       1        Both A & B at once, I think.
+ *                                            Check the SR bit.
+ *   D          1            0       0        SIE has seen the host change
+ *                                            direction, implying Status phase.
+ *   E          1            0       1        Out descriptor is updated, and
+ *                                            SIE has seen an IN following it.
+ *                                            This is probably the Status phase
+ *                                            for a Control Write, but could be
+ *                                            an early SETUP for a Control Read
+ *                                            instead. Maybe. The documentation
+ *                                            is unclear. Check the SR bit
+ *                                            anyway.
+ */
+static enum table_case decode_table_10_7(uint32_t doepint)
+{
+	enum table_case val = BAD_0;
+
+	/* Bits: SI, SPD, IOC */
+	if (doepint & DOEPINT_XFERCOMPL)
+		val += 1;
+	if (doepint & DOEPINT_SETUP)
+		val += 2;
+	if (doepint & DOEPINT_STSPHSERCVD)
+		val += 4;
+
+	return val;
+}
 
-static int set_addr;
-/* remaining size of descriptor data to transfer */
-static int desc_left;
-/* pointer to descriptor data if any */
-static const uint8_t *desc_ptr;
+/* For STATUS/OUT: Use two DMA descriptors, each with one-packet buffers */
+#define NUM_OUT_BUFFERS 2
+static uint8_t ep0_out_buf[NUM_OUT_BUFFERS][USB_MAX_PACKET_SIZE];
+static struct g_usb_desc ep0_out_desc[NUM_OUT_BUFFERS];
+static int cur_out_idx;				/* latest with xfercompl=1 */
+static const struct g_usb_desc *cur_out_desc;
+static int next_out_idx;			/* next packet will go here */
+static struct g_usb_desc *next_out_desc;
+
+/* For IN: Several DMA descriptors, all pointing into one large buffer, so that
+ * we can return the configuration descriptor as one big blob. */
+#define NUM_IN_PACKETS_AT_ONCE 4
+#define IN_BUF_SIZE (NUM_IN_PACKETS_AT_ONCE * USB_MAX_PACKET_SIZE)
+static uint8_t ep0_in_buf[IN_BUF_SIZE];
+static struct g_usb_desc ep0_in_desc[NUM_IN_PACKETS_AT_ONCE];
+static struct g_usb_desc *cur_in_desc;
 
 /* Reset all this to a good starting state. */
 static void initialize_dma_buffers(void)
 {
-	ep0_out_desc.flags = DOEPDMA_RXBYTES(64) | DOEPDMA_LAST |
-			     DOEPDMA_BS_HOST_RDY | DOEPDMA_IOC;
-	ep0_out_desc.addr = ep0_buf_rx;
-	ep0_in_desc.flags = DIEPDMA_TXBYTES(0) | DIEPDMA_LAST |
-			    DIEPDMA_BS_HOST_RDY | DIEPDMA_IOC;
-	ep0_in_desc.addr = ep0_buf_tx;
-	GR_USB_DIEPDMA(0) = (uint32_t)&ep0_in_desc;
-	GR_USB_DOEPDMA(0) = (uint32_t)&ep0_out_desc;
+	int i;
+
+	print_later("initialize_dma_buffers()", 0, 0, 0, 0, 0);
+
+	for (i = 0; i < NUM_OUT_BUFFERS; i++) {
+		ep0_out_desc[i].addr = ep0_out_buf[i];
+		ep0_out_desc[i].flags = DOEPDMA_BS_HOST_BSY;
+	}
+	next_out_idx = 0;
+	next_out_desc = ep0_out_desc + next_out_idx;
+	GR_USB_DOEPDMA(0) = (uint32_t)next_out_desc;
+	/* cur_out_* will be updated when we get the first RX packet */
+
+	for (i = 0; i < NUM_IN_PACKETS_AT_ONCE; i++) {
+		ep0_in_desc[i].addr = ep0_in_buf + i * USB_MAX_PACKET_SIZE;
+		ep0_in_desc[i].flags = DIEPDMA_BS_HOST_BSY;
+	}
+	cur_in_desc = ep0_in_desc;
+	GR_USB_DIEPDMA(0) = (uint32_t)(cur_in_desc);
+};
+
+/* Change the RX descriptors after each SETUP/OUT packet is received so we can
+ * prepare to receive another without losing track of this one. */
+static void got_RX_packet(void)
+{
+	cur_out_idx = next_out_idx;
+	cur_out_desc = ep0_out_desc + cur_out_idx;
+	next_out_idx = (next_out_idx + 1) % NUM_OUT_BUFFERS;
+	next_out_desc = ep0_out_desc + next_out_idx;
+	GR_USB_DOEPDMA(0) = (uint32_t)next_out_desc;
 }
 
 /* Load the EP0 IN FIFO buffer with some data (zero-length works too). Returns
  * len, or negative on error. */
 int load_in_fifo(const void *source, uint32_t len)
 {
-	if (len > sizeof(ep0_buf_tx))
+	uint8_t *buffer = ep0_in_buf;
+	int zero_packet = !len;
+	int d, l;
+
+	/* Copy the data into our FIFO buffer */
+	if (len >= IN_BUF_SIZE) {
+		report_error();
 		return -1;
+	}
+	if (len)
+		memcpy(buffer, source, len);
+
+	/* Set up the descriptors */
+	for (d = l = 0; len >= USB_MAX_PACKET_SIZE; d++) {
+		ep0_in_desc[d].addr = buffer + d * USB_MAX_PACKET_SIZE;
+		ep0_in_desc[d].flags = DIEPDMA_TXBYTES(USB_MAX_PACKET_SIZE);
+		len -= USB_MAX_PACKET_SIZE;
+		l = d;
+	}
+	/* Maybe one short packet left? */
+	if (len || zero_packet) {
+		ep0_in_desc[d].addr = buffer + d * USB_MAX_PACKET_SIZE;
+		ep0_in_desc[d].flags = DIEPDMA_TXBYTES(len) | DIEPDMA_SP;
+		l = d;
+	}
+	/* Mark the last descriptor as last. */
+	ep0_in_desc[l].flags |= (DIEPDMA_LAST | DIEPDMA_IOC);
 
-	memcpy(ep0_buf_tx, source, len);
-	ep0_in_desc.flags = DIEPDMA_LAST | DIEPDMA_BS_HOST_RDY |
-		DIEPDMA_IOC | DIEPDMA_TXBYTES(len);
-	GR_USB_DIEPCTL(0) |= DXEPCTL_CNAK | DXEPCTL_EPENA;
+	/* Point to the first in the chain */
+	cur_in_desc = ep0_in_desc;
 
 	return len;
 }
@@ -279,244 +384,557 @@ int load_in_fifo(const void *source, uint32_t len)
 int accept_out_fifo(uint32_t len)
 {
 	/* TODO: This is not yet implemented */
+	report_error();
 	return -1;
 }
 
-/*
- * Requests on the control endpoint (aka EP0). The USB spec mandates that all
- * values are little-endian over the wire. Since this file is intentionally
- * chip-specific and we're a little-endian architecture, we can just cast the
- * buffer into the correct struct.
- */
-static void ep0_rx(void)
+static void flush_in_fifo(void)
 {
-	uint32_t epint = GR_USB_DOEPINT(0);
-	struct usb_setup_packet *req = (struct usb_setup_packet *)ep0_buf_rx;
+	/* TODO: Programmer's Guide p167 suggests lots more stuff */
+	GR_USB_GRSTCTL = GRSTCTL_TXFNUM(0) | GRSTCTL_TXFFLSH;
+	while (GR_USB_GRSTCTL & GRSTCTL_TXFFLSH)
+		;				/* timeout? */
+}
 
-	print_later("ep0_rx: DOEPINT(0) is 0x%x", epint, 0, 0, 0, 0);
+/* We're complaining about something by stalling both IN and OUT packets,
+ * but a SETUP packet will get through anyway, so prepare for it. */
+static void stall_both_fifos(void)
+{
+	print_later("stall_both_fifos()", 0, 0, 0, 0, 0);
 
-	GR_USB_DOEPINT(0) = epint; /* clear IT */
+	what_am_i_doing = WAITING_FOR_SETUP_PACKET;
 
-	print_later("R: %02x %02x %04x %04x %04x",
-		    req->bmRequestType, req->bRequest, req->wValue,
-		    req->wIndex, req->wLength);
+	next_out_desc->flags =
+		DOEPDMA_RXBYTES(USB_MAX_PACKET_SIZE)
+		| DOEPDMA_IOC | DOEPDMA_LAST;
 
-	/* reset any incomplete descriptor transfer */
-	desc_ptr = NULL;
+	/* We don't care about IN packets right now, only OUT. */
+	GR_USB_DAINTMSK |= DAINT_OUTEP(0);
+	GR_USB_DAINTMSK &= ~DAINT_INEP(0);
 
-	/* interface specific requests */
-	if ((req->bmRequestType & USB_RECIP_MASK) == USB_RECIP_INTERFACE) {
-		uint8_t iface = req->wIndex & 0xff;
-		int bytes;
+	/* Stall both IN and OUT. The hardware will reset them when the next
+	 * SETUP comes along. */
+	GR_USB_DOEPCTL(0) = DXEPCTL_STALL | DXEPCTL_EPENA;
+	flush_in_fifo();
+	GR_USB_DIEPCTL(0) = DXEPCTL_STALL | DXEPCTL_EPENA;
+}
 
-		if (iface >= USB_IFACE_COUNT)
-			goto unknown_req;
+/* The next packet from the host should be a Setup packet. Get ready for it. */
+static void expect_setup_packet(void)
+{
+	print_later("expect_setup_packet()", 0, 0, 0, 0, 0);
 
-		bytes = usb_iface_request[iface](req);
-		if (bytes < 0)
-			goto unknown_req;
+	what_am_i_doing = WAITING_FOR_SETUP_PACKET;
 
-		ep0_out_desc.flags = DOEPDMA_RXBYTES(64) | DOEPDMA_LAST
-			| DOEPDMA_BS_HOST_RDY | DOEPDMA_IOC;
-		GR_USB_DOEPCTL(0) |= DXEPCTL_CNAK | DXEPCTL_EPENA;
-		return;
-	}
+	next_out_desc->flags =
+		DOEPDMA_RXBYTES(USB_MAX_PACKET_SIZE)
+		| DOEPDMA_IOC | DOEPDMA_LAST;
+
+	/* We don't care about IN packets right now, only OUT. */
+	GR_USB_DAINTMSK |= DAINT_OUTEP(0);
+	GR_USB_DAINTMSK &= ~DAINT_INEP(0);
+
+	/* Let it run. We might need CNAK if we just got an OUT for status */
+	GR_USB_DOEPCTL(0) = DXEPCTL_CNAK | DXEPCTL_EPENA;
+}
+
+/* The TX FIFO buffer is loaded. Start the Data phase. */
+static void expect_data_phase_in(enum table_case tc)
+{
+	print_later("expect_data_phase_in(%c)", "0ABCDE67"[tc], 0, 0, 0, 0);
 
-	if (req->bmRequestType == USB_DIR_IN &&
-	    req->bRequest == USB_REQ_GET_DESCRIPTOR) {
+	what_am_i_doing = DATA_STAGE_IN;
+
+	/* We apparently have to do this every time we transmit anything */
+	flush_in_fifo();
+
+	/* I don't think we have to do this every time, but the Programmer's
+	 * Guide says to, so...	*/
+	GR_USB_DIEPDMA(0) = (uint32_t)(cur_in_desc);
+
+	/* Blindly following instructions here, too. */
+	if (tc == TABLE_CASE_C)
+		GR_USB_DIEPCTL(0) = DXEPCTL_CNAK | DXEPCTL_EPENA;
+	else
+		GR_USB_DIEPCTL(0) = DXEPCTL_EPENA;
+
+	/*
+	 * When the IN is done, we expect a zero-length OUT for the status
+	 * phase but it could be an early SETUP instead. We'll have to deal
+	 * with either one when it arrives.
+	 */
+	next_out_desc->flags =
+		DOEPDMA_RXBYTES(USB_MAX_PACKET_SIZE)
+		| DOEPDMA_IOC | DOEPDMA_LAST;
+
+	/* And here's this jimmy rustler again... */
+	if (tc == TABLE_CASE_C)
+		GR_USB_DOEPCTL(0) = DXEPCTL_CNAK | DXEPCTL_EPENA;
+	else
+		GR_USB_DOEPCTL(0) = DXEPCTL_EPENA;
+
+	/* Get an interrupt when either IN or OUT arrives */
+	GR_USB_DAINTMSK |= (DAINT_OUTEP(0) | DAINT_INEP(0));
+
+}
+
+static void expect_data_phase_out(enum table_case tc)
+{
+	print_later("expect_data_phase_out(%c)", "0ABCDE67"[tc], 0, 0, 0, 0);
+	/* TODO: This is not yet supported */
+	report_error();
+	expect_setup_packet();
+}
+
+/* No Data phase, just Status phase (which is IN, since Setup is OUT) */
+static void expect_status_phase_in(enum table_case tc)
+{
+	print_later("expect_status_phase_in(%c)", "0ABCDE67"[tc], 0, 0, 0, 0);
+
+	what_am_i_doing = NO_DATA_STAGE;
+
+	/* Expect a zero-length IN for the Status phase */
+	(void) load_in_fifo(0, 0);
+
+	/* We apparently have to do this every time we transmit anything */
+	flush_in_fifo();
+
+	/* I don't think we have to do this every time, but the Programmer's
+	 * Guide says to, so...	*/
+	GR_USB_DIEPDMA(0) = (uint32_t)(cur_in_desc);
+
+	/* Blindly following instructions here, too. */
+	if (tc == TABLE_CASE_C)
+		GR_USB_DIEPCTL(0) = DXEPCTL_CNAK | DXEPCTL_EPENA;
+	else
+		GR_USB_DIEPCTL(0) = DXEPCTL_EPENA;
+
+	/* The Programmer's Guide instructions for the Normal Two-Stage Control
+	 * Transfer leave this next bit out, so we only need it if we intend to
+	 * process an Exceptional Two-Stage Control Transfer. Because obviously
+	 * we always know in advance what the host is going to do. Idiots. */
+
+	/* Be prepared to get a new Setup packet during the Status phase */
+	next_out_desc->flags =
+		DOEPDMA_RXBYTES(USB_MAX_PACKET_SIZE)
+		| DOEPDMA_IOC | DOEPDMA_LAST;
+
+	/* We've already set GR_USB_DOEPDMA(0), so just enable it. */
+	if (tc == TABLE_CASE_C)
+		GR_USB_DOEPCTL(0) = DXEPCTL_CNAK | DXEPCTL_EPENA;
+	else
+		GR_USB_DOEPCTL(0) = DXEPCTL_EPENA;
+
+	/* Get an interrupt when either IN or OUT arrives */
+	GR_USB_DAINTMSK |= (DAINT_OUTEP(0) | DAINT_INEP(0));
+}
+
+/* Handle a Setup packet that expects us to send back data in reply. Return the
+ * length of the data we're returning, or negative to indicate an error. */
+static int handle_setup_with_in_stage(enum table_case tc,
+				      struct usb_setup_packet *req)
+{
+	const void *data = 0;
+	uint32_t len = 0;
+	int ugly_hack = 0;
+	static const uint16_t zero;		/* == 0 */
+
+	print_later("handle_setup_with_in_stage(%c)", "0ABCDE67"[tc],
+		    0, 0, 0, 0);
+
+	switch (req->bRequest) {
+	case USB_REQ_GET_DESCRIPTOR: {
 		uint8_t type = req->wValue >> 8;
 		uint8_t idx = req->wValue & 0xff;
-		const uint8_t *desc;
-		int len;
 
 		switch (type) {
-		case USB_DT_DEVICE: /* Setup : Get device descriptor */
-			desc = (void *)&dev_desc;
+		case USB_DT_DEVICE:
+			data = &dev_desc;
 			len = sizeof(dev_desc);
 			break;
-		case USB_DT_CONFIGURATION: /* Setup : Get configuration desc */
-			desc = __usb_desc;
+		case USB_DT_CONFIGURATION:
+			data = __usb_desc;
 			len = USB_DESC_SIZE;
+			ugly_hack = 1;		/* see below */
 			break;
 #ifdef CONFIG_USB_BOS
-		case USB_DT_BOS: /* Setup : Get BOS descriptor */
-			desc = bos_ctx.descp;
+		case USB_DT_BOS:
+			data = bos_ctx.descp;
 			len = bos_ctx.size;
 			break;
 #endif
-		case USB_DT_STRING: /* Setup : Get string descriptor */
+		case USB_DT_STRING:
 			if (idx >= USB_STR_COUNT)
-				/* The string does not exist : STALL */
-				goto unknown_req;
-			desc = usb_strings[idx];
-			len = desc[0];
+				return -1;
+			data = usb_strings[idx];
+			len = *(uint8_t *)data;
 			break;
-		case USB_DT_DEVICE_QUALIFIER: /* Get device qualifier desc */
-			/* Not high speed : STALL next IN used as handshake */
-			goto unknown_req;
-		default: /* unhandled descriptor */
-			goto unknown_req;
+		case USB_DT_DEVICE_QUALIFIER:
+			/* We're not high speed */
+			return -1;
+		default:
+			report_error();
+			return -1;
 		}
-		/* do not send more than what the host asked for */
-		len = MIN(req->wLength, len);
+		break;
+	}
+	case USB_REQ_GET_STATUS: {
+		/* TODO: Device Status: Remote Wakeup? Self Powered? */
+		data = &zero;
+		len = sizeof(zero);
+		break;
+	}
+	case USB_REQ_GET_CONFIGURATION:
+		/* TODO: We might need this to handle USB suspend properly */
+		return -1;
+
+	case USB_REQ_SYNCH_FRAME:
+		/* Unimplemented */
+		return -1;
+
+	default:
+		report_error();
+		return -1;
+	}
+
+	/* Don't send back more than we were asked for. */
+	len = MIN(req->wLength, len);
+
+	/* Prepare the TX FIFO. If we haven't preallocated enough room in the
+	 * TX FIFO for the largest reply, we'll have to stall. This is a bug in
+	 * our code, but detecting it easily at compile time is related to the
+	 * ugly_hack directly below. */
+	if (load_in_fifo(data, len) < 0)
+		return -1;
+
+	if (ugly_hack) {
+		/*
+		 * TODO: Somebody figure out how to fix this, please.
+		 *
+		 * The USB configuration descriptor request is unique in that
+		 * it not only returns the configuration descriptor, but also
+		 * all the interface descriptors and all their endpoint
+		 * descriptors as one enormous blob. We've set up some macros
+		 * so we can declare and implement separate interfaces in
+		 * separate files just by compiling them, and all the relevant
+		 * descriptors are sorted and bundled up by the linker. But the
+		 * total length of the entire blob needs to appear in the first
+		 * configuration descriptor struct and because we don't know
+		 * that value until after linking, it can't be initialized as a
+		 * constant. So we have to compute it at run-time and shove it
+		 * in here, which also means that we have to copy the whole
+		 * blob into our TX FIFO buffer so that it's mutable. Otherwise
+		 * we could just point at it (or pretty much any other constant
+		 * struct that we wanted to send to the host). Bah.
+		 */
+		struct usb_config_descriptor *cfg =
+			(struct usb_config_descriptor *)ep0_in_buf;
+		/* set the real descriptor size */
+		cfg->wTotalLength = USB_DESC_SIZE;
+	}
+
+	return len;
+}
+
+/* Handle a Setup that comes with additional data for us. */
+static int handle_setup_with_out_stage(enum table_case tc,
+				       struct usb_setup_packet *req)
+{
+	print_later("handle_setup_with_out_stage(%c)", "0ABCDE67"[tc],
+		    0, 0, 0, 0);
+
+	/* TODO: We don't support any of these. We should. */
+	return -1;
+}
+
+/* Some Setup packets don't have a data stage at all. */
+static int handle_setup_with_no_data_stage(enum table_case tc,
+					   struct usb_setup_packet *req)
+{
+	uint8_t set_addr;
+
+	print_later("handle_setup_with_no_data_stage(%c)", "0ABCDE67"[tc],
+		    0, 0, 0, 0);
+
+	switch (req->bRequest) {
+	case USB_REQ_SET_ADDRESS:
+		/*
+		 * Set the address after the IN packet handshake.
+		 *
+		 * From the USB 2.0 spec, section 9.4.6:
+		 *
+		 * As noted elsewhere, requests actually may result in
+		 * up to three stages. In the first stage, the Setup
+		 * packet is sent to the device. In the optional second
+		 * stage, data is transferred between the host and the
+		 * device. In the final stage, status is transferred
+		 * between the host and the device. The direction of
+		 * data and status transfer depends on whether the host
+		 * is sending data to the device or the device is
+		 * sending data to the host. The Status stage transfer
+		 * is always in the opposite direction of the Data
+		 * stage. If there is no Data stage, the Status stage
+		 * is from the device to the host.
+		 *
+		 * Stages after the initial Setup packet assume the
+		 * same device address as the Setup packet. The USB
+		 * device does not change its device address until
+		 * after the Status stage of this request is completed
+		 * successfully. Note that this is a difference between
+		 * this request and all other requests. For all other
+		 * requests, the operation indicated must be completed
+		 * before the Status stage
+		 */
+		set_addr = req->wValue & 0xff;
 		/*
-		 * if we cannot transmit everything at once,
-		 * keep the remainder for the next IN packet
+		 * NOTE: Now that we've said that, we don't do it. The
+		 * hardware for this SoC knows that an IN packet will
+		 * be following the SET ADDRESS, so it waits until it
+		 * sees that happen before the address change takes
+		 * effect. If we wait until after the IN packet to
+		 * change the register, the hardware gets confused and
+		 * doesn't respond to anything.
 		 */
-		if (len >= USB_MAX_PACKET_SIZE) {
-			desc_left = len - USB_MAX_PACKET_SIZE;
-			desc_ptr = desc + USB_MAX_PACKET_SIZE;
-			len = USB_MAX_PACKET_SIZE;
+		GWRITE_FIELD(USB, DCFG, DEVADDR, set_addr);
+		print_later("SETAD 0x%02x (%d)", set_addr, set_addr, 0, 0, 0);
+		break;
+
+	case USB_REQ_SET_CONFIGURATION:
+		/* TODO: Sanity-check this? We only have one config, right? */
+		print_later("SETCFG 0x%x", req->wValue, 0, 0, 0, 0);
+		break;
+
+	case USB_REQ_CLEAR_FEATURE:
+	case USB_REQ_SET_FEATURE:
+		/* TODO: Handle DEVICE_REMOTE_WAKEUP, ENDPOINT_HALT? */
+		print_later("SET_FEATURE/CLEAR_FEATURE. Whatever...",
+			    0, 0, 0, 0, 0);
+		break;
+
+	default:
+		/* Anything else is unsupported */
+		return -1;
+	}
+
+	/* No data to transfer, go straight to the Status phase. */
+	return 0;
+}
+
+/* Dispatch an incoming Setup packet according to its type */
+static void handle_setup(enum table_case tc)
+{
+	struct usb_setup_packet *req = cur_out_desc->addr;
+	int data_phase_in = req->bmRequestType & USB_DIR_IN;
+	int data_phase_out = !data_phase_in && req->wLength;
+	int bytes = -1;				/* default is to stall */
+
+	print_later("R: %02x %02x %04x %04x %04x",
+		    req->bmRequestType, req->bRequest,
+		    req->wValue, req->wIndex, req->wLength);
+
+	if (0 == (req->bmRequestType & (USB_TYPE_MASK | USB_RECIP_MASK))) {
+		/* Standard Device requests */
+		if (data_phase_in)
+			bytes = handle_setup_with_in_stage(tc, req);
+		else if (data_phase_out)
+			bytes = handle_setup_with_out_stage(tc, req);
+		else
+			bytes = handle_setup_with_no_data_stage(tc, req);
+	} else if (USB_RECIP_INTERFACE ==
+		   (req->bmRequestType & USB_RECIP_MASK)) {
+		/* Interface-specific requests */
+		uint8_t iface = req->wIndex & 0xff;
+
+		print_later("iface %d request (vs %d)",
+			    iface, USB_IFACE_COUNT, 0, 0, 0);
+		if (iface < USB_IFACE_COUNT) {
+			bytes = usb_iface_request[iface](req);
+			print_later("  iface returned %d", bytes, 0, 0, 0, 0);
 		}
-		memcpy(ep0_buf_tx, desc, len);
-		if (type == USB_DT_CONFIGURATION) {
-			struct usb_config_descriptor *cfg =
-				(struct usb_config_descriptor *)ep0_buf_tx;
-			/* set the real descriptor size */
-			cfg->wTotalLength = USB_DESC_SIZE;
+	} else {
+		/* Something we need to add support for? */
+		report_error();
+	}
+
+	print_later("data_phase_in %d data_phase_out %d bytes %d",
+		    !!data_phase_in, !!data_phase_out, bytes, 0, 0);
+
+	/* We say "no" to unsupported and intentionally unhandled requests by
+	 * stalling the Data and/or Status stage. */
+	if (bytes < 0) {
+		/* Stall both IN and OUT. SETUP will come through anyway. */
+		stall_both_fifos();
+	} else {
+		if (data_phase_in)
+			expect_data_phase_in(tc);
+		else if (data_phase_out)
+			expect_data_phase_out(tc);
+		else
+			expect_status_phase_in(tc);
+	}
+}
+
+/* This handles both IN and OUT interrupts for EP0 */
+static void ep0_interrupt(uint32_t intr_on_out, uint32_t intr_on_in)
+{
+	uint32_t doepint, diepint;
+	enum table_case tc;
+	int sr;
+
+	/* Determine the interrupt cause and clear the bits quickly, but only
+	 * if they really apply. I don't think they're trustworthy if we didn't
+	 * actually get an interrupt. */
+	doepint = GR_USB_DOEPINT(0);
+	if (intr_on_out)
+		GR_USB_DOEPINT(0) = doepint;
+	diepint = GR_USB_DIEPINT(0);
+	if (intr_on_in)
+		GR_USB_DIEPINT(0) = diepint;
+
+	print_later("doepint%c 0x%08x diepint%c 0x%08x what %d",
+		    intr_on_out ? '!' : '_', doepint,
+		    intr_on_in ? '!' : '_', diepint,
+		    what_am_i_doing);
+
+	/* Update current and pending RX FIFO buffers */
+	if (intr_on_out && (doepint & DOEPINT_XFERCOMPL))
+		got_RX_packet();
+
+	/* Decode the situation according to Table 10-7 */
+	tc = decode_table_10_7(doepint);
+	sr = cur_out_desc->flags & DOEPDMA_SR;
+
+	print_later("cur_out_idx %d flags 0x%08x case=%c SR=%d",
+		    cur_out_idx, cur_out_desc->flags,
+		    "0ABCDE67"[tc], !!sr, 0);
+
+	switch (what_am_i_doing) {
+	case WAITING_FOR_SETUP_PACKET:
+		if (tc == TABLE_CASE_A || tc == TABLE_CASE_C) {
+			if (sr) {
+				handle_setup(tc);
+			} else {
+				report_error();
+				print_later("next_out_idx %d flags 0x%08x",
+					    next_out_idx, next_out_desc->flags,
+					    0, 0, 0);
+				expect_setup_packet();
+			}
 		}
-		ep0_in_desc.flags = DIEPDMA_LAST | DIEPDMA_BS_HOST_RDY |
-				    DIEPDMA_IOC | DIEPDMA_TXBYTES(len);
-
-		GR_USB_DIEPCTL(0) |= DXEPCTL_CNAK | DXEPCTL_EPENA;
-		ep0_out_desc.flags = DOEPDMA_RXBYTES(64) | DOEPDMA_LAST
-				   | DOEPDMA_BS_HOST_RDY | DOEPDMA_IOC;
-		GR_USB_DOEPCTL(0) |= DXEPCTL_CNAK | DXEPCTL_EPENA;
-		/* send the null OUT transaction if the transfer is complete */
-	} else if (req->bmRequestType == USB_DIR_IN &&
-		   req->bRequest == USB_REQ_GET_STATUS) {
-		uint16_t zero = 0;
-		/* Get status */
-		memcpy(ep0_buf_tx, &zero, 2);
-		ep0_in_desc.flags = DIEPDMA_LAST | DIEPDMA_BS_HOST_RDY | DIEPDMA_IOC |
-				    DIEPDMA_TXBYTES(2);
-		GR_USB_DIEPCTL(0) |= DXEPCTL_CNAK | DXEPCTL_EPENA;
-		ep0_out_desc.flags = DOEPDMA_RXBYTES(64) | DOEPDMA_LAST
-				   | DOEPDMA_BS_HOST_RDY | DOEPDMA_IOC;
-		GR_USB_DOEPCTL(0) |= DXEPCTL_CNAK | DXEPCTL_EPENA;
-	} else if (req->bmRequestType == USB_DIR_OUT) {
-		switch (req->bRequest) {
-		case USB_REQ_SET_ADDRESS:
-			/*
-			 * Set the address after the IN packet handshake.
-			 *
-			 * From the USB 2.0 spec, section 9.4.6:
-			 *
-			 * As noted elsewhere, requests actually may result in
-			 * up to three stages. In the first stage, the Setup
-			 * packet is sent to the device. In the optional second
-			 * stage, data is transferred between the host and the
-			 * device. In the final stage, status is transferred
-			 * between the host and the device. The direction of
-			 * data and status transfer depends on whether the host
-			 * is sending data to the device or the device is
-			 * sending data to the host. The Status stage transfer
-			 * is always in the opposite direction of the Data
-			 * stage. If there is no Data stage, the Status stage
-			 * is from the device to the host.
-			 *
-			 * Stages after the initial Setup packet assume the
-			 * same device address as the Setup packet. The USB
-			 * device does not change its device address until
-			 * after the Status stage of this request is completed
-			 * successfully. Note that this is a difference between
-			 * this request and all other requests. For all other
-			 * requests, the operation indicated must be completed
-			 * before the Status stage
-			 */
-			set_addr = req->wValue & 0xff;
+		/* This only happens if we're stalling, so keep doing it. */
+		if (tc == TABLE_CASE_B) {
+			print_later("Still waiting for Setup...",
+				    0, 0, 0, 0, 0);
+			stall_both_fifos();
+		}
+		break;
+
+	case DATA_STAGE_IN:
+		if (intr_on_in && (diepint & DIEPINT_XFERCOMPL)) {
+			print_later("IN is complete? Maybe? How do we know?",
+				    0, 0, 0, 0, 0);
+			/* I don't *think* we need to do this, unless we need
+			 * to transfer more data. Customer support agrees and
+			 * it shouldn't matter if the host is well-behaved, but
+			 * seems like we had issues without it.
+			 * TODO: Test this case until we know for sure. */
+			GR_USB_DIEPCTL(0) = DXEPCTL_EPENA;
+
 			/*
-			 * NOTE: Now that we've said that, we don't do it. The
-			 * hardware for this SoC knows that an IN packet will
-			 * be following the SET ADDRESS, so it waits until it
-			 * sees that happen before the address change takes
-			 * effect. If we wait until after the IN packet to
-			 * change the register, the hardware gets confused and
-			 * doesn't respond to anything.
+			 * The Programmer's Guide says (p291) to stall any
+			 * further INs, but that's stupid because it'll destroy
+			 * the packet we just tranferred to SPRAM, so don't do
+			 * that (we tried it anyway, and Bad Things happened).
+			 * Also don't stop here, but keep looking at stuff.
 			 */
-			GWRITE_FIELD(USB, DCFG, DEVADDR, set_addr);
-			print_later("SETAD 0x%02x (%d)",
-				    set_addr, set_addr, 0, 0, 0);
-			/* still need a null IN transaction -> TX Valid */
-			ep0_in_desc.flags = DIEPDMA_LAST | DIEPDMA_BS_HOST_RDY
-				| DIEPDMA_IOC | DIEPDMA_TXBYTES(0) | DIEPDMA_SP;
-			GR_USB_DIEPCTL(0) |= DXEPCTL_CNAK | DXEPCTL_EPENA;
-			ep0_out_desc.flags = DOEPDMA_RXBYTES(64) | DOEPDMA_LAST
-					   | DOEPDMA_BS_HOST_RDY | DOEPDMA_IOC;
-			GR_USB_DOEPCTL(0) |= DXEPCTL_CNAK | DXEPCTL_EPENA;
+		}
+
+		/* But we should ignore the OUT endpoint if we didn't actually
+		 * get an OUT interrupt. */
+		if (!intr_on_out)
+			break;
+
+		if (tc == TABLE_CASE_B) {
+			print_later("IN has been detected...", 0, 0, 0, 0, 0);
+			/* The first IN packet has been seen. Keep going. */
+			GR_USB_DIEPCTL(0) = DXEPCTL_CNAK | DXEPCTL_EPENA;
+			GR_USB_DOEPCTL(0) = DXEPCTL_CNAK | DXEPCTL_EPENA;
+			break;
+		}
+		if (tc == TABLE_CASE_A) {
+			if (!sr) {
+				/* We've handled the Status phase. All done. */
+				print_later("Status phase complete",
+					    0, 0, 0, 0, 0);
+				expect_setup_packet();
+				break;
+			}
+			/* We expected an OUT, but got a Setup. Deal with it. */
+			print_later("Early Setup", 0, 0, 0, 0, 0);
+			handle_setup(tc);
 			break;
-		case USB_REQ_SET_CONFIGURATION:
-			/* uint8_t cfg = req->wValue & 0xff; */
-			print_later("SETCFG 0x%x", req->wValue, 0, 0, 0, 0);
-			/* null IN for handshake */
-			ep0_in_desc.flags = DIEPDMA_LAST | DIEPDMA_BS_HOST_RDY | DIEPDMA_IOC |
-					    DIEPDMA_TXBYTES(0) | DIEPDMA_SP;
-			GR_USB_DIEPCTL(0) |= DXEPCTL_CNAK | DXEPCTL_EPENA;
-			ep0_out_desc.flags = DOEPDMA_RXBYTES(64) | DOEPDMA_LAST
-					   | DOEPDMA_BS_HOST_RDY | DOEPDMA_IOC;
-			GR_USB_DOEPCTL(0) |= DXEPCTL_CNAK | DXEPCTL_EPENA;
+		}
+		/* From the Exceptional Control Read Transfer section ... */
+		if (tc == TABLE_CASE_C) {
+			if (sr) {
+				print_later("Early Setup w/Data packet seen",
+					    0, 0, 0, 0, 0);
+				handle_setup(tc);
+				break;
+			}
+			print_later("Status phase complete. I think...",
+				    0, 0, 0, 0, 0);
+			expect_setup_packet();
 			break;
-		default: /* unhandled request */
-			goto unknown_req;
 		}
 
-	} else {
-		goto unknown_req;
-	}
+		/* Anything else should be ignorable. Right? */
+		break;
 
-	return;
-unknown_req:
-	print_later("unknown req", 0, 0, 0, 0, 0);
-	ep0_out_desc.flags = DOEPDMA_RXBYTES(64) | DOEPDMA_LAST |
-			     DOEPDMA_BS_HOST_RDY | DOEPDMA_IOC;
-	GR_USB_DOEPCTL(0) |= DXEPCTL_CNAK | DXEPCTL_EPENA;
-	GR_USB_DIEPCTL(0) |= DXEPCTL_STALL | DXEPCTL_EPENA;
-	return;
-}
+	case NO_DATA_STAGE:
+		if (intr_on_in && (diepint & DIEPINT_XFERCOMPL)) {
+			print_later("Status phase complete", 0, 0, 0, 0, 0);
+			/* Let the IN proceed */
+			GR_USB_DIEPCTL(0) = DXEPCTL_EPENA;
+			/* We've already prepared the OUT descriptor. */
+			what_am_i_doing = WAITING_FOR_SETUP_PACKET;
+		}
 
-static void ep0_tx(void)
-{
-	uint32_t epint = GR_USB_DIEPINT(0);
+		/* Done unless we got an OUT interrupt */
+		if (!intr_on_out)
+			break;
+
+		if (tc == TABLE_CASE_B) {
+			print_later("IN has been detected...", 0, 0, 0, 0, 0);
+			/* Reenable the previously prepared OUT descriptor. */
+			GR_USB_DOEPCTL(0) = DXEPCTL_CNAK | DXEPCTL_EPENA;
+			break;
+		}
 
-	GR_USB_DIEPINT(0) = epint; /* clear IT */
+		if (tc == TABLE_CASE_A || tc == TABLE_CASE_C) {
+			if (sr) {
+				/* We expected an IN, but got a Setup. */
+				print_later("Early Setup", 0, 0, 0, 0, 0);
+				handle_setup(tc);
+				break;
+			}
+		}
 
-	/* This is where most SoCs would change the address.
-	 * We don't. See the note above. */
-	if (set_addr) {
-		print_later("STATUS SETAD 0x%02x (%d)",
-			    set_addr, set_addr, 0, 0, 0);
-		set_addr = 0;
-	}
-	if (desc_ptr) {
-		/* we have an on-going descriptor transfer */
-		int len = MIN(desc_left, USB_MAX_PACKET_SIZE);
-		memcpy(ep0_buf_tx, desc_ptr, len);
-		ep0_in_desc.flags = DIEPDMA_LAST | DIEPDMA_BS_HOST_RDY |
-				    DIEPDMA_IOC | DIEPDMA_TXBYTES(len);
-		desc_left -= len;
-		desc_ptr += len;
-		/* send the null OUT transaction if the transfer is complete */
-		GR_USB_DIEPCTL(0) |= DXEPCTL_CNAK | DXEPCTL_EPENA;
-		/* TODO set Data PID in DIEPCTL */
-		return;
+		/* Some other kind of OUT interrupt instead. */
+		report_error();
+		expect_setup_packet();
+		break;
 	}
 }
 
+/* Endpoint-specific callback for when the USB device recognizes a reset */
 static void ep0_reset(void)
 {
 	/* Reset EP0 address */
 	GWRITE_FIELD(USB, DCFG, DEVADDR, 0);
 	initialize_dma_buffers();
-
-	GR_USB_DOEPCTL(0) = DXEPCTL_MPS64 | DXEPCTL_USBACTEP |
-			    DXEPCTL_EPTYPE_CTRL |
-			    DXEPCTL_CNAK | DXEPCTL_EPENA;
-	GR_USB_DIEPCTL(0) = DXEPCTL_MPS64 | DXEPCTL_USBACTEP |
-			    DXEPCTL_EPTYPE_CTRL;
-	GR_USB_DAINTMSK = DAINT_OUTEP(0) | DAINT_INEP(0);
-
+	expect_setup_packet();
 }
-USB_DECLARE_EP(0, ep0_tx, ep0_rx, ep0_reset);
 
 /****************************************************************************/
 /* USB device initialization and shutdown routines */
@@ -602,7 +1020,8 @@ static void usb_reset(void)
 
 	print_later("usb_reset()", 0, 0, 0, 0, 0);
 
-	for (ep = 0; ep < USB_EP_COUNT; ep++)
+	ep0_reset();
+	for (ep = 1; ep < USB_EP_COUNT; ep++)
 		usb_ep_reset[ep]();
 }
 
@@ -657,7 +1076,24 @@ void usb_interrupt(void)
 		 * OEPINT and IEPINT bits from GINTSTS. */
 		uint32_t daint = GR_USB_DAINT;
 
-		for (ep = 0; ep < USB_EP_COUNT; ep++) {
+		print_later("  oepint%c iepint%c daint 0x%08x",
+			    oepint ? '!' : '_', iepint ? '!' : '_',
+			    daint, 0, 0);
+
+		/* EP0 has a combined IN/OUT handler. Only call it once, but
+		 * let it know which direction(s) had an interrupt. */
+		if (daint & (DAINT_OUTEP(0) | DAINT_INEP(0))) {
+			uint32_t intr_on_out = (oepint &&
+						(daint & DAINT_OUTEP(0)));
+			uint32_t intr_on_in = (iepint &&
+					       (daint & DAINT_INEP(0)));
+			ep0_interrupt(intr_on_out, intr_on_in);
+		}
+
+		/* Invoke the unidirectional IN and OUT functions for the other
+		 * endpoints. Each handler must clear their own bits in
+		 * DIEPINTn/DOEPINTn. */
+		for (ep = 1; ep < USB_EP_COUNT; ep++) {
 			if (oepint && (daint & DAINT_OUTEP(ep)))
 				usb_ep_rx[ep]();
 			if (iepint && (daint & DAINT_INEP(ep)))