USB polling patches - 1/3 - Fix HC polling implementations. For OHCI/UHCI/EHCI: Catches any interrupt that occurs while polling for transaction completion, and preserves the status rather than simply throwing it away. This permits us to catch transaction completion on i386 while cold (and not splhigh) and allows us to poll for completion even when interrupts are enabled. For OHCI/EHCI: Remove the dependency on hz from the timeout logic in *hci_waitintr. For OHCI: Catch a missing call to ohci_waitintr for bulk transactions. When we receive a RHSC interrupt while polling, temporarily disable it to avoid an interrupt storm. For UHCI/EHCI: When processing transactions in polling mode, temporarily remove them from the active intr lists, otherwise they will be processed twice by other transactions that are started as a result of our transaction. diff -rup dev/usb-0/ehci.c dev/usb/ehci.c --- dev/usb-0/ehci.c Wed Dec 29 11:52:27 2004 +++ dev/usb/ehci.c Fri Feb 25 15:31:11 2005 @@ -252,6 +252,11 @@ Static void ehci_dump_exfer(struct ehci #define ehci_add_intr_list(sc, ex) \ LIST_INSERT_HEAD(&(sc)->sc_intrhead, (ex), inext); +#define ehci_done_intr_list(sc, ex) \ + do { \ + LIST_REMOVE((ex), inext); \ + LIST_INSERT_HEAD(&(sc)->sc_donehead, (ex), inext); \ + } while (0) #define ehci_del_intr_list(ex) \ do { \ LIST_REMOVE((ex), inext); \ @@ -538,9 +543,14 @@ ehci_intr(void *v) if (sc->sc_bus.use_polling) { u_int32_t intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)); - if (intrs) - EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */ - return (0); + if (!(intrs & sc->sc_eintrs)) + return (0); +#ifdef DIAGNOSTIC + DPRINTFN(16, ("ehci_intr: ignored interrupt while polling\n")); +#endif + sc->sc_pintrs |= intrs; + EOWRITE4(sc, EHCI_USBSTS, intrs); + return (1); } return (ehci_intr1(sc)); @@ -562,6 +572,7 @@ ehci_intr1(ehci_softc_t *sc) } intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)); + intrs |= sc->sc_pintrs; if (!intrs) return (0); @@ -573,6 +584,7 @@ ehci_intr1(ehci_softc_t *sc) return (0); EOWRITE4(sc, EHCI_USBSTS, intrs); /* Acknowledge */ + sc->sc_pintrs = 0; sc->sc_bus.intr_context++; sc->sc_bus.no_intrs++; if (eintrs & EHCI_STS_IAA) { @@ -743,7 +755,20 @@ ehci_check_intr(ehci_softc_t *sc, struct done: DPRINTFN(12, ("ehci_check_intr: ex=%p done\n", ex)); usb_uncallout(ex->xfer.timeout_handle, ehci_timeout, ex); - ehci_idone(ex); + if (sc->sc_bus.use_polling) { + /* + * Move the transaction off the active list, otherwise other + * synchronous callbacks will try to process it again. Re-add + * it only if it is still active after processing. + */ + ehci_done_intr_list(sc, ex); + ehci_idone(ex); + if (ehci_active_intr_list(ex)) { + DPRINTF(("ehci_check_intr reactivating %p\n", ex)); + ehci_add_intr_list(sc, ex); + } + } else + ehci_idone(ex); } void @@ -861,24 +886,22 @@ ehci_idone(struct ehci_xfer *ex) } /* - * Wait here until controller claims to have an interrupt. - * Then call ehci_intr and return. Use timeout to avoid waiting - * too long. + * Poll (with timeout) for completion of the given transfer. Used for + * transactions during boot and shutdown, when interrupts are not enabled. */ void ehci_waitintr(ehci_softc_t *sc, usbd_xfer_handle xfer) { int timo = xfer->timeout; - int usecs; u_int32_t intrs; xfer->status = USBD_IN_PROGRESS; - for (usecs = timo * 1000000 / hz; usecs > 0; usecs -= 1000) { + for (; timo >= 0; timo--) { usb_delay_ms(&sc->sc_bus, 1); if (sc->sc_dying) break; - intrs = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)) & - sc->sc_eintrs; + intrs = (EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)) | + sc->sc_pintrs) & sc->sc_eintrs; DPRINTFN(15,("ehci_waitintr: 0x%04x\n", intrs)); #ifdef EHCI_DEBUG if (ehcidebug > 15) @@ -905,14 +928,15 @@ ehci_poll(struct usbd_bus *bus) #ifdef EHCI_DEBUG static int last; int new; - new = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)); + new = EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)) | sc->sc_pintrs; if (new != last) { DPRINTFN(10,("ehci_poll: intrs=0x%04x\n", new)); last = new; } #endif - if (EOREAD4(sc, EHCI_USBSTS) & sc->sc_eintrs) + if ((EHCI_STS_INTRS(EOREAD4(sc, EHCI_USBSTS)) | sc->sc_pintrs) & + sc->sc_eintrs) ehci_intr1(sc); } diff -rup dev/usb-0/ehcivar.h dev/usb/ehcivar.h --- dev/usb-0/ehcivar.h Wed Dec 29 11:52:27 2004 +++ dev/usb/ehcivar.h Fri Feb 25 15:31:11 2005 @@ -112,6 +112,7 @@ typedef struct ehci_softc { struct ehci_soft_islot sc_islots[EHCI_INTRQHS]; LIST_HEAD(, ehci_xfer) sc_intrhead; + LIST_HEAD(, ehci_xfer) sc_donehead; ehci_soft_qh_t *sc_freeqhs; ehci_soft_qtd_t *sc_freeqtds; @@ -124,6 +125,7 @@ typedef struct ehci_softc { char sc_softwake; u_int32_t sc_eintrs; + u_int32_t sc_pintrs; /* interrupts caught while polling */ ehci_soft_qh_t *sc_async_head; SIMPLEQ_HEAD(, usbd_xfer) sc_free_xfers; /* free xfers */ diff -rup dev/usb-0/ohci.c dev/usb/ohci.c --- dev/usb-0/ohci.c Mon Dec 27 08:41:40 2004 +++ dev/usb/ohci.c Fri Feb 25 15:31:11 2005 @@ -1088,15 +1088,28 @@ ohci_intr(void *p) return (0); /* If we get an interrupt while polling, then just ignore it. */ - if (!cold && sc->sc_bus.use_polling) { + if (sc->sc_bus.use_polling) { + u_int32_t intrs = OREAD4(sc, OHCI_INTERRUPT_STATUS); #ifdef DIAGNOSTIC static struct timeval ohci_intr_tv; - if ((OREAD4(sc, OHCI_INTERRUPT_STATUS) & sc->sc_eintrs) && - usbd_ratecheck(&ohci_intr_tv)) + if ((intrs & sc->sc_eintrs) && usbd_ratecheck(&ohci_intr_tv)) DPRINTFN(16, ("ohci_intr: ignored interrupt while polling\n")); #endif - return (0); + if (!(intrs & sc->sc_eintrs)) + return (0); + + /* + * Disable the RHSC interrupt to avoid an interrupt storm. + * It will be re-enabled when polling continues. + */ + if (intrs & OHCI_RHSC) + OWRITE4(sc, OHCI_INTERRUPT_DISABLE, OHCI_RHSC); + + sc->sc_pintrs |= intrs; + intrs &= ~OHCI_MIE; + OWRITE4(sc, OHCI_INTERRUPT_STATUS, intrs); + return (1); } return (ohci_intr1(sc)); @@ -1123,11 +1136,14 @@ ohci_intr1(ohci_softc_t *sc) if (done != 0) { if (done & ~OHCI_DONE_INTRS) intrs = OHCI_WDH; - if (done & OHCI_DONE_INTRS) + if (done & OHCI_DONE_INTRS) { intrs |= OREAD4(sc, OHCI_INTERRUPT_STATUS); + intrs |= sc->sc_pintrs; + } sc->sc_hcca->hcca_done_head = 0; } else { intrs = OREAD4(sc, OHCI_INTERRUPT_STATUS); + intrs |= sc->sc_pintrs; /* If we've flushed out a WDH then reread */ if (intrs & OHCI_WDH) { done = le32toh(sc->sc_hcca->hcca_done_head); @@ -1140,6 +1156,7 @@ ohci_intr1(ohci_softc_t *sc) intrs &= ~OHCI_MIE; OWRITE4(sc, OHCI_INTERRUPT_STATUS, intrs); /* Acknowledge */ + sc->sc_pintrs = 0; eintrs = intrs & sc->sc_eintrs; if (!eintrs) return (0); @@ -1593,15 +1610,15 @@ void ohci_waitintr(ohci_softc_t *sc, usbd_xfer_handle xfer) { int timo = xfer->timeout; - int usecs; u_int32_t intrs; xfer->status = USBD_IN_PROGRESS; - for (usecs = timo * 1000000 / hz; usecs > 0; usecs -= 1000) { + for (; timo >= 0; timo--) { usb_delay_ms(&sc->sc_bus, 1); if (sc->sc_dying) break; - intrs = OREAD4(sc, OHCI_INTERRUPT_STATUS) & sc->sc_eintrs; + intrs = (OREAD4(sc, OHCI_INTERRUPT_STATUS) | sc->sc_pintrs) + & sc->sc_eintrs; DPRINTFN(15,("ohci_waitintr: 0x%04x\n", intrs)); #ifdef OHCI_DEBUG if (ohcidebug > 15) @@ -1628,14 +1645,14 @@ ohci_poll(struct usbd_bus *bus) #ifdef OHCI_DEBUG static int last; int new; - new = OREAD4(sc, OHCI_INTERRUPT_STATUS); + new = OREAD4(sc, OHCI_INTERRUPT_STATUS) | sc->sc_pintrs; if (new != last) { DPRINTFN(10,("ohci_poll: intrs=0x%04x\n", new)); last = new; } #endif - if (OREAD4(sc, OHCI_INTERRUPT_STATUS) & sc->sc_eintrs) + if ((OREAD4(sc, OHCI_INTERRUPT_STATUS) | sc->sc_pintrs) & sc->sc_eintrs) ohci_intr1(sc); } @@ -2929,6 +2946,9 @@ ohci_device_bulk_start(usbd_xfer_handle #endif splx(s); + + if (sc->sc_bus.use_polling) + ohci_waitintr(sc, xfer); return (USBD_IN_PROGRESS); } diff -rup dev/usb-0/ohcivar.h dev/usb/ohcivar.h --- dev/usb-0/ohcivar.h Tue Jul 8 23:19:09 2003 +++ dev/usb/ohcivar.h Fri Feb 25 15:31:11 2005 @@ -96,6 +96,7 @@ typedef struct ohci_softc { u_int sc_bws[OHCI_NO_INTRS]; u_int32_t sc_eintrs; + u_int32_t sc_pintrs; /* interrupts caught while polling */ ohci_soft_ed_t *sc_isoc_head; ohci_soft_ed_t *sc_ctrl_head; ohci_soft_ed_t *sc_bulk_head; diff -rup dev/usb-0/uhci.c dev/usb/uhci.c --- dev/usb-0/uhci.c Thu Nov 11 22:15:48 2004 +++ dev/usb/uhci.c Fri Feb 25 15:31:11 2005 @@ -360,6 +360,11 @@ struct usbd_pipe_methods uhci_device_iso #define uhci_add_intr_info(sc, ii) \ LIST_INSERT_HEAD(&(sc)->sc_intrhead, (ii), list) +#define uhci_done_intr_info(sc, ii) \ + do { \ + LIST_REMOVE((ii), list); \ + LIST_INSERT_HEAD(&(sc)->sc_donehead, (ii), list); \ + } while (0) #define uhci_del_intr_info(ii) \ do { \ LIST_REMOVE((ii), list); \ @@ -506,6 +511,7 @@ uhci_init(uhci_softc_t *sc) } LIST_INIT(&sc->sc_intrhead); + LIST_INIT(&sc->sc_donehead); SIMPLEQ_INIT(&sc->sc_free_xfers); @@ -1178,10 +1184,16 @@ uhci_intr(void *arg) return (0); if (sc->sc_bus.use_polling) { + int status = UREAD2(sc, UHCI_STS) & UHCI_STS_ALLINTRS; + + if (! status) + return (0); #ifdef DIAGNOSTIC DPRINTFN(16, ("uhci_intr: ignored interrupt while polling\n")); #endif - return (0); + sc->sc_pintrs |= status; + UWRITE2(sc, UHCI_STS, status); /* acknowledge the ints */ + return (1); } return (uhci_intr1(sc)); } @@ -1200,6 +1212,7 @@ uhci_intr1(uhci_softc_t *sc) #endif status = UREAD2(sc, UHCI_STS) & UHCI_STS_ALLINTRS; + status |= sc->sc_pintrs; if (status == 0) /* The interrupt was not for us. */ return (0); @@ -1207,6 +1220,7 @@ uhci_intr1(uhci_softc_t *sc) printf("%s: interrupt while not operating ignored\n", USBDEVNAME(sc->sc_bus.bdev)); UWRITE2(sc, UHCI_STS, status); /* acknowledge the ints */ + sc->sc_pintrs = 0; return (0); } @@ -1245,6 +1259,7 @@ uhci_intr1(uhci_softc_t *sc) if (!ack) return (0); /* nothing to acknowledge */ UWRITE2(sc, UHCI_STS, ack); /* acknowledge the ints */ + sc->sc_pintrs = 0; sc->sc_bus.no_intrs++; usb_schedsoftintr(&sc->sc_bus); @@ -1348,7 +1363,20 @@ uhci_check_intr(uhci_softc_t *sc, uhci_i done: DPRINTFN(12, ("uhci_check_intr: ii=%p done\n", ii)); usb_uncallout(ii->xfer->timeout_handle, uhci_timeout, ii); - uhci_idone(ii); + if (sc->sc_bus.use_polling) { + /* + * Move the transaction off the active list, otherwise other + * synchronous callbacks will try to process it again. Re-add + * it only if it is still active after processing. + */ + uhci_done_intr_info(sc, ii); + uhci_idone(ii); + if (uhci_active_intr_info(ii)) { + DPRINTF(("uhci_check_intr reactivating %p\n", ii)); + uhci_add_intr_info(sc, ii); + } + } else + uhci_idone(ii); } /* Called at splusb() */ @@ -1514,10 +1542,8 @@ uhci_timeout_task(void *addr) } /* - * Wait here until controller claims to have an interrupt. - * Then call uhci_intr and return. Use timeout to avoid waiting - * too long. - * Only used during boot when interrupts are not enabled yet. + * Poll (with timeout) for completion of the given transfer. Used for + * transactions during boot and shutdown, when interrupts are not enabled. */ void uhci_waitintr(uhci_softc_t *sc, usbd_xfer_handle xfer) @@ -1530,12 +1556,11 @@ uhci_waitintr(uhci_softc_t *sc, usbd_xfe xfer->status = USBD_IN_PROGRESS; for (; timo >= 0; timo--) { usb_delay_ms(&sc->sc_bus, 1); - DPRINTFN(20,("uhci_waitintr: 0x%04x\n", UREAD2(sc, UHCI_STS))); - if (UREAD2(sc, UHCI_STS) & UHCI_STS_ALLINTRS) { - uhci_intr1(sc); - if (xfer->status != USBD_IN_PROGRESS) - return; - } + if (sc->sc_dying) + break; + uhci_intr1(sc); + if (xfer->status != USBD_IN_PROGRESS) + return; } /* Timeout */ @@ -1556,7 +1581,7 @@ uhci_poll(struct usbd_bus *bus) { uhci_softc_t *sc = (uhci_softc_t *)bus; - if (UREAD2(sc, UHCI_STS) & UHCI_STS_ALLINTRS) + if (sc->sc_pintrs != 0 || UREAD2(sc, UHCI_STS) & UHCI_STS_ALLINTRS) uhci_intr1(sc); } diff -rup dev/usb-0/uhcivar.h dev/usb/uhcivar.h --- dev/usb-0/uhcivar.h Tue Jul 8 23:19:09 2003 +++ dev/usb/uhcivar.h Fri Feb 25 15:31:11 2005 @@ -170,7 +170,10 @@ typedef struct uhci_softc { char sc_suspend; char sc_dying; + int sc_pintrs; /* Interrupts caught while polling */ + LIST_HEAD(, uhci_intr_info) sc_intrhead; + LIST_HEAD(, uhci_intr_info) sc_donehead; /* Info for the root hub interrupt "pipe". */ int sc_ival; /* time between root hub intrs */