Source-Changes-HG archive
[Date Prev][Date Next][Thread Prev][Thread Next][Date Index][Thread Index][Old Index]
[src/trunk]: src/sys/dev/pci Add multiqueue support, vioif(4)
details: https://anonhg.NetBSD.org/src/rev/4252876e88c9
branches: trunk
changeset: 838384:4252876e88c9
user: yamaguchi <yamaguchi%NetBSD.org@localhost>
date: Mon Jan 14 15:00:27 2019 +0000
description:
Add multiqueue support, vioif(4)
diffstat:
sys/dev/pci/if_vioif.c | 872 +++++++++++++++++++++++++++++++++++-------------
1 files changed, 627 insertions(+), 245 deletions(-)
diffs (truncated from 1438 to 300 lines):
diff -r fedfda9b8e5b -r 4252876e88c9 sys/dev/pci/if_vioif.c
--- a/sys/dev/pci/if_vioif.c Mon Jan 14 14:57:25 2019 +0000
+++ b/sys/dev/pci/if_vioif.c Mon Jan 14 15:00:27 2019 +0000
@@ -1,4 +1,4 @@
-/* $NetBSD: if_vioif.c,v 1.45 2019/01/14 14:57:25 yamaguchi Exp $ */
+/* $NetBSD: if_vioif.c,v 1.46 2019/01/14 15:00:27 yamaguchi Exp $ */
/*
* Copyright (c) 2010 Minoura Makoto.
@@ -26,7 +26,7 @@
*/
#include <sys/cdefs.h>
-__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.45 2019/01/14 14:57:25 yamaguchi Exp $");
+__KERNEL_RCSID(0, "$NetBSD: if_vioif.c,v 1.46 2019/01/14 15:00:27 yamaguchi Exp $");
#ifdef _KERNEL_OPT
#include "opt_net_mpsafe.h"
@@ -45,6 +45,7 @@
#include <sys/sockio.h>
#include <sys/cpu.h>
#include <sys/module.h>
+#include <sys/pcq.h>
#include <dev/pci/virtioreg.h>
#include <dev/pci/virtiovar.h>
@@ -59,6 +60,7 @@
#ifdef NET_MPSAFE
#define VIOIF_MPSAFE 1
+#define VIOIF_MULTIQ 1
#endif
#ifdef SOFTINT_INTR
@@ -71,28 +73,35 @@
/* Configuration registers */
#define VIRTIO_NET_CONFIG_MAC 0 /* 8bit x 6byte */
#define VIRTIO_NET_CONFIG_STATUS 6 /* 16bit */
+#define VIRTIO_NET_CONFIG_MAX_VQ_PAIRS 8 /* 16bit */
/* Feature bits */
-#define VIRTIO_NET_F_CSUM (1<<0)
-#define VIRTIO_NET_F_GUEST_CSUM (1<<1)
-#define VIRTIO_NET_F_MAC (1<<5)
-#define VIRTIO_NET_F_GSO (1<<6)
-#define VIRTIO_NET_F_GUEST_TSO4 (1<<7)
-#define VIRTIO_NET_F_GUEST_TSO6 (1<<8)
-#define VIRTIO_NET_F_GUEST_ECN (1<<9)
-#define VIRTIO_NET_F_GUEST_UFO (1<<10)
-#define VIRTIO_NET_F_HOST_TSO4 (1<<11)
-#define VIRTIO_NET_F_HOST_TSO6 (1<<12)
-#define VIRTIO_NET_F_HOST_ECN (1<<13)
-#define VIRTIO_NET_F_HOST_UFO (1<<14)
-#define VIRTIO_NET_F_MRG_RXBUF (1<<15)
-#define VIRTIO_NET_F_STATUS (1<<16)
-#define VIRTIO_NET_F_CTRL_VQ (1<<17)
-#define VIRTIO_NET_F_CTRL_RX (1<<18)
-#define VIRTIO_NET_F_CTRL_VLAN (1<<19)
+#define VIRTIO_NET_F_CSUM __BIT(0)
+#define VIRTIO_NET_F_GUEST_CSUM __BIT(1)
+#define VIRTIO_NET_F_MAC __BIT(5)
+#define VIRTIO_NET_F_GSO __BIT(6)
+#define VIRTIO_NET_F_GUEST_TSO4 __BIT(7)
+#define VIRTIO_NET_F_GUEST_TSO6 __BIT(8)
+#define VIRTIO_NET_F_GUEST_ECN __BIT(9)
+#define VIRTIO_NET_F_GUEST_UFO __BIT(10)
+#define VIRTIO_NET_F_HOST_TSO4 __BIT(11)
+#define VIRTIO_NET_F_HOST_TSO6 __BIT(12)
+#define VIRTIO_NET_F_HOST_ECN __BIT(13)
+#define VIRTIO_NET_F_HOST_UFO __BIT(14)
+#define VIRTIO_NET_F_MRG_RXBUF __BIT(15)
+#define VIRTIO_NET_F_STATUS __BIT(16)
+#define VIRTIO_NET_F_CTRL_VQ __BIT(17)
+#define VIRTIO_NET_F_CTRL_RX __BIT(18)
+#define VIRTIO_NET_F_CTRL_VLAN __BIT(19)
+#define VIRTIO_NET_F_CTRL_RX_EXTRA __BIT(20)
+#define VIRTIO_NET_F_GUEST_ANNOUNCE __BIT(21)
+#define VIRTIO_NET_F_MQ __BIT(22)
#define VIRTIO_NET_FLAG_BITS \
VIRTIO_COMMON_FLAG_BITS \
+ "\x17""MQ" \
+ "\x16""GUEST_ANNOUNCE" \
+ "\x15""CTRL_RX_EXTRA" \
"\x14""CTRL_VLAN" \
"\x13""CTRL_RX" \
"\x12""CTRL_VQ" \
@@ -152,6 +161,11 @@
# define VIRTIO_NET_CTRL_VLAN_ADD 0
# define VIRTIO_NET_CTRL_VLAN_DEL 1
+#define VIRTIO_NET_CTRL_MQ 4
+# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_SET 0
+# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MIN 1
+# define VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX 0x8000
+
struct virtio_net_ctrl_status {
uint8_t ack;
} __packed;
@@ -171,6 +185,10 @@
uint16_t id;
} __packed;
+struct virtio_net_ctrl_mq {
+ uint16_t virtqueue_pairs;
+} __packed;
+
struct vioif_ctrl_cmdspec {
bus_dmamap_t dmamap;
void *buf;
@@ -197,12 +215,15 @@
struct virtqueue *txq_vq;
bool txq_stopping;
bool txq_link_active;
+ pcq_t *txq_intrq;
struct virtio_net_hdr *txq_hdrs;
bus_dmamap_t *txq_hdr_dmamaps;
struct mbuf **txq_mbufs;
bus_dmamap_t *txq_dmamaps;
+
+ void *txq_deferred_transmit;
};
struct vioif_rxqueue {
@@ -234,30 +255,33 @@
struct virtio_net_ctrl_rx *ctrlq_rx;
struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_uc;
struct virtio_net_ctrl_mac_tbl *ctrlq_mac_tbl_mc;
+ struct virtio_net_ctrl_mq *ctrlq_mq;
bus_dmamap_t ctrlq_cmd_dmamap;
bus_dmamap_t ctrlq_status_dmamap;
bus_dmamap_t ctrlq_rx_dmamap;
bus_dmamap_t ctrlq_tbl_uc_dmamap;
bus_dmamap_t ctrlq_tbl_mc_dmamap;
+ bus_dmamap_t ctrlq_mq_dmamap;
};
struct vioif_softc {
device_t sc_dev;
struct virtio_softc *sc_virtio;
- struct virtqueue sc_vq[3];
-#define VQ_RX 0
-#define VQ_TX 1
-#define VQ_CTRL 2
+ struct virtqueue *sc_vqs;
+
+ int sc_max_nvq_pairs;
+ int sc_req_nvq_pairs;
+ int sc_act_nvq_pairs;
uint8_t sc_mac[ETHER_ADDR_LEN];
struct ethercom sc_ethercom;
short sc_deferred_init_done;
bool sc_link_active;
- struct vioif_txqueue sc_txq;
- struct vioif_rxqueue sc_rxq;
+ struct vioif_txqueue *sc_txq;
+ struct vioif_rxqueue *sc_rxq;
bool sc_has_ctrl;
struct vioif_ctrlqueue sc_ctrlq;
@@ -271,14 +295,6 @@
#define VIRTIO_NET_TX_MAXNSEGS (16) /* XXX */
#define VIRTIO_NET_CTRL_MAC_MAXENTRIES (64) /* XXX */
-#define VIOIF_TXQ_LOCK(_q) mutex_enter((_q)->txq_lock)
-#define VIOIF_TXQ_UNLOCK(_q) mutex_exit((_q)->txq_lock)
-#define VIOIF_TXQ_LOCKED(_q) mutex_owned((_q)->txq_lock)
-
-#define VIOIF_RXQ_LOCK(_q) mutex_enter((_q)->rxq_lock)
-#define VIOIF_RXQ_UNLOCK(_q) mutex_exit((_q)->rxq_lock)
-#define VIOIF_RXQ_LOCKED(_q) mutex_owned((_q)->rxq_lock)
-
/* cfattach interface functions */
static int vioif_match(device_t, cfdata_t, void *);
static void vioif_attach(device_t, device_t, void *);
@@ -288,24 +304,28 @@
static int vioif_init(struct ifnet *);
static void vioif_stop(struct ifnet *, int);
static void vioif_start(struct ifnet *);
+static void vioif_start_locked(struct ifnet *, struct vioif_txqueue *);
+static int vioif_transmit(struct ifnet *, struct mbuf *);
+static void vioif_transmit_locked(struct ifnet *, struct vioif_txqueue *);
static int vioif_ioctl(struct ifnet *, u_long, void *);
static void vioif_watchdog(struct ifnet *);
/* rx */
-static int vioif_add_rx_mbuf(struct vioif_softc *, int);
-static void vioif_free_rx_mbuf(struct vioif_softc *, int);
-static void vioif_populate_rx_mbufs(struct vioif_softc *);
-static void vioif_populate_rx_mbufs_locked(struct vioif_softc *);
-static int vioif_rx_deq(struct vioif_softc *);
-static int vioif_rx_deq_locked(struct vioif_softc *);
+static int vioif_add_rx_mbuf(struct vioif_rxqueue *, int);
+static void vioif_free_rx_mbuf(struct vioif_rxqueue *, int);
+static void vioif_populate_rx_mbufs(struct vioif_rxqueue *);
+static void vioif_populate_rx_mbufs_locked(struct vioif_rxqueue *);
+static int vioif_rx_deq(struct vioif_rxqueue *);
+static int vioif_rx_deq_locked(struct vioif_rxqueue *);
static int vioif_rx_vq_done(struct virtqueue *);
static void vioif_rx_softint(void *);
-static void vioif_rx_drain(struct vioif_softc *);
+static void vioif_rx_drain(struct vioif_rxqueue *);
/* tx */
static int vioif_tx_vq_done(struct virtqueue *);
static int vioif_tx_vq_done_locked(struct virtqueue *);
-static void vioif_tx_drain(struct vioif_softc *);
+static void vioif_tx_drain(struct vioif_txqueue *);
+static void vioif_deferred_transmit(void *);
/* other control */
static bool vioif_is_link_up(struct vioif_softc *);
@@ -318,6 +338,9 @@
static int vioif_ctrl_vq_done(struct virtqueue *);
static int vioif_config_change(struct virtio_softc *);
static void vioif_ctl_softint(void *);
+static int vioif_ctrl_mq_vq_pairs_set(struct vioif_softc *, int);
+static void vioif_enable_interrupt_vqpairs(struct vioif_softc *);
+static void vioif_disable_interrupt_vqpairs(struct vioif_softc *);
CFATTACH_DECL_NEW(vioif, sizeof(struct vioif_softc),
vioif_match, vioif_attach, NULL, NULL);
@@ -333,6 +356,69 @@
return 0;
}
+static int
+vioif_alloc_queues(struct vioif_softc *sc)
+{
+ int nvq_pairs = sc->sc_max_nvq_pairs;
+ int nvqs = nvq_pairs * 2;
+ int i;
+
+ KASSERT(nvq_pairs <= VIRTIO_NET_CTRL_MQ_VQ_PAIRS_MAX);
+
+ sc->sc_rxq = kmem_zalloc(sizeof(sc->sc_rxq[0]) * nvq_pairs,
+ KM_NOSLEEP);
+ if (sc->sc_rxq == NULL)
+ return -1;
+
+ sc->sc_txq = kmem_zalloc(sizeof(sc->sc_txq[0]) * nvq_pairs,
+ KM_NOSLEEP);
+ if (sc->sc_txq == NULL)
+ return -1;
+
+ if (sc->sc_has_ctrl)
+ nvqs++;
+
+ sc->sc_vqs = kmem_zalloc(sizeof(sc->sc_vqs[0]) * nvqs, KM_NOSLEEP);
+ if (sc->sc_vqs == NULL)
+ return -1;
+
+ nvqs = 0;
+ for (i = 0; i < nvq_pairs; i++) {
+ sc->sc_rxq[i].rxq_vq = &sc->sc_vqs[nvqs++];
+ sc->sc_txq[i].txq_vq = &sc->sc_vqs[nvqs++];
+ }
+
+ if (sc->sc_has_ctrl)
+ sc->sc_ctrlq.ctrlq_vq = &sc->sc_vqs[nvqs++];
+
+ return 0;
+}
+
+static void
+vioif_free_queues(struct vioif_softc *sc)
+{
+ int nvq_pairs = sc->sc_max_nvq_pairs;
+ int nvqs = nvq_pairs * 2;
+
+ if (sc->sc_ctrlq.ctrlq_vq)
+ nvqs++;
+
+ if (sc->sc_txq) {
+ kmem_free(sc->sc_txq, sizeof(sc->sc_txq[0]) * nvq_pairs);
+ sc->sc_txq = NULL;
+ }
+
+ if (sc->sc_rxq) {
+ kmem_free(sc->sc_rxq, sizeof(sc->sc_rxq[0]) * nvq_pairs);
+ sc->sc_rxq = NULL;
+ }
+
+ if (sc->sc_vqs) {
+ kmem_free(sc->sc_vqs, sizeof(sc->sc_vqs[0]) * nvqs);
+ sc->sc_vqs = NULL;
+ }
+}
+
/* allocate memory */
/*
* dma memory is used for:
@@ -362,19 +448,23 @@
vioif_alloc_mems(struct vioif_softc *sc)
Home |
Main Index |
Thread Index |
Old Index