dev/virtio/net/virtio-net.c - trusty/lk/common - Git at Google

 /*
  * Copyright (c) 2015 Travis Geiselbrecht
  *
  * Permission is hereby granted, free of charge, to any person obtaining
  * a copy of this software and associated documentation files
  * (the "Software"), to deal in the Software without restriction,
  * including without limitation the rights to use, copy, modify, merge,
  * publish, distribute, sublicense, and/or sell copies of the Software,
  * and to permit persons to whom the Software is furnished to do so,
  * subject to the following conditions:
  *
  * The above copyright notice and this permission notice shall be
  * included in all copies or substantial portions of the Software.
  *
  * THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
  * EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
  * MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
  * IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
  * CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
  * TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
  * SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
  */
 #include <dev/virtio/net.h>

 #include <debug.h>
 #include <assert.h>
 #include <trace.h>
 #include <compiler.h>
 #include <list.h>
 #include <string.h>
 #include <err.h>
 #include <kernel/thread.h>
 #include <kernel/event.h>
 #include <kernel/spinlock.h>
 #include <kernel/vm.h>
 #include <lib/pktbuf.h>
 #include <lib/minip.h>

 #define LOCAL_TRACE 0

 struct virtio_net_config {
     uint8_t mac[6];
     uint16_t status;
     uint16_t max_virtqueue_pairs;
 } __PACKED;

 struct virtio_net_hdr {
     uint8_t  flags;
     uint8_t  gso_type;
     uint16_t hdr_len;
     uint16_t gso_size;
     uint16_t csum_start;
     uint16_t csum_offset;
     uint16_t num_buffers; // unused in tx
 } __PACKED;

 #define VIRTIO_NET_F_CSUM                   (1<<0)
 #define VIRTIO_NET_F_GUEST_CSUM             (1<<1)
 #define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS    (1<<2)
 #define VIRTIO_NET_F_MAC                    (1<<5)
 #define VIRTIO_NET_F_GSO                    (1<<6)
 #define VIRTIO_NET_F_GUEST_TSO4             (1<<7)
 #define VIRTIO_NET_F_GUEST_TSO6             (1<<8)
 #define VIRTIO_NET_F_GUEST_ECN              (1<<9)
 #define VIRTIO_NET_F_GUEST_UFO              (1<<10)
 #define VIRTIO_NET_F_HOST_TSO4              (1<<11)
 #define VIRTIO_NET_F_HOST_TSO6              (1<<12)
 #define VIRTIO_NET_F_HOST_ECN               (1<<13)
 #define VIRTIO_NET_F_HOST_UFO               (1<<14)
 #define VIRTIO_NET_F_MRG_RXBUF              (1<<15)
 #define VIRTIO_NET_F_STATUS                 (1<<16)
 #define VIRTIO_NET_F_CTRL_VQ                (1<<17)
 #define VIRTIO_NET_F_CTRL_RX                (1<<18)
 #define VIRTIO_NET_F_CTRL_VLAN              (1<<19)
 #define VIRTIO_NET_F_GUEST_ANNOUNCE         (1<<21)
 #define VIRTIO_NET_F_MQ                     (1<<22)
 #define VIRTIO_NET_F_CTRL_MAC_ADDR          (1<<23)

 #define VIRTIO_NET_S_LINK_UP                (1<<0)
 #define VIRTIO_NET_S_ANNOUNCE               (1<<1)

 #define TX_RING_SIZE 16
 #define RX_RING_SIZE 16

 #define RING_RX 0
 #define RING_TX 1

 #define VIRTIO_NET_MSS 1514

 struct virtio_net_dev {
     struct virtio_device *dev;
     bool started;

     struct virtio_net_config *config;

     spin_lock_t lock;
     event_t rx_event;

     /* list of active tx/rx packets to be freed at irq time */
     pktbuf_t *pending_tx_packet[TX_RING_SIZE];
     pktbuf_t *pending_rx_packet[RX_RING_SIZE];

     uint tx_pending_count;
     struct list_node completed_rx_queue;
 };

 static enum handler_return virtio_net_irq_driver_callback(struct virtio_device *dev, uint ring, const struct vring_used_elem *e);
 static int virtio_net_rx_worker(void *arg);
 static status_t virtio_net_queue_rx(struct virtio_net_dev *ndev, pktbuf_t *p);

 // XXX remove need for this
 static struct virtio_net_dev *the_ndev;

 static void dump_feature_bits(uint32_t feature)
 {
     printf("virtio-net host features (0x%x):", feature);
     if (feature & VIRTIO_NET_F_CSUM) printf(" CSUM");
     if (feature & VIRTIO_NET_F_GUEST_CSUM) printf(" GUEST_CSUM");
     if (feature & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) printf(" CTRL_GUEST_OFFLOADS");
     if (feature & VIRTIO_NET_F_MAC) printf(" MAC");
     if (feature & VIRTIO_NET_F_GSO) printf(" GSO");
     if (feature & VIRTIO_NET_F_GUEST_TSO4) printf(" GUEST_TSO4");
     if (feature & VIRTIO_NET_F_GUEST_TSO6) printf(" GUEST_TSO6");
     if (feature & VIRTIO_NET_F_GUEST_ECN) printf(" GUEST_ECN");
     if (feature & VIRTIO_NET_F_GUEST_UFO) printf(" GUEST_UFO");
     if (feature & VIRTIO_NET_F_HOST_TSO4) printf(" HOST_TSO4");
     if (feature & VIRTIO_NET_F_HOST_TSO6) printf(" HOST_TSO6");
     if (feature & VIRTIO_NET_F_HOST_ECN) printf(" HOST_ECN");
     if (feature & VIRTIO_NET_F_HOST_UFO) printf(" HOST_UFO");
     if (feature & VIRTIO_NET_F_MRG_RXBUF) printf(" MRG_RXBUF");
     if (feature & VIRTIO_NET_F_STATUS) printf(" STATUS");
     if (feature & VIRTIO_NET_F_CTRL_VQ) printf(" CTRL_VQ");
     if (feature & VIRTIO_NET_F_CTRL_RX) printf(" CTRL_RX");
     if (feature & VIRTIO_NET_F_CTRL_VLAN) printf(" CTRL_VLAN");
     if (feature & VIRTIO_NET_F_GUEST_ANNOUNCE) printf(" GUEST_ANNOUNCE");
     if (feature & VIRTIO_NET_F_MQ) printf(" MQ");
     if (feature & VIRTIO_NET_F_CTRL_MAC_ADDR) printf(" CTRL_MAC_ADDR");
     printf("\n");
 }

 status_t virtio_net_init(struct virtio_device *dev, uint32_t host_features)
 {
     LTRACEF("dev %p, host_features 0x%x\n", dev, host_features);

     /* allocate a new net device */
     struct virtio_net_dev *ndev = calloc(1, sizeof(struct virtio_net_dev));
     if (!ndev)
         return ERR_NO_MEMORY;

     ndev->dev = dev;
     dev->priv = ndev;
     ndev->started = false;

     ndev->lock = SPIN_LOCK_INITIAL_VALUE;
     event_init(&ndev->rx_event, false, EVENT_FLAG_AUTOUNSIGNAL);
     list_initialize(&ndev->completed_rx_queue);

     ndev->config = (struct virtio_net_config *)dev->config_ptr;

     /* ack and set the driver status bit */
     virtio_status_acknowledge_driver(dev);

     // XXX check features bits and ack/nak them
     dump_feature_bits(host_features);

     /* set our irq handler */
     dev->irq_driver_callback = &virtio_net_irq_driver_callback;

     /* set DRIVER_OK */
     virtio_status_driver_ok(dev);

     /* allocate a pair of virtio rings */
     virtio_alloc_ring(dev, RING_RX, RX_RING_SIZE); // rx
     virtio_alloc_ring(dev, RING_TX, TX_RING_SIZE); // tx

     the_ndev = ndev;

     return NO_ERROR;
 }

 status_t virtio_net_start(void)
 {
     if (the_ndev->started)
         return ERR_ALREADY_STARTED;

     the_ndev->started = true;

     /* start the rx worker thread */
     thread_resume(thread_create("virtio_net_rx", &virtio_net_rx_worker, (void *)the_ndev, HIGH_PRIORITY, DEFAULT_STACK_SIZE));

     /* queue up a bunch of rxes */
     for (uint i = 0; i < RX_RING_SIZE - 1; i++) {
         pktbuf_t *p = pktbuf_alloc();
         if (p) {
             virtio_net_queue_rx(the_ndev, p);
         }
     }

     return NO_ERROR;
 }

 static status_t virtio_net_queue_tx_pktbuf(struct virtio_net_dev *ndev, pktbuf_t *p2)
 {
     struct virtio_device *vdev = ndev->dev;

     uint16_t i;
     pktbuf_t *p;

     DEBUG_ASSERT(ndev);

     p = pktbuf_alloc();
     if (!p)
         return ERR_NO_MEMORY;

     /* point our header to the base of the first pktbuf */
     struct virtio_net_hdr *hdr = pktbuf_append(p, sizeof(struct virtio_net_hdr) - 2);
     memset(hdr, 0, p->dlen);

     spin_lock_saved_state_t state;
     spin_lock_irqsave(&ndev->lock, state);

     /* only queue if we have enough tx descriptors */
     if (ndev->tx_pending_count + 2 > TX_RING_SIZE)
         goto nodesc;

     /* allocate a chain of descriptors for our transfer */
     struct vring_desc *desc = virtio_alloc_desc_chain(vdev, RING_TX, 2, &i);
     if (!desc) {
         spin_unlock_irqrestore(&ndev->lock, state);

 nodesc:
         TRACEF("out of virtio tx descriptors, tx_pending_count %u\n", ndev->tx_pending_count);
         pktbuf_free(p, true);

         return ERR_NO_MEMORY;
     }

     ndev->tx_pending_count += 2;

     /* save a pointer to our pktbufs for the irq handler to free */
     LTRACEF("saving pointer to pkt in index %u and %u\n", i, desc->next);
     DEBUG_ASSERT(ndev->pending_tx_packet[i] == NULL);
     DEBUG_ASSERT(ndev->pending_tx_packet[desc->next] == NULL);
     ndev->pending_tx_packet[i] = p;
     ndev->pending_tx_packet[desc->next] = p2;

     /* set up the descriptor pointing to the header */
     desc->addr = pktbuf_data_phys(p);
     desc->len = p->dlen;
     desc->flags |= VRING_DESC_F_NEXT;

     /* set up the descriptor pointing to the buffer */
     desc = virtio_desc_index_to_desc(vdev, RING_TX, desc->next);
     desc->addr = pktbuf_data_phys(p2);
     desc->len = p2->dlen;
     desc->flags = 0;

     /* submit the transfer */
     virtio_submit_chain(vdev, RING_TX, i);

     /* kick it off */
     virtio_kick(vdev, RING_TX);

     spin_unlock_irqrestore(&ndev->lock, state);

     return NO_ERROR;
 }

 /* variant of the above function that copies the buffer into a pktbuf before sending */
 static status_t virtio_net_queue_tx(struct virtio_net_dev *ndev, const void *buf, size_t len)
 {
     DEBUG_ASSERT(ndev);
     DEBUG_ASSERT(buf);

     pktbuf_t *p = pktbuf_alloc();
     if (!p)
         return ERR_NO_MEMORY;

     /* copy the outgoing packet into the pktbuf */
     p->data = p->buffer;
     p->dlen = len;
     memcpy(p->data, buf, len);

     /* call through to the variant of the function that takes a pre-populated pktbuf */
     status_t err = virtio_net_queue_tx_pktbuf(ndev, p);
     if (err < 0) {
         pktbuf_free(p, true);
     }

     return err;
 }

 static status_t virtio_net_queue_rx(struct virtio_net_dev *ndev, pktbuf_t *p)
 {
     struct virtio_device *vdev = ndev->dev;

     DEBUG_ASSERT(ndev);
     DEBUG_ASSERT(p);

     /* point our header to the base of the pktbuf */
     p->data = p->buffer;
     struct virtio_net_hdr *hdr = (struct virtio_net_hdr *)p->data;
     memset(hdr, 0, sizeof(struct virtio_net_hdr) - 2);

     p->dlen = sizeof(struct virtio_net_hdr) - 2 + VIRTIO_NET_MSS;

     spin_lock_saved_state_t state;
     spin_lock_irqsave(&ndev->lock, state);

     /* allocate a chain of descriptors for our transfer */
     uint16_t i;
     struct vring_desc *desc = virtio_alloc_desc_chain(vdev, RING_RX, 1, &i);
     DEBUG_ASSERT(desc); /* shouldn't be possible not to have a descriptor ready */

     /* save a pointer to our pktbufs for the irq handler to use */
     DEBUG_ASSERT(ndev->pending_rx_packet[i] == NULL);
     ndev->pending_rx_packet[i] = p;

     /* set up the descriptor pointing to the header */
     desc->addr = pktbuf_data_phys(p);
     desc->len = p->dlen;
     desc->flags = VRING_DESC_F_WRITE;

     /* submit the transfer */
     virtio_submit_chain(vdev, RING_RX, i);

     /* kick it off */
     virtio_kick(vdev, RING_RX);

     spin_unlock_irqrestore(&ndev->lock, state);

     return NO_ERROR;
 }

 static enum handler_return virtio_net_irq_driver_callback(struct virtio_device *dev, uint ring, const struct vring_used_elem *e)
 {
     struct virtio_net_dev *ndev = (struct virtio_net_dev *)dev->priv;

     LTRACEF("dev %p, ring %u, e %p, id %u, len %u\n", dev, ring, e, e->id, e->len);

     spin_lock(&ndev->lock);

     /* parse our descriptor chain, add back to the free queue */
     uint16_t i = e->id;
     for (;;) {
         int next;
         struct vring_desc *desc = virtio_desc_index_to_desc(dev, ring, i);

         if (desc->flags & VRING_DESC_F_NEXT) {
             next = desc->next;
         } else {
             /* end of chain */
             next = -1;
         }

         virtio_free_desc(dev, ring, i);

         if (ring == RING_RX) {
             /* put the freed rx buffer in a queue */
             pktbuf_t *p = ndev->pending_rx_packet[i];
             ndev->pending_rx_packet[i] = NULL;

             DEBUG_ASSERT(p);
             LTRACEF("rx pktbuf %p filled\n", p);

             /* trim the pktbuf according to the written length in the used element descriptor */
             if (e->len > (sizeof(struct virtio_net_hdr) - 2 + VIRTIO_NET_MSS)) {
                 TRACEF("bad used len on RX %u\n", e->len);
                 p->dlen = 0;
             } else {
                 p->dlen = e->len;
             }

             list_add_tail(&ndev->completed_rx_queue, &p->list);
         } else { // ring == RING_TX
             /* free the pktbuf associated with the tx packet we just consumed */
             pktbuf_t *p = ndev->pending_tx_packet[i];
             ndev->pending_tx_packet[i] = NULL;
             ndev->tx_pending_count--;

             DEBUG_ASSERT(p);
             LTRACEF("freeing pktbuf %p\n", p);

             pktbuf_free(p, false);
         }

         if (next < 0)
             break;
         i = next;
     }

     spin_unlock(&ndev->lock);

     /* if rx ring, signal our event */
     if (ring == 0) {
         event_signal(&ndev->rx_event, false);
     }

     return INT_RESCHEDULE;
 }

 static int virtio_net_rx_worker(void *arg)
 {
     struct virtio_net_dev *ndev = (struct virtio_net_dev *)arg;

     for (;;) {
         event_wait(&ndev->rx_event);

         /* pull some packets from the received queue */
         for (;;) {
             spin_lock_saved_state_t state;
             spin_lock_irqsave(&ndev->lock, state);

             pktbuf_t *p = list_remove_head_type(&ndev->completed_rx_queue, pktbuf_t, list);

             spin_unlock_irqrestore(&ndev->lock, state);

             if (!p)
                 break; /* nothing left in the queue, go back to waiting */

             LTRACEF("got packet len %u\n", p->dlen);

             /* process our packet */
             struct virtio_net_hdr *hdr = pktbuf_consume(p, sizeof(struct virtio_net_hdr) - 2);
             if (hdr) {
                 /* call up into the stack */
                 minip_rx_driver_callback(p);
             }

             /* requeue the pktbuf in the rx queue */
             virtio_net_queue_rx(ndev, p);
         }
     }
     return 0;
 }

 int virtio_net_found(void)
 {
     return the_ndev ? 1 : 0;
 }

 status_t virtio_net_get_mac_addr(uint8_t mac_addr[6])
 {
     if (!the_ndev)
         return ERR_NOT_FOUND;

     memcpy(mac_addr, the_ndev->config->mac, 6);

     return NO_ERROR;
 }

 status_t virtio_net_send_minip_pkt(pktbuf_t *p)
 {
     LTRACEF("p %p, dlen %u, flags 0x%x\n", p, p->dlen, p->flags);

     DEBUG_ASSERT(p && p->dlen);

     if ((p->flags & PKTBUF_FLAG_EOF) == 0) {
         /* can't handle multi part packets yet */
         PANIC_UNIMPLEMENTED;

         return ERR_NOT_IMPLEMENTED;
     }

     /* hand the pktbuf off to the nic, it owns the pktbuf from now on out unless it fails */
     status_t err = virtio_net_queue_tx_pktbuf(the_ndev, p);
     if (err < 0) {
         pktbuf_free(p, true);
     }

     return err;
 }
	/*
	* Copyright (c) 2015 Travis Geiselbrecht
	*
	* Permission is hereby granted, free of charge, to any person obtaining
	* a copy of this software and associated documentation files
	* (the "Software"), to deal in the Software without restriction,
	* including without limitation the rights to use, copy, modify, merge,
	* publish, distribute, sublicense, and/or sell copies of the Software,
	* and to permit persons to whom the Software is furnished to do so,
	* subject to the following conditions:
	*
	* The above copyright notice and this permission notice shall be
	* included in all copies or substantial portions of the Software.
	*
	* THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND,
	* EXPRESS OR IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF
	* MERCHANTABILITY, FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT.
	* IN NO EVENT SHALL THE AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY
	* CLAIM, DAMAGES OR OTHER LIABILITY, WHETHER IN AN ACTION OF CONTRACT,
	* TORT OR OTHERWISE, ARISING FROM, OUT OF OR IN CONNECTION WITH THE
	* SOFTWARE OR THE USE OR OTHER DEALINGS IN THE SOFTWARE.
	*/
	#include <dev/virtio/net.h>

	#include <debug.h>
	#include <assert.h>
	#include <trace.h>
	#include <compiler.h>
	#include <list.h>
	#include <string.h>
	#include <err.h>
	#include <kernel/thread.h>
	#include <kernel/event.h>
	#include <kernel/spinlock.h>
	#include <kernel/vm.h>
	#include <lib/pktbuf.h>
	#include <lib/minip.h>

	#define LOCAL_TRACE 0

	struct virtio_net_config {
	uint8_t mac[6];
	uint16_t status;
	uint16_t max_virtqueue_pairs;
	} __PACKED;

	struct virtio_net_hdr {
	uint8_t flags;
	uint8_t gso_type;
	uint16_t hdr_len;
	uint16_t gso_size;
	uint16_t csum_start;
	uint16_t csum_offset;
	uint16_t num_buffers; // unused in tx
	} __PACKED;

	#define VIRTIO_NET_F_CSUM (1<<0)
	#define VIRTIO_NET_F_GUEST_CSUM (1<<1)
	#define VIRTIO_NET_F_CTRL_GUEST_OFFLOADS (1<<2)
	#define VIRTIO_NET_F_MAC (1<<5)
	#define VIRTIO_NET_F_GSO (1<<6)
	#define VIRTIO_NET_F_GUEST_TSO4 (1<<7)
	#define VIRTIO_NET_F_GUEST_TSO6 (1<<8)
	#define VIRTIO_NET_F_GUEST_ECN (1<<9)
	#define VIRTIO_NET_F_GUEST_UFO (1<<10)
	#define VIRTIO_NET_F_HOST_TSO4 (1<<11)
	#define VIRTIO_NET_F_HOST_TSO6 (1<<12)
	#define VIRTIO_NET_F_HOST_ECN (1<<13)
	#define VIRTIO_NET_F_HOST_UFO (1<<14)
	#define VIRTIO_NET_F_MRG_RXBUF (1<<15)
	#define VIRTIO_NET_F_STATUS (1<<16)
	#define VIRTIO_NET_F_CTRL_VQ (1<<17)
	#define VIRTIO_NET_F_CTRL_RX (1<<18)
	#define VIRTIO_NET_F_CTRL_VLAN (1<<19)
	#define VIRTIO_NET_F_GUEST_ANNOUNCE (1<<21)
	#define VIRTIO_NET_F_MQ (1<<22)
	#define VIRTIO_NET_F_CTRL_MAC_ADDR (1<<23)

	#define VIRTIO_NET_S_LINK_UP (1<<0)
	#define VIRTIO_NET_S_ANNOUNCE (1<<1)

	#define TX_RING_SIZE 16
	#define RX_RING_SIZE 16

	#define RING_RX 0
	#define RING_TX 1

	#define VIRTIO_NET_MSS 1514

	struct virtio_net_dev {
	struct virtio_device *dev;
	bool started;

	struct virtio_net_config *config;

	spin_lock_t lock;
	event_t rx_event;

	/* list of active tx/rx packets to be freed at irq time */
	pktbuf_t *pending_tx_packet[TX_RING_SIZE];
	pktbuf_t *pending_rx_packet[RX_RING_SIZE];

	uint tx_pending_count;
	struct list_node completed_rx_queue;
	};

	static enum handler_return virtio_net_irq_driver_callback(struct virtio_device dev, uint ring, const struct vring_used_elem e);
	static int virtio_net_rx_worker(void *arg);
	static status_t virtio_net_queue_rx(struct virtio_net_dev ndev, pktbuf_t p);

	// XXX remove need for this
	static struct virtio_net_dev *the_ndev;

	static void dump_feature_bits(uint32_t feature)
	{
	printf("virtio-net host features (0x%x):", feature);
	if (feature & VIRTIO_NET_F_CSUM) printf(" CSUM");
	if (feature & VIRTIO_NET_F_GUEST_CSUM) printf(" GUEST_CSUM");
	if (feature & VIRTIO_NET_F_CTRL_GUEST_OFFLOADS) printf(" CTRL_GUEST_OFFLOADS");
	if (feature & VIRTIO_NET_F_MAC) printf(" MAC");
	if (feature & VIRTIO_NET_F_GSO) printf(" GSO");
	if (feature & VIRTIO_NET_F_GUEST_TSO4) printf(" GUEST_TSO4");
	if (feature & VIRTIO_NET_F_GUEST_TSO6) printf(" GUEST_TSO6");
	if (feature & VIRTIO_NET_F_GUEST_ECN) printf(" GUEST_ECN");
	if (feature & VIRTIO_NET_F_GUEST_UFO) printf(" GUEST_UFO");
	if (feature & VIRTIO_NET_F_HOST_TSO4) printf(" HOST_TSO4");
	if (feature & VIRTIO_NET_F_HOST_TSO6) printf(" HOST_TSO6");
	if (feature & VIRTIO_NET_F_HOST_ECN) printf(" HOST_ECN");
	if (feature & VIRTIO_NET_F_HOST_UFO) printf(" HOST_UFO");
	if (feature & VIRTIO_NET_F_MRG_RXBUF) printf(" MRG_RXBUF");
	if (feature & VIRTIO_NET_F_STATUS) printf(" STATUS");
	if (feature & VIRTIO_NET_F_CTRL_VQ) printf(" CTRL_VQ");
	if (feature & VIRTIO_NET_F_CTRL_RX) printf(" CTRL_RX");
	if (feature & VIRTIO_NET_F_CTRL_VLAN) printf(" CTRL_VLAN");
	if (feature & VIRTIO_NET_F_GUEST_ANNOUNCE) printf(" GUEST_ANNOUNCE");
	if (feature & VIRTIO_NET_F_MQ) printf(" MQ");
	if (feature & VIRTIO_NET_F_CTRL_MAC_ADDR) printf(" CTRL_MAC_ADDR");
	printf("\n");
	}

	status_t virtio_net_init(struct virtio_device *dev, uint32_t host_features)
	{
	LTRACEF("dev %p, host_features 0x%x\n", dev, host_features);

	/* allocate a new net device */
	struct virtio_net_dev *ndev = calloc(1, sizeof(struct virtio_net_dev));
	if (!ndev)
	return ERR_NO_MEMORY;

	ndev->dev = dev;
	dev->priv = ndev;
	ndev->started = false;

	ndev->lock = SPIN_LOCK_INITIAL_VALUE;
	event_init(&ndev->rx_event, false, EVENT_FLAG_AUTOUNSIGNAL);
	list_initialize(&ndev->completed_rx_queue);

	ndev->config = (struct virtio_net_config *)dev->config_ptr;

	/* ack and set the driver status bit */
	virtio_status_acknowledge_driver(dev);

	// XXX check features bits and ack/nak them
	dump_feature_bits(host_features);

	/* set our irq handler */
	dev->irq_driver_callback = &virtio_net_irq_driver_callback;

	/* set DRIVER_OK */
	virtio_status_driver_ok(dev);

	/* allocate a pair of virtio rings */
	virtio_alloc_ring(dev, RING_RX, RX_RING_SIZE); // rx
	virtio_alloc_ring(dev, RING_TX, TX_RING_SIZE); // tx

	the_ndev = ndev;

	return NO_ERROR;
	}

	status_t virtio_net_start(void)
	{
	if (the_ndev->started)
	return ERR_ALREADY_STARTED;

	the_ndev->started = true;

	/* start the rx worker thread */
	thread_resume(thread_create("virtio_net_rx", &virtio_net_rx_worker, (void *)the_ndev, HIGH_PRIORITY, DEFAULT_STACK_SIZE));

	/* queue up a bunch of rxes */
	for (uint i = 0; i < RX_RING_SIZE - 1; i++) {
	pktbuf_t *p = pktbuf_alloc();
	if (p) {
	virtio_net_queue_rx(the_ndev, p);
	}
	}

	return NO_ERROR;
	}

	static status_t virtio_net_queue_tx_pktbuf(struct virtio_net_dev ndev, pktbuf_t p2)
	{
	struct virtio_device *vdev = ndev->dev;

	uint16_t i;
	pktbuf_t *p;

	DEBUG_ASSERT(ndev);

	p = pktbuf_alloc();
	if (!p)
	return ERR_NO_MEMORY;

	/* point our header to the base of the first pktbuf */
	struct virtio_net_hdr *hdr = pktbuf_append(p, sizeof(struct virtio_net_hdr) - 2);
	memset(hdr, 0, p->dlen);

	spin_lock_saved_state_t state;
	spin_lock_irqsave(&ndev->lock, state);

	/* only queue if we have enough tx descriptors */
	if (ndev->tx_pending_count + 2 > TX_RING_SIZE)
	goto nodesc;

	/* allocate a chain of descriptors for our transfer */
	struct vring_desc *desc = virtio_alloc_desc_chain(vdev, RING_TX, 2, &i);
	if (!desc) {
	spin_unlock_irqrestore(&ndev->lock, state);

	nodesc:
	TRACEF("out of virtio tx descriptors, tx_pending_count %u\n", ndev->tx_pending_count);
	pktbuf_free(p, true);

	return ERR_NO_MEMORY;
	}

	ndev->tx_pending_count += 2;

	/* save a pointer to our pktbufs for the irq handler to free */
	LTRACEF("saving pointer to pkt in index %u and %u\n", i, desc->next);
	DEBUG_ASSERT(ndev->pending_tx_packet[i] == NULL);
	DEBUG_ASSERT(ndev->pending_tx_packet[desc->next] == NULL);
	ndev->pending_tx_packet[i] = p;
	ndev->pending_tx_packet[desc->next] = p2;

	/* set up the descriptor pointing to the header */
	desc->addr = pktbuf_data_phys(p);
	desc->len = p->dlen;
	desc->flags \|= VRING_DESC_F_NEXT;

	/* set up the descriptor pointing to the buffer */
	desc = virtio_desc_index_to_desc(vdev, RING_TX, desc->next);
	desc->addr = pktbuf_data_phys(p2);
	desc->len = p2->dlen;
	desc->flags = 0;

	/* submit the transfer */
	virtio_submit_chain(vdev, RING_TX, i);

	/* kick it off */
	virtio_kick(vdev, RING_TX);

	spin_unlock_irqrestore(&ndev->lock, state);

	return NO_ERROR;
	}

	/* variant of the above function that copies the buffer into a pktbuf before sending */
	static status_t virtio_net_queue_tx(struct virtio_net_dev ndev, const void buf, size_t len)
	{
	DEBUG_ASSERT(ndev);
	DEBUG_ASSERT(buf);

	pktbuf_t *p = pktbuf_alloc();
	if (!p)
	return ERR_NO_MEMORY;

	/* copy the outgoing packet into the pktbuf */
	p->data = p->buffer;
	p->dlen = len;
	memcpy(p->data, buf, len);

	/* call through to the variant of the function that takes a pre-populated pktbuf */
	status_t err = virtio_net_queue_tx_pktbuf(ndev, p);
	if (err < 0) {
	pktbuf_free(p, true);
	}

	return err;
	}

	static status_t virtio_net_queue_rx(struct virtio_net_dev ndev, pktbuf_t p)
	{
	struct virtio_device *vdev = ndev->dev;

	DEBUG_ASSERT(ndev);
	DEBUG_ASSERT(p);

	/* point our header to the base of the pktbuf */
	p->data = p->buffer;
	struct virtio_net_hdr hdr = (struct virtio_net_hdr )p->data;
	memset(hdr, 0, sizeof(struct virtio_net_hdr) - 2);

	p->dlen = sizeof(struct virtio_net_hdr) - 2 + VIRTIO_NET_MSS;

	spin_lock_saved_state_t state;
	spin_lock_irqsave(&ndev->lock, state);

	/* allocate a chain of descriptors for our transfer */
	uint16_t i;
	struct vring_desc *desc = virtio_alloc_desc_chain(vdev, RING_RX, 1, &i);
	DEBUG_ASSERT(desc); /* shouldn't be possible not to have a descriptor ready */

	/* save a pointer to our pktbufs for the irq handler to use */
	DEBUG_ASSERT(ndev->pending_rx_packet[i] == NULL);
	ndev->pending_rx_packet[i] = p;

	/* set up the descriptor pointing to the header */
	desc->addr = pktbuf_data_phys(p);
	desc->len = p->dlen;
	desc->flags = VRING_DESC_F_WRITE;

	/* submit the transfer */
	virtio_submit_chain(vdev, RING_RX, i);

	/* kick it off */
	virtio_kick(vdev, RING_RX);

	spin_unlock_irqrestore(&ndev->lock, state);

	return NO_ERROR;
	}

	static enum handler_return virtio_net_irq_driver_callback(struct virtio_device dev, uint ring, const struct vring_used_elem e)
	{
	struct virtio_net_dev ndev = (struct virtio_net_dev )dev->priv;

	LTRACEF("dev %p, ring %u, e %p, id %u, len %u\n", dev, ring, e, e->id, e->len);

	spin_lock(&ndev->lock);

	/* parse our descriptor chain, add back to the free queue */
	uint16_t i = e->id;
	for (;;) {
	int next;
	struct vring_desc *desc = virtio_desc_index_to_desc(dev, ring, i);

	if (desc->flags & VRING_DESC_F_NEXT) {
	next = desc->next;
	} else {
	/* end of chain */
	next = -1;
	}

	virtio_free_desc(dev, ring, i);

	if (ring == RING_RX) {
	/* put the freed rx buffer in a queue */
	pktbuf_t *p = ndev->pending_rx_packet[i];
	ndev->pending_rx_packet[i] = NULL;

	DEBUG_ASSERT(p);
	LTRACEF("rx pktbuf %p filled\n", p);

	/* trim the pktbuf according to the written length in the used element descriptor */
	if (e->len > (sizeof(struct virtio_net_hdr) - 2 + VIRTIO_NET_MSS)) {
	TRACEF("bad used len on RX %u\n", e->len);
	p->dlen = 0;
	} else {
	p->dlen = e->len;
	}

	list_add_tail(&ndev->completed_rx_queue, &p->list);
	} else { // ring == RING_TX
	/* free the pktbuf associated with the tx packet we just consumed */
	pktbuf_t *p = ndev->pending_tx_packet[i];
	ndev->pending_tx_packet[i] = NULL;
	ndev->tx_pending_count--;

	DEBUG_ASSERT(p);
	LTRACEF("freeing pktbuf %p\n", p);

	pktbuf_free(p, false);
	}

	if (next < 0)
	break;
	i = next;
	}

	spin_unlock(&ndev->lock);

	/* if rx ring, signal our event */
	if (ring == 0) {
	event_signal(&ndev->rx_event, false);
	}

	return INT_RESCHEDULE;
	}

	static int virtio_net_rx_worker(void *arg)
	{
	struct virtio_net_dev ndev = (struct virtio_net_dev )arg;

	for (;;) {
	event_wait(&ndev->rx_event);

	/* pull some packets from the received queue */
	for (;;) {
	spin_lock_saved_state_t state;
	spin_lock_irqsave(&ndev->lock, state);

	pktbuf_t *p = list_remove_head_type(&ndev->completed_rx_queue, pktbuf_t, list);

	spin_unlock_irqrestore(&ndev->lock, state);

	if (!p)
	break; /* nothing left in the queue, go back to waiting */

	LTRACEF("got packet len %u\n", p->dlen);

	/* process our packet */
	struct virtio_net_hdr *hdr = pktbuf_consume(p, sizeof(struct virtio_net_hdr) - 2);
	if (hdr) {
	/* call up into the stack */
	minip_rx_driver_callback(p);
	}

	/* requeue the pktbuf in the rx queue */
	virtio_net_queue_rx(ndev, p);
	}
	}
	return 0;
	}

	int virtio_net_found(void)
	{
	return the_ndev ? 1 : 0;
	}

	status_t virtio_net_get_mac_addr(uint8_t mac_addr[6])
	{
	if (!the_ndev)
	return ERR_NOT_FOUND;

	memcpy(mac_addr, the_ndev->config->mac, 6);

	return NO_ERROR;
	}

	status_t virtio_net_send_minip_pkt(pktbuf_t *p)
	{
	LTRACEF("p %p, dlen %u, flags 0x%x\n", p, p->dlen, p->flags);

	DEBUG_ASSERT(p && p->dlen);

	if ((p->flags & PKTBUF_FLAG_EOF) == 0) {
	/* can't handle multi part packets yet */
	PANIC_UNIMPLEMENTED;

	return ERR_NOT_IMPLEMENTED;
	}

	/* hand the pktbuf off to the nic, it owns the pktbuf from now on out unless it fails */
	status_t err = virtio_net_queue_tx_pktbuf(the_ndev, p);
	if (err < 0) {
	pktbuf_free(p, true);
	}

	return err;
	}