phunix/minix/net/lwip/bpfdev.c

/* LWIP service - bpfdev.c - Berkeley Packet Filter (/dev/bpf) interface */
/*
 * BPF is a cloning device: opening /dev/bpf returns a new BPF device which is
 * independent from any other opened BPF devices.  We assume that each BPF
 * device is used by one single user process, and this implementation therefore
 * does not support multiple concurrent device calls on the same BPF device.
 *
 * Packet buffering basically follows the BSD model: each BPF device that is
 * configured (that is, it has been attached to an interface) has two buffers,
 * each of the configured size: a store buffer, where new packets are stored,
 * and a hold buffer, which is typically full and awaiting retrieval through a
 * read call from userland.  The buffers are swapped ("rotated") when the store
 * buffer is filled up and the hold buffer is empty - if the hold buffer is not
 * empty is not empty either, additional packets are dropped.
 *
 * These buffers are allocated when the BPF device is attached to an interface.
 * The interface may later disappear, in which case the BPF device is detached
 * from it, allowing any final packets to be read before read requests start
 * returning I/O errors.  The buffers are freed only when the device is closed.
 */

#include "lwip.h"
#include "bpfdev.h"

#include <minix/chardriver.h>
#include <net/if.h>
#include <net/bpfdesc.h>
#include <minix/bpf.h>
#include <sys/mman.h>

/*
 * Make sure that our implementation matches the BPF version in the NetBSD
 * headers.  If they change the version number, we may have to make changes
 * here accordingly.
 */
#if BPF_MAJOR_VERSION != 1 || BPF_MINOR_VERSION != 1
#error "NetBSD BPF version has changed"
#endif

/* The number of BPF devices. */
#define NR_BPFDEV		16

/* BPF receive buffer size: allowed range and default. */
#define BPF_BUF_MIN		BPF_WORDALIGN(sizeof(struct bpf_hdr))
#define BPF_BUF_DEF		32768
#define BPF_BUF_MAX		262144

/*
 * By opening /dev/bpf, one will obtain a cloned device with a different minor
 * number, which maps to one of the BPF devices.
 */
#define BPFDEV_MINOR		0	/* minor number of /dev/bpf */
#define BPFDEV_BASE_MINOR	1	/* base minor number for BPF devices */

static struct bpfdev {
	struct bpfdev_link bpf_link;	/* structure link, MUST be first */
	TAILQ_ENTRY(bpfdev) bpf_next;	/* next on free or interface list */
	struct ifdev *bpf_ifdev;	/* associated interface, or NULL */
	unsigned int bpf_flags;		/* flags (BPFF_) */
	size_t bpf_size;		/* size of packet buffers */
	char *bpf_sbuf;			/* store buffer (mmap'd, or NULL) */
	char *bpf_hbuf;			/* hold buffer (mmap'd, or NULL) */
	size_t bpf_slen;		/* used part of store buffer */
	size_t bpf_hlen;		/* used part of hold buffer */
	struct bpf_insn *bpf_filter;	/* verified BPF filter, or NULL */
	size_t bpf_filterlen;		/* length of filter, for munmap */
	pid_t bpf_pid;			/* process ID of last using process */
	clock_t bpf_timeout;		/* timeout for read calls (0 = none) */
	struct {			/* state for pending read request */
		endpoint_t br_endpt;	/* reading endpoint, or NONE */
		cp_grant_id_t br_grant;	/* grant for reader's buffer */
		cdev_id_t br_id;	/* read request identifier */
		minix_timer_t br_timer;	/* timer for read timeout */
	} bpf_read;
	struct {			/* state for pending select request */
		endpoint_t bs_endpt;	/* selecting endpoint, or NONE */
		unsigned int bs_selops;	/* pending select operations */
	} bpf_select;
	struct {			/* packet capture statistics */
		uint64_t bs_recv;	/* # of packets run through filter */
		uint64_t bs_drop;	/* # of packets dropped: buffer full */
		uint64_t bs_capt;	/* # of packets accepted by filter */
	} bpf_stat;
} bpf_array[NR_BPFDEV];

#define BPFF_IN_USE	0x01		/* this BPF device object is in use */
#define BPFF_PROMISC	0x02		/* promiscuous mode enabled */
#define BPFF_IMMEDIATE	0x04		/* immediate mode is enabled */
#define BPFF_SEESENT	0x08		/* also process host-sent packets */
#define BPFF_HDRCMPLT	0x10		/* do not fill in link-layer source */
#define BPFF_FEEDBACK	0x20		/* feed back written packet as input */

static TAILQ_HEAD(, bpfdev_link) bpfl_freelist;	/* list of free BPF devices */

static struct bpf_stat bpf_stat;

static ssize_t bpfdev_peers(struct rmib_call *, struct rmib_node *,
	struct rmib_oldp *, struct rmib_newp *);

/* The CTL_NET NET_BPF subtree.  All nodes are dynamically numbered. */
static struct rmib_node net_bpf_table[] = {
	RMIB_INT(RMIB_RO, BPF_BUF_MAX, "maxbufsize",
	    "Maximum size for data capture buffer"), /* TODO: read-write */
	RMIB_STRUCT(RMIB_RO, sizeof(bpf_stat), &bpf_stat, "stats",
	    "BPF stats"),
	RMIB_FUNC(RMIB_RO | CTLTYPE_NODE, 0, bpfdev_peers, "peers",
	    "BPF peers"),
};

static struct rmib_node net_bpf_node =
    RMIB_NODE(RMIB_RO, net_bpf_table, "bpf", "BPF options");

/*
 * Initialize the BPF module.
 */
void
bpfdev_init(void)
{
	const int mib[] = { CTL_NET, NET_BPF };
	unsigned int slot;
	int r;

	/* Initialize data structures. */
	TAILQ_INIT(&bpfl_freelist);

	for (slot = 0; slot < __arraycount(bpf_array); slot++) {
		bpf_array[slot].bpf_flags = 0;

		TAILQ_INSERT_TAIL(&bpfl_freelist, &bpf_array[slot].bpf_link,
		    bpfl_next);
	}

	memset(&bpf_stat, 0, sizeof(bpf_stat));

	/* Register the "net.bpf" subtree with the MIB service. */
	if ((r = rmib_register(mib, __arraycount(mib), &net_bpf_node)) != OK)
		panic("unable to register net.bpf RMIB tree: %d", r);
}

/*
 * Given a BPF device object, return the corresponding minor number.
 */
static devminor_t
bpfdev_get_minor(struct bpfdev * bpfdev)
{

	assert(bpfdev != NULL);

	return BPFDEV_BASE_MINOR + (devminor_t)(bpfdev - bpf_array);
}

/*
 * Given a minor number, return the corresponding BPF device object, or NULL if
 * the minor number does not identify a BPF device.
 */
static struct bpfdev *
bpfdev_get_by_minor(devminor_t minor)
{

	if (minor < BPFDEV_BASE_MINOR ||
	    (unsigned int)minor >= BPFDEV_BASE_MINOR + __arraycount(bpf_array))
		return NULL;

	return &bpf_array[minor - BPFDEV_BASE_MINOR];
}

/*
 * Open a BPF device, returning a cloned device instance.
 */
static int
bpfdev_open(devminor_t minor, int access __unused, endpoint_t user_endpt)
{
	struct bpfdev_link *bpfl;
	struct bpfdev *bpf;

	/* Disallow opening cloned devices through device nodes. */
	if (minor != BPFDEV_MINOR)
		return ENXIO;

	if (TAILQ_EMPTY(&bpfl_freelist))
		return ENOBUFS;

	bpfl = TAILQ_FIRST(&bpfl_freelist);
	TAILQ_REMOVE(&bpfl_freelist, bpfl, bpfl_next);

	bpf = (struct bpfdev *)bpfl;

	memset(bpf, 0, sizeof(*bpf));

	bpf->bpf_flags = BPFF_IN_USE | BPFF_SEESENT;
	bpf->bpf_size = BPF_BUF_DEF;
	bpf->bpf_pid = getnpid(user_endpt);
	bpf->bpf_read.br_endpt = NONE;
	bpf->bpf_select.bs_endpt = NONE;

	return CDEV_CLONED | bpfdev_get_minor(bpf);
}

/*
 * Close a BPF device.
 */
static int
bpfdev_close(devminor_t minor)
{
	struct bpfdev *bpf;

	if ((bpf = bpfdev_get_by_minor(minor)) == NULL)
		return EINVAL;

	/*
	 * There cannot possibly be a pending read request, so we never need to
	 * cancel the read timer from here either.
	 */
	assert(bpf->bpf_read.br_endpt == NONE);

	if (bpf->bpf_sbuf != NULL) {
		assert(bpf->bpf_hbuf != NULL);

		if (munmap(bpf->bpf_sbuf, bpf->bpf_size) != 0)
			panic("munmap failed: %d", -errno);
		if (munmap(bpf->bpf_hbuf, bpf->bpf_size) != 0)
			panic("munmap failed: %d", -errno);

		bpf->bpf_sbuf = NULL;
		bpf->bpf_hbuf = NULL;
	} else
		assert(bpf->bpf_hbuf == NULL);

	if (bpf->bpf_filter != NULL) {
		assert(bpf->bpf_filterlen > 0);

		if (munmap(bpf->bpf_filter, bpf->bpf_filterlen) != 0)
			panic("munmap failed: %d", -errno);

		bpf->bpf_filter = NULL;
	}

	/*
	 * If the BPF device was attached to an interface, and that interface
	 * has not disappeared in the meantime, detach from it now.
	 */
	if (bpf->bpf_ifdev != NULL) {
		if (bpf->bpf_flags & BPFF_PROMISC)
			ifdev_clear_promisc(bpf->bpf_ifdev);

		ifdev_detach_bpf(bpf->bpf_ifdev, &bpf->bpf_link);

		bpf->bpf_ifdev = NULL;
	}

	bpf->bpf_flags = 0;		/* mark as no longer in use */

	TAILQ_INSERT_HEAD(&bpfl_freelist, &bpf->bpf_link, bpfl_next);

	return OK;
}

/*
 * Rotate buffers for the BPF device, by swapping the store buffer and the hold
 * buffer.
 */
static void
bpfdev_rotate(struct bpfdev * bpf)
{
	char *buf;
	size_t len;

	/*
	 * When rotating, the store buffer may or may not be empty, but the
	 * hold buffer must always be empty.
	 */
	assert(bpf->bpf_hlen == 0);

	buf = bpf->bpf_sbuf;
	len = bpf->bpf_slen;
	bpf->bpf_sbuf = bpf->bpf_hbuf;
	bpf->bpf_slen = bpf->bpf_hlen;
	bpf->bpf_hbuf = buf;
	bpf->bpf_hlen = len;
}

/*
 * Test whether any of the given select operations are ready on the BPF device,
 * and return the set of ready operations.
 */
static unsigned int
bpfdev_test_select(struct bpfdev * bpf, unsigned int ops)
{
	unsigned int ready_ops;

	ready_ops = 0;

	/*
	 * The BPF device is ready for reading if the hold buffer is not empty
	 * (i.e.: the store buffer has been filled up completely and was
	 * therefore rotated) or if immediate mode is set and the store buffer
	 * is not empty (i.e.: any packet is available at all).  In the latter
	 * case, the buffers will be rotated during the read.  We do not
	 * support applying the read timeout to selects and maintaining state
	 * between the select and the following read, because despite that
	 * libpcap claims that it is the right behavior, that is just insane.
	 */
	if (ops & CDEV_OP_RD) {
		if (bpf->bpf_ifdev == NULL)
			ready_ops |= CDEV_OP_RD;
		else if (bpf->bpf_hlen > 0)
			ready_ops |= CDEV_OP_RD;
		else if ((bpf->bpf_flags & BPFF_IMMEDIATE) &&
		    bpf->bpf_slen > 0)
			ready_ops |= CDEV_OP_RD;
	}

	if (ops & CDEV_OP_WR)
		ready_ops |= CDEV_OP_WR;

	return ready_ops;
}

/*
 * There has been a state change on the BPF device.  If now possible, resume a
 * pending select query, if any.
 */
static void
bpfdev_resume_select(struct bpfdev * bpf)
{
	unsigned int ops, ready_ops;
	endpoint_t endpt;

	/* First see if there is a pending select request at all. */
	if ((endpt = bpf->bpf_select.bs_endpt) == NONE)
		return;
	ops = bpf->bpf_select.bs_selops;

	assert(ops != 0);

	/* Then see if any of the pending operations are now ready. */
	if ((ready_ops = bpfdev_test_select(bpf, ops)) == 0)
		return;

	/* If so, notify VFS about the ready operations. */
	chardriver_reply_select(bpf->bpf_select.bs_endpt,
	    bpfdev_get_minor(bpf), ready_ops);

	/*
	 * Forget about the ready operations.  If that leaves no pending
	 * operations, forget about the select request altogether.
	 */
	if ((bpf->bpf_select.bs_selops &= ~ready_ops) == 0)
		bpf->bpf_select.bs_endpt = NONE;
}

/*
 * There has been a state change on the BPF device.  If now possible, resume a
 * pending read request, if any.  If the call is a result of a timeout,
 * 'is_timeout' is set.  In that case, the read request must be resumed with an
 * EAGAIN error if no packets are available, and the running timer must be
 * canceled.  Otherwise, the resumption is due to a full buffer or a
 * disappeared interface, and 'is_timeout' is not set.  In this case, the read
 * request must be resumed with an I/O error if no packets are available.
 */
static void
bpfdev_resume_read(struct bpfdev * bpf, int is_timeout)
{
	ssize_t r;

	assert(bpf->bpf_read.br_endpt != NONE);

	/*
	 * If the hold buffer is still empty, see if the store buffer has
	 * any packets to copy out.
	 */
	if (bpf->bpf_hlen == 0)
		bpfdev_rotate(bpf);

	/* Return any available packets, or otherwise an error. */
	if (bpf->bpf_hlen > 0) {
		assert(bpf->bpf_hlen <= bpf->bpf_size);

		r = sys_safecopyto(bpf->bpf_read.br_endpt,
		    bpf->bpf_read.br_grant, 0, (vir_bytes)bpf->bpf_hbuf,
		    bpf->bpf_hlen);

		if (r == OK) {
			r = (ssize_t)bpf->bpf_hlen;

			bpf->bpf_hlen = 0;

			assert(bpf->bpf_slen != bpf->bpf_size);

			/*
			 * Allow readers to get the last packets after the
			 * interface has disappeared, before getting errors.
			 */
			if (bpf->bpf_ifdev == NULL)
				bpfdev_rotate(bpf);
		}
	} else
		r = (is_timeout) ? EAGAIN : EIO;

	chardriver_reply_task(bpf->bpf_read.br_endpt, bpf->bpf_read.br_id, r);

	bpf->bpf_read.br_endpt = NONE;

	/* Was there still a timer running?  Then cancel it now. */
	if (bpf->bpf_timeout > 0 && !is_timeout)
		cancel_timer(&bpf->bpf_read.br_timer);
}

/*
 * A read timeout has triggered for the BPF device.  Wake up the pending read
 * request.
 */
static void
bpfdev_timeout(int arg)
{
	struct bpfdev *bpf;

	assert(arg >= 0 && (unsigned int)arg < __arraycount(bpf_array));

	bpf = &bpf_array[arg];

	assert(bpf->bpf_read.br_endpt != NONE);

	bpfdev_resume_read(bpf, TRUE /*is_timeout*/);
}

/*
 * Read from a BPF device.
 */
static ssize_t
bpfdev_read(devminor_t minor, uint64_t position, endpoint_t endpt,
	cp_grant_id_t grant, size_t size, int flags, cdev_id_t id)
{
	struct bpfdev *bpf;
	ssize_t r;
	int suspend;

	if ((bpf = bpfdev_get_by_minor(minor)) == NULL)
		return EINVAL;

	/* Allow only one read call at a time. */
	if (bpf->bpf_read.br_endpt != NONE)
		return EIO;

	/* Has this BPF device been configured at all yet? */
	if (bpf->bpf_sbuf == NULL)
		return EINVAL;

	/*
	 * Does the read call size match the entire buffer size?  This is a
	 * ridiculous requirement but it makes our job quite a bit easier..
	 */
	if (size != bpf->bpf_size)
		return EINVAL;

	/*
	 * Following standard receive semantics, if the interface is gone,
	 * return all the packets that were pending before returning an error.
	 * This requires extra buffer rotations after read completion, too.
	 */
	if (bpf->bpf_ifdev == NULL && bpf->bpf_hlen == 0)
		return EIO;

	/*
	 * If immediate mode is not enabled, we should always suspend the read
	 * call if the hold buffer is empty.  If immediate mode is enabled, we
	 * should only suspend the read call if both buffers are empty, and
	 * return data from the hold buffer or otherwise the store buffer,
	 * whichever is not empty.  A non-blocking call behaves as though
	 * immediate mode is enabled, except it will return EAGAIN instead of
	 * suspending the read call if both buffers are empty.  Thus, we may
	 * have to rotate buffers for both immediate mode and non-blocking
	 * calls.  The latter is necessary for libpcap to behave correctly.
	 */
	if ((flags & CDEV_NONBLOCK) || (bpf->bpf_flags & BPFF_IMMEDIATE))
		suspend = (bpf->bpf_hlen == 0 && bpf->bpf_slen == 0);
	else
		suspend = (bpf->bpf_hlen == 0);

	if (suspend) {
		if (flags & CDEV_NONBLOCK)
			return EAGAIN;

		/* Suspend the read call for later. */
		bpf->bpf_read.br_endpt = endpt;
		bpf->bpf_read.br_grant = grant;
		bpf->bpf_read.br_id = id;

		/* Set a timer if requested. */
		if (bpf->bpf_timeout > 0)
			set_timer(&bpf->bpf_read.br_timer, bpf->bpf_timeout,
			    bpfdev_timeout, (int)(bpf - bpf_array));

		return EDONTREPLY;
	}

	/* If we get here, either buffer has data; rotate buffers if needed. */
	if (bpf->bpf_hlen == 0)
		bpfdev_rotate(bpf);
	assert(bpf->bpf_hlen > 0);

	if ((r = sys_safecopyto(endpt, grant, 0, (vir_bytes)bpf->bpf_hbuf,
	    bpf->bpf_hlen)) != OK)
		return r;

	r = (ssize_t)bpf->bpf_hlen;

	bpf->bpf_hlen = 0;

	/*
	 * If the store buffer is exactly full, rotate it now.  Also, if the
	 * interface has disappeared, the store buffer will never fill up.
	 * Rotate it so that the application will get any remaining data before
	 * getting errors about the interface being gone.
	 */
	if (bpf->bpf_slen == bpf->bpf_size || bpf->bpf_ifdev == NULL)
		bpfdev_rotate(bpf);

	return r;
}

/*
 * Write to a BPF device.
 */
static ssize_t
bpfdev_write(devminor_t minor, uint64_t position, endpoint_t endpt,
	cp_grant_id_t grant, size_t size, int flags, cdev_id_t id)
{
	struct bpfdev *bpf;
	struct pbuf *pbuf, *pptr, *pcopy;
	size_t off;
	err_t err;
	int r;

	if ((bpf = bpfdev_get_by_minor(minor)) == NULL)
		return EINVAL;

	if (bpf->bpf_ifdev == NULL)
		return EINVAL;

	/* VFS skips zero-sized I/O calls right now, but that may change. */
	if (size == 0)
		return 0;	/* nothing to do */

	if (size > ifdev_get_hdrlen(bpf->bpf_ifdev) +
	    ifdev_get_mtu(bpf->bpf_ifdev))
		return EMSGSIZE;

	if ((pbuf = pchain_alloc(PBUF_LINK, size)) == NULL)
		return ENOMEM;

	/* TODO: turn this into a series of vector copies. */
	off = 0;
	for (pptr = pbuf; pptr != NULL; pptr = pptr->next) {
		if ((r = sys_safecopyfrom(endpt, grant, off,
		    (vir_bytes)pptr->payload, pptr->len)) != OK) {
			pbuf_free(pbuf);

			return r;
		}
		off += pptr->len;
	}
	assert(off == size);

	/*
	 * In feedback mode, we cannot use the same packet buffers for both
	 * output and input, so make a copy.  We do this before calling the
	 * output function, which may change part of the buffers, because the
	 * BSDs take this approach as well.
	 */
	if (bpf->bpf_flags & BPFF_FEEDBACK) {
		if ((pcopy = pchain_alloc(PBUF_LINK, size)) == NULL) {
			pbuf_free(pbuf);

			return ENOMEM;
		}

		if (pbuf_copy(pcopy, pbuf) != ERR_OK)
			panic("unexpected pbuf copy failure");
	} else
		pcopy = NULL;

	/* Pass in the packet as output, and free it again. */
	err = ifdev_output(bpf->bpf_ifdev, pbuf, NULL /*netif*/,
	    TRUE /*to_bpf*/, !!(bpf->bpf_flags & BPFF_HDRCMPLT));

	pbuf_free(pbuf);

	/* In feedback mode, pass in the copy as input, if output succeeded. */
	if (err == ERR_OK && (bpf->bpf_flags & BPFF_FEEDBACK))
		ifdev_input(bpf->bpf_ifdev, pcopy, NULL /*netif*/,
		    FALSE /*to_bpf*/);
	else if (pcopy != NULL)
		pbuf_free(pcopy);

	return (err == ERR_OK) ? (ssize_t)size : util_convert_err(err);
}

/*
 * Attach a BPF device to a network interface, using the interface name given
 * in an ifreq structure.  As side effect, allocate hold and store buffers for
 * the device.  These buffers will stay allocated until the device is closed,
 * even though the interface may disappear before that.  Return OK if the BPF
 * device was successfully attached to the interface, or a negative error code
 * otherwise.
 */
static int
bpfdev_attach(struct bpfdev * bpf, struct ifreq * ifr)
{
	struct ifdev *ifdev;
	void *sbuf, *hbuf;

	/* Find the interface with the given name. */
	ifr->ifr_name[sizeof(ifr->ifr_name) - 1] = '\0';
	if ((ifdev = ifdev_find_by_name(ifr->ifr_name)) == NULL)
		return ENXIO;

	/*
	 * Allocate a store buffer and a hold buffer.  Preallocate the memory,
	 * or we might get killed later during low-memory conditions.
	 */
	if ((sbuf = (char *)mmap(NULL, bpf->bpf_size, PROT_READ | PROT_WRITE,
	    MAP_ANON | MAP_PRIVATE | MAP_PREALLOC, -1, 0)) == MAP_FAILED)
		return ENOMEM;

	if ((hbuf = (char *)mmap(NULL, bpf->bpf_size, PROT_READ | PROT_WRITE,
	    MAP_ANON | MAP_PRIVATE | MAP_PREALLOC, -1, 0)) == MAP_FAILED) {
		(void)munmap(sbuf, bpf->bpf_size);

		return ENOMEM;
	}

	bpf->bpf_ifdev = ifdev;
	bpf->bpf_sbuf = sbuf;
	bpf->bpf_hbuf = hbuf;
	assert(bpf->bpf_slen == 0);
	assert(bpf->bpf_hlen == 0);

	ifdev_attach_bpf(ifdev, &bpf->bpf_link);

	return OK;
}

/*
 * Detach the BPF device from its interface, which is about to disappear.
 */
void
bpfdev_detach(struct bpfdev_link * bpfl)
{
	struct bpfdev *bpf = (struct bpfdev *)bpfl;

	assert(bpf->bpf_flags & BPFF_IN_USE);
	assert(bpf->bpf_ifdev != NULL);

	/*
	 * We deliberately leave the buffers allocated here, for two reasons:
	 *
	 * 1) it lets applications to read any last packets in the buffers;
	 * 2) it prevents reattaching the BPF device to another interface.
	 */
	bpf->bpf_ifdev = NULL;

	/*
	 * Resume pending read and select requests, returning any data left,
	 * or an error if none.
	 */
	if (bpf->bpf_hlen == 0)
		bpfdev_rotate(bpf);

	if (bpf->bpf_read.br_endpt != NONE)
		bpfdev_resume_read(bpf, FALSE /*is_timeout*/);

	bpfdev_resume_select(bpf);
}

/*
 * Flush the given BPF device, resetting its buffer contents and statistics
 * counters.
 */
static void
bpfdev_flush(struct bpfdev * bpf)
{

	bpf->bpf_slen = 0;
	bpf->bpf_hlen = 0;

	bpf->bpf_stat.bs_recv = 0;
	bpf->bpf_stat.bs_drop = 0;
	bpf->bpf_stat.bs_capt = 0;
}

/*
 * Install a filter program on the BPF device.  A new filter replaces any old
 * one.  A zero-sized filter simply clears a previous filter.  On success,
 * perform a flush and return OK.  On failure, return a negative error code
 * without making any modifications to the current filter.
 */
static int
bpfdev_setfilter(struct bpfdev * bpf, endpoint_t endpt, cp_grant_id_t grant)
{
	struct bpf_insn *filter;
	unsigned int count;
	size_t len;
	int r;

	if ((r = sys_safecopyfrom(endpt, grant,
	    offsetof(struct minix_bpf_program, mbf_len), (vir_bytes)&count,
	    sizeof(count))) != OK)
		return r;

	if (count > BPF_MAXINSNS)
		return EINVAL;
	len = count * sizeof(struct bpf_insn);

	if (len > 0) {
		if ((filter = (struct bpf_insn *)mmap(NULL, len,
		    PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0)) ==
		    MAP_FAILED)
			return ENOMEM;

		if ((r = sys_safecopyfrom(endpt, grant,
		    offsetof(struct minix_bpf_program, mbf_insns),
		    (vir_bytes)filter, len)) != OK) {
			(void)munmap(filter, len);

			return r;
		}

		if (!bpf_validate(filter, count)) {
			(void)munmap(filter, len);

			return EINVAL;
		}
	} else
		filter = NULL;

	if (bpf->bpf_filter != NULL)
		(void)munmap(bpf->bpf_filter, bpf->bpf_filterlen);

	bpf->bpf_filter = filter;
	bpf->bpf_filterlen = len;

	bpfdev_flush(bpf);

	return OK;
}

/*
 * Process an I/O control request on the BPF device.
 */
static int
bpfdev_ioctl(devminor_t minor, unsigned long request, endpoint_t endpt,
	cp_grant_id_t grant, int flags, endpoint_t user_endpt, cdev_id_t id)
{
	struct bpfdev *bpf;
	struct bpf_stat bs;
	struct bpf_version bv;
	struct bpf_dltlist bfl;
	struct timeval tv;
	struct ifreq ifr;
	unsigned int uval;
	int r, val;

	if ((bpf = bpfdev_get_by_minor(minor)) == NULL)
		return EINVAL;

	/*
	 * We do not support multiple concurrent requests in this module.  That
	 * not only means that we forbid a read(2) call on a BPF device object
	 * while another read(2) is already pending: we also disallow IOCTL
	 * IOCTL calls while such a read(2) call is in progress.  This
	 * restriction should never be a problem for user programs, and allows
	 * us to rely on the fact that that no settings can change between the
	 * start and end of any read call.  As a side note, pending select(2)
	 * queries may be similarly affected, and will also not be fully
	 * accurate if any options are changed while pending.
	 */
	if (bpf->bpf_read.br_endpt != NONE)
		return EIO;

	bpf->bpf_pid = getnpid(user_endpt);

	/* These are in order of the NetBSD BIOC.. IOCTL numbers. */
	switch (request) {
	case BIOCGBLEN:
		uval = bpf->bpf_size;

		return sys_safecopyto(endpt, grant, 0, (vir_bytes)&uval,
		    sizeof(uval));

	case BIOCSBLEN:
		if (bpf->bpf_sbuf != NULL)
			return EINVAL;

		if ((r = sys_safecopyfrom(endpt, grant, 0, (vir_bytes)&uval,
		    sizeof(uval))) != OK)
			return r;

		if (uval < BPF_BUF_MIN)
			uval = BPF_BUF_MIN;
		else if (uval > BPF_BUF_MAX)
			uval = BPF_BUF_MAX;

		/* Is this the right thing to do?  It doesn't matter for us. */
		uval = BPF_WORDALIGN(uval);

		if ((r = sys_safecopyto(endpt, grant, 0, (vir_bytes)&uval,
		    sizeof(uval))) != OK)
			return r;

		bpf->bpf_size = uval;

		return OK;

	case MINIX_BIOCSETF:
		return bpfdev_setfilter(bpf, endpt, grant);

	case BIOCPROMISC:
		if (bpf->bpf_ifdev == NULL)
			return EINVAL;

		if (!(bpf->bpf_flags & BPFF_PROMISC)) {
			if (!ifdev_set_promisc(bpf->bpf_ifdev))
				return EINVAL;

			bpf->bpf_flags |= BPFF_PROMISC;
		}

		return OK;

	case BIOCFLUSH:
		bpfdev_flush(bpf);

		return OK;

	case BIOCGDLT:
		if (bpf->bpf_ifdev == NULL)
			return EINVAL;

		/* TODO: support for type configuration per BPF device. */
		uval = ifdev_get_dlt(bpf->bpf_ifdev);

		return sys_safecopyto(endpt, grant, 0, (vir_bytes)&uval,
		    sizeof(uval));

	case BIOCGETIF:
		if (bpf->bpf_ifdev == NULL)
			return EINVAL;

		memset(&ifr, 0, sizeof(ifr));
		strlcpy(ifr.ifr_name, ifdev_get_name(bpf->bpf_ifdev),
		    sizeof(ifr.ifr_name));

		return sys_safecopyto(endpt, grant, 0, (vir_bytes)&ifr,
		    sizeof(ifr));

	case BIOCSETIF:
		/*
		 * Test on the presence of a buffer rather than on an interface
		 * since the latter may disappear and thus be reset to NULL, in
		 * which case we do not want to allow rebinding to another.
		 */
		if (bpf->bpf_sbuf != NULL)
			return EINVAL;

		if ((r = sys_safecopyfrom(endpt, grant, 0, (vir_bytes)&ifr,
		    sizeof(ifr))) != OK)
			return r;

		return bpfdev_attach(bpf, &ifr);

	case BIOCGSTATS:
		/*
		 * Why do we not embed a bpf_stat structure directly in the
		 * BPF device structure?  Well, bpf_stat has massive padding..
		 */
		memset(&bs, 0, sizeof(bs));
		bs.bs_recv = bpf->bpf_stat.bs_recv;
		bs.bs_drop = bpf->bpf_stat.bs_drop;
		bs.bs_capt = bpf->bpf_stat.bs_capt;

		return sys_safecopyto(endpt, grant, 0, (vir_bytes)&bs,
		    sizeof(bs));

	case BIOCIMMEDIATE:
		if ((r = sys_safecopyfrom(endpt, grant, 0, (vir_bytes)&uval,
		    sizeof(uval))) != OK)
			return r;

		if (uval)
			bpf->bpf_flags |= BPFF_IMMEDIATE;
		else
			bpf->bpf_flags &= ~BPFF_IMMEDIATE;

		return OK;

	case BIOCVERSION:
		memset(&bv, 0, sizeof(bv));
		bv.bv_major = BPF_MAJOR_VERSION;
		bv.bv_minor = BPF_MINOR_VERSION;

		return sys_safecopyto(endpt, grant, 0, (vir_bytes)&bv,
		    sizeof(bv));

	case BIOCGHDRCMPLT:
		uval = !!(bpf->bpf_flags & BPFF_HDRCMPLT);

		return sys_safecopyto(endpt, grant, 0, (vir_bytes)&uval,
		    sizeof(uval));

	case BIOCSHDRCMPLT:
		if ((r = sys_safecopyfrom(endpt, grant, 0, (vir_bytes)&uval,
		    sizeof(uval))) != OK)
			return r;

		if (uval)
			bpf->bpf_flags |= BPFF_HDRCMPLT;
		else
			bpf->bpf_flags &= ~BPFF_HDRCMPLT;

		return OK;

	case BIOCSDLT:
		if (bpf->bpf_ifdev == NULL)
			return EINVAL;

		if ((r = sys_safecopyfrom(endpt, grant, 0, (vir_bytes)&uval,
		    sizeof(uval))) != OK)
			return r;

		/* TODO: support for type configuration per BPF device. */
		if (uval != ifdev_get_dlt(bpf->bpf_ifdev))
			return EINVAL;

		return OK;

	case MINIX_BIOCGDLTLIST:
		if (bpf->bpf_ifdev == NULL)
			return EINVAL;

		if ((r = sys_safecopyfrom(endpt, grant, 0, (vir_bytes)&bfl,
		    sizeof(bfl))) != OK)
			return r;

		if (bfl.bfl_list != NULL) {
			if (bfl.bfl_len < 1)
				return ENOMEM;

			/*
			 * Copy out the 'list', which consists of one entry.
			 * If we were to produce multiple entries, we would
			 * have to check against the MINIX_BPF_MAXDLT limit.
			 */
			uval = ifdev_get_dlt(bpf->bpf_ifdev);

			if ((r = sys_safecopyto(endpt, grant,
			    offsetof(struct minix_bpf_dltlist, mbfl_list),
			    (vir_bytes)&uval, sizeof(uval))) != OK)
				return r;
		}
		bfl.bfl_len = 1;

		return sys_safecopyto(endpt, grant, 0, (vir_bytes)&bfl,
		    sizeof(bfl));

	case BIOCGSEESENT:
		uval = !!(bpf->bpf_flags & BPFF_SEESENT);

		return sys_safecopyto(endpt, grant, 0, (vir_bytes)&uval,
		    sizeof(uval));

	case BIOCSSEESENT:
		if ((r = sys_safecopyfrom(endpt, grant, 0, (vir_bytes)&uval,
		    sizeof(uval))) != OK)
			return r;

		if (uval)
			bpf->bpf_flags |= BPFF_SEESENT;
		else
			bpf->bpf_flags &= ~BPFF_SEESENT;

		return OK;

	case BIOCSRTIMEOUT:
		if ((r = sys_safecopyfrom(endpt, grant, 0, (vir_bytes)&tv,
		    sizeof(tv))) != OK)
			return r;

		if ((r = util_timeval_to_ticks(&tv, &bpf->bpf_timeout)) != OK)
			return r;

		return OK;

	case BIOCGRTIMEOUT:
		util_ticks_to_timeval(bpf->bpf_timeout, &tv);

		return sys_safecopyto(endpt, grant, 0, (vir_bytes)&tv,
		    sizeof(tv));

	case BIOCGFEEDBACK:
		uval = !!(bpf->bpf_flags & BPFF_FEEDBACK);

		return sys_safecopyto(endpt, grant, 0, (vir_bytes)&uval,
		    sizeof(uval));

	case BIOCSFEEDBACK:
		if ((r = sys_safecopyfrom(endpt, grant, 0, (vir_bytes)&uval,
		    sizeof(uval))) != OK)
			return r;

		if (uval)
			bpf->bpf_flags |= BPFF_FEEDBACK;
		else
			bpf->bpf_flags &= ~BPFF_FEEDBACK;

		return OK;

	case FIONREAD:
		val = 0;
		if (bpf->bpf_hlen > 0)
			val = bpf->bpf_hlen;
		else if ((bpf->bpf_flags & BPFF_IMMEDIATE) &&
		    bpf->bpf_slen > 0)
			val = bpf->bpf_slen;
		else
			val = 0;

		return sys_safecopyto(endpt, grant, 0, (vir_bytes)&val,
		    sizeof(val));

	default:
		return ENOTTY;
	}
}

/*
 * Cancel a previously suspended request on a BPF device.  Since only read
 * requests may be suspended (select is handled differently), the cancel
 * request must be for a read request.  Note that character devices currently
 * (still) behave slightly differently from socket devices here: while socket
 * drivers are supposed to respond to the original request, character drivers
 * must respond to the original request from the cancel callback.
 */
static int
bpfdev_cancel(devminor_t minor, endpoint_t endpt, cdev_id_t id)
{
	struct bpfdev *bpf;

	if ((bpf = bpfdev_get_by_minor(minor)) == NULL)
		return EDONTREPLY;

	/* Is this a cancel request for the currently pending read request? */
	if (bpf->bpf_read.br_endpt != endpt || bpf->bpf_read.br_id != id)
		return EDONTREPLY;

	/* If so, cancel the read request. */
	if (bpf->bpf_timeout > 0)
		cancel_timer(&bpf->bpf_read.br_timer);

	bpf->bpf_read.br_endpt = NONE;

	return EINTR; /* the return value for the canceled read request */
}

/*
 * Perform a select query on a BPF device.
 */
static int
bpfdev_select(devminor_t minor, unsigned int ops, endpoint_t endpt)
{
	struct bpfdev *bpf;
	unsigned int r, notify;

	if ((bpf = bpfdev_get_by_minor(minor)) == NULL)
		return EINVAL;

	notify = (ops & CDEV_NOTIFY);
	ops &= (CDEV_OP_RD | CDEV_OP_WR | CDEV_OP_ERR);

	r = bpfdev_test_select(bpf, ops);

	/*
	 * For the operations that were not immediately ready, if requested,
	 * save the select request for later.
	 */
	ops &= ~r;

	if (ops != 0 && notify) {
		if (bpf->bpf_select.bs_endpt != NONE) {
			/* Merge in the operations with any earlier request. */
			if (bpf->bpf_select.bs_endpt != endpt)
				return EIO;
			bpf->bpf_select.bs_selops |= ops;
		} else {
			bpf->bpf_select.bs_endpt = endpt;
			bpf->bpf_select.bs_selops = ops;
		}
	}

	return r;
}

/*
 * Process an incoming packet on the interface to which the given BPF device is
 * attached.  If the packet passes the filter (if any), store as much as
 * requested of it in the store buffer, rotating buffers if needed and resuming
 * suspended read and select requests as appropriate.  This function is also
 * called through bpfdev_output() below.
 */
void
bpfdev_input(struct bpfdev_link * bpfl, const struct pbuf * pbuf)
{
	struct bpfdev *bpf = (struct bpfdev *)bpfl;
	struct timespec ts;
	struct bpf_hdr bh;
	const struct pbuf *pptr;
	size_t caplen, hdrlen, totlen, off, chunk;
	int hfull;

	/*
	 * Apparently bs_recv is the counter of packets that were run through
	 * the filter, not the number of packets that were or could be received
	 * by the user (which is what I got from the manual page.. oh well).
	 */
	bpf->bpf_stat.bs_recv++;
	bpf_stat.bs_recv++;

	/*
	 * Run the packet through the BPF device's filter to see whether the
	 * packet should be stored and if so, how much of it.  If no filter is
	 * set, all packets will be stored in their entirety.
	 */
	caplen = bpf_filter_ext(bpf->bpf_filter, pbuf, (u_char *)pbuf->payload,
	    pbuf->tot_len, pbuf->len);

	if (caplen == 0)
		return;		/* no match; ignore packet */

	if (caplen > pbuf->tot_len)
		caplen = pbuf->tot_len;

	/* Truncate packet entries to the full size of the buffers. */
	hdrlen = BPF_WORDALIGN(sizeof(bh));
	totlen = BPF_WORDALIGN(hdrlen + caplen);

	if (totlen > bpf->bpf_size) {
		totlen = bpf->bpf_size;
		caplen = totlen - hdrlen;
	}
	assert(totlen >= hdrlen);

	bpf->bpf_stat.bs_capt++;
	bpf_stat.bs_capt++;

	assert(bpf->bpf_sbuf != NULL);
	if (totlen > bpf->bpf_size - bpf->bpf_slen) {
		/*
		 * If the store buffer is full and the hold buffer is not
		 * empty, we cannot swap the two buffers, and so we must drop
		 * the current packet.
		 */
		if (bpf->bpf_hlen > 0) {
			bpf->bpf_stat.bs_drop++;
			bpf_stat.bs_drop++;

			return;
		}

		/*
		 * Rotate the buffers: the hold buffer will now be "full" and
		 * ready to be read - it may not actually be entirely full, but
		 * we could not fit this packet and we are not going to deliver
		 * packets out of order..
		 */
		bpfdev_rotate(bpf);

		hfull = TRUE;
	} else
		hfull = FALSE;

	/*
	 * Retrieve the capture time for the packet.  Ideally this would be
	 * done only once per accepted packet, but we do not expect many BPF
	 * devices to be receiving the same packets often enough to make that
	 * worth it.
	 */
	clock_time(&ts);

	/*
	 * Copy the packet into the store buffer, including a newly generated
	 * header.  Zero any padding areas, even if strictly not necessary.
	 */
	memset(&bh, 0, sizeof(bh));
	bh.bh_tstamp.tv_sec = ts.tv_sec;
	bh.bh_tstamp.tv_usec = ts.tv_nsec / 1000;
	bh.bh_caplen = caplen;
	bh.bh_datalen = pbuf->tot_len;
	bh.bh_hdrlen = hdrlen;

	assert(bpf->bpf_sbuf != NULL);
	off = bpf->bpf_slen;

	memcpy(&bpf->bpf_sbuf[off], &bh, sizeof(bh));
	if (hdrlen > sizeof(bh))
		memset(&bpf->bpf_sbuf[off + sizeof(bh)], 0,
		    hdrlen - sizeof(bh));
	off += hdrlen;

	for (pptr = pbuf; pptr != NULL && caplen > 0; pptr = pptr->next) {
		chunk = pptr->len;
		if (chunk > caplen)
			chunk = caplen;

		memcpy(&bpf->bpf_sbuf[off], pptr->payload, chunk);

		off += chunk;
		caplen -= chunk;
	}

	assert(off <= bpf->bpf_slen + totlen);
	if (bpf->bpf_slen + totlen > off)
		memset(&bpf->bpf_sbuf[off], 0, bpf->bpf_slen + totlen - off);

	bpf->bpf_slen += totlen;

	/*
	 * Edge case: if the hold buffer is empty and the store buffer is now
	 * exactly full, rotate buffers so that the packets can be read
	 * immediately, without waiting for the next packet to cause rotation.
	 */
	if (bpf->bpf_hlen == 0 && bpf->bpf_slen == bpf->bpf_size) {
		bpfdev_rotate(bpf);

		hfull = TRUE;
	}

	/*
	 * If the hold buffer is now full, or if immediate mode is enabled,
	 * then we now have data to deliver to userland.  See if we can wake up
	 * any read or select call (either but not both here).
	 */
	if (hfull || (bpf->bpf_flags & BPFF_IMMEDIATE)) {
		if (bpf->bpf_read.br_endpt != NONE)
			bpfdev_resume_read(bpf, FALSE /*is_timeout*/);
		else
			bpfdev_resume_select(bpf);
	}
}

/*
 * Process an outgoing packet on the interface to which the given BPF device is
 * attached.  If the BPF device is configured to capture outgoing packets as
 * well, attempt to capture the packet as per bpfdev_input().
 */
void
bpfdev_output(struct bpfdev_link * bpfl, const struct pbuf * pbuf)
{
	struct bpfdev *bpf = (struct bpfdev *)bpfl;

	if (bpf->bpf_flags & BPFF_SEESENT)
		bpfdev_input(bpfl, pbuf);
}

/*
 * Fill the given 'bde' structure with information about BPF device 'bpf'.
 */
static void
bpfdev_get_info(struct bpf_d_ext * bde, const struct bpfdev * bpf)
{

	bde->bde_bufsize = bpf->bpf_size;
	bde->bde_promisc = !!(bpf->bpf_flags & BPFF_PROMISC);
	bde->bde_state = BPF_IDLE;
	bde->bde_immediate = !!(bpf->bpf_flags & BPFF_IMMEDIATE);
	bde->bde_hdrcmplt = !!(bpf->bpf_flags & BPFF_HDRCMPLT);
	bde->bde_seesent = !!(bpf->bpf_flags & BPFF_SEESENT);
	/*
	 * NetBSD updates the process ID upon device open, close, ioctl, and
	 * poll.  From those, only open and ioctl make sense for us.  Sadly
	 * there is no way to indicate "no known PID" to netstat(1), so we
	 * cannot even save just the endpoint and look up the corresponding PID
	 * later, since the user process may be gone by then.
	 */
	bde->bde_pid = bpf->bpf_pid;
	bde->bde_rcount = bpf->bpf_stat.bs_recv;
	bde->bde_dcount = bpf->bpf_stat.bs_drop;
	bde->bde_ccount = bpf->bpf_stat.bs_capt;
	if (bpf->bpf_ifdev != NULL)
		strlcpy(bde->bde_ifname, ifdev_get_name(bpf->bpf_ifdev),
		    sizeof(bde->bde_ifname));
}

/*
 * Obtain statistics about open BPF devices ("peers").  This node may be
 * accessed by the superuser only.  Used by netstat(1).
 */
static ssize_t
bpfdev_peers(struct rmib_call * call, struct rmib_node * node __unused,
	struct rmib_oldp * oldp, struct rmib_newp * newp __unused)
{
	struct bpfdev *bpf;
	struct bpf_d_ext bde;
	unsigned int slot;
	ssize_t off;
	int r, size, max;

	if (!(call->call_flags & RMIB_FLAG_AUTH))
		return EPERM;

	if (call->call_namelen != 2)
		return EINVAL;

	size = call->call_name[0];
	if (size < 0 || (size_t)size > sizeof(bde))
		return EINVAL;
	if (size == 0)
		size = sizeof(bde);
	max = call->call_name[1];

	off = 0;

	for (slot = 0; slot < __arraycount(bpf_array); slot++) {
		bpf = &bpf_array[slot];

		if (!(bpf->bpf_flags & BPFF_IN_USE))
			continue;

		if (rmib_inrange(oldp, off)) {
			memset(&bde, 0, sizeof(bde));

			bpfdev_get_info(&bde, bpf);

			if ((r = rmib_copyout(oldp, off, &bde, size)) < 0)
				return r;
		}

		off += sizeof(bde);
		if (max > 0 && --max == 0)
			break;
	}

	/* No slack needed: netstat(1) resizes its buffer as needed. */
	return off;
}

static const struct chardriver bpfdev_tab = {
	.cdr_open		= bpfdev_open,
	.cdr_close		= bpfdev_close,
	.cdr_read		= bpfdev_read,
	.cdr_write		= bpfdev_write,
	.cdr_ioctl		= bpfdev_ioctl,
	.cdr_cancel		= bpfdev_cancel,
	.cdr_select		= bpfdev_select
};

/*
 * Process a character driver request.  Since the LWIP service offers character
 * devices for BPF only, it must be a request for a BPF device.
 */
void
bpfdev_process(message * m_ptr, int ipc_status)
{

	chardriver_process(&bpfdev_tab, m_ptr, ipc_status);
}