armada_bufmgr.c

#include <assert.h>
#include <stddef.h>
#include <stdint.h>
#include <stdlib.h>
#include <string.h>
#include <errno.h>
#include <sys/fcntl.h>
#include <sys/mman.h>
#include <time.h>

#include <xf86drm.h>

#include "libdrm_lists.h"
#include "armada_bufmgr.h"
#include "armada_ioctl.h"

#ifndef container_of
#define container_of(ptr, type, member) ({ \
    const typeof( ((type *)0)->member ) *__mptr = (ptr); \
    (type *)( (char *)__mptr - offsetof(type,member) );})
#endif

/* The interval in seconds between cache cleans */
#define BO_CACHE_CLEAN_INTERVAL	1
/* The maximum age in seconds of a BO in the cache */
#define BO_CACHE_MAX_AGE	2
/* Number of buckets in the BO cache */
#define NUM_BUCKETS		(3*9)

/*
 * These sizes come from the i915 DRM backend - which uses roughly
 * for n = 2..
 *   (4096 << n) + (4096 << n) * 1 / 4
 *   (4096 << n) + (4096 << n) * 2 / 4
 *   (4096 << n) + (4096 << n) * 3 / 4
 * The reasoning being that powers of two are too wasteful in X.
 */
static size_t bucket_size[NUM_BUCKETS] = {
	   4096,	   8192,	  12288,
	  20480,	  24576,	  28672,
	  40960,	  49152,	  57344,
	  81920,	  98304,	 114688,
	 163840,	 196608,	 229376,
	 327680,	 393216,	 458752,
	 655360,	 786432,	 917504,
	1310720,	1572864,	1835008,
	2621440,	3145728,	3670016,
};

struct armada_bucket {
	drmMMListHead head;	/* LRU list of bos in this size */
	size_t size;
};

struct armada_bo_cache {
	struct armada_bucket buckets[NUM_BUCKETS];
	drmMMListHead head;	/* LRU list of all freed bos */
	time_t last_cleaned;
};

struct drm_armada_bufmgr {
	struct armada_bo_cache cache;
	void *handle_hash;	/* Hash of DRM handles */
	void *name_hash;	/* Hash of DRM global names */
	int fd;
};

struct armada_bo {
	struct drm_armada_bo bo;
	struct drm_armada_bufmgr *mgr; /* manager associated with this bo */
	drmMMListHead bucket;        /* Cache bucket list */
	drmMMListHead free;          /* Free list */
	time_t free_time;            /* Time this bo was freed */
	size_t alloc_size;           /* Allocated size */
	uint32_t ref;                /* Reference count */
	uint32_t name;               /* Global name */
	uint8_t reusable;
};

#define to_armada_bo(_bo) container_of(_bo, struct armada_bo, bo)

#ifndef DRM_IOCTL_MODE_CREATE_DUMB
/* create a dumb scanout buffer */
struct drm_mode_create_dumb {
        uint32_t height;
        uint32_t width;
        uint32_t bpp;
        uint32_t flags;
        /* handle, pitch, size will be returned */
        uint32_t handle;
        uint32_t pitch;
        uint64_t size;
};
#define DRM_IOCTL_MODE_CREATE_DUMB DRM_IOWR(0xB2, struct drm_mode_create_dumb)
#endif

#ifndef DRM_IOCTL_MODE_DESTROY_DUMB
struct drm_mode_destroy_dumb {
        uint32_t handle;
};
#define DRM_IOCTL_MODE_DESTROY_DUMB    DRM_IOWR(0xB4, struct drm_mode_destroy_dumb)
#endif

#ifndef DRM_IOCTL_MODE_MAP_DUMB
struct drm_mode_map_dumb {
	uint32_t handle;
	uint32_t pad;
	uint64_t offset;
};
#define DRM_IOCTL_MODE_MAP_DUMB    DRM_IOWR(0xB3, struct drm_mode_map_dumb)
#endif

/* Given a width and bpp, return the pitch of a bo */
static unsigned armada_bo_pitch(unsigned width, unsigned bpp)
{
    unsigned pitch = bpp != 4 ? width * ((bpp + 7) / 8) : width / 2;

    /* 88AP510 spec recommends pitch be a multiple of 128 */
    return (pitch + 127) & ~127;
}

/* Given the pitch and height, return the allocated size in bytes of a bo */
static size_t armada_bo_size(unsigned pitch, unsigned height)
{
    return pitch * height;
}

static size_t armada_bo_round_size(size_t size)
{
    if (size > 1048576)
        size = (size + 1048575) & ~1048575;
    else if (size > 65536)
        size = (size + 65535) & ~65535;
    else
        size = (size + 4095) & ~4095;
    return size;
}

static int armada_gem_handle_close(int fd, uint32_t handle)
{
    struct drm_gem_close close;

    memset(&close, 0, sizeof(close));
    close.handle = handle;
    return ioctl(fd, DRM_IOCTL_GEM_CLOSE, &close);
}

static void armada_bo_free(struct armada_bo *bo)
{
    int ret, fd = bo->mgr->fd;

    if (bo->bo.ptr) {
        munmap(bo->bo.ptr, bo->alloc_size);
        bo->bo.ptr = NULL;
    }

    assert(drmHashDelete(bo->mgr->handle_hash, bo->bo.handle) == 0);
    if (bo->name)
        assert(drmHashDelete(bo->mgr->name_hash, bo->name) == 0);

    if (bo->bo.type == DRM_ARMADA_BO_DUMB) {
        struct drm_mode_destroy_dumb arg;

        memset(&arg, 0, sizeof(arg));
        arg.handle = bo->bo.handle;
        ret = drmIoctl(fd, DRM_IOCTL_MODE_DESTROY_DUMB, &arg);
    } else {
        ret = armada_gem_handle_close(fd, bo->bo.handle);
    }

    if (ret == 0)
        free(bo);
}

static void armada_bo_cache_init(struct armada_bo_cache *cache)
{
    struct timespec time;
    unsigned i;

    clock_gettime(CLOCK_MONOTONIC, &time);

    cache->last_cleaned = time.tv_sec;
    DRMINITLISTHEAD(&cache->head);

    for (i = 0; i < NUM_BUCKETS; i++) {
        DRMINITLISTHEAD(&cache->buckets[i].head);
        cache->buckets[i].size = bucket_size[i];
    }
}

static void armada_bo_cache_fini(struct armada_bo_cache *cache)
{
    while (!DRMLISTEMPTY(&cache->head)) {
        struct armada_bo *bo;

        bo = DRMLISTENTRY(struct armada_bo, cache->head.next, free);

        DRMLISTDEL(&bo->bucket);
        DRMLISTDEL(&bo->free);

        armada_bo_free(bo);
    }
}

static struct armada_bucket *armada_find_bucket(struct armada_bo_cache *cache, size_t size)
{
    unsigned i;

    for (i = 0; i < NUM_BUCKETS; i++) {
        struct armada_bucket *bucket = &cache->buckets[i];

        if (bucket->size >= size)
            return bucket;
    }

    return NULL;
}

static void armada_bo_cache_clean(struct armada_bo_cache *cache, time_t time)
{
    if (time - cache->last_cleaned < BO_CACHE_CLEAN_INTERVAL)
        return;

    cache->last_cleaned = time;

    while (!DRMLISTEMPTY(&cache->head)) {
        struct armada_bo *bo;

        bo = DRMLISTENTRY(struct armada_bo, cache->head.next, free);
        if (time - bo->free_time < BO_CACHE_MAX_AGE)
            break;

        DRMLISTDEL(&bo->bucket);
        DRMLISTDEL(&bo->free);

        armada_bo_free(bo);
    }
}

static struct armada_bo *armada_bo_bucket_get(struct armada_bucket *bucket, size_t size)
{
    struct armada_bo *bo = NULL;

    if (!DRMLISTEMPTY(&bucket->head)) {
        drmMMListHead *entry = bucket->head.next;

        bo = DRMLISTENTRY(struct armada_bo, entry, bucket);
        DRMLISTDEL(&bo->bucket);
        DRMLISTDEL(&bo->free);
    }
    return bo;
}

static void armada_bo_cache_put(struct armada_bo *bo)
{
    struct armada_bo_cache *cache = &bo->mgr->cache;
    struct armada_bucket *bucket = armada_find_bucket(cache, bo->alloc_size);

    if (bucket) {
        struct timespec time;

        clock_gettime(CLOCK_MONOTONIC, &time);

        bo->free_time = time.tv_sec;
        DRMLISTADDTAIL(&bo->bucket, &bucket->head);
        DRMLISTADDTAIL(&bo->free, &cache->head);

        armada_bo_cache_clean(cache, time.tv_sec);

        return;
    }
    armada_bo_free(bo);
}

struct drm_armada_bo *drm_armada_bo_create(struct drm_armada_bufmgr *mgr,
    unsigned w, unsigned h, unsigned bpp)
{
    struct drm_armada_gem_create arg;
    struct armada_bucket *bucket;
    struct armada_bo *bo;
    unsigned pitch;
    size_t alloc_size;
    int fd = mgr->fd;
    int ret;

    pitch = armada_bo_pitch(w, bpp);
    alloc_size = armada_bo_size(pitch, h);

    /* Try to find a bucket for this allocation */
    bucket = armada_find_bucket(&mgr->cache, alloc_size);
    if (bucket) {
        /* Can we allocate from our cache? */
        bo = armada_bo_bucket_get(bucket, alloc_size);
        if (bo) {
            bo->bo.size = pitch * h;
            bo->bo.pitch = pitch;
            bo->ref = 1;
            return &bo->bo;
        }

        /* Otherwise, allocate a bo of the bucket size */
        alloc_size = bucket->size;
    } else {
        /* No bucket, so round the size up according to our old rules */
        alloc_size = armada_bo_round_size(alloc_size);
    }

    /* No, create a new bo */
    bo = calloc(1, sizeof *bo);
    if (!bo)
        return NULL;

    memset(&arg, 0, sizeof(arg));
    arg.size = alloc_size;

    ret = drmIoctl(fd, DRM_IOCTL_ARMADA_GEM_CREATE, &arg);
    if (ret) {
        free(bo);
        return NULL;
    }

    bo->bo.ref = 1;
    bo->bo.handle = arg.handle;
    bo->bo.size = pitch * h;
    bo->bo.pitch = pitch;
    bo->bo.type = DRM_ARMADA_BO_SHMEM;
    bo->alloc_size = alloc_size;
    bo->ref = 1;
    bo->mgr = mgr;
    bo->reusable = 1;

    /* Add it to the handle hash table */
    assert(drmHashInsert(mgr->handle_hash, bo->bo.handle, bo) == 0);

    return &bo->bo;
}

struct drm_armada_bo *drm_armada_bo_create_size(struct drm_armada_bufmgr *mgr,
    size_t alloc_size)
{
    struct drm_armada_gem_create arg;
    struct armada_bucket *bucket;
    struct armada_bo *bo;
    int fd = mgr->fd;
    int ret;

    /* Try to find a bucket for this allocation */
    bucket = armada_find_bucket(&mgr->cache, alloc_size);
    if (bucket) {
        /* Can we allocate from our cache? */
        bo = armada_bo_bucket_get(bucket, alloc_size);
        if (bo) {
            bo->bo.size = alloc_size;
            bo->bo.pitch = 1;
            bo->ref = 1;
            return &bo->bo;
        }

        /* Otherwise, allocate a bo of the bucket size */
        alloc_size = bucket->size;
    } else {
        /* No bucket, so round the size up according to our old rules */
        alloc_size = armada_bo_round_size(alloc_size);
    }

    /* No, create a new bo */
    bo = calloc(1, sizeof *bo);
    if (!bo)
        return NULL;

    memset(&arg, 0, sizeof(arg));
    arg.size = alloc_size;

    ret = drmIoctl(fd, DRM_IOCTL_ARMADA_GEM_CREATE, &arg);
    if (ret) {
        free(bo);
        return NULL;
    }

    bo->bo.ref = 1;
    bo->bo.handle = arg.handle;
    bo->bo.size = alloc_size;
    bo->bo.pitch = 1;
    bo->bo.type = DRM_ARMADA_BO_SHMEM;
    bo->alloc_size = alloc_size;
    bo->ref = 1;
    bo->mgr = mgr;
    bo->reusable = 1;

    /* Add it to the handle hash table */
    assert(drmHashInsert(mgr->handle_hash, bo->bo.handle, bo) == 0);

    return &bo->bo;
}

static struct armada_bo *drm_armada_bo_lookup_or_create(
    struct drm_armada_bufmgr *mgr, uint32_t handle, size_t size)
{
    struct armada_bo *bo;

    /*
     * Lookup this handle in our hash of handles.  If it
     * already exists, increment the refcount and return it.
     */
    if (drmHashLookup(mgr->handle_hash, handle, (void **)&bo) == 0) {
        drm_armada_bo_get(&bo->bo);
        return bo;
    }

    bo = calloc(1, sizeof *bo);
    if (!bo)
        return NULL;

    bo->bo.ref = 1;
    bo->bo.handle = handle;
    bo->bo.size = size;
    bo->bo.type = DRM_ARMADA_BO_LINEAR; /* assumed */
    bo->alloc_size = size;
    bo->ref = 1;
    bo->mgr = mgr;

    /* Add it to the handle hash table */
    assert(drmHashInsert(mgr->handle_hash, bo->bo.handle, bo) == 0);

    return bo;
}

struct drm_armada_bo *drm_armada_bo_create_from_name(struct drm_armada_bufmgr *mgr,
    uint32_t name)
{
    struct drm_gem_open arg;
    struct armada_bo *bo;
    int ret, fd = mgr->fd;

    /*
     * Lookup this handle in our hash of names.  If it
     * already exists, increment the refcount and return it.
     */
    if (drmHashLookup(mgr->name_hash, name, (void **)&bo) == 0) {
        drm_armada_bo_get(&bo->bo);
        return &bo->bo;
    }

    memset(&arg, 0, sizeof(arg));
    arg.name = name;
    ret = drmIoctl(fd, DRM_IOCTL_GEM_OPEN, &arg);
    if (ret == -1)
        return NULL;

    bo = drm_armada_bo_lookup_or_create(mgr, arg.handle, arg.size);
    if (!bo) {
        armada_gem_handle_close(fd, arg.handle);
        return NULL;
    }

    bo->name = name;
    assert(drmHashInsert(mgr->name_hash, bo->name, bo) == 0);

    return &bo->bo;
}

struct drm_armada_bo *drm_armada_bo_from_fd(struct drm_armada_bufmgr *mgr,
    int prime_fd)
{
    int fd = mgr->fd;
    struct armada_bo *bo;
    uint32_t handle;
    off_t size;

    size = lseek(prime_fd, 0, SEEK_END);
    if (size == (off_t)-1)
        return NULL;

    if (drmPrimeFDToHandle(fd, prime_fd, &handle))
        return NULL;

    bo = drm_armada_bo_lookup_or_create(mgr, handle, size);
    if (!bo) {
        armada_gem_handle_close(fd, handle);
        return NULL;
    }

    return &bo->bo;
}

struct drm_armada_bo *drm_armada_bo_dumb_create(struct drm_armada_bufmgr *mgr,
    unsigned w, unsigned h, unsigned bpp)
{
    struct armada_bo *bo;
    int fd = mgr->fd;

    bo = calloc(1, sizeof *bo);
    if (bo) {
        struct drm_mode_create_dumb arg;
        int ret;

        memset(&arg, 0, sizeof(arg));
        arg.width = w;
        arg.height = h;
        arg.bpp = bpp;

        ret = drmIoctl(fd, DRM_IOCTL_MODE_CREATE_DUMB, &arg);
        if (ret) {
            free(bo);
            return NULL;
        }
        bo->bo.ref = 1;
        bo->bo.handle = arg.handle;
        bo->bo.size = arg.size;
        bo->bo.pitch = arg.pitch;
        bo->bo.type = DRM_ARMADA_BO_DUMB;
        bo->alloc_size = arg.size;
        bo->ref = 1;
        bo->mgr = mgr;

        /* Add it to the handle hash table */
        assert(drmHashInsert(mgr->handle_hash, bo->bo.handle, bo) == 0);
    }
    return &bo->bo;
}

void drm_armada_bo_get(struct drm_armada_bo *dbo)
{
    struct armada_bo *bo = to_armada_bo(dbo);
    bo->ref++;
}

void drm_armada_bo_put(struct drm_armada_bo *dbo)
{
    struct armada_bo *bo = to_armada_bo(dbo);

    if (bo->ref-- == 1) {
        int ret;

        if (bo->reusable)
            armada_bo_cache_put(bo);
        else
            armada_bo_free(bo);
    }
}

int drm_armada_bo_flink(struct drm_armada_bo *dbo, uint32_t *name)
{
    struct armada_bo *bo = to_armada_bo(dbo);
    int fd = bo->mgr->fd;

    if (!bo->name) {
        struct drm_gem_flink flink;
        int ret;

        memset(&flink, 0, sizeof(flink));
        flink.handle = bo->bo.handle;
        ret = ioctl(fd, DRM_IOCTL_GEM_FLINK, &flink);
        if (ret)
            return ret;
        bo->name = flink.name;
        bo->reusable = 0;

        assert(drmHashInsert(bo->mgr->name_hash, bo->name, bo) == 0);
    }
    *name = bo->name;
    return 0;
}

int drm_armada_bo_to_fd(struct drm_armada_bo *dbo, int *prime_fd)
{
    struct armada_bo *bo = to_armada_bo(dbo);
    int fd = bo->mgr->fd;

    if (drmPrimeHandleToFD(fd, bo->bo.handle, DRM_CLOEXEC, prime_fd))
        return -1;

    return 0;
}

int drm_armada_bo_map(struct drm_armada_bo *dbo)
{
    struct armada_bo *bo = to_armada_bo(dbo);
    void *map;
    int ret, fd = bo->mgr->fd;

    if (bo->bo.ptr)
        return 0;

    if (bo->bo.type == DRM_ARMADA_BO_DUMB) {
        struct drm_mode_map_dumb arg;

        memset(&arg, 0, sizeof(arg));
        arg.handle = bo->bo.handle;

        ret = drmIoctl(fd, DRM_IOCTL_MODE_MAP_DUMB, &arg);
        if (ret)
            return ret;

        map = mmap(0, bo->alloc_size, PROT_READ | PROT_WRITE, MAP_SHARED, fd,
                   arg.offset);

        if (map == MAP_FAILED)
            return -1;
    } else if (bo->bo.type == DRM_ARMADA_BO_SHMEM) {
        struct drm_armada_gem_mmap arg;

        memset(&arg, 0, sizeof(arg));
        arg.handle = bo->bo.handle;
        arg.offset = 0;
        arg.size = bo->alloc_size;

        ret = drmIoctl(fd, DRM_IOCTL_ARMADA_GEM_MMAP, &arg);
        if (ret)
            return -1;

        map = (void *)(uintptr_t)arg.addr;
    } else {
        errno = EINVAL;
        return -1;
    }

    bo->bo.ptr = map;

    return 0;
}

int drm_armada_bo_subdata(struct drm_armada_bo *dbo, unsigned long offset,
    unsigned long size, const void *data)
{
    struct armada_bo *bo = to_armada_bo(dbo);
    struct drm_armada_gem_pwrite arg;
    int fd = bo->mgr->fd;

    memset(&arg, 0, sizeof(arg));
    arg.ptr = (uint64_t)(uintptr_t)data;
    arg.handle = bo->bo.handle;
    arg.offset = offset;
    arg.size = size;

    return drmIoctl(fd, DRM_IOCTL_ARMADA_GEM_PWRITE, &arg);
}

int drm_armada_cache_reap(struct drm_armada_bufmgr *mgr)
{
    struct timespec time;

    if (!DRMLISTEMPTY(&mgr->cache.head)) {
        clock_gettime(CLOCK_MONOTONIC, &time);

        armada_bo_cache_clean(&mgr->cache, time.tv_sec);
    }

    return !DRMLISTEMPTY(&mgr->cache.head);
}

int drm_armada_init(int fd, struct drm_armada_bufmgr **mgrp)
{
    struct drm_armada_bufmgr *mgr;

    mgr = calloc(1, sizeof(*mgr));
    if (!mgr)
        return -1;

    mgr->handle_hash = drmHashCreate();
    if (!mgr->handle_hash) {
        free(mgr);
        return -1;
    }
    mgr->name_hash = drmHashCreate();
    if (!mgr->name_hash) {
        drmHashDestroy(mgr->handle_hash);
        free(mgr);
        return -1;
    }

    armada_bo_cache_init(&mgr->cache);
    mgr->fd = fd;
    *mgrp = mgr;

    return 0;
}

void drm_armada_fini(struct drm_armada_bufmgr *mgr)
{
    armada_bo_cache_fini(&mgr->cache);
    drmHashDestroy(mgr->handle_hash);
    drmHashDestroy(mgr->name_hash);
    free(mgr);
}