Protobufs

Overview

Protocol Buffers (protobufs) are a messaging protocol made by Google that allows for efficient encoding and decoding of complex data over a data stream (e.g. a network connection). The reason you need a protocol is that whenever you send data, both sides need to agree on the format of the data. We also want to send the minimal amount of information required to convey our message, (if you want to learn more about this field called information theory take EECS 126). Protobufs allow us to do both of these things easily.

It works by first defining a set of messages, similar to C structs, that can be sent. The definitions are done in the proto language which is explained in depth here https://developers.google.com/protocol-buffers/docs/proto3.

Then, in each language you need to create bindings from the .proto files to objects in that language. You'll notice that there's no option to choose C as a target language. Luckily, a group of people made a third-party extension of Google protobufs that can take .proto files and generate C source and header files that can be used for a C program to pack and unpack protobuf messages. A good place to understand how that works is the protobuf-c Github and their Wiki.

There are a few reasons why we chose Google Protocol Buffers for our communication with Dawn and Shepherd:

• Consistency with the old version of Runtime (old Runtime and Dawn used protobufs as well to communicate)
• Speed. It was decided the speed boost gained by using protobufs was worth the trouble of setting it up to work in C. Consider this: suppose we chose a protocol like JSON. To send a boolean, we might need to send the string: "{"switch0":false}". That's 19 characters (i.e. 19 bytes) sent over the network. Compare with protobufs, which would be literally 1 byte (0). Integers and complicated nested message types offer similar amounts of message size reductions. Combine it all together, and the network traffic reduction gained by using protobufs is substantial.
• Consistency between Runtime communication with Dawn and Shepherd. Shepherd uses JSON internally to send data around, and originally the plan was to communicate with Dawn using protobufs and communicate with Shepherd using JSON. But that's not very smart, because then Runtime would have to convert our internal data into two different message formats, which would be extremely ugly. So the decision was made to use protobufs for all of Runtime's network communications.

Tutorial & Usage

Below is a tutorial on using the protobuf-c library and the protobufs that the proto compiler generates. An incomplete tutorial can be found on the protobuf-c Github repo's Wiki, but these examples were what the original authors of Runtime used to learn how to use the library.

runmode_in.c

#include <stdio.h>
#include <stdlib.h>
#include "../pbc_gen/run_mode.pb-c.h"
#define MAX_MSG_SIZE 1024

static size_t read_buffer(unsigned max_length, uint8_t* out) {
    size_t cur_len = 0;
    size_t nread;
    while ((nread = fread(out + cur_len, 1, max_length - cur_len, stdin)) != 0) {
        cur_len += nread;
        if (cur_len == max_length) {
            fprintf(stderr, "max message length exceeded\n");
            exit(1);
        }
    }
    return cur_len;
}

int main() {
    RunMode* run_mode;

    // Read packed message from standard-input.
    uint8_t buf[MAX_MSG_SIZE];
    size_t msg_len = read_buffer(MAX_MSG_SIZE, buf);

    // Unpack the message using protobuf-c.
    run_mode = run_mode__unpack(NULL, msg_len, buf);
    if (run_mode == NULL) {
        fprintf(stderr, "error unpacking incoming message\n");
        exit(1);
    }

    // display the message's fields.
    printf("Received: mode = %u\n", run_mode->mode);  //comes in as unsigned int

    // Free the unpacked message
    run_mode__free_unpacked(run_mode, NULL);
    return 0;
}

runmode_out.c

#include <stdio.h>
#include <stdlib.h>
#include "../pbc_gen/run_mode.pb-c.h"

int main() {
    RunMode run_mode = RUN_MODE__INIT;  //iniitialize hooray
    void* buf;                          // Buffer to store serialized data
    unsigned len;                       // Length of serialized data

    //put some data
    run_mode.mode = MODE__AUTO;

    len = run_mode__get_packed_size(&run_mode);

    buf = malloc(len);
    run_mode__pack(&run_mode, buf);

    fprintf(stderr, "Writing %d serialized bytes\n", len);  // See the length of message
    fwrite(buf, len, 1, stdout);                            // Write to stdout to allow direct command line piping

    free(buf);  // Free the allocated serialized buffer
    return 0;
}

log_in.c

#include <stdio.h>
#include <stdlib.h>
#include "../pbc_gen/text.pb-c.h"
#define MAX_MSG_SIZE 1024

static size_t read_buffer(unsigned max_length, uint8_t* out) {
    size_t cur_len = 0;
    size_t nread;
    while ((nread = fread(out + cur_len, 1, max_length - cur_len, stdin)) != 0) {
        cur_len += nread;
        if (cur_len == max_length) {
            fprintf(stderr, "max message length exceeded\n");
            exit(1);
        }
    }
    return cur_len;
}

int main() {
    Text* log_msg;

    // Read packed message from standard-input.
    uint8_t buf[MAX_MSG_SIZE];
    size_t msg_len = read_buffer(MAX_MSG_SIZE, buf);

    // Unpack the message using protobuf-c.
    log_msg = text__unpack(NULL, msg_len, buf);
    if (log_msg == NULL) {
        fprintf(stderr, "error unpacking incoming message\n");
        exit(1);
    }

    // display the message's fields.
    for (int i = 0; i < log_msg->n_payload; i++) {
        printf("\t%s\n", log_msg->payload[i]);
    }

    // Free the unpacked message
    text__free_unpacked(log_msg, NULL);
    return 0;
}

log_out.c

#include <stdio.h>
#include <stdlib.h>
#include <string.h>
#include "../pbc_gen/text.pb-c.h"

#define MAX_STRLEN 100

char* strs[4] = {"hello", "beautiful", "precious", "world"};

int main() {
    Text log_msg = TEXT__INIT;  //iniitialize hooray
    void* buf;                  // Buffer to store serialized data
    unsigned len;               // Length of serialized data

    //put some data
    log_msg.n_payload = 4;
    log_msg.payload = (char**) malloc(sizeof(char*) * log_msg.n_payload);
    for (int i = 0; i < log_msg.n_payload; i++) {
        log_msg.payload[i] = (char*) malloc(sizeof(char) * strlen(strs[i]));
        strcpy(log_msg.payload[i], (const char*) strs[i]);
    }

    len = text__get_packed_size(&log_msg);

    buf = malloc(len);
    text__pack(&log_msg, buf);

    fprintf(stderr, "Writing %d serialized bytes\n", len);  // See the length of message
    fwrite(buf, len, 1, stdout);                            // Write to stdout to allow direct command line piping

    free(buf);  // Free the allocated serialized buffer
    for (int i = 0; i < log_msg.n_payload; i++) {
        free(log_msg.payload[i]);
    }
    free(log_msg.payload);
    return 0;
}

devdata_in.c

#include <stdio.h>
#include <stdlib.h>
#include "../pbc_gen/device.pb-c.h"
#define MAX_MSG_SIZE 1024

static size_t read_buffer(unsigned max_length, uint8_t* out) {
    size_t cur_len = 0;
    size_t nread;
    while ((nread = fread(out + cur_len, 1, max_length - cur_len, stdin)) != 0) {
        cur_len += nread;
        if (cur_len == max_length) {
            fprintf(stderr, "max message length exceeded\n");
            exit(1);
        }
    }
    return cur_len;
}

int main() {
    DevData* dev_data;

    // Read packed message from standard-input.
    uint8_t buf[MAX_MSG_SIZE];
    size_t msg_len = read_buffer(MAX_MSG_SIZE, buf);

    // Unpack the message using protobuf-c.
    dev_data = dev_data__unpack(NULL, msg_len, buf);
    if (dev_data == NULL) {
        fprintf(stderr, "error unpacking incoming message\n");
        exit(1);
    }

    // display the message's fields.
    printf("Received:\n");
    for (int i = 0; i < dev_data->n_devices; i++) {
        printf("Device No. %d: ", i);
        printf("\ttype = %s, uid = %llu, itype = %d\n", dev_data->devices[i]->name, dev_data->devices[i]->uid, dev_data->devices[i]->type);
        printf("\tParams:\n");
        for (int j = 0; j < dev_data->devices[i]->n_params; j++) {
            printf("\t\tparam \"%s\" has type ", dev_data->devices[i]->params[j]->name);
            switch (dev_data->devices[i]->params[j]->val_case) {
                case (PARAM__VAL_FVAL):
                    printf("FLOAT with value %f\n", dev_data->devices[i]->params[j]->fval);
                    break;
                case (PARAM__VAL_IVAL):
                    printf("INT with value %d\n", dev_data->devices[i]->params[j]->ival);
                    break;
                case (PARAM__VAL_BVAL):
                    printf("BOOL with value %d\n", dev_data->devices[i]->params[j]->bval);
                    break;
                default:
                    printf("UNKNOWN");
                    break;
            }
        }
    }

    // Free the unpacked message
    dev_data__free_unpacked(dev_data, NULL);
    return 0;
}

devdata_out.c

#include <stdio.h>
#include <stdlib.h>
#include "../pbc_gen/device.pb-c.h"

int main() {
    void* buf;     // Buffer to store serialized data
    unsigned len;  // Length of serialized data

    //initialize all the messages and submessages (let's send two devices, the first with 1 param and the second with 2 params)
    DevData dev_data = DEV_DATA__INIT;
    Device dev1 = DEVICE__INIT;
    Device dev2 = DEVICE__INIT;
    Param d1p1 = PARAM__INIT;
    Param d2p1 = PARAM__INIT;
    Param d2p2 = PARAM__INIT;

    //set all the fields .....
    d1p1.name = "switch0";
    d1p1.val_case = PARAM__VAL_FVAL;
    d1p1.fval = 0.3;

    d2p1.name = "sensor0";
    d2p1.val_case = PARAM__VAL_IVAL;
    d2p1.ival = 42;

    d2p2.name = "bogus";
    d2p2.val_case = PARAM__VAL_BVAL;
    d2p2.bval = 1;

    dev1.name = "LimitSwitch";
    dev1.uid = 984789478297;
    dev1.type = 12;
    dev1.n_params = 1;
    dev1.params = (Param**) malloc(dev1.n_params * sizeof(Param*));
    dev1.params[0] = &d1p1;

    dev2.name = "LineFollower";
    dev2.uid = 47834674267;
    dev2.type = 13;
    dev2.n_params = 2;
    dev2.params = (Param**) malloc(dev2.n_params * sizeof(Param*));
    dev2.params[0] = &d2p1;
    dev2.params[1] = &d2p2;

    dev_data.n_devices = 2;
    dev_data.devices = (Device**) malloc(dev_data.n_devices * sizeof(Device*));
    dev_data.devices[0] = &dev1;
    dev_data.devices[1] = &dev2;
    //done setting all fields!

    len = dev_data__get_packed_size(&dev_data);

    buf = malloc(len);
    dev_data__pack(&dev_data, buf);

    fprintf(stderr, "Writing %d serialized bytes\n", len);  // See the length of message
    fwrite(buf, len, 1, stdout);                            // Write to stdout to allow direct command line piping

    free(buf);  // Free the allocated serialized buffer
    free(dev1.params);
    free(dev2.params);
    free(dev_data.devices);
    return 0;
}