Skip to content

Commit

Permalink
Local sync
Browse files Browse the repository at this point in the history
  • Loading branch information
0x19 committed Sep 17, 2024
1 parent b947650 commit b45fe33
Show file tree
Hide file tree
Showing 29 changed files with 1,152 additions and 92 deletions.
36 changes: 15 additions & 21 deletions Makefile
Original file line number Diff line number Diff line change
@@ -1,3 +1,5 @@
# Main Makefile

.DEFAULT_GOAL := help

BIN_NAME := build/fdb
Expand All @@ -8,6 +10,9 @@ UNAME_S_LOWERCASE := $(shell echo $(UNAME_S) | tr A-Z a-z)
BUILD_TARGET := build-$(UNAME_S_LOWERCASE)
COMMIT_HASH := $(shell git rev-parse HEAD)

# Include the eBPF Makefile (adjust the path as needed)
include c/Makefile

.PHONY: submodule
submodule: ## Update submodules
git submodule update --init --recursive
Expand All @@ -16,24 +21,16 @@ submodule: ## Update submodules
help: ## Display this help
@awk 'BEGIN {FS = ":.*##"; printf "\nUsage:\n make \033[36m<target>\033[0m\n"} /^[a-zA-Z_-]+:.*?##/ { printf " \033[36m%-15s\033[0m %s\n", $$1, $$2 } /^##@/ { printf "\n\033[1m%s\033[0m\n", substr($$0, 5) } ' $(MAKEFILE_LIST)

# Other commands as before...

.PHONY: deps
deps: ## Install dependencies
ifeq ($(UNAME_S),Linux)
sudo apt-get update && sudo apt-get install -y golang sqlite3 redis-server pipx
sudo -v ; curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b $(go env GOPATH)/bin v1.58.1
sudo -v ; curl https://rclone.org/install.sh | sudo bash
pipx ensurepath
pipx install slither-analyzer
endif
ifeq ($(UNAME_S),Darwin)
brew install go sqlite golangci-lint redis pipx
sudo apt-get update && sudo apt-get install -y golang sqlite3 redis-server pipx clang llvm libelf-dev gcc make linux-tools-$(uname -r) sudo apt install linux-tools-common iproute2
##sudo -v ; curl -sSfL https://raw.githubusercontent.com/golangci/golangci-lint/master/install.sh | sudo sh -s -- -b $(go env GOPATH)/bin v1.58.1
sudo -v ; curl https://rclone.org/install.sh | sudo bash
pipx ensurepath
pipx install slither-analyzer
endif
ifeq ($(OS),Windows_NT)
choco install golang sqlite golangci-lint redis
endif
# MacOS and Windows dependencies...

.PHONY: lint
lint: ## Lint the Go code using golangci-lint
Expand All @@ -46,13 +43,7 @@ build: build-linux ## Build the binary for the current OS/Arch
build-linux: ## Build the binary for Linux
@GOOS=linux GOARCH=amd64 go build -o ./$(BIN_NAME) -ldflags "-X main.Version=$(VERSION) -X main.CommitHash=$(COMMIT_HASH)" ./entrypoint/main.go

.PHONY: build-darwin
build-darwin: ## Build the binary for MacOS
GOOS=darwin GOARCH=amd64 go build -o ./$(BIN_NAME) -ldflags "-X main.Version=$(VERSION) -X main.CommitHash=$(COMMIT_HASH)" ./entrypoint/main.go

.PHONY: build-windows
build-windows: ## Build the binary for Windows
GOOS=windows GOARCH=amd64 go build -o ./$(BIN_NAME).exe -ldflags "-X main.Version=$(VERSION) -X main.CommitHash=$(COMMIT_HASH)" ./entrypoint/main.go
# More build targets...

.PHONY: run
run: build ## Run the binary
Expand Down Expand Up @@ -80,4 +71,7 @@ ifeq ($(OS),Windows_NT) # Windows
del /Q $(BIN_NAME).exe
else
rm -f $(BIN_NAME)
endif
endif

# eBPF-specific commands wrapped under ebpf namespace
.PHONY: ebpf-build ebpf-load ebpf-unload ebpf-clean
126 changes: 108 additions & 18 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,19 +10,77 @@

# (f)db

**NOTE: At this moment I am adding all possible faster ways, including TCP, to be able to do proper benchmarking first.
In the future, I may drop transports that prove to be inefficient based on benchmark results. At the same time I will slowly start to write wrappers around the packages for convenient usage incl. deployments.**
**NOTE: At the moment, I am exploring and integrating all potential high-performance transport options, including
TCP, to establish a solid foundation for benchmarking. In the future, I may discard any transport methods that
prove inefficient based on the benchmarking results. Concurrently, I will begin writing convenient wrappers around
the underlying packages to streamline usage and deployments.**

**UNDER ACTIVE DEVELOPMENT**
**CURRENTLY UNDER ACTIVE DEVELOPMENT**

This is currently a prototype, with the idea of building incredibly fast transport layers on
top of key-value (KV) databases. The goal is to allow one or multiple instances of these
databases to be started and cross-shared in user space or accessed remotely bypassing general locks
that are enforced in KV databases or some of the OLAP databases such as DuckDB.
**Meaning everything about this repository can be changed. Best to keep track, not use the code yet besides to play with it...**

(f)db is born from frustrations experienced while working with key-value (KV) databases.
The goal is to build extremely fast transport layers on top of KV databases.

This project is going to be either f**k databases or fast database... There is no third solution...

Though this will be hard to achieve without DPDK. Will not overkill the prototype with it for now...
### Why is this necessary, and is it just another over-engineered solution in the web world?

It might not be essential for every use case, but it becomes particularly interesting if you require a distributed key-value database with replicated data across multiple nodes.
Additionally, it offers the ability to access and manage data efficiently across multiple servers, each sharing a single connection endpoint,
to offload data to different databases and/or nodes.

For instance, imagine using the same client to seamlessly push data to either an MDBX or DuckDB instance,
regardless of whether they are hosted on the same or different servers.
This abstraction provides flexibility in data management and replication across distributed systems.

### Why I need this?

I require a solution that enables different services to write to and read from different databases while using a unified transport layer. Achieving this with existing tools is complex, if not impossible, without a dedicated transport layer on top of the database. This transport layer allows for cross-service communication and interaction with the underlying database.

This project addresses that gap by building a high-performance, flexible transport layer that enables seamless interaction with various databases in a distributed environment.

### Why Multiple Transports?

One core feature of (f)db is its support for multiple transport protocols, including UDP, UDS, TCP, and QUIC. The reason for this is simple: no single transport protocol is perfect. Each excels in different areas, such as throughput, latency, reliability, or efficiency under specific network conditions.

By offering multiple transport options, (f)db provides the flexibility to fine-tune and optimize performance depending on your specific needs and environment. This allows users to select the most appropriate transport based on their use case, whether it's high-throughput, low-latency, or optimized for specific infrastructure.

### Networking

This project uses [gnet](https://github.com/panjf2000/gnet), an event-driven, high-performance networking library built for Go. gnet is much faster than Go’s standard net package for several reasons:

- Event-Driven Model: gnet uses an event-driven approach, meaning it reacts to specific network events (like data being available to read or connections being closed) instead of continually polling or blocking threads like the standard net package. This reduces CPU overhead and leads to faster processing.
- Efficient Use of Goroutines: Unlike Go’s net package, gnet minimizes the use of goroutines, which helps reduce the context-switching overhead that can slow down highly concurrent applications. Instead, gnet directly handles network I/O in an optimized way.
- Zero-Copy Data Handling: gnet offers zero-copy mechanisms for data processing, meaning that memory is not repeatedly copied between kernel and user space, significantly improving throughput for high-performance networking applications.
- Multi-Event Processing: gnet allows the processing of multiple events within a single loop, which can lead to higher efficiency when handling a large number of simultaneous connections. This feature is particularly valuable for real-time, high-frequency applications that need to handle many clients at once.

By leveraging gnet’s capabilities, (f)db can react quickly to network events, optimize throughput, and minimize latencies, making it ideal for distributed systems where performance is critical.


### eBPF Integration

As part of the ongoing development of high-performance transport layers for (f)db, eBPF (Extended Berkeley Packet Filter) has been integrated to enhance packet processing and monitoring capabilities within the network stack.

By leveraging eBPF, we can build efficient, scalable, and secure transport layers that directly interact with network traffic, providing real-time insights and optimizations for packet flow, without adding latency or reducing throughput.

- Real-Time Packet Filtering: With eBPF, (f)db can selectively filter out unnecessary traffic at the kernel level, significantly reducing the amount of data that needs to be processed in user space.
- Network Performance Insights: eBPF programs can monitor performance metrics, such as packet drops, latencies, and bandwidth usage, allowing for dynamic tuning of transport protocols based on real-time traffic conditions.
- Low-Latency Processing: eBPF’s ability to operate within the kernel ensures that packet processing happens as close to the hardware as possible, minimizing delays and improving overall system responsiveness.
- Ring Buffer for Data Handling: eBPF uses a ring buffer to store and transfer network data efficiently to user-space applications, ensuring fast and reliable communication between the kernel and (f)db.

**Right now some basic program exists that can be loaded but it does not do anything more then writing into ring buffer**

This is something to be dealt with later on...

For example, can be used for mass writes without ACK, different types of discoveries or for example DDoS detection or non-whitelisted server ip access, idk...

### Future Considerations

While achieving all these goals without using advanced techniques like DPDK (Data Plane Development Kit) will be challenging, the initial prototype will focus on building a solid foundation. Advanced optimizations can be layered on top later, depending on the project's needs and performance demands.

This approach ensures that (f)db remains both adaptable and scalable, with the potential to handle a variety of use cases while maintaining high performance.


## Diagram

Expand Down Expand Up @@ -168,6 +226,48 @@ docker-compose up -d
This will bring up the [(f)db](https://github.com/unpackdev/fdb) instance, [OpenTelemetry](https://opentelemetry.io/docs/languages/go/) collector, and [Jaeger](https://www.jaegertracing.io/) for tracing, making the system ready for production-level monitoring and telemetry.


### Loading the eBPF Program

To get started, you'll first need to download and update your local environment by installing the necessary dependencies.
Additionally, ensure that you have the (f)db project built to properly load and unload the eBPF program.

```
make deps
make build
sudo setcap cap_bpf+ep c/obj/ebpf_program.o
```

Next, compile the eBPF program located at [program](./c/src/ebpf_program.c).

```
make ebpf-build
```

Before loading the program, identify the network interface you want to bind the eBPF program to.
The default interface is typically eth0, but it can vary based on your system configuration.
To inspect your available network interfaces, use the following command:

```
./build/fdb ebpf interfaces
```

Finally, load the eBPF program onto your desired network interface:

```
sudo make ebpf-load INTERFACE={interface-name}
```

Once you’ve completed the above steps, your output should look something like this:

```
xxx:xx$ make ebpf-build && sudo make ebpf-load INTERFACE=xxx
eBPF program compiled successfully.
sudo ip link set dev enp73s0 xdp obj c/obj/ebpf_program.o sec xdp
eBPF program loaded onto interface xxx.
```

With that, your eBPF program is successfully loaded and running on the specified interface!

### Running the Binary

For a more custom or direct deployment, you can build and run the fdb binary manually.
Expand All @@ -190,16 +290,6 @@ make build
Ensure that your configuration file is tuned for production, including settings for transports, database paths, logging levels, and performance profiling.
By default, this will start the server with all the transports and services configured in the config.yaml, ready for high-performance and production use.

## GNET

gnet is a high-performance, lightweight, non-blocking, event-driven networking framework written in pure Go.

gnet is an event-driven networking framework that is ultra-fast and lightweight. It is built from scratch by exploiting epoll and kqueue and it can achieve much higher performance with lower memory consumption than Go net in many specific scenarios.

https://github.com/panjf2000/gnet



## QUIC (HTTP/3)

https://github.com/quic-go/quic-go/wiki/UDP-Buffer-Sizes
Expand Down
47 changes: 47 additions & 0 deletions c/Makefile
Original file line number Diff line number Diff line change
@@ -0,0 +1,47 @@
# eBPF Makefile for handling eBPF-related tasks

# Variables
CLANG := clang
BPFTOOL := bpftool
SRC_DIR := c/src
OBJ_DIR := c/obj
SRC := $(SRC_DIR)/ebpf_program.c
OBJ := $(OBJ_DIR)/ebpf_program.o
INTERFACE := eth0 # Replace with the network interface you want to attach to

# eBPF targets
.PHONY: ebpf-all ebpf-build ebpf-load ebpf-unload ebpf-status ebpf-clean

# Default eBPF target
ebpf-all: ebpf-build

# Create object directory if it doesn't exist
$(OBJ_DIR):
mkdir -p $(OBJ_DIR)

# Compile the eBPF program
$(OBJ): $(SRC) | $(OBJ_DIR)
$(CLANG) -O2 -g -target bpf -c $(SRC) -o $(OBJ)

# Build the eBPF program
ebpf-build: $(OBJ)
@echo "eBPF program compiled successfully."

# Load the eBPF program onto the interface
ebpf-load: $(OBJ)
sudo ip link set dev $(INTERFACE) xdp obj $(OBJ) sec xdp
@echo "eBPF program loaded onto interface $(INTERFACE)."

# Unload the eBPF program from the interface
ebpf-unload:
sudo ip link set dev $(INTERFACE) xdp off
@echo "eBPF program unloaded from interface $(INTERFACE)."

# Check if the eBPF program is loaded
ebpf-status:
$(BPFTOOL) prog

# Clean up the compiled objects
ebpf-clean:
rm -rf $(OBJ_DIR)
@echo "Cleaned up eBPF compiled files."
Binary file added c/obj/ebpf_program.o
Binary file not shown.
70 changes: 70 additions & 0 deletions c/src/ebpf_program.c
Original file line number Diff line number Diff line change
@@ -0,0 +1,70 @@
#include <linux/bpf.h>
#include <linux/if_ether.h>
#include <linux/ip.h>
#include <linux/tcp.h>
#include <linux/udp.h>
#include <linux/in.h>
#include <linux/types.h>
#include <stdint.h>
#include <bpf/bpf_helpers.h>

#define ETHERNET_MTU 1500 // Ethernet MTU size

struct {
__uint(type, BPF_MAP_TYPE_RINGBUF);
__uint(max_entries, 1 << 24); // 16MB ring buffer size
} msg_ringbuf SEC(".maps");

SEC("xdp")
int handle_packet(struct xdp_md *ctx) {
// Get the start and end pointers for the packet data
unsigned char *data = (unsigned char *)(long)ctx->data;
unsigned char *data_end = (unsigned char *)(long)ctx->data_end;

// Calculate the packet length directly from ctx
uint32_t total_len = data_end - data;

// Bound the packet length by Ethernet MTU
if (total_len == 0 || total_len > ETHERNET_MTU)
return XDP_PASS; // We pass the packet instead of dropping it

// Ensure the packet has enough data for the Ethernet header
if (data + sizeof(struct ethhdr) > data_end)
return XDP_PASS; // Not enough data for Ethernet header

struct ethhdr *eth = (struct ethhdr *)data;

// Only process IPv4 packets
if (__constant_htons(eth->h_proto) != ETH_P_IP)
return XDP_PASS; // We only handle IPv4, pass the rest

// Ensure the packet has enough data for the IP header
struct iphdr *ip = (struct iphdr *)(data + sizeof(struct ethhdr));
if ((unsigned char *)ip + sizeof(struct iphdr) > data_end)
return XDP_PASS; // Not enough data for IP header

// Check if the packet has enough data for TCP or UDP headers
unsigned char *transport_header = data + sizeof(struct ethhdr) + sizeof(struct iphdr);
if (ip->protocol == IPPROTO_TCP && (transport_header + sizeof(struct tcphdr) > data_end))
return XDP_PASS; // Not enough data for TCP header
if (ip->protocol == IPPROTO_UDP && (transport_header + sizeof(struct udphdr) > data_end))
return XDP_PASS; // Not enough data for UDP header

// Reserve space in the ring buffer for the packet data
void *ringbuf_space = bpf_ringbuf_reserve(&msg_ringbuf, ETHERNET_MTU, 0);
if (!ringbuf_space)
return XDP_PASS; // If we can't reserve space, pass the packet

// Copy the entire packet into the ring buffer
if (bpf_probe_read_kernel(ringbuf_space, ETHERNET_MTU, data)) {
bpf_ringbuf_discard(ringbuf_space, 0); // Discard the reserved space if reading fails
return XDP_ABORTED; // Abort if copying the packet fails
}

// Submit the packet to the ring buffer
bpf_ringbuf_submit(ringbuf_space, 0);

return XDP_PASS; // Pass the packet after processing
}

char _license[] SEC("license") = "GPL";
Empty file removed client/.gitkeep
Empty file.
Loading

0 comments on commit b45fe33

Please sign in to comment.