Skip to content

Commit

Permalink
Merge branch 'theseus_main' into task-cancel
Browse files Browse the repository at this point in the history
  • Loading branch information
tsoutsman committed Sep 9, 2023
2 parents adb3b0e + b521678 commit 9849e9c
Show file tree
Hide file tree
Showing 131 changed files with 5,585 additions and 4,093 deletions.
311 changes: 158 additions & 153 deletions Cargo.lock

Large diffs are not rendered by default.

4 changes: 2 additions & 2 deletions Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -79,10 +79,12 @@ exclude = [

## Exclude application crates used for testing specific Theseus functionality.
## TODO: move these to a specific "tests" folder so we can exclude that entire folder.
"applications/test_aligned_page_allocation",
"applications/test_backtrace",
"applications/test_block_io",
"applications/test_channel",
"applications/test_filerw",
"applications/test_identity_mapping",
"applications/test_ixgbe",
"applications/test_libc",
"applications/test_mlx5",
Expand All @@ -104,8 +106,6 @@ exclude = [
volatile = { git = "https://github.com/theseus-os/volatile" }
### use our own no_std-compatilbe getopts
getopts = { git = "https://github.com/theseus-os/getopts" }
### use the latest version of smoltcp from github; the one on crates.io is out of date
smoltcp = { git = "https://github.com/m-labs/smoltcp" }

### Patch `libc` so we can use libc-specific types when using `cfg(target_os = "theseus")`.
libc = { git = "https://github.com/theseus-os/libc", branch = "theseus" }
Expand Down
17 changes: 15 additions & 2 deletions Makefile
Original file line number Diff line number Diff line change
Expand Up @@ -67,11 +67,15 @@ LIMINE_DIR := $(ROOT_DIR)/limine-prebuilt
### Set up tool names/locations for cross-compiling on a Mac OS / macOS host (Darwin).
UNAME = $(shell uname -s)
ifeq ($(UNAME),Darwin)
CROSS = x86_64-elf-
CROSS = $(ARCH)-elf-
## macOS uses a different unmounting utility
UNMOUNT = diskutil unmount
USB_DRIVES = $(shell diskutil list external | grep -s "/dev/" | awk '{print $$1}')
else
## Handle building for aarch64 on x86_64 Linux/WSL
ifeq ($(ARCH),aarch64)
CROSS = aarch64-linux-gnu-
endif
## Just use normal umount on Linux/WSL
UNMOUNT = umount
USB_DRIVES = $(shell lsblk -O | grep -i usb | awk '{print $$2}' | grep --color=never '[^0-9]$$')
Expand Down Expand Up @@ -220,7 +224,7 @@ build: $(nano_core_binary)
## Note: we skip "normal" .rlib archives that have 2 members: a single .o object file and a single .rmeta file.
## Note: the below line with `cut` simply removes the `lib` prefix and the `.rlib` suffix from the file name.
@for f in $(shell find $(TARGET_DEPS_DIR)/ -name "*.rlib"); do \
if [ "`$(CROSS)ar -t $${f} | wc -l`" != "2" ]; then \
if [ `$(CROSS)ar -t $${f} | wc -l` != "2" ]; then \
echo -e "\033[1;34mUnarchiving multi-file rlib: \033[0m $${f}" \
&& mkdir -p "$(BUILD_DIR)/extracted_rlibs/`basename $${f}`-unpacked/" \
&& $(CROSS)ar -xo --output "$(BUILD_DIR)/extracted_rlibs/`basename $${f}`-unpacked/" $${f} \
Expand Down Expand Up @@ -295,6 +299,10 @@ else
$(error Error: unsupported option "debug=$(debug)". Options are 'full', 'none', or 'base')
endif

## Sixth, fix up CPU local sections.
@echo -e "Parsing CPU local sections"
@cargo run --release --manifest-path $(ROOT_DIR)/tools/elf_cls/Cargo.toml -- $(ARCH) --dir $(OBJECT_FILES_BUILD_DIR)

#############################
### end of "build" target ###
#############################
Expand Down Expand Up @@ -349,6 +357,8 @@ endif
## This builds the nano_core binary itself, which is the fully-linked code that first runs right after the bootloader
$(nano_core_binary): cargo $(nano_core_static_lib) $(linker_script)
$(CROSS)ld -n -T $(linker_script) -o $(nano_core_binary) $(compiled_nano_core_asm) $(nano_core_static_lib)
## Fix up CLS sections.
cargo run --release --manifest-path $(ROOT_DIR)/tools/elf_cls/Cargo.toml -- $(ARCH) --file $(nano_core_binary)
## Dump readelf output for verification. See pull request #542 for more details:
## @RUSTFLAGS="" cargo run --release --manifest-path $(ROOT_DIR)/tools/demangle_readelf_file/Cargo.toml \
## <($(CROSS)readelf -s -W $(nano_core_binary) | sed '/OBJECT LOCAL .* str\./d;/NOTYPE LOCAL /d;/FILE LOCAL /d;/SECTION LOCAL /d;') \
Expand Down Expand Up @@ -933,6 +943,9 @@ else ifeq ($(ARCH),aarch64)
QEMU_FLAGS += -machine virt,gic-version=3
QEMU_FLAGS += -device ramfb
QEMU_FLAGS += -cpu cortex-a72
QEMU_FLAGS += -usb
QEMU_FLAGS += -device usb-ehci,id=ehci
QEMU_FLAGS += -device usb-kbd
else
QEMU_FLAGS += -cpu Broadwell
endif
Expand Down
29 changes: 20 additions & 9 deletions README.md
Original file line number Diff line number Diff line change
Expand Up @@ -10,7 +10,7 @@ Theseus is a new OS written from scratch in [Rust](https://www.rust-lang.org/) t
For more info, check out Theseus's [documentation](#Documentation) or our [published academic papers](https://theseus-os.github.io/Theseus/book/misc/papers_presentations.html), which describe Theseus's design and implementation.

Theseus is under active development, and although it is not yet mature, we envision that Theseus will be useful in high-end embedded systems or edge datacenter environments.
We are continually working to improve the OS, including its fault recovery abilities for higher system availability without redundancy, as well as easier and more arbitrary live evolution and runtime flexbility.
We are continually working to improve the OS, including its fault recovery abilities for higher system availability without redundancy, as well as easier and more arbitrary live evolution and runtime flexibility.


# Quick start
Expand Down Expand Up @@ -47,7 +47,7 @@ Currently, we support building Theseus on the following platforms:
* Linux, 64-bit Debian-based distributions like Ubuntu, tested on Ubuntu 16.04, 18.04, 20.04.
- Arch Linux and Fedora have also been reported to work correctly.
* Windows, using the Windows Subsystem for Linux (WSL), tested on the Ubuntu version of WSL and WSL2.
* MacOS, tested on versions High Sierra (10.13) and Catalina (10.15.2).
* MacOS, tested on versions High Sierra (v10.13), Catalina (v10.15), and Ventura (v13.5).
* Docker, atop any host OS that can run a Docker container.
Expand Down Expand Up @@ -100,7 +100,7 @@ If you're on WSL, also do the following steps:
rm -rf /tmp/theseus_tools_src
```

* If you're building Theseus on an M1-based Mac, you may need to use x86 emulation
* If you're building Theseus on an M1-based Mac, you may need to use `gmake` instead of `make` for build commands. Alternatively, you can use `bash` with x86 emulation, but this is generally not necessary.
```sh
arch -x86_64 bash # or another shell of your choice
```
Expand All @@ -121,7 +121,7 @@ Note: building and running Theseus within a Docker container may be slower than
```
* After docker installs, enable your user account to run docker without root privileges:
`sudo groupadd docker; sudo usermod -aG docker $USER`
Then, **log out and log back in** (or restart your computer) for the user/group changes to take effet.
Then, **log out and log back in** (or restart your computer) for the user/group changes to take effect.
3. Build the docker image:
```
Expand Down Expand Up @@ -164,7 +164,7 @@ Feel free to try newer versions, however they may not work.
## Targeting ARMv8 (aarch64)
There is an ongoing effort to port Theseus to ARMv8 (aarch64); this is currently a work-in-progress.
Support for Theseus on aarch64 is an ongoing effort, but most of the core subsystems are complete.
To build and run Theseus on aarch64, first install the required dependencies:
* On Debian-like Linux (Ubuntu, etc):
Expand All @@ -175,16 +175,24 @@ To build and run Theseus on aarch64, first install the required dependencies:
```bash
sudo pacman -S aarch64-linux-gnu-gcc qemu-system-aarch64
```
* On macOS, the `mac_os_build_setup` script should have already installed this for you, but if not:
```zsh
brew install aarch64-elf-gcc aarch64-elf-binutils
```
Then, build Theseus and run it in QEMU:
```bash
make ARCH=aarch64 FEATURES= CROSS=aarch64-linux-gnu- run
make ARCH=aarch64 run
```
Doing a "full" build of all Theseus crates isn't yet supported on aarch64,
as our aarch64 support in Theseus doesn't yet cover *all* crates in the entire repo.
## Using QEMU
QEMU allows us to run Theseus quickly and easily in its own virtual machine.
To release, press <kbd>Ctrl</kbd> + <kbd>Alt</kbd> + <kbd>G</kbd> (or just <kbd>Ctrl</kbd> + <kbd>Alt</kbd> on some systems), which releases your keyboard and mouse focus from the QEMU window.
To release your keyboard and mouse focus from the QEMU window, press <kbd>Ctrl</kbd> + <kbd>Alt</kbd> + <kbd>G</kbd>, or <kbd>Ctrl</kbd> + <kbd>Alt</kbd> on some systems, or just <kbd>Cmd</kbd> + <kbd>Tab</kbd> out to another app on macOS.
To exit QEMU, in the terminal window that you originally ran `make run`, press <kbd>Ctrl</kbd> + <kbd>A</kbd> then <kbd>X</kbd>, or you can also click the GUI `ⓧ` button on the title bar if running QEMU in graphical mode.
To investigate the hardware/machine state of the running QEMU VM, you can switch to the QEMU console by pressing <kbd>Ctrl</kbd> + <kbd>Alt</kbd> + <kbd>2</kbd>.
Expand All @@ -209,6 +217,9 @@ To enable KVM support, add `host=yes` to your make command, e.g.,
make run host=yes
```
Note that KVM acceleration is only supported on native Linux hosts.
# Documentation
Theseus includes two forms of documentation:
1. The [source-level documentation](https://theseus-os.github.io/Theseus/doc/___Theseus_Crates___/index.html), generated from code and inline comments (via *rustdoc*).
Expand All @@ -226,7 +237,7 @@ make view-book ## for the Theseus book
## Booting on Real Hardware
We have tested Theseus on a variety of real machines, including Intel NUC devices, various Thinkpad laptops, and Supermicro servers.
Currently, the only limiting factor is that the device support booting via USB or PXE using traditional BIOS rather than UEFI; support for UEFI is a work-in-progress.
Currently, we have only tested booting Theseus via USB or PXE using a traditional BIOS bootloader rather than UEFI, but UEFI is fully supported so it should work.
To boot over USB, simply run `make boot usb=sdc`, in which `sdc` is the device node for the USB disk itself *(**not a partition** like sdc2)* to which you want to write the OS image.
On WSL or other host environments where `/dev` device nodes don't exist, you can simply run `make iso` and burn the `.iso` file in the `build/` directory to a USB, e.g., using [Rufus](https://rufus.ie/) on Windows.
Expand Down Expand Up @@ -266,7 +277,7 @@ We don't yet have a patched version of GDB for aarch64 targets, but we can use t
2. Build Theseus for aarch64 and run it in QEMU:
```sh
make ARCH=aarch64 FEATURES= CROSS=aarch64-linux-gnu- run ## or use `run_pause`
make ARCH=aarch64 run ## or use `run_pause`
```
3. In another terminal window, run the following to start GDB and attach it to the running QEMU instance:
```sh
Expand Down
4 changes: 1 addition & 3 deletions applications/ping/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -149,9 +149,7 @@ fn _main(matches: Matches) -> Result<(), &'static str> {

// Poll the socket to send the packet. Once we have a custom socket type this
// won't be necessary.
interface
.poll()
.map_err(|_| "failed to poll interface after sending")?;
interface.poll();
num_sent += 1;
}

Expand Down
15 changes: 15 additions & 0 deletions applications/test_aligned_page_allocation/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
[package]
name = "test_aligned_page_allocation"
version = "0.1.0"
description = "Tests the `AllocationRequest::AlignedTo` variant, which is needed for huge pages"
authors = ["Kevin Boos <kevinaboos@gmail.com>"]
edition = "2021"

[dependencies]
log = "0.4.8"

[dependencies.memory]
path = "../../kernel/memory"

[dependencies.app_io]
path = "../../kernel/app_io"
45 changes: 45 additions & 0 deletions applications/test_aligned_page_allocation/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,45 @@
//! A set of basic tests for the [`AllocationRequest::AlignedTo`] variant.

#![no_std]

extern crate alloc;

use alloc::{
vec::Vec,
string::String,
};
use app_io::println;
use memory::AllocationRequest;

static TEST_SET: [usize; 9] = [1, 2, 4, 8, 27, 48, 256, 512, 1024];

pub fn main(_args: Vec<String>) -> isize {
match rmain() {
Ok(_) => 0,
Err(e) => {
println!("Error: {}", e);
-1
}
}
}

fn rmain() -> Result<(), &'static str> {
for num_pages in TEST_SET.into_iter() {
for alignment in TEST_SET.into_iter() {
println!("Attempting to allocate {num_pages} pages with alignment of {alignment} 4K pages...");
match memory::allocate_pages_deferred(
AllocationRequest::AlignedTo { alignment_4k_pages: alignment },
num_pages,
) {
Ok((ap, _action)) => {
assert_eq!(ap.start().number() % alignment, 0);
assert_eq!(ap.size_in_pages(), num_pages);
println!(" Success: {ap:?}");
}
Err(e) => println!(" !! FAILURE: {e:?}"),
}
}
}

Ok(())
}
15 changes: 15 additions & 0 deletions applications/test_identity_mapping/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,15 @@
[package]
name = "test_identity_mapping"
version = "0.1.0"
description = "Tests the `memory::create_identity_mapping()` function"
authors = ["Kevin Boos <kevinaboos@gmail.com>"]
edition = "2021"

[dependencies]
log = "0.4.8"

[dependencies.memory]
path = "../../kernel/memory"

[dependencies.app_io]
path = "../../kernel/app_io"
39 changes: 39 additions & 0 deletions applications/test_identity_mapping/src/lib.rs
Original file line number Diff line number Diff line change
@@ -0,0 +1,39 @@
//! A set of basic tests for [`memory::create_identity_mapping()`].

#![no_std]

extern crate alloc;

use alloc::{
vec::Vec,
string::String,
};
use app_io::println;

static TEST_SET: [usize; 8] = [1, 2, 4, 8, 25, 48, 256, 1024];

pub fn main(_args: Vec<String>) -> isize {
match rmain() {
Ok(_) => 0,
Err(e) => {
println!("Error: {}", e);
-1
}
}
}

fn rmain() -> Result<(), &'static str> {
let flags = memory::PteFlags::new().valid(true);
for num_pages in TEST_SET.into_iter() {
println!("Attempting to create identity mapping of {num_pages} pages...");
match memory::create_identity_mapping(num_pages, flags) {
Ok(mp) => {
assert_eq!(mp.size_in_pages(), num_pages);
println!(" Success: {mp:?}");
}
Err(e) => println!(" !! FAILURE: {e:?}"),
}
}

Ok(())
}
7 changes: 3 additions & 4 deletions applications/test_ixgbe/Cargo.toml
Original file line number Diff line number Diff line change
Expand Up @@ -2,6 +2,7 @@
name = "test_ixgbe"
version = "0.1.0"
authors = ["Ramla Ijaz <ijazramla@gmail.com>"]
edition = "2021"

[dependencies]
getopts = "0.2.21"
Expand All @@ -18,7 +19,5 @@ path = "../../kernel/ixgbe"
[dependencies.spawn]
path = "../../kernel/spawn"

[dependencies.network_interface_card]
path = "../../kernel/network_interface_card"


[dependencies.net]
path = "../../kernel/net"
23 changes: 8 additions & 15 deletions applications/test_ixgbe/src/lib.rs
Original file line number Diff line number Diff line change
Expand Up @@ -17,24 +17,18 @@
//! e.g."sudo arp -i eno1 -s 192.168.0.20 0c:c4:7a:d2:ee:1a"

#![no_std]
#[macro_use] extern crate alloc;
// #[macro_use] extern crate log;
#[macro_use] extern crate app_io;
extern crate network_interface_card;
extern crate ixgbe;
extern crate spawn;
extern crate getopts;

use alloc::vec::Vec;
use alloc::string::String;

extern crate alloc;

use alloc::{string::String, vec::Vec, vec};
use ixgbe::{
virtual_function, get_ixgbe_nics_list,
test_packets::create_raw_packet,
IXGBE_NUM_RX_QUEUES_ENABLED, IXGBE_NUM_TX_QUEUES_ENABLED, tx_send_mq, rx_poll_mq
};
use network_interface_card::NetworkInterfaceCard;
use getopts::{Matches, Options};

use app_io::println;
use net::NetworkDevice;

const DEST_MAC_ADDR: [u8; 6] = [0xa0, 0x36, 0x9f, 0x1d, 0x94, 0x4c];

Expand Down Expand Up @@ -99,14 +93,13 @@ fn rmain(matches: &Matches, _opts: &Options) -> Result<(), &'static str> {

for nic in &mut nics {
let buffer = create_raw_packet(&DEST_MAC_ADDR, &mac_address, &[1;46])?;
nic.send_packet(buffer)?;
nic.send(buffer);
}

if matches.opt_present("r") {
loop {
for nic in &mut nics {
nic.poll_receive()?;
if let Some(_buffer) = nic.get_received_frame() {
if let Some(_buffer) = nic.receive() {
println!("Received packet on vnic {}", nic.id());
}
}
Expand Down
10 changes: 10 additions & 0 deletions applications/test_preemption_counter/Cargo.toml
Original file line number Diff line number Diff line change
@@ -0,0 +1,10 @@
[package]
name = "test_preemption_counter"
version = "0.1.0"
authors = ["Klim Tsoutsman <klim@tsoutsman.com>"]
description = "Application for testing the preemption counter"
edition = "2021"

[dependencies]
app_io = { path = "../../kernel/app_io" }
preemption = { path = "../../kernel/preemption" }
Loading

0 comments on commit 9849e9c

Please sign in to comment.