Add model download and header prep as part of Makefile.uk

Signed-off-by: Alexander Jung <alex@unikraft.io>
unikraft · Oct 22, 2023 · dbcde60 · dbcde60
1 parent abae8a5
commit dbcde60
Show file tree

Hide file tree

Showing 2 changed files with 13 additions and 0 deletions.
diff --git a/.gitignore b/.gitignore
@@ -9,3 +9,4 @@ model.h
 workdir/
 build/
 stories15M.bin
+model.bin
diff --git a/Makefile.uk b/Makefile.uk
@@ -3,3 +3,15 @@ L2E_SRCS-y += $(L2E_BASE)/run.c
 L2E_SRCS-y += $(L2E_BASE)/model.h
 L2E_SRCS-y += $(L2E_BASE)/tokenizer.h
 L2E_CFLAGS-y += -DUNIK
+
+$(L2E_BASE)/model.h: $(L2E_BASE)/strlit $(L2E_BASE)/model.bin
+	$(L2E_BASE)/strlit -i emb_Model_data $(L2E_BASE)/model.bin $@
+
+$(L2E_BASE)/model.bin:
+	$(WGET) -O $@ https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.bin
+
+$(L2E_BASE)/tokenizer.h: $(L2E_BASE)/strlit
+	$(L2E_BASE)/strlit -i emb_Model_data $(L2E_BASE)/tokenizer.bin $@
+
+$(L2E_BASE)/strlit:
+	$(CC) -Ofast $(L2E_BASE)/strliteral.c -o $@