From e1931986b7392509bb5c3201892466f91a8d5588 Mon Sep 17 00:00:00 2001 From: Alexander Jung Date: Sun, 22 Oct 2023 10:34:05 +0000 Subject: [PATCH] Add model download and header prep as part of `Makefile.uk` Signed-off-by: Alexander Jung --- .gitignore | 1 + Makefile.uk | 12 ++++++++++++ 2 files changed, 13 insertions(+) diff --git a/.gitignore b/.gitignore index 42992af3..809dfca5 100644 --- a/.gitignore +++ b/.gitignore @@ -9,3 +9,4 @@ model.h workdir/ build/ stories15M.bin +model.bin diff --git a/Makefile.uk b/Makefile.uk index 09c0bdaa..d2ffe72c 100644 --- a/Makefile.uk +++ b/Makefile.uk @@ -3,3 +3,15 @@ L2E_SRCS-y += $(L2E_BASE)/run.c L2E_SRCS-y += $(L2E_BASE)/model.h L2E_SRCS-y += $(L2E_BASE)/tokenizer.h L2E_CFLAGS-y += -DUNIK + +$(L2E_BASE)/model.h: $(L2E_BASE)/strlit $(L2E_BASE)/model.bin + $(L2E_BASE)/strlit -i emb_Model_data $(L2E_BASE)/model.bin $@ + +$(L2E_BASE)/model.bin: + $(WGET) -O $@ https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.bin + +$(L2E_BASE)/tokenizer.h: $(L2E_BASE)/strlit + $(L2E_BASE)/strlit -i emb_Tokenizer_data $(L2E_BASE)/tokenizer.bin $@ + +$(L2E_BASE)/strlit: + $(CC) -Ofast $(L2E_BASE)/strliteral.c -o $@