diff --git a/targets.bzl b/targets.bzl deleted file mode 100644 index 0dcb023..0000000 --- a/targets.bzl +++ /dev/null @@ -1,212 +0,0 @@ -load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "get_executorch_supported_platforms", "runtime") -load("@fbsource//xplat/executorch/third-party:glob_defs.bzl", "subdir_glob") - -PLATFORMS = get_executorch_supported_platforms() - -def define_common_targets(): - """Defines targets that should be shared between fbcode and xplat. - - The directory containing this targets.bzl file should also contain both - TARGETS and BUCK files that call this function. - """ - - runtime.cxx_library( - name = "headers", - exported_headers = subdir_glob([ - ("include", "pytorch/tokenizers/*.h"), - ]), - visibility = [ - "@EXECUTORCH_CLIENTS", - "//pytorch/tokenizers/...", - ], - header_namespace = "", - platforms = PLATFORMS, - ) - - runtime.cxx_library( - name = "regex", - srcs = [ - "src/re2_regex.cpp", - "src/regex.cpp", - ], - exported_deps = [ - "fbsource//third-party/re2:re2", - ":headers", - ], - visibility = ["//pytorch/tokenizers/..."], - header_namespace = "", - platforms = PLATFORMS, - ) - - runtime.cxx_library( - name = "regex_lookahead", - srcs = [ - "src/pcre2_regex.cpp", - "src/regex_lookahead.cpp", - "src/std_regex.cpp", - ], - exported_deps = [ - ":regex", - ":headers", - ], - compiler_flags = [ - "-Wno-global-constructors", - "-Wno-missing-prototypes", - ], - exported_external_deps = [ - "pcre2", - ], - # Making sure this library is not being stripped by linker. - # @lint-ignore BUCKLINT: Avoid link_whole=True - link_whole = True, - visibility = [ - "@EXECUTORCH_CLIENTS", - "//pytorch/tokenizers/...", - ], - header_namespace = "", - platforms = PLATFORMS, - ) - - runtime.cxx_library( - name = "bpe_tokenizer_base", - srcs = [ - "src/bpe_tokenizer_base.cpp", - ], - exported_deps = [ - "fbsource//third-party/re2:re2", - ":headers", - ], - visibility = [ - "//pytorch/tokenizers/...", - ], - platforms = PLATFORMS, - ) - - runtime.cxx_library( - name = "sentencepiece", - srcs = [ - "src/sentencepiece.cpp", - ], - deps = [ - ":regex", - ], - exported_deps = [ - ":headers", - ], - visibility = [ - "@EXECUTORCH_CLIENTS", - "//pytorch/tokenizers/...", - ], - exported_external_deps = [ - "sentencepiece", - "abseil-cpp", - ], - platforms = PLATFORMS, - ) - - runtime.cxx_library( - name = "tiktoken", - srcs = [ - "src/tiktoken.cpp", - ], - deps = [ - ":regex", - ], - exported_deps = [ - "fbsource//third-party/re2:re2", - ":bpe_tokenizer_base", - ":headers", - ], - visibility = [ - "@EXECUTORCH_CLIENTS", - "//pytorch/tokenizers/...", - ], - platforms = PLATFORMS, - ) - - runtime.cxx_library( - name = "hf_tokenizer", - srcs = [ - "src/hf_tokenizer.cpp", - "src/pre_tokenizer.cpp", - "src/token_decoder.cpp", - "src/normalizer.cpp", - ], - deps = [ - ":regex", - ], - exported_deps = [ - "fbsource//third-party/re2:re2", - ":bpe_tokenizer_base", - ":headers", - "//pytorch/tokenizers/third-party:unicode", - ], - visibility = [ - "@EXECUTORCH_CLIENTS", - "//pytorch/tokenizers/...", - ], - exported_external_deps = [ - "nlohmann_json", - ], - platforms = PLATFORMS, - ) - - runtime.cxx_library( - name = "llama2c_tokenizer", - srcs = [ - "src/llama2c_tokenizer.cpp", - ], - exported_deps = [ - ":headers", - ], - visibility = [ - "@EXECUTORCH_CLIENTS", - "//pytorch/tokenizers/...", - ], - platforms = PLATFORMS, - ) - - runtime.cxx_library( - name = "tekken", - srcs = [ - "src/tekken.cpp", - ], - deps = [ - ":regex", - ], - exported_deps = [ - "fbsource//third-party/re2:re2", - ":bpe_tokenizer_base", - ":headers", - ], - exported_external_deps = [ - "nlohmann_json", - ], - visibility = [ - "@EXECUTORCH_CLIENTS", - "//pytorch/tokenizers/...", - ], - platforms = PLATFORMS, - ) - - runtime.cxx_python_extension( - name = "pytorch_tokenizers_cpp", - srcs = [ - "src/python_bindings.cpp", - ], - visibility = [ - "@EXECUTORCH_CLIENTS", - "//pytorch/tokenizers/...", - ], - base_module = "pytorch_tokenizers", - deps = [ - ":hf_tokenizer", - ":llama2c_tokenizer", - ":sentencepiece", - ":tekken", - ":tiktoken", - ], - external_deps = [ - "pybind11", - ], - ) diff --git a/third-party/targets.bzl b/third-party/targets.bzl index 9bdcd8c..5e98b91 100644 --- a/third-party/targets.bzl +++ b/third-party/targets.bzl @@ -1,23 +1,6 @@ load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") -load("@fbsource//xplat/executorch/third-party:glob_defs.bzl", "subdir_glob") def define_common_targets(): - runtime.cxx_library( - name = "unicode", - srcs = [ - "llama.cpp-unicode/src/unicode.cpp", - "llama.cpp-unicode/src/unicode-data.cpp", - ], - header_namespace = "", - exported_headers = subdir_glob([ - ("llama.cpp-unicode/include", "*.h"), - ]), - compiler_flags = [ - "-Wno-error=deprecated-declarations", - ], - visibility = ["@EXECUTORCH_CLIENTS", "//pytorch/tokenizers/..."], - ) - if runtime.is_oss: runtime.cxx_library( name = "abseil",