1load("@fbsource//xplat/executorch/build:runtime_wrapper.bzl", "runtime") 2 3def define_common_targets(): 4 """Defines targets that should be shared between fbcode and xplat. 5 6 The directory containing this targets.bzl file should also contain both 7 TARGETS and BUCK files that call this function. 8 """ 9 runtime.python_library( 10 name = "tokenizer_py_lib", 11 srcs = [ 12 "__init__.py", 13 "tokenizer.py", 14 "utils.py", 15 ], 16 base_module = "executorch.extension.llm.tokenizer", 17 visibility = [ 18 "//executorch/examples/...", 19 "//executorch/extension/llm/tokenizer/...", 20 "//executorch/extension/llm/export/...", 21 "//bento/...", 22 "//bento_kernels/...", 23 "@EXECUTORCH_CLIENTS", 24 ], 25 _is_external_target = True, 26 deps = [ 27 "//executorch/examples/models/llama/tokenizer:tiktoken_py", 28 ], 29 external_deps = [ 30 "sentencepiece-py", 31 ], 32 ) 33 34 runtime.python_binary( 35 name = "tokenizer_py", 36 main_module = "executorch.extension.llm.tokenizer.tokenizer", 37 visibility = [ 38 "//executorch/examples/...", 39 "fbsource//xplat/executorch/examples/...", 40 ], 41 _is_external_target = True, 42 deps = [ 43 ":tokenizer_py_lib", 44 ], 45 ) 46 47 runtime.cxx_library( 48 name = "tokenizer_header", 49 exported_headers = [ 50 "tokenizer.h", 51 ], 52 exported_deps = [ 53 "//executorch/runtime/core:core", 54 ], 55 visibility = [ 56 "@EXECUTORCH_CLIENTS", 57 ], 58 ) 59 60 runtime.cxx_library( 61 name = "bpe_tokenizer", 62 srcs = [ 63 "bpe_tokenizer.cpp", 64 ], 65 exported_headers = [ 66 "bpe_tokenizer.h", 67 ], 68 exported_deps = [ 69 ":tokenizer_header", 70 "//executorch/runtime/core:core", 71 ], 72 visibility = [ 73 "@EXECUTORCH_CLIENTS", 74 ], 75 ) 76 77 runtime.cxx_library( 78 name = "tiktoken", 79 srcs = [ 80 "tiktoken.cpp", 81 ], 82 exported_headers = [ 83 "tiktoken.h", 84 "base64.h", 85 ], 86 exported_deps = [ 87 ":tokenizer_header", 88 "//executorch/runtime/core:core", 89 ], 90 visibility = [ 91 "@EXECUTORCH_CLIENTS", 92 ], 93 exported_external_deps = [ 94 "re2", 95 ], 96 ) 97