{"name":"py314-tokenizers","portdir":"python/py-tokenizers","version":"0.22.2","license":"Apache-2","platforms":"darwin","epoch":0,"replaced_by":null,"homepage":"https://github.com/huggingface/tokenizers","description":"Fast and Customizable Tokenizers","long_description":"Tokenizers provides an implementation of today's most used tokenizers, with a focus on performance and versatility. Includes BPE, WordPiece, and Unigram tokenizer implementations.","active":true,"categories":["python"],"maintainers":[],"variants":["universal"],"dependencies":[{"type":"build","ports":["py314-maturin","py314-build","py314-installer","py314-setuptools-rust","cargo","legacy-support","rust","clang-20"]},{"type":"lib","ports":["python314","py314-huggingface_hub","libunwind"]}],"depends_on":[{"type":"lib","ports":["py-tokenizers","py314-transformers"]}]}