sentence piece model from gguf

A developer shared a Python script that extracts a SentencePiece tokenizer model from GGUF files, enabling the use of the tokenizer with the sentencepiece library. The approach reads token, score, and token type fields from the GGUF metadata and reconstructs a ModelProto, allowing direct encoding and decoding of text without loading the full model.

python from gguf.gguf reader import GGUFReader python def get string array field gguf reader: GGUFReader, key: str : f = gguf reader.get field key return bytes f.parts d .decode "utf-8" for d in f.data def get field gguf reader: GGUFReader, key: str : f = gguf reader.get field key if f is None: return 0 if len f.data = 1: raise NotImplementedError f"multiple data is not supported" part = f.parts f.data 0 if len part = 1: raise NotImplementedError f"multiple parts are not supported" value = part 0 if isinstance value, np.float32 : return float value elif isinstance value, np.uint32 : return int value return value gguf path = "/path/to/mistral-7b-v0.1.Q6 K.gguf" gguf reader = GGUFReader str gguf path tokens = get string array field gguf reader, "tokenizer.ggml.tokens" merges = get string array field gguf reader, "tokenizer.ggml.merges" merges = tuple m.split " " for m in merges scores = p.item for p in gguf reader.get field "tokenizer.ggml.scores" .parts 5: why 5 ? token types = p.item for p in gguf reader.get field "tokenizer.ggml.token type" .parts 5: python import sentencepiece as spm python import sentencepiece.sentencepiece model pb2 as model normalizer spec = model.NormalizerSpec name="identity", precompiled charsmap=b"", add dummy prefix=True, remove extra whitespaces=False, normalization rule tsv=b"", trainer spec = model.TrainerSpec model type="BPE", vocab size=32000, input format="text", split by unicode script=True, split by whitespace=True, split by number=True, treat whitespace as suffix=False, split digits=True, allow whitespace only pieces=True, vocabulary output piece score=True, byte fallback=True, unk id=0, bos id=1, eos id=2, pad id=-1, unk piece="<unk ", bos piece="<s ", eos piece="</s ", pad piece="<pad ", pretokenization delimiter="", m = model.ModelProto trainer spec=trainer spec, normalizer spec=normalizer spec for token, score, token type in zip tokens, scores, token types : m.pieces.append model.ModelProto.SentencePiece piece=token, score=score, type=token type tokenizer = spm.SentencePieceProcessor model proto=m.SerializeToString tokenizer.encode "how tall is obama" 910, 9369, 349, 818, 2786 tokenizer.decode 910, 9369, 349, 818, 2786 'how tall is obama' tokenizer.id to piece 910 '▁how'