Sukharev
b98b892ba1
* update major version --------- Co-authored-by: E Einowski <5124085+EwiththeBowtie@users.noreply.github.com> Co-authored-by: Max Zavodniuk <lovemaxonly@gmail.com> Co-authored-by: Malthe Poulsen <30603252+malpou@users.noreply.github.com> Co-authored-by: Sébastien Fichot <fichot.sebastien@gmail.com>
14 lines
380 B
TypeScript
14 lines
380 B
TypeScript
import cl100k_base from '@dqbd/tiktoken/encoders/cl100k_base.json';
|
|
import { Tiktoken } from '@dqbd/tiktoken/lite';
|
|
|
|
export function tokenCount(content: string): number {
|
|
const encoding = new Tiktoken(
|
|
cl100k_base.bpe_ranks,
|
|
cl100k_base.special_tokens,
|
|
cl100k_base.pat_str
|
|
);
|
|
const tokens = encoding.encode(content);
|
|
encoding.free();
|
|
return tokens.length;
|
|
}
|