Trez/opencommit-gitea
4
0
Fork 0
Code Issues Pull Requests 1 Actions Packages Projects Releases Wiki Activity

refactor(tokenCount.ts): format code with consistent indentation and quotes

Browse Source 
The code has been reformatted to use consistent indentation and quotes. This improves the readability and maintainability of the code. No functionality has been changed.
This commit is contained in:
di-sukharev
2023-03-29 11:25:48 +08:00
parent 0e1ad33179
commit 853662acc4
1 changed files with 10 additions and 10 deletions
Download Patch File Download Diff File Expand all files Collapse all files
src/utils/tokenCount.ts
+10 -10
View File
@@ -1,14 +1,14 @@
import { Tiktoken } from "@dqbd/tiktoken/lite" import { Tiktoken } from '@dqbd/tiktoken/lite';
import cl100k_base from "@dqbd/tiktoken/encoders/cl100k_base.json" assert{type: "json"} import cl100k_base from '@dqbd/tiktoken/encoders/cl100k_base.json' assert { type: 'json' };
export function tokenCount(content: string): number { export function tokenCount(content: string): number {
const encoding = new Tiktoken( const encoding = new Tiktoken(
cl100k_base.bpe_ranks, cl100k_base.bpe_ranks,
cl100k_base.special_tokens, cl100k_base.special_tokens,
cl100k_base.pat_str cl100k_base.pat_str
); );
const tokens = encoding.encode(content); const tokens = encoding.encode(content);
encoding.free(); encoding.free();
return tokens.length; return tokens.length;
} }