Tiktoken: in the future, show tokens

Shivak11 · May 16, 2024 · d5c3f50 · d5c3f50
1 parent 21d045b
commit d5c3f50
Showing 1 changed file with 3 additions and 0 deletions.
diff --git a/src/common/util/token-counter.ts b/src/common/util/token-counter.ts
@@ -6,6 6,9 @@ import { DLLMId, findLLMOrThrow } from '~/modules/llms/store-llms';
 // Do not set this to true in production, it's very verbose
 const DEBUG_TOKEN_COUNT = false;
 
 // Globals
 // const tokenEncodings: string[] = ['gpt2', 'r50k_base', 'p50k_base', 'p50k_edit', 'cl100k_base', 'o200k_base'] satisfies TiktokenEncoding[];
 
 // Global symbols to dynamically load the Tiktoken library
 let get_encoding: ((encoding: TiktokenEncoding) => Tiktoken) | null = null;
 let encoding_for_model: ((model: TiktokenModel) => Tiktoken) | null = null;