aardio 范例: 大模型 token 数快速估算工具

调用 transformers 计算 token 数

//调用 transformers 计算 token 数: https://www.aardio.com/zh-cn/doc/example/Languages/Python/Python%203.x/tokenizer.html
import win.ui;
/*DSG{{*/
var winform = win.form(text="大模型 token 数快速估算工具";right=759;bottom=469)
winform.add(
edit={cls="richedit";left=12;top=9;right=747;bottom=418;db=1;dl=1;dr=1;dt=1;edge=1;hscroll=1;multiline=1;vscroll=1;z=1};
radioGpt={cls="radiobutton";text="GPT";left=465;top=428;right=532;bottom=456;z=3};
radioMmseg={cls="radiobutton";text="通用";left=554;top=428;right=624;bottom=454;checked=1;z=4};
richedit={cls="richedit";text="0 tokens";left=18;top=432;right=393;bottom=451;db=1;dl=1;dr=1;multiline=1;transparent=1;z=2}
)
/*}}*/

import mmseg; 
import string.gpt;
import win.debounce;
import math.format;
winform.edit.onChange = win.debounce( function(){ 
    var str = winform.edit.text;
    if(!#str) return;

    var tokens;
    if(winform.radioGpt.checked){
        tokens = string.gpt.tokens(winform.edit.text);
    }
    else{
        tokens = mmseg.llmTokens(winform.edit.text);
    }

    winform.richedit.text = math.format.size(tokens,"≈ %s%s tokens");

},500);

winform.radioMmseg.oncommand = winform.edit.onChange;
winform.radioGpt.oncommand = winform.edit.onChange;

winform.edit.limit = -1;

winform.edit.enablePopMenu();

winform.show();
win.loopMessage();

/*
// 调用 Gemini 接口计算 token 数
var http = web.rest.jsonClient();
http.setHeaders({"x-goog-api-key":'api_key'}); 
var gemini = http.api("https://generativelanguage.googleapis.com/v1beta/{resource}/{model}:{method}");

var resp,err = gemini.models["gemini-2.5-flash"].countTokens({
    contents = {
        {
            parts = {
                {text = prompt}
            }
        }
    },
});

var totalTokens = resp[["totalTokens"]]
*/

/*
// 调用 GLM 接口计算 token 数
import web.rest.jsonClient;
var http = web.rest.jsonClient();
http.setAuthToken('api_key');

var glm = http.api("https://open.bigmodel.cn/api/paas/v4");
var resp,err = glm.tokenizer({
  "model": "glm-4-plus",    
  "messages": [
    {
      "role": "user",
      "content": prompt
    }
  ]
})

var totalTokens = resp[["usage"]][["prompt_tokens"]]
*/