使用llama.cpp本地部署 ：Qwen3.8-27B-GGUF
使用：https://github.com/ggml-org/llama.cpp
ubuntu 下载文件：llama-b11780-bin-ubuntu-vulkan-x64.tar.gz
模型下载地址：
https://huggingface.co/unsloth/Qwen3.8-27B-GGUF/tree/main
下载 ：
aria2c -x 15 -s 15 https://hf-mirror.com/unsloth/Qwen3.8-27B-GGUF/resolve/main/Qwen3.8-27B-Q4_K_M.gguf
模板文件：qwen38_fixed_template.jinja
{%- set image_count = namespace(value=0) %}
{%- set video_count = namespace(value=0) %}
{%- macro render_content(content, do_vision_count, is_system_content=false) %}
{%- if content is string %}
{{- content }}
{%- elif content is iterable and content is not mapping %}
{%- for item in content %}
{%- if 'image' in item or 'image_url' in item or item.type == 'image' %}
{%- if is_system_content %}
{{- raise_exception('System message cannot contain images.') }}
{%- endif %}
{%- if do_vision_count %}
{%- set image_count.value = image_count.value + 1 %}
{%- endif %}
{%- if add_vision_id %}
{{- 'Picture ' ~ image_count.value ~ ': ' }}
{%- endif %}
{{- '<|vision_start|><|image_pad|><|vision_end|>' }}
{%- elif 'video' in item or item.type == 'video' %}
{%- if is_system_content %}
{{- raise_exception('System message cannot contain videos.') }}
{%- endif %}
{%- if do_vision_count %}
{%- set video_count.value = video_count.value + 1 %}
{%- endif %}
{%- if add_vision_id %}
{{- 'Video ' ~ video_count.value ~ ': ' }}
{%- endif %}
{{- '<|vision_start|><|video_pad|><|vision_end|>' }}
{%- elif 'text' in item %}
{{- item.text }}
{%- else %}
{{- raise_exception('Unexpected item type in content.') }}
{%- endif %}
{%- endfor %}
{%- elif content is none or content is undefined %}
{{- '' }}
{%- else %}
{{- raise_exception('Unexpected content type.') }}
{%- endif %}
{%- endmacro %}
{%- if not messages %}
{{- raise_exception('No messages provided.') }}
{%- endif %}
{%- set reasoning_instructions = '' %}
{%- if enable_thinking is undefined or enable_thinking is true %}
{%- set resolved_reasoning_effort = reasoning_effort|default('xhigh') %}
{%- if resolved_reasoning_effort == 'high' %}
{%- set resolved_reasoning_effort = 'xhigh' %}
{%- endif %}
{%- if resolved_reasoning_effort not in ('xhigh', 'medium', 'low') %}
{{- raise_exception('Unexpected reasoning effort ' ~ reasoning_effort ~ '. Supported types are xhigh (default), medium, and low.') }}
{%- endif %}
{%- if resolved_reasoning_effort == 'xhigh' %}
{%- set reasoning_instructions = 'Reasoning effort is set to xhigh. Please think carefully through the task, validate key assumptions, consider plausible alternatives, and prioritize correctness, consistency, and clarity in the final answer.' %}
{%- elif resolved_reasoning_effort == 'low' %}
{%- set reasoning_instructions = 'Reasoning effort is set to low. Keep your thinking brief and focused, moving directly to the conclusion without unnecessary elaboration.' %}
{%- endif %}
{%- endif %}
{%- if tools and tools is iterable and tools is not mapping %}
{{- '<|im_start|>system\n' }}
{%- if reasoning_instructions %}
{{- reasoning_instructions + '\n\n' }}
{%- endif %}
{{- "# Tools\n\nYou have access to the following functions:\n\n<tools>" }}
{%- for tool in tools %}
{{- "\n" }}
{{- tool | tojson }}
{%- endfor %}
{{- "\n</tools>" }}
{{- '\n\nIf you choose to call a function ONLY reply in the following format with NO suffix:\n\n<tool_call>\n<function=example_function_name>\n<parameter=example_parameter_1>\nvalue_1\n</parameter>\n<parameter=example_parameter_2>\nThis is the value for the second parameter\nthat can span\nmultiple lines\n</parameter>\n</function>\n</tool_call>\n\n<IMPORTANT>\nReminder:\n- Function calls MUST follow the specified format: an inner <function=...></function> block must be nested within <tool_call></tool_call> XML tags\n- Required parameters MUST be specified\n- You may provide optional reasoning for your function call in natural language BEFORE the function call, but NOT after\n- If there is no function call available, answer the question like normal with your current knowledge and do not tell the user about function calls\n</IMPORTANT>' }}
{%- if messages[0].role == 'system' or messages[0].role == 'developer' %}
{%- set content = render_content(messages[0].content, false, true)|trim %}
{%- if content %}
{{- '\n\n' + content }}
{%- endif %}
{%- endif %}
{{- '<|im_end|>\n' }}
{%- else %}
{%- if messages[0].role == 'system' or messages[0].role == 'developer' %}
{%- set content = render_content(messages[0].content, false, true)|trim %}
{%- if content %}
{{- '<|im_start|>system\n' + (reasoning_instructions + '\n\n' if reasoning_instructions else '') + content + '<|im_end|>\n' }}
{%- elif reasoning_instructions %}
{{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
{%- endif %}
{%- elif reasoning_instructions %}
{{- '<|im_start|>system\n' + reasoning_instructions + '<|im_end|>\n' }}
{%- endif %}
{%- endif %}
{%- set ns = namespace(multi_step_tool=true, last_query_index=messages|length - 1) %}
{%- for message in messages[::-1] %}
{%- set index = (messages|length - 1) - loop.index0 %}
{%- if ns.multi_step_tool and message.role == "user" %}
{%- set content = render_content(message.content, false)|trim %}
{%- if not(content.startswith('<tool_response>') and content.endswith('</tool_response>')) %}
{%- set ns.multi_step_tool = false %}
{%- set ns.last_query_index = index %}
{%- endif %}
{%- endif %}
{%- endfor %}
{%- if ns.multi_step_tool %}
{{- raise_exception('No user query found in messages.') }}
{%- endif %}
{%- for message in messages %}
{%- set content = render_content(message.content, true)|trim %}
{%- if message.role == "system" or message.role == "developer" %}
{%- if not loop.first %}
{{- '<|im_start|>system\n' + content + '<|im_end|>' + '\n' }}
{%- endif %}
{%- elif message.role == "user" %}
{{- '<|im_start|>' + message.role + '\n' + content + '<|im_end|>' + '\n' }}
{%- elif message.role == "assistant" %}
{%- set reasoning_content = '' %}
{%- if message.reasoning_content is string %}
{%- set reasoning_content = message.reasoning_content %}
{%- endif %}
{%- set reasoning_content = reasoning_content|trim %}
{%- if preserve_thinking is undefined or preserve_thinking is true or loop.index0 > ns.last_query_index %}
{{- '<|im_start|>' + message.role + '\n<think>\n' + reasoning_content + '\n</think>\n\n' + content }}
{%- else %}
{{- '<|im_start|>' + message.role + '\n' + content }}
{%- endif %}
{%- if message.tool_calls and message.tool_calls is iterable and message.tool_calls is not mapping %}
{%- for tool_call in message.tool_calls %}
{%- if tool_call.function is defined %}
{%- set tool_call = tool_call.function %}
{%- endif %}
{%- if tool_call.name is not defined or tool_call.name is none %}
{{- raise_exception('Tool call is missing a function name.') }}
{%- endif %}
{%- if loop.first %}
{%- if content|trim %}
{{- '\n\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
{%- else %}
{{- '<tool_call>\n<function=' + tool_call.name + '>\n' }}
{%- endif %}
{%- else %}
{{- '\n<tool_call>\n<function=' + tool_call.name + '>\n' }}
{%- endif %}
{%- if tool_call.arguments is mapping %}
{%- for args_name, args_value in tool_call.arguments|items %}
{{- '<parameter=' + args_name + '>\n' }}
{%- set args_value = args_value | string if args_value is string else args_value | tojson | safe %}
{{- args_value }}
{{- '\n</parameter>\n' }}
{%- endfor %}
{%- elif tool_call.arguments is string %}
{%- if tool_call.arguments|trim %}
{{- raise_exception('Tool call arguments for function "' + (tool_call.name | string) + '" were passed as a JSON string. Parse them into an object before calling apply_chat_template.') }}
{%- endif %}
{%- elif tool_call.arguments is defined and tool_call.arguments is not none %}
{{- raise_exception('Tool call arguments for function "' + (tool_call.name | string) + '" must be an object/mapping or a JSON string.') }}
{%- endif %}
{{- '</function>\n</tool_call>' }}
{%- endfor %}
{%- endif %}
{{- '<|im_end|>\n' }}
{%- elif message.role == "tool" %}
{%- if loop.previtem and loop.previtem.role != "tool" %}
{{- '<|im_start|>user' }}
{%- endif %}
{{- '\n<tool_response>\n' }}
{{- content }}
{{- '\n</tool_response>' }}
{%- if not loop.last and loop.nextitem.role != "tool" %}
{{- '<|im_end|>\n' }}
{%- elif loop.last %}
{{- '<|im_end|>\n' }}
{%- endif %}
{%- else %}
{{- raise_exception('Unexpected message role.') }}
{%- endif %}
{%- endfor %}
{%- if add_generation_prompt %}
{{- '<|im_start|>assistant\n' }}
{%- if enable_thinking is defined and enable_thinking is false %}
{{- '<think>\n\n</think>\n\n' }}
{%- else %}
{{- '<think>\n' }}
{%- endif %}
{%- endif %}
{#- Unsloth fixes - developer role, tool calling, thinking level #}
{#- qmnp fixes - mid conversation system messages #}
启动方式：128K
./llama-server -m ./models/Qwen3.8-27B-UD-Q4_K_M.gguf --port 9011 --host 0.0.0.0 --chat-template-file ./qwen38_fixed_template.jinja --jinja --api-key-file api-keys.txt -c 131072
1.50.723.373 I slot print_timing: id 0 | task 0 | n_gen = 510, tg = 27.21 t/s, tg_3s = 27.20 t/s
1.53.751.377 I slot print_timing: id 0 | task 0 | n_gen = 591, tg = 27.15 t/s, tg_3s = 26.75 t/s
1.56.779.359 I slot print_timing: id 0 | task 0 | n_gen = 673, tg = 27.14 t/s, tg_3s = 27.08 t/s
1.59.788.336 I slot print_timing: id 0 | task 0 | n_gen = 752, tg = 27.04 t/s, tg_3s = 26.25 t/s
启动方式：256K 非常慢，几乎不用
./llama-server -m ./models/Qwen3.8-27B-UD-Q4_K_M.gguf --port 9011 --host 0.0.0.0 --chat-template-file ./qwen38_fixed_template.jinja --jinja --api-key-file api-keys.txt -c 262100
11.24.486.083 I slot print_timing: id 0 | task 742 | n_gen = 880, tg = 3.42 t/s, tg_3s = 3.41 t/s
11.27.741.306 I slot print_timing: id 0 | task 742 | n_gen = 891, tg = 3.42 t/s, tg_3s = 3.38 t/s
11.30.978.808 I slot print_timing: id 0 | task 742 | n_gen = 902, tg = 3.42 t/s, tg_3s = 3.40 t/s
11.34.223.746 I slot print_timing: id 0 | task 742 | n_gen = 913, tg = 3.42 t/s, tg_3s = 3.39 t/s
11.37.460.968 I slot print_timing: id 0 | task 742 | n_gen = 924, tg = 3.42 t/s, tg_3s = 3.40 t/s
11.40.698.952 I slot print_timing: id 0 | task 742 | n_gen = 935, tg = 3.41 t/s, tg_3s = 3.40 t/s
3.43.494.088 I slot print_timing: id 0 | task 226 | n_gen = 100, tg = 5.59 t/s, tg_3s = 5.65 t/s
3.46.537.341 I slot print_timing: id 0 | task 226 | n_gen = 117, tg = 5.59 t/s, tg_3s = 5.59 t/s
3.49.604.693 I slot print_timing: id 0 | task 226 | n_gen = 134, tg = 5.58 t/s, tg_3s = 5.54 t/s
api-keys.txt 内容：
sk-llmkey111111111 sk-llmkey222222222
关闭思考模式：启动方式：128K
./llama-server -m ./models/Qwen3.8-27B-UD-Q4_K_M.gguf --port 9011 --host 0.0.0.0 --chat-template-file ./qwen38_fixed_template.jinja --jinja --api-key-file api-keys.txt -c 131072 --chat-template-kwargs '{"enable_thinking": false}'
量化档位从好到差
| BF16/FP16 | ~55GB | 无损 | 70GB+ |
| Q8_0 | ~29GB | 几乎无感（<1%） | 32GB |
| Q6_K | ~22GB | 很小（~1-2%） | 24GB |
| Q5_K_M | ~19GB | 小（~2-3%） | 24GB |
| Q4_K_M（你现在的） | ~16GB | 可感知（~3-5%） | 20GB |
| Q3_K_M 及以下 | ~13GB | 明显变笨 | 16GB |
参考启动方式：文件路径自行调整：
./llama-server -m ../llama-data/models/Qwen3.8-27B-UD-Q4_K_M.gguf --port 9011 --host 0.0.0.0 --chat-template-file ../llama-data/qwen38_fixed_template.jinja --jinja --api-key-file ../llama-data/api-keys.txt -c 131072 --chat-template-kwargs '{"enable_thinking": false}'
0.00.000.938 W Setting 'enable_thinking' via --chat-template-kwargs is deprecated. Use --reasoning on / --reasoning off instead.
0.00.011.894 I srv llama_server: initializing ...
0.00.077.663 I cmn common_param: common_params_print_info: verbosity = 3 (adjust with the `-lv N` CLI arg)
0.00.079.363 I srv load_model: loading model '../llama-data/models/Qwen3.8-27B-UD-Q4_K_M.gguf'
0.00.625.084 W model has unused tensor blk.64.attn_norm.weight (size = 20480 bytes) -- ignoring
0.00.625.090 W model has unused tensor blk.64.post_attention_norm.weight (size = 20480 bytes) -- ignoring
0.00.625.096 W model has unused tensor blk.64.attn_q.weight (size = 51609600 bytes) -- ignoring
0.00.625.099 W model has unused tensor blk.64.attn_k.weight (size = 5570560 bytes) -- ignoring
0.00.625.101 W model has unused tensor blk.64.attn_v.weight (size = 5570560 bytes) -- ignoring
0.00.625.109 W model has unused tensor blk.64.attn_output.weight (size = 25804800 bytes) -- ignoring
0.00.625.111 W model has unused tensor blk.64.attn_q_norm.weight (size = 1024 bytes) -- ignoring
0.00.625.113 W model has unused tensor blk.64.attn_k_norm.weight (size = 1024 bytes) -- ignoring
0.00.625.116 W model has unused tensor blk.64.ffn_gate.weight (size = 73113600 bytes) -- ignoring
0.00.625.118 W model has unused tensor blk.64.ffn_down.weight (size = 73113600 bytes) -- ignoring
0.00.625.120 W model has unused tensor blk.64.ffn_up.weight (size = 73113600 bytes) -- ignoring
0.00.625.123 W model has unused tensor blk.64.nextn.eh_proj.weight (size = 43008000 bytes) -- ignoring
0.00.625.125 W model has unused tensor blk.64.nextn.enorm.weight (size = 20480 bytes) -- ignoring
0.00.625.128 W model has unused tensor blk.64.nextn.hnorm.weight (size = 20480 bytes) -- ignoring
0.00.625.134 W model has unused tensor blk.64.nextn.shared_head_norm.weight (size = 20480 bytes) -- ignoring
0.05.801.003 I cmn init: llama threadpool init, n_threads = 6
0.07.953.390 I srv load_model: initializing, n_slots = 4, n_ctx_slot = 131072, kv_unified = 'true'
0.07.957.876 W srv init: chat template supports preserving reasoning, it is enabled by default (may use more tokens, disable via --no-reasoning-preserve)
0.07.957.914 I srv server_run_i: model loaded
0.07.957.918 I srv server_on_li: listening on http://0.0.0.0:9011
GPU：
