Int8 quant for optimized performance on Ampere.

usage with sglang

Currently, upstream sglang doesn't load this quant correctly due to a few minor issues. Until upstream is fixed, a working fork is available at https://github.com/nytopop/sglang/tree/qwen-30b-a3b:

uv venv --python 3.12

# use patched sglang from git
uv pip install "git+https://github.com/nytopop/sglang.git@qwen-30b-a3b#subdirectory=python[all]" --find-links https://flashinfer.ai/whl/cu124/torch2.5/flashinfer-python

# run
uv run python -m sglang.launch_server --model-path nytopop/Qwen3-30B-A3B.w8a8 --quantization w8a8_int8 --reasoning-parser qwen3

creation

from transformers import AutoModelForCausalLM
from llmcompressor import oneshot
from llmcompressor.modifiers.quantization import QuantizationModifier
from llmcompressor.transformers.compression.helpers import calculate_offload_device_map

model_id  = "Qwen/Qwen3-30B-A3B"
model_out = model_id.split("/")[1] + ".w8a8"

device_map = calculate_offload_device_map(
    model_id, reserve_for_hessians=False, num_gpus=1, torch_dtype="bfloat16"
)

for k, v in device_map.items():
  if v == 'disk':
    device_map[k] = 'cpu'

model = AutoModelForCausalLM.from_pretrained(
  model_id,
  device_map=device_map,
  torch_dtype="bfloat16",
)

recipe = QuantizationModifier(targets="Linear", scheme="W8A8", ignore=["lm_head", "re:.*mlp.gate$"])

oneshot(model=model, recipe=recipe, output_dir=model_out)