it's time to update your Gemma 4 GGUFs

#!/usr/bin/env python3 """ Replace the chat template in a GGUF file. Usage: python replace_chat_template.py input.gguf output.gguf --template chat_template.jinja """ import argparse import numpy as np from gguf import GGUFReader, GGUFWriter, GGUFValueType def decode_bytes(b): """Decode bytes from a GGUF field, falling back to latin-1 for binary data like BPE merges.""" try: return b.tobytes().decode("utf-8") except UnicodeDecodeError: return b.tobytes().decode("latin-1") def main(): parser = argparse.ArgumentParser(description="Replace the chat template in a GGUF file") parser.add_argument("input", help="Path to the input GGUF file") parser.add_argument("output", help="Path to write the modified GGUF file") parser.add_argument("--template", required=True, help="Path to the Jinja2 chat template file") args = parser.parse_args() with open(args.template, "r", encoding="utf-8") as f: new_template = f.read() reader = GGUFReader(args.input) arch = reader.fields["general.architecture"].parts[-1].tobytes().decode() writer = GGUFWriter(args.output, arch=arch) # --- Copy KV metadata --- replaced = False skipped = [] for key, field in reader.fields.items(): # Skip — already set by GGUFWriter constructor if key == "general.architecture": continue # Replace the chat template if key == "tokenizer.chat_template": writer.add_string("tokenizer.chat_template", new_template) replaced = True continue val_type = field.types[0] try: if val_type == GGUFValueType.STRING: writer.add_string(key, decode_bytes(field.parts[-1])) elif val_type == GGUFValueType.UINT8: writer.add_uint8(key, field.parts[-1].item()) elif val_type == GGUFValueType.INT8: writer.add_int8(key, field.parts[-1].item()) elif val_type == GGUFValueType.UINT16: writer.add_uint16(key, field.parts[-1].item()) elif val_type == GGUFValueType.INT16: writer.add_int16(key, field.parts[-1].item()) elif val_type == GGUFValueType.UINT32: writer.add_uint32(key, field.parts[-1].item()) elif val_type == GGUFValueType.INT32: writer.add_int32(key, field.parts[-1].item()) elif val_type == GGUFValueType.UINT64: writer.add_uint64(key, field.parts[-1].item()) elif val_type == GGUFValueType.INT64: writer.add_int64(key, field.parts[-1].item()) elif val_type == GGUFValueType.FLOAT32: writer.add_float32(key, field.parts[-1].item()) elif val_type == GGUFValueType.FLOAT64: writer.add_float64(key, field.parts[-1].item()) elif val_type == GGUFValueType.BOOL: writer.add_bool(key, bool(field.parts[-1].item())) elif val_type == GGUFValueType.ARRAY: arr_type = field.types[1] # field.data contains indices into field.parts for the actual value elements, # filtering out the metadata parts (counts, type tags, etc.) data_parts = [field.parts[i] for i in field.data] if arr_type == GGUFValueType.STRING: writer.add_array(key, [decode_bytes(p) for p in data_parts]) else: arr = np.concatenate(data_parts) if len(data_parts) > 1 else data_parts[0] # Convert numpy array to Python list — _pack_val requires a Sequence writer.add_array(key, arr.tolist()) else: skipped.append((key, f"unknown type {val_type}")) except Exception as e: skipped.append((key, str(e))) if not replaced: print("Note: no existing chat template found — adding new one") writer.add_string("tokenizer.chat_template", new_template) if skipped: print(f"Warning: {len(skipped)} field(s) could not be copied:") for key, reason in skipped: print(f" - {key}: {reason}") # --- Copy tensors --- # Use raw_dtype to preserve the original quantization type (Q4_K_M, Q5_K_M, etc.) # without GGUFWriter rejecting them via its dtype-based type check. print(f"Copying {len(reader.tensors)} tensors...") for tensor in reader.tensors: # Don't pass raw_shape — tensor.shape is the logical shape but # add_tensor_info expects the byte shape when raw_dtype is set. # Letting it default to tensor.data.shape gives the correct byte shape. writer.add_tensor( tensor.name, tensor.data, raw_dtype=tensor.tensor_type, ) # --- Write everything --- writer.write_header_to_file() writer.write_kv_data_to_file() writer.write_tensors_to_file() writer.close() print(f"Done — written to {args.output}") if __name__ == "__main__": main()