gguf-quantization-with-imatrix-and-k-quantization.ipynb

{"metadata":{"kernelspec":{"language":"python","display_name":"Python 3","name":"python3"},"language_info":{"name":"python","version":"3.10.14","mimetype":"text/x-python","codemirror_mode":{"name":"ipython","version":3},"pygments_lexer":"ipython3","nbconvert_exporter":"python","file_extension":".py"},"kaggle":{"accelerator":"gpu","dataSources":[],"dockerImageVersionId":30762,"isInternetEnabled":true,"language":"python","sourceType":"notebook","isGpuEnabled":true}},"nbformat_minor":4,"nbformat":4,"cells":[{"cell_type":"markdown","source":"# Introduction\n\nThis note implement the concept in https://towardsdatascience.com/gguf-quantization-with-imatrix-and-k-quantization-to-run-llms-on-your-cpu-02356b531926\n\nThis note will quantize Meta-Llama with Imatrix and K-quantization will llama.cpp\n\n# Install llama.cpp\n\nwe use llama-cpp-python binding instead of the directly using llama-cpp\n","metadata":{}},{"cell_type":"code","source":"!git clone https://github.com/ggerganov/llama.cpp\n%cd llama.cpp\n# !export GGML_CUDA=1 \n!cp -r /usr/local/cuda-12.3/targets /usr/local/nvidia/ \n!make GGML_CUDA=1 CUDA_PATH=/usr/local/nvidia  > make.log 2>&1\n","metadata":{"execution":{"iopub.status.busy":"2024-09-24T05:47:36.115593Z","iopub.execute_input":"2024-09-24T05:47:36.115896Z","iopub.status.idle":"2024-09-24T06:09:17.652308Z","shell.execute_reply.started":"2024-09-24T05:47:36.115863Z","shell.execute_reply":"2024-09-24T06:09:17.651002Z"},"trusted":true},"execution_count":1,"outputs":[{"name":"stdout","text":"Cloning into 'llama.cpp'...\nremote: Enumerating objects: 34606, done.\u001b[K\nremote: Counting objects: 100% (133/133), done.\u001b[K\nremote: Compressing objects: 100% (84/84), done.\u001b[K\nremote: Total 34606 (delta 63), reused 92 (delta 46), pack-reused 34473 (from 1)\u001b[K\nReceiving objects: 100% (34606/34606), 59.39 MiB | 23.78 MiB/s, done.\nResolving deltas: 100% (24989/24989), done.\n/kaggle/working/llama.cpp\n","output_type":"stream"}]},{"cell_type":"code","source":"!tail make.log","metadata":{"execution":{"iopub.status.busy":"2024-09-24T06:09:17.655006Z","iopub.execute_input":"2024-09-24T06:09:17.655754Z","iopub.status.idle":"2024-09-24T06:09:18.631108Z","shell.execute_reply.started":"2024-09-24T06:09:17.655706Z","shell.execute_reply":"2024-09-24T06:09:18.630012Z"},"trusted":true},"execution_count":2,"outputs":[{"name":"stdout","text":"c++ -std=c++11 -fPIC -O3 -g -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread -fopenmp  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -Iggml/include -Iggml/src -Iinclude -Isrc -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_OPENMP -DGGML_USE_LLAMAFILE -DGGML_USE_CUDA -DGGML_CUDA_USE_GRAPHS -I/usr/local/nvidia/include -I/usr/local/nvidia/targets/x86_64-linux/include  -c examples/cvector-generator/cvector-generator.cpp -o examples/cvector-generator/cvector-generator.o\nc++ -std=c++11 -fPIC -O3 -g -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread -fopenmp  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -Iggml/include -Iggml/src -Iinclude -Isrc -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_OPENMP -DGGML_USE_LLAMAFILE -DGGML_USE_CUDA -DGGML_CUDA_USE_GRAPHS -I/usr/local/nvidia/include -I/usr/local/nvidia/targets/x86_64-linux/include  ggml/src/llamafile/sgemm.o ggml/src/ggml-cuda.o ggml/src/ggml-cuda/acc.o ggml/src/ggml-cuda/arange.o ggml/src/ggml-cuda/argsort.o ggml/src/ggml-cuda/binbcast.o ggml/src/ggml-cuda/clamp.o ggml/src/ggml-cuda/concat.o ggml/src/ggml-cuda/conv-transpose-1d.o ggml/src/ggml-cuda/convert.o ggml/src/ggml-cuda/cpy.o ggml/src/ggml-cuda/cross-entropy-loss.o ggml/src/ggml-cuda/diagmask.o ggml/src/ggml-cuda/dmmv.o ggml/src/ggml-cuda/fattn-tile-f16.o ggml/src/ggml-cuda/fattn-tile-f32.o ggml/src/ggml-cuda/fattn.o ggml/src/ggml-cuda/getrows.o ggml/src/ggml-cuda/im2col.o ggml/src/ggml-cuda/mmq.o ggml/src/ggml-cuda/mmvq.o ggml/src/ggml-cuda/norm.o ggml/src/ggml-cuda/opt-step-adamw.o ggml/src/ggml-cuda/out-prod.o ggml/src/ggml-cuda/pad.o ggml/src/ggml-cuda/pool2d.o ggml/src/ggml-cuda/quantize.o ggml/src/ggml-cuda/rope.o ggml/src/ggml-cuda/rwkv-wkv.o ggml/src/ggml-cuda/scale.o ggml/src/ggml-cuda/softmax.o ggml/src/ggml-cuda/sum.o ggml/src/ggml-cuda/sumrows.o ggml/src/ggml-cuda/tsembd.o ggml/src/ggml-cuda/unary.o ggml/src/ggml-cuda/upscale.o ggml/src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.o ggml/src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.o ggml/src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.o ggml/src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.o ggml/src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.o ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.o ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.o ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.o ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.o ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.o ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.o ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.o ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.o ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.o ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.o ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.o ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.o ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.o ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.o ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.o ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.o ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.o ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.o ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.o ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.o ggml/src/ggml.o ggml/src/ggml-alloc.o ggml/src/ggml-backend.o ggml/src/ggml-quants.o ggml/src/ggml-aarch64.o src/llama.o src/llama-vocab.o src/llama-grammar.o src/llama-sampling.o src/unicode.o src/unicode-data.o common/common.o common/arg.o common/log.o common/console.o common/ngram-cache.o common/sampling.o common/train.o common/build-info.o common/json-schema-to-grammar.o examples/cvector-generator/cvector-generator.o -o llama-cvector-generator -lcuda -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/nvidia/lib64 -L/usr/lib64 -L/usr/local/nvidia/targets/x86_64-linux/lib -L/usr/local/nvidia/lib64/stubs -L/usr/lib/wsl/lib \nc++ -std=c++11 -fPIC -O3 -g -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread -fopenmp  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -Iggml/include -Iggml/src -Iinclude -Isrc -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_OPENMP -DGGML_USE_LLAMAFILE -DGGML_USE_CUDA -DGGML_CUDA_USE_GRAPHS -I/usr/local/nvidia/include -I/usr/local/nvidia/targets/x86_64-linux/include  -c examples/gen-docs/gen-docs.cpp -o examples/gen-docs/gen-docs.o\nc++ -std=c++11 -fPIC -O3 -g -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread -fopenmp  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -Iggml/include -Iggml/src -Iinclude -Isrc -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_OPENMP -DGGML_USE_LLAMAFILE -DGGML_USE_CUDA -DGGML_CUDA_USE_GRAPHS -I/usr/local/nvidia/include -I/usr/local/nvidia/targets/x86_64-linux/include  ggml/src/llamafile/sgemm.o ggml/src/ggml-cuda.o ggml/src/ggml-cuda/acc.o ggml/src/ggml-cuda/arange.o ggml/src/ggml-cuda/argsort.o ggml/src/ggml-cuda/binbcast.o ggml/src/ggml-cuda/clamp.o ggml/src/ggml-cuda/concat.o ggml/src/ggml-cuda/conv-transpose-1d.o ggml/src/ggml-cuda/convert.o ggml/src/ggml-cuda/cpy.o ggml/src/ggml-cuda/cross-entropy-loss.o ggml/src/ggml-cuda/diagmask.o ggml/src/ggml-cuda/dmmv.o ggml/src/ggml-cuda/fattn-tile-f16.o ggml/src/ggml-cuda/fattn-tile-f32.o ggml/src/ggml-cuda/fattn.o ggml/src/ggml-cuda/getrows.o ggml/src/ggml-cuda/im2col.o ggml/src/ggml-cuda/mmq.o ggml/src/ggml-cuda/mmvq.o ggml/src/ggml-cuda/norm.o ggml/src/ggml-cuda/opt-step-adamw.o ggml/src/ggml-cuda/out-prod.o ggml/src/ggml-cuda/pad.o ggml/src/ggml-cuda/pool2d.o ggml/src/ggml-cuda/quantize.o ggml/src/ggml-cuda/rope.o ggml/src/ggml-cuda/rwkv-wkv.o ggml/src/ggml-cuda/scale.o ggml/src/ggml-cuda/softmax.o ggml/src/ggml-cuda/sum.o ggml/src/ggml-cuda/sumrows.o ggml/src/ggml-cuda/tsembd.o ggml/src/ggml-cuda/unary.o ggml/src/ggml-cuda/upscale.o ggml/src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb16.o ggml/src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqfloat-cpb32.o ggml/src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb16.o ggml/src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb32.o ggml/src/ggml-cuda/template-instances/fattn-wmma-f16-instance-kqhalf-cpb8.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq1_s.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_s.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xs.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq2_xxs.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_s.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq3_xxs.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_nl.o ggml/src/ggml-cuda/template-instances/mmq-instance-iq4_xs.o ggml/src/ggml-cuda/template-instances/mmq-instance-q2_k.o ggml/src/ggml-cuda/template-instances/mmq-instance-q3_k.o ggml/src/ggml-cuda/template-instances/mmq-instance-q4_0.o ggml/src/ggml-cuda/template-instances/mmq-instance-q4_1.o ggml/src/ggml-cuda/template-instances/mmq-instance-q4_k.o ggml/src/ggml-cuda/template-instances/mmq-instance-q5_0.o ggml/src/ggml-cuda/template-instances/mmq-instance-q5_1.o ggml/src/ggml-cuda/template-instances/mmq-instance-q5_k.o ggml/src/ggml-cuda/template-instances/mmq-instance-q6_k.o ggml/src/ggml-cuda/template-instances/mmq-instance-q8_0.o ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q4_0-q4_0.o ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q4_0-q4_0.o ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-q8_0-q8_0.o ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-q8_0-q8_0.o ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs128-f16-f16.o ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs256-f16-f16.o ggml/src/ggml-cuda/template-instances/fattn-vec-f16-instance-hs64-f16-f16.o ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs128-f16-f16.o ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs256-f16-f16.o ggml/src/ggml-cuda/template-instances/fattn-vec-f32-instance-hs64-f16-f16.o ggml/src/ggml.o ggml/src/ggml-alloc.o ggml/src/ggml-backend.o ggml/src/ggml-quants.o ggml/src/ggml-aarch64.o src/llama.o src/llama-vocab.o src/llama-grammar.o src/llama-sampling.o src/unicode.o src/unicode-data.o common/common.o common/arg.o common/log.o common/console.o common/ngram-cache.o common/sampling.o common/train.o common/build-info.o common/json-schema-to-grammar.o examples/gen-docs/gen-docs.o -o llama-gen-docs -lcuda -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/nvidia/lib64 -L/usr/lib64 -L/usr/local/nvidia/targets/x86_64-linux/lib -L/usr/local/nvidia/lib64/stubs -L/usr/lib/wsl/lib \ncc -Iggml/include -Iggml/src -Iinclude -Isrc -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_OPENMP -DGGML_USE_LLAMAFILE -DGGML_USE_CUDA -DGGML_CUDA_USE_GRAPHS -I/usr/local/nvidia/include -I/usr/local/nvidia/targets/x86_64-linux/include  -std=c11   -fPIC -O3 -g -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wshadow -Wstrict-prototypes -Wpointer-arith -Wmissing-prototypes -Werror=implicit-int -Werror=implicit-function-declaration -pthread -march=native -mtune=native -fopenmp -Wdouble-promotion  -c tests/test-c.c -o tests/test-c.o\nc++ -std=c++11 -fPIC -O3 -g -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread -fopenmp  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -Iggml/include -Iggml/src -Iinclude -Isrc -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_OPENMP -DGGML_USE_LLAMAFILE -DGGML_USE_CUDA -DGGML_CUDA_USE_GRAPHS -I/usr/local/nvidia/include -I/usr/local/nvidia/targets/x86_64-linux/include  -c examples/deprecation-warning/deprecation-warning.cpp -o examples/deprecation-warning/deprecation-warning.o\nc++ -std=c++11 -fPIC -O3 -g -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread -fopenmp  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -Iggml/include -Iggml/src -Iinclude -Isrc -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_OPENMP -DGGML_USE_LLAMAFILE -DGGML_USE_CUDA -DGGML_CUDA_USE_GRAPHS -I/usr/local/nvidia/include -I/usr/local/nvidia/targets/x86_64-linux/include  examples/deprecation-warning/deprecation-warning.o -o main -lcuda -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/nvidia/lib64 -L/usr/lib64 -L/usr/local/nvidia/targets/x86_64-linux/lib -L/usr/local/nvidia/lib64/stubs -L/usr/lib/wsl/lib \nNOTICE: The 'main' binary is deprecated. Please use 'llama-cli' instead.\nc++ -std=c++11 -fPIC -O3 -g -Wall -Wextra -Wpedantic -Wcast-qual -Wno-unused-function -Wmissing-declarations -Wmissing-noreturn -pthread -fopenmp  -march=native -mtune=native -Wno-array-bounds -Wno-format-truncation -Wextra-semi -Iggml/include -Iggml/src -Iinclude -Isrc -Icommon -D_XOPEN_SOURCE=600 -D_GNU_SOURCE -DNDEBUG -DGGML_USE_OPENMP -DGGML_USE_LLAMAFILE -DGGML_USE_CUDA -DGGML_CUDA_USE_GRAPHS -I/usr/local/nvidia/include -I/usr/local/nvidia/targets/x86_64-linux/include  examples/deprecation-warning/deprecation-warning.o -o server -lcuda -lcublas -lculibos -lcudart -lcublasLt -lpthread -ldl -lrt -L/usr/local/nvidia/lib64 -L/usr/lib64 -L/usr/local/nvidia/targets/x86_64-linux/lib -L/usr/local/nvidia/lib64/stubs -L/usr/lib/wsl/lib \nNOTICE: The 'server' binary is deprecated. Please use 'llama-server' instead.\n","output_type":"stream"}]},{"cell_type":"code","source":"!pwd\n!pip install --force-reinstall -r requirements.txt > pip_install.log 2>&1\n!tail pip_install.log","metadata":{"execution":{"iopub.status.busy":"2024-09-24T06:09:18.632565Z","iopub.execute_input":"2024-09-24T06:09:18.632891Z","iopub.status.idle":"2024-09-24T06:11:04.509003Z","shell.execute_reply.started":"2024-09-24T06:09:18.632857Z","shell.execute_reply":"2024-09-24T06:11:04.508093Z"},"trusted":true},"execution_count":3,"outputs":[{"name":"stdout","text":"/kaggle/working/llama.cpp\npointpats 2.5.0 requires shapely>=2, but you have shapely 1.8.5.post1 which is incompatible.\nrapids-dask-dependency 24.8.0a0 requires dask==2024.7.1, but you have dask 2024.8.1 which is incompatible.\nrmm 24.8.2 requires cuda-python<12.0a0,>=11.7.1, but you have cuda-python 12.6.0 which is incompatible.\ns3fs 2024.6.1 requires fsspec==2024.6.1.*, but you have fsspec 2024.9.0 which is incompatible.\nspaghetti 1.7.6 requires shapely>=2.0.1, but you have shapely 1.8.5.post1 which is incompatible.\nspopt 0.6.1 requires shapely>=2.0.1, but you have shapely 1.8.5.post1 which is incompatible.\ntensorflow-metadata 0.14.0 requires protobuf<4,>=3.7, but you have protobuf 4.25.5 which is incompatible.\ntensorflow-transform 0.14.0 requires protobuf<4,>=3.7, but you have protobuf 4.25.5 which is incompatible.\nydata-profiling 4.9.0 requires scipy<1.14,>=1.4.1, but you have scipy 1.14.0 which is incompatible.\u001b[0m\u001b[31m\n\u001b[0mSuccessfully installed MarkupSafe-2.1.5 certifi-2024.8.30 charset-normalizer-3.3.2 filelock-3.16.1 fsspec-2024.6.1 gguf-0.10.0 huggingface-hub-0.25.1 idna-3.10 jinja2-3.1.4 mpmath-1.3.0 networkx-3.3 numpy-1.26.4 packaging-24.1 protobuf-4.25.3 pyyaml-6.0.2 regex-2024.9.11 requests-2.32.3 safetensors-0.4.5 sentencepiece-0.2.0 sympy-1.13.3 tokenizers-0.19.1 torch-2.2.2+cpu tqdm-4.66.5 transformers-4.44.2 typing-extensions-4.12.2 urllib3-2.2.1\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Download model\n\n","metadata":{}},{"cell_type":"code","source":"!pwd\n%cd ..\n!mkdir \"./quantized_model_gemma2-2b/\"","metadata":{"execution":{"iopub.status.busy":"2024-09-24T06:11:04.510444Z","iopub.execute_input":"2024-09-24T06:11:04.510777Z","iopub.status.idle":"2024-09-24T06:11:06.456944Z","shell.execute_reply.started":"2024-09-24T06:11:04.510742Z","shell.execute_reply":"2024-09-24T06:11:06.455828Z"},"trusted":true},"execution_count":4,"outputs":[{"name":"stdout","text":"/kaggle/working/llama.cpp\n/kaggle/working\n","output_type":"stream"}]},{"cell_type":"code","source":"import os\nfrom huggingface_hub import login\nfrom kaggle_secrets import UserSecretsClient\nfrom huggingface_hub import snapshot_download\n\nuser_secrets = UserSecretsClient()\n\nos.environ[\"HF_TOKEN\"]=user_secrets.get_secret(\"HUGGINGFACE_TOKEN\")\n\nos.environ[\"WANDB_API_KEY\"]=user_secrets.get_secret(\"WANDB_API_KEY\")\nos.environ[\"MODEL_NAME\"] = \"google/gemma-2-2b-it\"\nos.environ[\"DATASET\"] = \"HuggingFaceH4/ultrafeedback_binarized\"\n\nlogin(os.environ[\"HF_TOKEN\"])\n\nmodel_name = \"google/gemma-2-2b-it\" # the model we want to quantize\nmethods = ['Q4_K_S','Q4_K_M'] #the methods to be used for quantization\nbase_model = \"./original_model_gemma2-2b/\" # where the FP16 GGUF model will be stored\nquantized_path = \"./quantized_model_gemma2-2b/\" #where the quantized GGUF model will be stored\noriginal_model = quantized_path + 'FP16.gguf'\n\nsnapshot_download(repo_id=model_name, local_dir=base_model , local_dir_use_symlinks=False)","metadata":{"execution":{"iopub.status.busy":"2024-09-24T06:11:06.460269Z","iopub.execute_input":"2024-09-24T06:11:06.460997Z","iopub.status.idle":"2024-09-24T06:11:32.967919Z","shell.execute_reply.started":"2024-09-24T06:11:06.460960Z","shell.execute_reply":"2024-09-24T06:11:32.966798Z"},"trusted":true},"execution_count":5,"outputs":[{"name":"stdout","text":"The token has not been saved to the git credentials helper. Pass `add_to_git_credential=True` in this function directly or `--add-to-git-credential` if using via `huggingface-cli` if you want to set the git credential as well.\nToken is valid (permission: read).\nYour token has been saved to /root/.cache/huggingface/token\nLogin successful\n","output_type":"stream"},{"name":"stderr","text":"/opt/conda/lib/python3.10/site-packages/huggingface_hub/file_download.py:1204: UserWarning: `local_dir_use_symlinks` parameter is deprecated and will be ignored. The process to download files to a local folder has been updated and do not rely on symlinks anymore. You only need to pass a destination folder as`local_dir`.\nFor more details, check out https://huggingface.co/docs/huggingface_hub/main/en/guides/download#download-files-to-local-folder.\n  warnings.warn(\n","output_type":"stream"},{"output_type":"display_data","data":{"text/plain":"Fetching 11 files:   0%|          | 0/11 [00:00<?, ?it/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"77e20e51eca94680989984b64cda2aaa"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model-00001-of-00002.safetensors:   0%|          | 0.00/4.99G [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"6b52784b65f443839c6c2f709d7098df"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model-00002-of-00002.safetensors:   0%|          | 0.00/241M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"f984c76a167d4d7ba4806d8097115bd4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"README.md:   0%|          | 0.00/29.1k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e2f46e4413a745f8a38f3832d61537a4"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"config.json:   0%|          | 0.00/838 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"b65d9958c222484cb0347eb9051947f3"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":".gitattributes:   0%|          | 0.00/1.57k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e8e791ff2c2541a593f1ac27aaa91055"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"special_tokens_map.json:   0%|          | 0.00/636 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"0f53340d0839409ab1a53ee821c1e1c8"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"model.safetensors.index.json:   0%|          | 0.00/24.2k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"09c79fa033c24a95b17ca9b5f1085a52"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"generation_config.json:   0%|          | 0.00/187 [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"2c75e7f806144654b842874250ca5f57"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.model:   0%|          | 0.00/4.24M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"875428c2b07849ee809afd454dfcef04"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer.json:   0%|          | 0.00/17.5M [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"9415eb9720de4125bd2c710859562905"}},"metadata":{}},{"output_type":"display_data","data":{"text/plain":"tokenizer_config.json:   0%|          | 0.00/47.0k [00:00<?, ?B/s]","application/vnd.jupyter.widget-view+json":{"version_major":2,"version_minor":0,"model_id":"e3f71904054a4f168446dfea2c371620"}},"metadata":{}},{"execution_count":5,"output_type":"execute_result","data":{"text/plain":"'/kaggle/working/original_model_gemma2-2b'"},"metadata":{}}]},{"cell_type":"markdown","source":"## Convert model to gguf","metadata":{}},{"cell_type":"code","source":"!python llama.cpp/convert_hf_to_gguf.py \"./original_model_gemma2-2b/\" --outfile \"./quantized_model_gemma2-2b/FP16.gguf\"","metadata":{"execution":{"iopub.status.busy":"2024-09-24T06:11:32.969130Z","iopub.execute_input":"2024-09-24T06:11:32.969442Z","iopub.status.idle":"2024-09-24T06:12:04.823659Z","shell.execute_reply.started":"2024-09-24T06:11:32.969410Z","shell.execute_reply":"2024-09-24T06:12:04.822509Z"},"trusted":true},"execution_count":6,"outputs":[{"name":"stdout","text":"Writing: 100%|███████████████████████████| 5.23G/5.23G [00:22<00:00, 234Mbyte/s]\n","output_type":"stream"}]},{"cell_type":"markdown","source":"## Download the datasets that will be used for calibration and evaluation:","metadata":{}},{"cell_type":"code","source":"!wget https://object.pouta.csc.fi/OPUS-Wikipedia/v1.0/mono/en.txt.gz\n!gunzip en.txt.gz\n!head -n 10000 en.txt > en-h10000.txt\n!sh llama.cpp/scripts/get-wikitext-2.sh","metadata":{"execution":{"iopub.status.busy":"2024-09-24T06:12:04.825212Z","iopub.execute_input":"2024-09-24T06:12:04.825550Z","iopub.status.idle":"2024-09-24T06:12:51.835981Z","shell.execute_reply.started":"2024-09-24T06:12:04.825515Z","shell.execute_reply":"2024-09-24T06:12:51.834859Z"},"trusted":true},"execution_count":7,"outputs":[{"name":"stdout","text":"--2024-09-24 06:12:05--  https://object.pouta.csc.fi/OPUS-Wikipedia/v1.0/mono/en.txt.gz\nResolving object.pouta.csc.fi (object.pouta.csc.fi)... 86.50.254.18, 86.50.254.19\nConnecting to object.pouta.csc.fi (object.pouta.csc.fi)|86.50.254.18|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 532958396 (508M) [application/gzip]\nSaving to: 'en.txt.gz'\n\nen.txt.gz           100%[===================>] 508.27M  18.4MB/s    in 29s     \n\n2024-09-24 06:12:35 (17.8 MB/s) - 'en.txt.gz' saved [532958396/532958396]\n\n--2024-09-24 06:12:51--  https://huggingface.co/datasets/ggml-org/ci/resolve/main/wikitext-2-raw-v1.zip\nResolving huggingface.co (huggingface.co)... 3.165.160.11, 3.165.160.59, 3.165.160.12, ...\nConnecting to huggingface.co (huggingface.co)|3.165.160.11|:443... connected.\nHTTP request sent, awaiting response... 302 Found\nLocation: https://cdn-lfs-us-1.huggingface.co/repos/c6/78/c67802fcd48fa6f6a86773410b21cc6db1c5c546b20683b6c30b95f327a66922/ef7edb566e3e2b2d31b29c1fdb0c89a4cc683597484c3dc2517919c615435a11?response-content-disposition=inline%3B+filename*%3DUTF-8%27%27wikitext-2-raw-v1.zip%3B+filename%3D%22wikitext-2-raw-v1.zip%22%3B&response-content-type=application%2Fzip&Expires=1727417571&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyNzQxNzU3MX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zL2M2Lzc4L2M2NzgwMmZjZDQ4ZmE2ZjZhODY3NzM0MTBiMjFjYzZkYjFjNWM1NDZiMjA2ODNiNmMzMGI5NWYzMjdhNjY5MjIvZWY3ZWRiNTY2ZTNlMmIyZDMxYjI5YzFmZGIwYzg5YTRjYzY4MzU5NzQ4NGMzZGMyNTE3OTE5YzYxNTQzNWExMT9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSomcmVzcG9uc2UtY29udGVudC10eXBlPSoifV19&Signature=AD-KzanEDdJoutt5n0pMSUv1T9mXPiuHExWWKaX8WQ39lMPrkAIXTsdJflc0UmSiyAtC82dhcltjDxI9GYkDNHCl1SxbJ15yyMCJfIDzlahmUEP8tMxuyAXDDyE%7E3jXS8R5E2h2WTAu-QATD2%7ErKSzEz2VLobOzCsCXhNj7GhoosnB55Zn1fNhOCuPiPHe9ykVFBhjPsUdvGvnSxilzNt42T0Lyjwa1z630J4j%7EYg-SmWi%7EicuhmIEPTZzx8PpWlbXiCs%7E-EdN36DX-rmIQuC7l7A3%7EEoA6e7MhaIQnLJ-DpY4VjMF5LHMXO8eNy-xqSYErQTpRp9yryoSRd-zrAug__&Key-Pair-Id=K24J24Z295AEI9 [following]\n--2024-09-24 06:12:51--  https://cdn-lfs-us-1.huggingface.co/repos/c6/78/c67802fcd48fa6f6a86773410b21cc6db1c5c546b20683b6c30b95f327a66922/ef7edb566e3e2b2d31b29c1fdb0c89a4cc683597484c3dc2517919c615435a11?response-content-disposition=inline%3B+filename*%3DUTF-8''wikitext-2-raw-v1.zip%3B+filename%3D%22wikitext-2-raw-v1.zip%22%3B&response-content-type=application%2Fzip&Expires=1727417571&Policy=eyJTdGF0ZW1lbnQiOlt7IkNvbmRpdGlvbiI6eyJEYXRlTGVzc1RoYW4iOnsiQVdTOkVwb2NoVGltZSI6MTcyNzQxNzU3MX19LCJSZXNvdXJjZSI6Imh0dHBzOi8vY2RuLWxmcy11cy0xLmh1Z2dpbmdmYWNlLmNvL3JlcG9zL2M2Lzc4L2M2NzgwMmZjZDQ4ZmE2ZjZhODY3NzM0MTBiMjFjYzZkYjFjNWM1NDZiMjA2ODNiNmMzMGI5NWYzMjdhNjY5MjIvZWY3ZWRiNTY2ZTNlMmIyZDMxYjI5YzFmZGIwYzg5YTRjYzY4MzU5NzQ4NGMzZGMyNTE3OTE5YzYxNTQzNWExMT9yZXNwb25zZS1jb250ZW50LWRpc3Bvc2l0aW9uPSomcmVzcG9uc2UtY29udGVudC10eXBlPSoifV19&Signature=AD-KzanEDdJoutt5n0pMSUv1T9mXPiuHExWWKaX8WQ39lMPrkAIXTsdJflc0UmSiyAtC82dhcltjDxI9GYkDNHCl1SxbJ15yyMCJfIDzlahmUEP8tMxuyAXDDyE~3jXS8R5E2h2WTAu-QATD2~rKSzEz2VLobOzCsCXhNj7GhoosnB55Zn1fNhOCuPiPHe9ykVFBhjPsUdvGvnSxilzNt42T0Lyjwa1z630J4j~Yg-SmWi~icuhmIEPTZzx8PpWlbXiCs~-EdN36DX-rmIQuC7l7A3~EoA6e7MhaIQnLJ-DpY4VjMF5LHMXO8eNy-xqSYErQTpRp9yryoSRd-zrAug__&Key-Pair-Id=K24J24Z295AEI9\nResolving cdn-lfs-us-1.huggingface.co (cdn-lfs-us-1.huggingface.co)... 3.163.189.127, 3.163.189.91, 3.163.189.20, ...\nConnecting to cdn-lfs-us-1.huggingface.co (cdn-lfs-us-1.huggingface.co)|3.163.189.127|:443... connected.\nHTTP request sent, awaiting response... 200 OK\nLength: 4721645 (4.5M) [application/zip]\nSaving to: 'wikitext-2-raw-v1.zip'\n\nwikitext-2-raw-v1.z 100%[===================>]   4.50M  --.-KB/s    in 0.06s   \n\n2024-09-24 06:12:51 (77.2 MB/s) - 'wikitext-2-raw-v1.zip' saved [4721645/4721645]\n\nArchive:  wikitext-2-raw-v1.zip\n   creating: wikitext-2-raw/\n  inflating: wikitext-2-raw/wiki.test.raw  \n  inflating: wikitext-2-raw/wiki.valid.raw  \n  inflating: wikitext-2-raw/wiki.train.raw  \nUsage:\n\n  ./llama-perplexity -m model.gguf -f wikitext-2-raw/wiki.test.raw [other params]\n\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Compute the importance matrix","metadata":{}},{"cell_type":"code","source":"!./llama.cpp/llama-imatrix -m \"./quantized_model_gemma2-2b/FP16.gguf\"  -f en-h10000.txt -o quantized_model_gemma2-2b/imatrix.dat --verbosity 1 -ngl 99","metadata":{"execution":{"iopub.status.busy":"2024-09-24T06:12:51.837407Z","iopub.execute_input":"2024-09-24T06:12:51.837753Z","iopub.status.idle":"2024-09-24T06:29:41.137520Z","shell.execute_reply.started":"2024-09-24T06:12:51.837716Z","shell.execute_reply":"2024-09-24T06:29:41.136392Z"},"trusted":true},"execution_count":8,"outputs":[{"name":"stdout","text":"build: 3814 (c087b6f1) with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\nllama_model_loader: loaded meta data with 38 key-value pairs and 288 tensors from ./quantized_model_gemma2-2b/FP16.gguf (version GGUF V3 (latest))\nllama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\nllama_model_loader: - kv   0:                       general.architecture str              = gemma2\nllama_model_loader: - kv   1:                               general.type str              = model\nllama_model_loader: - kv   2:                               general.name str              = Original_Model_Gemma2 2b\nllama_model_loader: - kv   3:                           general.basename str              = original_model_gemma2\nllama_model_loader: - kv   4:                         general.size_label str              = 2B\nllama_model_loader: - kv   5:                            general.license str              = gemma\nllama_model_loader: - kv   6:                   general.base_model.count u32              = 1\nllama_model_loader: - kv   7:                  general.base_model.0.name str              = Gemma 2 2b\nllama_model_loader: - kv   8:          general.base_model.0.organization str              = Google\nllama_model_loader: - kv   9:              general.base_model.0.repo_url str              = https://huggingface.co/google/gemma-2-2b\nllama_model_loader: - kv  10:                               general.tags arr[str,2]       = [\"conversational\", \"text-generation\"]\nllama_model_loader: - kv  11:                      gemma2.context_length u32              = 8192\nllama_model_loader: - kv  12:                    gemma2.embedding_length u32              = 2304\nllama_model_loader: - kv  13:                         gemma2.block_count u32              = 26\nllama_model_loader: - kv  14:                 gemma2.feed_forward_length u32              = 9216\nllama_model_loader: - kv  15:                gemma2.attention.head_count u32              = 8\nllama_model_loader: - kv  16:             gemma2.attention.head_count_kv u32              = 4\nllama_model_loader: - kv  17:    gemma2.attention.layer_norm_rms_epsilon f32              = 0.000001\nllama_model_loader: - kv  18:                gemma2.attention.key_length u32              = 256\nllama_model_loader: - kv  19:              gemma2.attention.value_length u32              = 256\nllama_model_loader: - kv  20:                          general.file_type u32              = 1\nllama_model_loader: - kv  21:              gemma2.attn_logit_softcapping f32              = 50.000000\nllama_model_loader: - kv  22:             gemma2.final_logit_softcapping f32              = 30.000000\nllama_model_loader: - kv  23:            gemma2.attention.sliding_window u32              = 4096\nllama_model_loader: - kv  24:                       tokenizer.ggml.model str              = llama\nllama_model_loader: - kv  25:                         tokenizer.ggml.pre str              = default\nllama_model_loader: - kv  26:                      tokenizer.ggml.tokens arr[str,256000]  = [\"<pad>\", \"<eos>\", \"<bos>\", \"<unk>\", ...\nllama_model_loader: - kv  27:                      tokenizer.ggml.scores arr[f32,256000]  = [-1000.000000, -1000.000000, -1000.00...\nllama_model_loader: - kv  28:                  tokenizer.ggml.token_type arr[i32,256000]  = [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, ...\nllama_model_loader: - kv  29:                tokenizer.ggml.bos_token_id u32              = 2\nllama_model_loader: - kv  30:                tokenizer.ggml.eos_token_id u32              = 1\nllama_model_loader: - kv  31:            tokenizer.ggml.unknown_token_id u32              = 3\nllama_model_loader: - kv  32:            tokenizer.ggml.padding_token_id u32              = 0\nllama_model_loader: - kv  33:               tokenizer.ggml.add_bos_token bool             = true\nllama_model_loader: - kv  34:               tokenizer.ggml.add_eos_token bool             = false\nllama_model_loader: - kv  35:                    tokenizer.chat_template str              = {{ bos_token }}{% if messages[0]['rol...\nllama_model_loader: - kv  36:            tokenizer.ggml.add_space_prefix bool             = false\nllama_model_loader: - kv  37:               general.quantization_version u32              = 2\nllama_model_loader: - type  f32:  105 tensors\nllama_model_loader: - type  f16:  183 tensors\nllm_load_vocab: special tokens cache size = 249\nllm_load_vocab: token to piece cache size = 1.6014 MB\nllm_load_print_meta: format           = GGUF V3 (latest)\nllm_load_print_meta: arch             = gemma2\nllm_load_print_meta: vocab type       = SPM\nllm_load_print_meta: n_vocab          = 256000\nllm_load_print_meta: n_merges         = 0\nllm_load_print_meta: vocab_only       = 0\nllm_load_print_meta: n_ctx_train      = 8192\nllm_load_print_meta: n_embd           = 2304\nllm_load_print_meta: n_layer          = 26\nllm_load_print_meta: n_head           = 8\nllm_load_print_meta: n_head_kv        = 4\nllm_load_print_meta: n_rot            = 256\nllm_load_print_meta: n_swa            = 4096\nllm_load_print_meta: n_embd_head_k    = 256\nllm_load_print_meta: n_embd_head_v    = 256\nllm_load_print_meta: n_gqa            = 2\nllm_load_print_meta: n_embd_k_gqa     = 1024\nllm_load_print_meta: n_embd_v_gqa     = 1024\nllm_load_print_meta: f_norm_eps       = 0.0e+00\nllm_load_print_meta: f_norm_rms_eps   = 1.0e-06\nllm_load_print_meta: f_clamp_kqv      = 0.0e+00\nllm_load_print_meta: f_max_alibi_bias = 0.0e+00\nllm_load_print_meta: f_logit_scale    = 0.0e+00\nllm_load_print_meta: n_ff             = 9216\nllm_load_print_meta: n_expert         = 0\nllm_load_print_meta: n_expert_used    = 0\nllm_load_print_meta: causal attn      = 1\nllm_load_print_meta: pooling type     = 0\nllm_load_print_meta: rope type        = 2\nllm_load_print_meta: rope scaling     = linear\nllm_load_print_meta: freq_base_train  = 10000.0\nllm_load_print_meta: freq_scale_train = 1\nllm_load_print_meta: n_ctx_orig_yarn  = 8192\nllm_load_print_meta: rope_finetuned   = unknown\nllm_load_print_meta: ssm_d_conv       = 0\nllm_load_print_meta: ssm_d_inner      = 0\nllm_load_print_meta: ssm_d_state      = 0\nllm_load_print_meta: ssm_dt_rank      = 0\nllm_load_print_meta: ssm_dt_b_c_rms   = 0\nllm_load_print_meta: model type       = 2B\nllm_load_print_meta: model ftype      = F16\nllm_load_print_meta: model params     = 2.61 B\nllm_load_print_meta: model size       = 4.87 GiB (16.00 BPW) \nllm_load_print_meta: general.name     = Original_Model_Gemma2 2b\nllm_load_print_meta: BOS token        = 2 '<bos>'\nllm_load_print_meta: EOS token        = 1 '<eos>'\nllm_load_print_meta: UNK token        = 3 '<unk>'\nllm_load_print_meta: PAD token        = 0 '<pad>'\nllm_load_print_meta: LF token         = 227 '<0x0A>'\nllm_load_print_meta: EOT token        = 107 '<end_of_turn>'\nllm_load_print_meta: max token length = 48\nggml_cuda_init: GGML_CUDA_FORCE_MMQ:    no\nggml_cuda_init: GGML_CUDA_FORCE_CUBLAS: no\nggml_cuda_init: found 1 CUDA devices:\n  Device 0: Tesla P100-PCIE-16GB, compute capability 6.0, VMM: yes\nllm_load_tensors: ggml ctx size =    0.26 MiB\nllm_load_tensors: offloading 26 repeating layers to GPU\nllm_load_tensors: offloading non-repeating layers to GPU\nllm_load_tensors: offloaded 27/27 layers to GPU\nllm_load_tensors:        CPU buffer size =  1125.00 MiB\nllm_load_tensors:      CUDA0 buffer size =  4986.92 MiB\n..................................................................\nllama_new_context_with_model: n_ctx      = 512\nllama_new_context_with_model: n_batch    = 512\nllama_new_context_with_model: n_ubatch   = 512\nllama_new_context_with_model: flash_attn = 0\nllama_new_context_with_model: freq_base  = 10000.0\nllama_new_context_with_model: freq_scale = 1\nllama_kv_cache_init:      CUDA0 KV buffer size =    52.00 MiB\nllama_new_context_with_model: KV self size  =   52.00 MiB, K (f16):   26.00 MiB, V (f16):   26.00 MiB\nllama_new_context_with_model:  CUDA_Host  output buffer size =     0.98 MiB\nllama_new_context_with_model:      CUDA0 compute buffer size =   504.50 MiB\nllama_new_context_with_model:  CUDA_Host compute buffer size =     6.51 MiB\nllama_new_context_with_model: graph nodes  = 1050\nllama_new_context_with_model: graph splits = 2\n\nsystem_info: n_threads = 2 (n_threads_batch = 2) / 4 | AVX = 1 | AVX_VNNI = 0 | AVX2 = 1 | AVX512 = 1 | AVX512_VBMI = 0 | AVX512_VNNI = 0 | AVX512_BF16 = 0 | FMA = 1 | NEON = 0 | SVE = 0 | ARM_FMA = 0 | F16C = 1 | FP16_VA = 0 | RISCV_VECT = 0 | WASM_SIMD = 0 | BLAS = 1 | SSE3 = 1 | SSSE3 = 1 | VSX = 0 | MATMUL_INT8 = 0 | LLAMAFILE = 1 | \ncompute_imatrix: tokenizing the input ..\ncompute_imatrix: tokenization took 2359.86 ms\ncompute_imatrix: computing over 696 chunks with batch_size 512\ncompute_imatrix: 1.97 seconds per pass - ETA 22.80 minutes\n[1]8.6770,[2]12.3273,[3]16.5609,[4]21.8836,[5]20.4886,[6]19.0793,[7]17.1510,[8]15.8433,[9]16.3802,\nsave_imatrix: stored collected data after 10 chunks in quantized_model_gemma2-2b/imatrix.dat\n[10]15.8274,[11]15.8316,[12]15.7493,[13]15.5668,[14]14.7952,[15]15.7660,[16]16.0228,[17]15.8303,[18]15.2348,[19]15.0015,\nsave_imatrix: stored collected data after 20 chunks in quantized_model_gemma2-2b/imatrix.dat\n[20]13.7488,[21]13.9516,[22]13.5287,[23]13.6752,[24]13.4761,[25]13.6356,[26]13.3252,[27]13.1602,[28]13.0602,[29]12.7336,\nsave_imatrix: stored collected data after 30 chunks in quantized_model_gemma2-2b/imatrix.dat\n[30]12.4564,[31]12.6380,[32]12.6140,[33]12.4972,[34]12.1728,[35]12.2149,[36]12.2531,[37]12.2916,[38]12.2690,[39]12.3680,\nsave_imatrix: stored collected data after 40 chunks in quantized_model_gemma2-2b/imatrix.dat\n[40]12.3018,[41]12.1604,[42]12.2772,[43]12.2023,[44]12.2840,[45]12.1491,[46]12.3551,[47]12.2668,[48]12.2038,[49]12.1995,\nsave_imatrix: stored collected data after 50 chunks in quantized_model_gemma2-2b/imatrix.dat\n[50]12.3215,[51]12.2419,[52]12.2102,[53]12.2107,[54]12.1225,[55]12.1210,[56]12.1935,[57]12.1855,[58]12.2019,[59]12.1583,\nsave_imatrix: stored collected data after 60 chunks in quantized_model_gemma2-2b/imatrix.dat\n[60]11.8310,[61]11.5849,[62]11.4720,[63]11.5315,[64]11.4517,[65]11.4914,[66]11.4232,[67]11.4489,[68]11.5094,[69]11.5301,\nsave_imatrix: stored collected data after 70 chunks in quantized_model_gemma2-2b/imatrix.dat\n[70]11.4260,[71]11.4387,[72]11.4396,[73]11.3928,[74]11.3414,[75]11.4019,[76]11.4182,[77]11.4747,[78]11.4849,[79]11.4182,\nsave_imatrix: stored collected data after 80 chunks in quantized_model_gemma2-2b/imatrix.dat\n[80]11.3828,[81]11.3250,[82]11.3236,[83]11.3472,[84]11.2685,[85]11.2809,[86]11.2589,[87]11.1906,[88]11.1792,[89]11.2110,\nsave_imatrix: stored collected data after 90 chunks in quantized_model_gemma2-2b/imatrix.dat\n[90]11.1672,[91]11.1862,[92]11.2088,[93]11.1563,[94]11.1740,[95]11.0856,[96]11.0484,[97]11.0520,[98]11.0137,[99]11.0172,\nsave_imatrix: stored collected data after 100 chunks in quantized_model_gemma2-2b/imatrix.dat\n[100]11.0779,[101]11.0905,[102]10.9605,[103]10.8199,[104]10.6662,[105]10.5691,[106]10.6100,[107]10.6241,[108]10.6722,[109]10.6712,\nsave_imatrix: stored collected data after 110 chunks in quantized_model_gemma2-2b/imatrix.dat\n[110]10.6946,[111]10.7639,[112]10.8148,[113]10.7595,[114]10.6559,[115]10.7006,[116]10.8286,[117]10.9089,[118]10.8973,[119]10.9121,\nsave_imatrix: stored collected data after 120 chunks in quantized_model_gemma2-2b/imatrix.dat\n[120]10.9889,[121]11.0909,[122]11.1301,[123]11.1878,[124]11.2451,[125]11.2474,[126]11.2318,[127]11.2432,[128]11.2033,[129]11.2606,\nsave_imatrix: stored collected data after 130 chunks in quantized_model_gemma2-2b/imatrix.dat\n[130]11.2382,[131]11.2586,[132]11.2770,[133]11.3357,[134]11.3350,[135]11.3879,[136]11.3942,[137]11.4180,[138]11.3672,[139]11.3168,\nsave_imatrix: stored collected data after 140 chunks in quantized_model_gemma2-2b/imatrix.dat\n[140]11.3021,[141]11.1918,[142]11.1930,[143]11.1992,[144]11.2292,[145]11.2587,[146]11.1915,[147]11.2177,[148]11.2350,[149]11.2465,\nsave_imatrix: stored collected data after 150 chunks in quantized_model_gemma2-2b/imatrix.dat\n[150]11.2193,[151]11.2516,[152]11.2739,[153]11.2838,[154]11.3009,[155]11.2815,[156]11.3158,[157]11.2802,[158]11.3005,[159]11.3176,\nsave_imatrix: stored collected data after 160 chunks in quantized_model_gemma2-2b/imatrix.dat\n[160]11.2517,[161]11.2288,[162]11.2319,[163]11.2720,[164]11.2566,[165]11.2112,[166]11.1702,[167]11.1581,[168]11.1399,[169]11.1556,\nsave_imatrix: stored collected data after 170 chunks in quantized_model_gemma2-2b/imatrix.dat\n[170]11.1574,[171]11.1146,[172]11.1572,[173]11.1547,[174]11.1600,[175]11.1771,[176]11.2209,[177]11.2123,[178]11.2253,[179]11.2332,\nsave_imatrix: stored collected data after 180 chunks in quantized_model_gemma2-2b/imatrix.dat\n[180]11.2162,[181]11.2158,[182]11.2128,[183]11.1709,[184]11.1605,[185]11.1686,[186]11.1387,[187]11.0874,[188]11.1002,[189]11.1242,\nsave_imatrix: stored collected data after 190 chunks in quantized_model_gemma2-2b/imatrix.dat\n[190]11.1744,[191]11.2189,[192]11.2542,[193]11.2542,[194]11.2713,[195]11.2648,[196]11.2421,[197]11.2036,[198]11.2389,[199]11.2467,\nsave_imatrix: stored collected data after 200 chunks in quantized_model_gemma2-2b/imatrix.dat\n[200]11.2043,[201]11.2300,[202]11.2387,[203]11.2467,[204]11.2584,[205]11.2946,[206]11.3204,[207]11.3345,[208]11.3525,[209]11.3738,\nsave_imatrix: stored collected data after 210 chunks in quantized_model_gemma2-2b/imatrix.dat\n[210]11.3823,[211]11.3677,[212]11.3686,[213]11.3804,[214]11.3707,[215]11.3538,[216]11.3968,[217]11.4375,[218]11.4577,[219]11.4530,\nsave_imatrix: stored collected data after 220 chunks in quantized_model_gemma2-2b/imatrix.dat\n[220]11.4345,[221]11.4188,[222]11.4088,[223]11.4092,[224]11.4272,[225]11.4332,[226]11.4197,[227]11.4396,[228]11.4339,[229]11.4368,\nsave_imatrix: stored collected data after 230 chunks in quantized_model_gemma2-2b/imatrix.dat\n[230]11.4394,[231]11.4224,[232]11.3942,[233]11.3812,[234]11.3780,[235]11.3851,[236]11.4332,[237]11.4912,[238]11.4910,[239]11.4829,\nsave_imatrix: stored collected data after 240 chunks in quantized_model_gemma2-2b/imatrix.dat\n[240]11.4629,[241]11.4408,[242]11.4247,[243]11.4272,[244]11.4300,[245]11.4595,[246]11.4720,[247]11.4728,[248]11.4625,[249]11.4777,\nsave_imatrix: stored collected data after 250 chunks in quantized_model_gemma2-2b/imatrix.dat\n[250]11.5188,[251]11.5714,[252]11.5777,[253]11.5415,[254]11.5145,[255]11.5605,[256]11.5867,[257]11.5668,[258]11.5945,[259]11.5956,\nsave_imatrix: stored collected data after 260 chunks in quantized_model_gemma2-2b/imatrix.dat\n[260]11.6058,[261]11.6133,[262]11.6177,[263]11.6174,[264]11.6182,[265]11.5916,[266]11.5685,[267]11.5811,[268]11.6043,[269]11.6081,\nsave_imatrix: stored collected data after 270 chunks in quantized_model_gemma2-2b/imatrix.dat\n[270]11.6055,[271]11.5940,[272]11.5927,[273]11.5727,[274]11.6031,[275]11.5879,[276]11.5799,[277]11.5975,[278]11.6109,[279]11.6164,\nsave_imatrix: stored collected data after 280 chunks in quantized_model_gemma2-2b/imatrix.dat\n[280]11.6278,[281]11.6459,[282]11.6507,[283]11.6448,[284]11.6378,[285]11.6594,[286]11.6598,[287]11.6520,[288]11.6434,[289]11.6516,\nsave_imatrix: stored collected data after 290 chunks in quantized_model_gemma2-2b/imatrix.dat\n[290]11.6494,[291]11.6613,[292]11.6420,[293]11.6506,[294]11.6787,[295]11.6945,[296]11.6965,[297]11.6741,[298]11.7053,[299]11.7081,\nsave_imatrix: stored collected data after 300 chunks in quantized_model_gemma2-2b/imatrix.dat\n[300]11.7165,[301]11.7351,[302]11.7321,[303]11.7321,[304]11.7589,[305]11.7398,[306]11.7368,[307]11.7341,[308]11.6939,[309]11.6907,\nsave_imatrix: stored collected data after 310 chunks in quantized_model_gemma2-2b/imatrix.dat\n[310]11.6360,[311]11.5738,[312]11.5930,[313]11.5961,[314]11.6286,[315]11.6203,[316]11.6331,[317]11.6251,[318]11.6236,[319]11.5986,\nsave_imatrix: stored collected data after 320 chunks in quantized_model_gemma2-2b/imatrix.dat\n[320]11.5782,[321]11.5488,[322]11.5066,[323]11.4933,[324]11.4845,[325]11.4733,[326]11.4444,[327]11.4021,[328]11.3680,[329]11.3347,\nsave_imatrix: stored collected data after 330 chunks in quantized_model_gemma2-2b/imatrix.dat\n[330]11.2936,[331]11.2658,[332]11.2285,[333]11.2160,[334]11.2086,[335]11.1733,[336]11.1493,[337]11.1423,[338]11.1416,[339]11.1474,\nsave_imatrix: stored collected data after 340 chunks in quantized_model_gemma2-2b/imatrix.dat\n[340]11.1491,[341]11.1148,[342]11.1091,[343]11.1086,[344]11.1089,[345]11.1099,[346]11.1357,[347]11.1175,[348]11.1240,[349]11.1226,\nsave_imatrix: stored collected data after 350 chunks in quantized_model_gemma2-2b/imatrix.dat\n[350]11.0897,[351]11.0824,[352]11.0702,[353]11.0563,[354]11.0927,[355]11.0727,[356]11.0776,[357]11.0948,[358]11.1037,[359]11.1224,\nsave_imatrix: stored collected data after 360 chunks in quantized_model_gemma2-2b/imatrix.dat\n[360]11.1183,[361]11.1099,[362]11.1191,[363]11.1417,[364]11.1378,[365]11.1432,[366]11.1690,[367]11.1781,[368]11.1936,[369]11.1958,\nsave_imatrix: stored collected data after 370 chunks in quantized_model_gemma2-2b/imatrix.dat\n[370]11.1987,[371]11.1745,[372]11.1787,[373]11.1928,[374]11.1865,[375]11.2022,[376]11.2118,[377]11.2176,[378]11.2201,[379]11.2083,\nsave_imatrix: stored collected data after 380 chunks in quantized_model_gemma2-2b/imatrix.dat\n[380]11.1883,[381]11.1845,[382]11.2023,[383]11.2058,[384]11.2186,[385]11.2394,[386]11.2524,[387]11.2637,[388]11.2608,[389]11.2537,\nsave_imatrix: stored collected data after 390 chunks in quantized_model_gemma2-2b/imatrix.dat\n[390]11.2593,[391]11.2478,[392]11.2511,[393]11.2355,[394]11.2122,[395]11.2076,[396]11.2143,[397]11.1925,[398]11.2003,[399]11.1762,\nsave_imatrix: stored collected data after 400 chunks in quantized_model_gemma2-2b/imatrix.dat\n[400]11.1826,[401]11.1659,[402]11.1566,[403]11.1496,[404]11.1502,[405]11.1630,[406]11.1528,[407]11.1604,[408]11.1801,[409]11.1966,\nsave_imatrix: stored collected data after 410 chunks in quantized_model_gemma2-2b/imatrix.dat\n[410]11.1873,[411]11.1950,[412]11.2118,[413]11.2209,[414]11.2470,[415]11.2625,[416]11.2954,[417]11.2904,[418]11.2864,[419]11.2941,\nsave_imatrix: stored collected data after 420 chunks in quantized_model_gemma2-2b/imatrix.dat\n[420]11.2810,[421]11.2737,[422]11.2704,[423]11.2849,[424]11.2848,[425]11.2819,[426]11.2793,[427]11.2708,[428]11.2669,[429]11.2893,\nsave_imatrix: stored collected data after 430 chunks in quantized_model_gemma2-2b/imatrix.dat\n[430]11.3036,[431]11.3407,[432]11.3416,[433]11.3448,[434]11.3445,[435]11.3588,[436]11.3540,[437]11.3639,[438]11.3732,[439]11.3755,\nsave_imatrix: stored collected data after 440 chunks in quantized_model_gemma2-2b/imatrix.dat\n[440]11.3638,[441]11.3689,[442]11.3776,[443]11.3812,[444]11.3828,[445]11.3819,[446]11.3906,[447]11.3828,[448]11.3704,[449]11.3594,\nsave_imatrix: stored collected data after 450 chunks in quantized_model_gemma2-2b/imatrix.dat\n[450]11.3558,[451]11.3481,[452]11.3681,[453]11.3522,[454]11.3416,[455]11.3414,[456]11.3373,[457]11.3293,[458]11.3557,[459]11.3591,\nsave_imatrix: stored collected data after 460 chunks in quantized_model_gemma2-2b/imatrix.dat\n[460]11.3586,[461]11.3649,[462]11.3757,[463]11.3796,[464]11.3809,[465]11.3756,[466]11.3657,[467]11.3686,[468]11.3691,[469]11.3663,\nsave_imatrix: stored collected data after 470 chunks in quantized_model_gemma2-2b/imatrix.dat\n[470]11.3454,[471]11.3397,[472]11.3288,[473]11.3533,[474]11.3608,[475]11.3707,[476]11.3681,[477]11.3754,[478]11.3925,[479]11.4037,\nsave_imatrix: stored collected data after 480 chunks in quantized_model_gemma2-2b/imatrix.dat\n[480]11.4087,[481]11.4090,[482]11.3937,[483]11.3918,[484]11.3954,[485]11.4062,[486]11.3856,[487]11.3698,[488]11.3459,[489]11.3413,\nsave_imatrix: stored collected data after 490 chunks in quantized_model_gemma2-2b/imatrix.dat\n[490]11.3359,[491]11.3421,[492]11.3498,[493]11.3497,[494]11.3584,[495]11.3840,[496]11.3742,[497]11.3797,[498]11.3950,[499]11.3998,\nsave_imatrix: stored collected data after 500 chunks in quantized_model_gemma2-2b/imatrix.dat\n[500]11.4050,[501]11.4164,[502]11.4369,[503]11.4281,[504]11.4164,[505]11.4178,[506]11.3874,[507]11.3769,[508]11.3872,[509]11.3874,\nsave_imatrix: stored collected data after 510 chunks in quantized_model_gemma2-2b/imatrix.dat\n[510]11.3921,[511]11.3882,[512]11.3812,[513]11.3888,[514]11.3965,[515]11.4192,[516]11.4192,[517]11.4090,[518]11.4095,[519]11.3788,\nsave_imatrix: stored collected data after 520 chunks in quantized_model_gemma2-2b/imatrix.dat\n[520]11.3790,[521]11.3833,[522]11.3932,[523]11.3858,[524]11.3899,[525]11.4011,[526]11.4091,[527]11.4081,[528]11.4106,[529]11.4103,\nsave_imatrix: stored collected data after 530 chunks in quantized_model_gemma2-2b/imatrix.dat\n[530]11.4058,[531]11.3926,[532]11.3918,[533]11.3950,[534]11.3880,[535]11.3740,[536]11.3696,[537]11.3601,[538]11.3542,[539]11.3421,\nsave_imatrix: stored collected data after 540 chunks in quantized_model_gemma2-2b/imatrix.dat\n[540]11.3463,[541]11.3516,[542]11.3404,[543]11.3082,[544]11.3021,[545]11.2915,[546]11.2995,[547]11.2911,[548]11.3026,[549]11.3114,\nsave_imatrix: stored collected data after 550 chunks in quantized_model_gemma2-2b/imatrix.dat\n[550]11.3301,[551]11.3288,[552]11.3465,[553]11.3479,[554]11.3662,[555]11.3783,[556]11.3951,[557]11.3996,[558]11.4106,[559]11.4266,\nsave_imatrix: stored collected data after 560 chunks in quantized_model_gemma2-2b/imatrix.dat\n[560]11.4391,[561]11.4534,[562]11.4625,[563]11.4668,[564]11.4515,[565]11.4441,[566]11.4369,[567]11.4341,[568]11.4345,[569]11.4252,\nsave_imatrix: stored collected data after 570 chunks in quantized_model_gemma2-2b/imatrix.dat\n[570]11.4236,[571]11.4498,[572]11.4515,[573]11.4578,[574]11.4608,[575]11.4714,[576]11.4817,[577]11.4874,[578]11.4779,[579]11.4657,\nsave_imatrix: stored collected data after 580 chunks in quantized_model_gemma2-2b/imatrix.dat\n[580]11.4573,[581]11.4515,[582]11.4473,[583]11.4486,[584]11.4634,[585]11.4726,[586]11.4733,[587]11.4665,[588]11.4716,[589]11.4863,\nsave_imatrix: stored collected data after 590 chunks in quantized_model_gemma2-2b/imatrix.dat\n[590]11.4839,[591]11.4793,[592]11.4797,[593]11.4859,[594]11.4832,[595]11.4843,[596]11.4868,[597]11.4931,[598]11.4798,[599]11.4636,\nsave_imatrix: stored collected data after 600 chunks in quantized_model_gemma2-2b/imatrix.dat\n[600]11.4661,[601]11.4704,[602]11.4702,[603]11.4665,[604]11.4836,[605]11.4813,[606]11.4815,[607]11.4552,[608]11.4719,[609]11.4477,\nsave_imatrix: stored collected data after 610 chunks in quantized_model_gemma2-2b/imatrix.dat\n[610]11.4297,[611]11.4062,[612]11.4072,[613]11.4123,[614]11.4106,[615]11.4152,[616]11.4092,[617]11.4066,[618]11.4077,[619]11.4133,\nsave_imatrix: stored collected data after 620 chunks in quantized_model_gemma2-2b/imatrix.dat\n[620]11.4216,[621]11.4196,[622]11.4089,[623]11.4036,[624]11.3952,[625]11.3927,[626]11.3791,[627]11.3767,[628]11.3612,[629]11.3588,\nsave_imatrix: stored collected data after 630 chunks in quantized_model_gemma2-2b/imatrix.dat\n[630]11.3694,[631]11.3731,[632]11.3763,[633]11.3829,[634]11.3913,[635]11.3846,[636]11.3875,[637]11.3793,[638]11.3986,[639]11.4114,\nsave_imatrix: stored collected data after 640 chunks in quantized_model_gemma2-2b/imatrix.dat\n[640]11.4213,[641]11.4253,[642]11.4263,[643]11.4324,[644]11.4421,[645]11.4296,[646]11.4134,[647]11.4171,[648]11.4240,[649]11.4293,\nsave_imatrix: stored collected data after 650 chunks in quantized_model_gemma2-2b/imatrix.dat\n[650]11.4357,[651]11.4354,[652]11.4412,[653]11.4410,[654]11.4447,[655]11.4532,[656]11.4531,[657]11.4506,[658]11.4441,[659]11.4450,\nsave_imatrix: stored collected data after 660 chunks in quantized_model_gemma2-2b/imatrix.dat\n[660]11.4535,[661]11.4492,[662]11.4736,[663]11.4770,[664]11.4757,[665]11.4820,[666]11.4742,[667]11.4754,[668]11.4876,[669]11.4874,\nsave_imatrix: stored collected data after 670 chunks in quantized_model_gemma2-2b/imatrix.dat\n[670]11.4773,[671]11.4738,[672]11.4736,[673]11.4719,[674]11.4666,[675]11.4704,[676]11.4729,[677]11.4620,[678]11.4633,[679]11.4670,\nsave_imatrix: stored collected data after 680 chunks in quantized_model_gemma2-2b/imatrix.dat\n[680]11.4617,[681]11.4598,[682]11.4657,[683]11.4680,[684]11.4723,[685]11.4788,[686]11.4749,[687]11.4646,[688]11.4674,[689]11.4536,\nsave_imatrix: stored collected data after 690 chunks in quantized_model_gemma2-2b/imatrix.dat\n[690]11.4614,[691]11.4601,[692]11.4658,[693]11.4778,[694]11.4685,[695]11.4574,[696]11.4733,\nFinal estimate: PPL = 11.4733 +/- 0.08400\n\nsave_imatrix: stored collected data after 696 chunks in quantized_model_gemma2-2b/imatrix.dat\n\nllama_perf_context_print:        load time =    6695.24 ms\nllama_perf_context_print: prompt eval time =  798122.18 ms / 356352 tokens (    2.24 ms per token,   446.49 tokens per second)\nllama_perf_context_print:        eval time =       0.00 ms /     1 runs   (    0.00 ms per token,      inf tokens per second)\nllama_perf_context_print:       total time = 1007556.62 ms / 356353 tokens\n","output_type":"stream"}]},{"cell_type":"markdown","source":"# Quantize the model with K-Quants","metadata":{}},{"cell_type":"code","source":"!./llama.cpp/llama-quantize --imatrix ./quantized_model_gemma2-2b/imatrix.dat \"./quantized_model_gemma2-2b/FP16.gguf\"  \"./quantized_model_gemma2-2b/FP16_I.gguf\" \"Q4_K_S\"","metadata":{"execution":{"iopub.status.busy":"2024-09-24T07:57:36.129192Z","iopub.execute_input":"2024-09-24T07:57:36.129579Z","iopub.status.idle":"2024-09-24T08:00:55.242844Z","shell.execute_reply.started":"2024-09-24T07:57:36.129547Z","shell.execute_reply":"2024-09-24T08:00:55.241753Z"},"trusted":true},"execution_count":11,"outputs":[{"name":"stdout","text":"load_imatrix: imatrix dataset='en-h10000.txt'\nload_imatrix: loaded 182 importance matrix entries from ./quantized_model_gemma2-2b/imatrix.dat computed on 696 chunks\nprepare_imatrix: have 182 importance matrix entries\nmain: build = 3814 (c087b6f1)\nmain: built with cc (Ubuntu 11.4.0-1ubuntu1~22.04) 11.4.0 for x86_64-linux-gnu\nmain: quantizing './quantized_model_gemma2-2b/FP16.gguf' to './quantized_model_gemma2-2b/FP16_I.gguf' as Q4_K_S\nllama_model_loader: loaded meta data with 38 key-value pairs and 288 tensors from ./quantized_model_gemma2-2b/FP16.gguf (version GGUF V3 (latest))\nllama_model_loader: Dumping metadata keys/values. Note: KV overrides do not apply in this output.\nllama_model_loader: - kv   0:                       general.architecture str              = gemma2\nllama_model_loader: - kv   1:                               general.type str              = model\nllama_model_loader: - kv   2:                               general.name str              = Original_Model_Gemma2 2b\nllama_model_loader: - kv   3:                           general.basename str              = original_model_gemma2\nllama_model_loader: - kv   4:                         general.size_label str              = 2B\nllama_model_loader: - kv   5:                            general.license str              = gemma\nllama_model_loader: - kv   6:                   general.base_model.count u32              = 1\nllama_model_loader: - kv   7:                  general.base_model.0.name str              = Gemma 2 2b\nllama_model_loader: - kv   8:          general.base_model.0.organization str              = Google\nllama_model_loader: - kv   9:              general.base_model.0.repo_url str              = https://huggingface.co/google/gemma-2-2b\nllama_model_loader: - kv  10:                               general.tags arr[str,2]       = [\"conversational\", \"text-generation\"]\nllama_model_loader: - kv  11:                      gemma2.context_length u32              = 8192\nllama_model_loader: - kv  12:                    gemma2.embedding_length u32              = 2304\nllama_model_loader: - kv  13:                         gemma2.block_count u32              = 26\nllama_model_loader: - kv  14:                 gemma2.feed_forward_length u32              = 9216\nllama_model_loader: - kv  15:                gemma2.attention.head_count u32              = 8\nllama_model_loader: - kv  16:             gemma2.attention.head_count_kv u32              = 4\nllama_model_loader: - kv  17:    gemma2.attention.layer_norm_rms_epsilon f32              = 0.000001\nllama_model_loader: - kv  18:                gemma2.attention.key_length u32              = 256\nllama_model_loader: - kv  19:              gemma2.attention.value_length u32              = 256\nllama_model_loader: - kv  20:                          general.file_type u32              = 1\nllama_model_loader: - kv  21:              gemma2.attn_logit_softcapping f32              = 50.000000\nllama_model_loader: - kv  22:             gemma2.final_logit_softcapping f32              = 30.000000\nllama_model_loader: - kv  23:            gemma2.attention.sliding_window u32              = 4096\nllama_model_loader: - kv  24:                       tokenizer.ggml.model str              = llama\nllama_model_loader: - kv  25:                         tokenizer.ggml.pre str              = default\nllama_model_loader: - kv  26:                      tokenizer.ggml.tokens arr[str,256000]  = [\"<pad>\", \"<eos>\", \"<bos>\", \"<unk>\", ...\nllama_model_loader: - kv  27:                      tokenizer.ggml.scores arr[f32,256000]  = [-1000.000000, -1000.000000, -1000.00...\nllama_model_loader: - kv  28:                  tokenizer.ggml.token_type arr[i32,256000]  = [3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, 3, ...\nllama_model_loader: - kv  29:                tokenizer.ggml.bos_token_id u32              = 2\nllama_model_loader: - kv  30:                tokenizer.ggml.eos_token_id u32              = 1\nllama_model_loader: - kv  31:            tokenizer.ggml.unknown_token_id u32              = 3\nllama_model_loader: - kv  32:            tokenizer.ggml.padding_token_id u32              = 0\nllama_model_loader: - kv  33:               tokenizer.ggml.add_bos_token bool             = true\nllama_model_loader: - kv  34:               tokenizer.ggml.add_eos_token bool             = false\nllama_model_loader: - kv  35:                    tokenizer.chat_template str              = {{ bos_token }}{% if messages[0]['rol...\nllama_model_loader: - kv  36:            tokenizer.ggml.add_space_prefix bool             = false\nllama_model_loader: - kv  37:               general.quantization_version u32              = 2\nllama_model_loader: - type  f32:  105 tensors\nllama_model_loader: - type  f16:  183 tensors\n================================ Have weights data with 182 entries\n[   1/ 288]                    token_embd.weight - [ 2304, 256000,     1,     1], type =    f16, \n====== llama_model_quantize_internal: did not find weights for token_embd.weight\nconverting to q6_K .. size =  1125.00 MiB ->   461.43 MiB\n[   2/ 288]               blk.0.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[   3/ 288]                blk.0.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q5_K .. size =    40.50 MiB ->    13.92 MiB\n[   4/ 288]                blk.0.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[   5/ 288]                  blk.0.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[   6/ 288]     blk.0.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[   7/ 288]           blk.0.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[   8/ 288]                blk.0.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[   9/ 288]                  blk.0.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  10/ 288]             blk.0.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  11/ 288]                  blk.0.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  12/ 288]                  blk.0.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q5_K .. size =     4.50 MiB ->     1.55 MiB\n[  13/ 288]               blk.1.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  14/ 288]                blk.1.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q5_K .. size =    40.50 MiB ->    13.92 MiB\n[  15/ 288]                blk.1.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  16/ 288]                  blk.1.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  17/ 288]     blk.1.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  18/ 288]           blk.1.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  19/ 288]                blk.1.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  20/ 288]                  blk.1.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  21/ 288]             blk.1.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  22/ 288]                  blk.1.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  23/ 288]                  blk.1.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q5_K .. size =     4.50 MiB ->     1.55 MiB\n[  24/ 288]              blk.10.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  25/ 288]               blk.10.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q5_K .. size =    40.50 MiB ->    13.92 MiB\n[  26/ 288]               blk.10.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  27/ 288]                 blk.10.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  28/ 288]    blk.10.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  29/ 288]          blk.10.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  30/ 288]               blk.10.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  31/ 288]                 blk.10.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  32/ 288]            blk.10.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  33/ 288]                 blk.10.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  34/ 288]                 blk.10.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q5_K .. size =     4.50 MiB ->     1.55 MiB\n[  35/ 288]              blk.11.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  36/ 288]               blk.11.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  37/ 288]               blk.11.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  38/ 288]                 blk.11.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  39/ 288]    blk.11.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  40/ 288]          blk.11.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  41/ 288]               blk.11.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  42/ 288]                 blk.11.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  43/ 288]            blk.11.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  44/ 288]                 blk.11.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  45/ 288]                 blk.11.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q5_K .. size =     4.50 MiB ->     1.55 MiB\n[  46/ 288]              blk.12.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  47/ 288]               blk.12.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  48/ 288]               blk.12.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  49/ 288]                 blk.12.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  50/ 288]    blk.12.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  51/ 288]          blk.12.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  52/ 288]               blk.12.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  53/ 288]                 blk.12.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  54/ 288]            blk.12.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  55/ 288]                 blk.12.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  56/ 288]                 blk.12.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  57/ 288]              blk.13.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  58/ 288]               blk.13.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  59/ 288]               blk.13.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  60/ 288]                 blk.13.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  61/ 288]    blk.13.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  62/ 288]          blk.13.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  63/ 288]               blk.13.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  64/ 288]                 blk.13.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  65/ 288]            blk.13.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  66/ 288]                 blk.13.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  67/ 288]                 blk.13.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  68/ 288]              blk.14.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  69/ 288]               blk.14.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  70/ 288]               blk.14.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  71/ 288]                 blk.14.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  72/ 288]    blk.14.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  73/ 288]          blk.14.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  74/ 288]               blk.14.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  75/ 288]                 blk.14.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  76/ 288]            blk.14.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  77/ 288]                 blk.14.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  78/ 288]                 blk.14.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  79/ 288]              blk.15.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  80/ 288]               blk.15.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  81/ 288]               blk.15.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  82/ 288]                 blk.15.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  83/ 288]    blk.15.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  84/ 288]          blk.15.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  85/ 288]               blk.15.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  86/ 288]                 blk.15.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  87/ 288]            blk.15.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  88/ 288]                 blk.15.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  89/ 288]                 blk.15.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  90/ 288]              blk.16.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  91/ 288]               blk.16.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  92/ 288]               blk.16.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  93/ 288]                 blk.16.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[  94/ 288]    blk.16.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  95/ 288]          blk.16.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  96/ 288]               blk.16.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[  97/ 288]                 blk.16.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[  98/ 288]            blk.16.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[  99/ 288]                 blk.16.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 100/ 288]                 blk.16.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 101/ 288]              blk.17.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 102/ 288]               blk.17.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 103/ 288]               blk.17.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 104/ 288]                 blk.17.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 105/ 288]    blk.17.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 106/ 288]          blk.17.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 107/ 288]               blk.17.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 108/ 288]                 blk.17.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 109/ 288]            blk.17.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 110/ 288]                 blk.17.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 111/ 288]                 blk.17.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 112/ 288]              blk.18.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 113/ 288]               blk.18.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 114/ 288]               blk.18.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 115/ 288]                 blk.18.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 116/ 288]    blk.18.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 117/ 288]          blk.18.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 118/ 288]               blk.18.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 119/ 288]                 blk.18.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 120/ 288]            blk.18.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 121/ 288]                 blk.18.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 122/ 288]                 blk.18.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 123/ 288]              blk.19.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 124/ 288]               blk.19.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 125/ 288]               blk.19.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 126/ 288]                 blk.19.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 127/ 288]    blk.19.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 128/ 288]          blk.19.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 129/ 288]               blk.19.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 130/ 288]                 blk.19.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 131/ 288]            blk.19.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 132/ 288]                 blk.19.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 133/ 288]                 blk.19.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 134/ 288]               blk.2.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 135/ 288]                blk.2.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 136/ 288]                blk.2.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 137/ 288]                  blk.2.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 138/ 288]     blk.2.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 139/ 288]           blk.2.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 140/ 288]                blk.2.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 141/ 288]                  blk.2.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 142/ 288]             blk.2.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 143/ 288]                  blk.2.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 144/ 288]                  blk.2.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 145/ 288]              blk.20.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 146/ 288]               blk.20.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 147/ 288]               blk.20.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 148/ 288]                 blk.20.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 149/ 288]    blk.20.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 150/ 288]          blk.20.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 151/ 288]               blk.20.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 152/ 288]                 blk.20.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 153/ 288]            blk.20.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 154/ 288]                 blk.20.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 155/ 288]                 blk.20.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 156/ 288]              blk.21.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 157/ 288]               blk.21.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 158/ 288]               blk.21.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 159/ 288]                 blk.21.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 160/ 288]    blk.21.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 161/ 288]          blk.21.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 162/ 288]               blk.21.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 163/ 288]                 blk.21.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 164/ 288]            blk.21.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 165/ 288]                 blk.21.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 166/ 288]                 blk.21.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 167/ 288]              blk.22.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 168/ 288]               blk.22.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 169/ 288]               blk.22.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 170/ 288]                 blk.22.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 171/ 288]    blk.22.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 172/ 288]          blk.22.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 173/ 288]               blk.22.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 174/ 288]                 blk.22.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 175/ 288]            blk.22.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 176/ 288]                 blk.22.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 177/ 288]                 blk.22.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 178/ 288]              blk.23.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 179/ 288]               blk.23.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 180/ 288]               blk.23.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 181/ 288]                 blk.23.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 182/ 288]    blk.23.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 183/ 288]          blk.23.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 184/ 288]               blk.23.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 185/ 288]                 blk.23.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 186/ 288]            blk.23.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 187/ 288]                 blk.23.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 188/ 288]                 blk.23.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 189/ 288]               blk.24.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 190/ 288]                 blk.24.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 191/ 288]            blk.24.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 192/ 288]                 blk.24.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 193/ 288]                 blk.24.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 194/ 288]               blk.3.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 195/ 288]                blk.3.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 196/ 288]                blk.3.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 197/ 288]                  blk.3.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 198/ 288]     blk.3.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 199/ 288]           blk.3.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 200/ 288]                blk.3.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 201/ 288]                  blk.3.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 202/ 288]             blk.3.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 203/ 288]                  blk.3.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 204/ 288]                  blk.3.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 205/ 288]               blk.4.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 206/ 288]                blk.4.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 207/ 288]                blk.4.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 208/ 288]                  blk.4.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 209/ 288]     blk.4.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 210/ 288]           blk.4.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 211/ 288]                blk.4.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 212/ 288]                  blk.4.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 213/ 288]             blk.4.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 214/ 288]                  blk.4.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 215/ 288]                  blk.4.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 216/ 288]               blk.5.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 217/ 288]                blk.5.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 218/ 288]                blk.5.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 219/ 288]                  blk.5.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 220/ 288]     blk.5.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 221/ 288]           blk.5.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 222/ 288]                blk.5.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 223/ 288]                  blk.5.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 224/ 288]             blk.5.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 225/ 288]                  blk.5.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 226/ 288]                  blk.5.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 227/ 288]               blk.6.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 228/ 288]                blk.6.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 229/ 288]                blk.6.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 230/ 288]                  blk.6.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 231/ 288]     blk.6.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 232/ 288]           blk.6.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 233/ 288]                blk.6.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 234/ 288]                  blk.6.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 235/ 288]             blk.6.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 236/ 288]                  blk.6.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 237/ 288]                  blk.6.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 238/ 288]               blk.7.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 239/ 288]                blk.7.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 240/ 288]                blk.7.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 241/ 288]                  blk.7.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 242/ 288]     blk.7.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 243/ 288]           blk.7.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 244/ 288]                blk.7.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 245/ 288]                  blk.7.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 246/ 288]             blk.7.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 247/ 288]                  blk.7.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 248/ 288]                  blk.7.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 249/ 288]               blk.8.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 250/ 288]                blk.8.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 251/ 288]                blk.8.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 252/ 288]                  blk.8.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 253/ 288]     blk.8.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 254/ 288]           blk.8.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 255/ 288]                blk.8.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 256/ 288]                  blk.8.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 257/ 288]             blk.8.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 258/ 288]                  blk.8.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 259/ 288]                  blk.8.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 260/ 288]               blk.9.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 261/ 288]                blk.9.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 262/ 288]                blk.9.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 263/ 288]                  blk.9.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 264/ 288]     blk.9.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 265/ 288]           blk.9.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 266/ 288]                blk.9.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 267/ 288]                  blk.9.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 268/ 288]             blk.9.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 269/ 288]                  blk.9.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 270/ 288]                  blk.9.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 271/ 288]              blk.24.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 272/ 288]               blk.24.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 273/ 288]                 blk.24.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 274/ 288]    blk.24.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 275/ 288]          blk.24.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 276/ 288]               blk.24.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 277/ 288]              blk.25.attn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 278/ 288]               blk.25.ffn_down.weight - [ 9216,  2304,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 279/ 288]               blk.25.ffn_gate.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 280/ 288]                 blk.25.ffn_up.weight - [ 2304,  9216,     1,     1], type =    f16, converting to q4_K .. size =    40.50 MiB ->    11.39 MiB\n[ 281/ 288]    blk.25.post_attention_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 282/ 288]          blk.25.post_ffw_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 283/ 288]               blk.25.ffn_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\n[ 284/ 288]                 blk.25.attn_k.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 285/ 288]            blk.25.attn_output.weight - [ 2048,  2304,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 286/ 288]                 blk.25.attn_q.weight - [ 2304,  2048,     1,     1], type =    f16, converting to q4_K .. size =     9.00 MiB ->     2.53 MiB\n[ 287/ 288]                 blk.25.attn_v.weight - [ 2304,  1024,     1,     1], type =    f16, converting to q4_K .. size =     4.50 MiB ->     1.27 MiB\n[ 288/ 288]                   output_norm.weight - [ 2304,     1,     1,     1], type =    f32, size =    0.009 MB\nllama_model_quantize_internal: model size  =  4986.92 MB\nllama_model_quantize_internal: quant size  =  1556.97 MB\n\nmain: quantize time = 197988.17 ms\nmain:    total time = 197988.17 ms\n","output_type":"stream"}]}]}