rename icpp_llama2 to llama2_c

.github/workflows/cicd.yml

@@ -4,10 +4,10 @@ on:
   workflow_dispatch:
   push:
     paths:
-      - "icpp_llama2/**"
-      - "Makefile"
-      - ".github/trigger.txt"
-      - ".github/workflows/cicd.yml"
+      - 'llama2_c/**'
+      - 'Makefile'
+      - '.github/trigger.txt'
+      - '.github/workflows/cicd.yml'
 
 env:
   PYTHON_VERSION: 3.11
@@ -18,13 +18,13 @@ jobs:
     runs-on: ubuntu-latest
     steps:
       # https://github.com/marketplace/actions/setup-miniconda
-      - uses: conda-incubator/setup-miniconda@v2
+      - uses: conda-incubator/setup-miniconda@v3
         with:
           python-version: ${{ env.PYTHON_VERSION }}
 
       # -------------------------------------------------------------------
-      # Checkout icpp-llm
-      - name: checkout icpp-llm
+      # Checkout icpp_llm
+      - name: checkout icpp_llm
         uses: actions/checkout@v3
         with:
           fetch-depth: 0
@@ -65,12 +65,6 @@ jobs:
           echo "Installing wasi-sdk"
           icpp install-wasi-sdk
 
-      - name: download models
-        shell: bash -l {0}
-        run: |
-          make icpp_llama2_get_stories15M
-          make icpp_llama2_get_stories260K
-
       - name: all-tests
         shell: bash -l {0}
         run: |

Makefile

@@ -49,19 +49,6 @@ CLANG_TIDY = $(ICPP_COMPILER_ROOT)/bin/clang-tidy
 .PHONY: all-tests
 all-tests: all-static test-all-llms 
 
-.PHONY: icpp_llama2_get_stories260K
-icpp_llama2_get_stories260K:
-	cd icpp_llama2 && \
-		mkdir -p stories260K && \
-		wget -P stories260K https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/stories260K.bin && \
-		wget -P stories260K https://huggingface.co/karpathy/tinyllamas/resolve/main/stories260K/tok512.bin
-
-.PHONY: icpp_llama2_get_stories15M
-icpp_llama2_get_stories15M:
-	cd icpp_llama2 && \
-		mkdir -p models && \
-		wget -P models https://huggingface.co/karpathy/tinyllamas/resolve/main/stories15M.bin
-
 .PHONY: summary
 summary:
 	@echo "-------------------------------------------------------------"
@@ -77,9 +64,9 @@ summary:
 test-all-llms:
 	dfx identity use default
 	@echo "#########################################"
-	@echo "####### testing icpp_llama2 #############"
+	@echo "####### testing llama2_c #############"
 	@echo "#########################################"
-	cd icpp_llama2 && \
+	cd llama2_c && \
 		icpp build-native && \
 		./build-native/mockic.exe && \
 		./demo.sh && \
@@ -92,8 +79,8 @@ all-static: \
 	python-format python-lint python-type
 
 CPP_AND_H_FILES = $(shell ls \
-icpp_llama2/src/*.cpp icpp_llama2/src/*.h \
-icpp_llama2/native/*.cpp icpp_llama2/native/*.h)
+llama2_c/src/*.cpp llama2_c/src/*.h \
+llama2_c/native/*.cpp llama2_c/native/*.h)
 
 .PHONY: cpp-format
 cpp-format:
@@ -107,7 +94,7 @@ cpp-lint:
 	@echo "cpp-lint"
 	@echo "TO IMPLEMENT with clang-tidy"
 
-PYTHON_DIRS ?= icpp_llama2
+PYTHON_DIRS ?= llama2_c
 
 .PHONY: python-format
 python-format:

README.md

@@ -1,36 +1,57 @@
 [![icpp_llm](https://github.com/icppWorld/icpp_llm/actions/workflows/cicd.yml/badge.svg)](https://github.com/icppWorld/icpp_llm/actions/workflows/cicd.yml)
 
 # LLMs for the Internet Computer
+
 <img src="./assets/icpp-llm-logo.png" alt="icpp-llm logo" width="200">
 
-  Try it out in [ICGPT](https://icgpt.icpp.world) !
+Try it out in [ICGPT](https://icgpt.icpp.world) !
 
-*The LLMs of this repo run in it's back-end canisters.*
+_The LLMs of this repo run in it's back-end canisters._
 
 # Getting Started
 
-A step-by-step guide to deploy your first LLM to the internet computer is provided in [icpp_llama2/README.md](https://github.com/icppWorld/icpp_llm/blob/main/icpp_llama2/README.md).
+A step-by-step guide to deploy your first LLM to the internet computer is provided in [llama2_c/README.md](https://github.com/icppWorld/icpp_llm/blob/main/llama2_c/README.md).
 
 # The Benefits of Running LLMs On-Chain
 
 The canisters within the Internet Computer have certain constraints. They come with memory restrictions, and there's a cap on the number of instructions one can execute per message, as discussed [here](https://forum.dfinity.org/t/instruction-limit-is-crushing-me/22070/10?u=icpp).
 
 This might lead one to question the rationale behind operating an LLM within an Internet Computer's canister.
 
-For me, the primary incentive is the unparalleled simplicity of using the IC in comparison to conventional cloud platforms. You develop, deploy & test using a local replica of the cloud, and when everything is ready, you deploy it to the IC with just one command. Everything becomes instantly and securely accessible online. You can very easily restrict access to the endpoints in case you don't want to make it fully public yet and want to share it with a smaller group instead. 
+For me, the primary incentive is the unparalleled simplicity of using the IC in comparison to conventional cloud platforms. You develop, deploy & test using a local replica of the cloud, and when everything is ready, you deploy it to the IC with just one command. Everything becomes instantly and securely accessible online. You can very easily restrict access to the endpoints in case you don't want to make it fully public yet and want to share it with a smaller group instead.
 
 Thanks to the Internet Computer's foundational cryptographic and blockchain technologies, concerns related to IT and security vanish. It's truly remarkable.
 
 With such user-friendliness, the IC canister runtime serves as an ideal environment for my research pursuits. It complements the type of research presented in this paper that offers a dataset designed to boost the creation, examination, and study of Language Models for areas with scarce resources or specific niches:
 
- > [TinyStories: How Small Can Language Models Be and Still Speak
-Coherent English?](https://arxiv.org/pdf/2305.07759.pdf)
+> [TinyStories: How Small Can Language Models Be and Still Speak
+> Coherent English?](https://arxiv.org/pdf/2305.07759.pdf)
 
 Besides the ease of use and the enhanced security, running LLMs directly on-chain also facilitates a seamless integration of tokenomics, eliminating the need to juggle between a complex blend of web3 and web2 components, and I believe it will lead to a new category of Generative AI based dApps.
 
+## QA
+
+We use MiniConda and run the QA locally like this:
+
+- Create a conda environment, and install icpp-pro and other python dependencies:
+
+  ```bash
+  conda create --name icpp_llm python=3.11
+  conda activate icpp_llm
+  pip install -r requirements.txt
+  ```
+
+- This installs icpp-pro. Next install wasi-sdk, dfx & clang++ as explained in [icpp-pro Installation](https://docs.icpp.world/installation.html)
+
+- Run the full QA via the Makefile:
+  ```bash
+  make all-tests
+  ```
+
+You can also peak in `.github/workflows/cicd.yml` to see how the QA is run as part of a GitHub actions workflow.
 
+More details are provided in the README of the sub-folders, which are standalone icpp-pro projects.
 
 ## Support
 
 For support, kindly create a GitHub Issue as outlined in the [Support](https://docs.icpp.world/support.html) documentation page.
-

icpp_llama2/README.md → llama2_c/README.md

@@ -13,10 +13,10 @@ The 15M parameter model is the backend of [ICGPT](https://icgpt.icpp.world/).
     conda create --name myllama2 python=3.11
     conda activate myllama2
     ```
-  - Clone this repo and enter the icpp_llama2 folder
+  - Clone this repo and enter the llama2_c folder
     ```bash
     git clone https://github.com/icppWorld/icpp_llm.git
-    cd icpp_llm/icpp_llama2
+    cd icpp_llm/llama2_c
     ```
   - Install the required python packages _(icpp-pro & ic-py)_:
     ```bash

llama2_c/README_llama2_c_resource_requirements.md

@@ -0,0 +1,66 @@
+# Canister resource requirements for llama2_c.
+
+Do not edit this file. It is created with the command:
+
+```bash
+python -m scripts.icpp_llama2_sizer
+```
+
+### Tokenizer Memory (per model)
+
+| Memory Type         | 260K<br>(MB) | 15M<br>(MB) | 42M<br>(MB) | 110M<br>(MB) |
+| ------------------- | ------------ | ----------- | ----------- | ------------ |
+| vocab_memory        | 0.00         | 0.12        | 0.12        | 0.12         |
+| vocab_scores_memory | 0.00         | 0.12        | 0.12        | 0.12         |
+| Total               | 0.00         | 0.24        | 0.24        | 0.24         |
+
+### TransformerWeights Memory (per model)
+
+| Memory Type           | 260K<br>(MB) | 15M<br>(MB) | 42M<br>(MB) | 110M<br>(MB) |
+| --------------------- | ------------ | ----------- | ----------- | ------------ |
+| token_embedding_table | 0.12         | 35.16       | 62.50       | 93.75        |
+| rms_att_weight        | 0.00         | 0.01        | 0.02        | 0.04         |
+| wq                    | 0.08         | 1.90        | 8.00        | 27.00        |
+| wk                    | 0.04         | 1.90        | 8.00        | 27.00        |
+| wv                    | 0.04         | 1.90        | 8.00        | 27.00        |
+| wo                    | 0.08         | 1.90        | 8.00        | 27.00        |
+| rms_ffn_weight        | 0.00         | 0.01        | 0.02        | 0.04         |
+| w1                    | 0.21         | 5.06        | 21.50       | 72.00        |
+| w2                    | 0.21         | 5.06        | 21.50       | 72.00        |
+| w3                    | 0.21         | 5.06        | 21.50       | 72.00        |
+| rms_final_weight      | 0.00         | 0.00        | 0.00        | 0.00         |
+| wcls                  | 0.12         | 35.16       | 62.50       | 93.75        |
+| Total                 | 1.12         | 93.11       | 221.53      | 511.57       |
+
+### RunState Memory (per user)
+
+| Memory Type | 260K<br>(MB) | 15M<br>(MB) | 42M<br>(MB) | 110M<br>(MB) |
+| ----------- | ------------ | ----------- | ----------- | ------------ |
+| x           | 0.00         | 0.00        | 0.00        | 0.00         |
+| xb          | 0.00         | 0.00        | 0.00        | 0.00         |
+| xb2         | 0.00         | 0.00        | 0.00        | 0.00         |
+| hb          | 0.00         | 0.00        | 0.01        | 0.01         |
+| hb2         | 0.00         | 0.00        | 0.01        | 0.01         |
+| q           | 0.00         | 0.00        | 0.00        | 0.00         |
+| k           | 0.00         | 0.00        | 0.00        | 0.00         |
+| v           | 0.00         | 0.00        | 0.00        | 0.00         |
+| att         | 0.02         | 0.01        | 0.03        | 0.05         |
+| logits      | 0.00         | 0.12        | 0.12        | 0.12         |
+| key_cache   | 0.31         | 1.69        | 16.00       | 36.00        |
+| value_cache | 0.31         | 1.69        | 16.00       | 36.00        |
+| Total       | 0.65         | 3.52        | 32.18       | 72.20        |
+
+### Total Memory
+
+| Memory Type                                 | 260K<br>(MB) | 15M<br>(MB) | 42M<br>(MB) | 110M<br>(MB) |
+| ------------------------------------------- | ------------ | ----------- | ----------- | ------------ |
+| Total Tokenizer Memory (per model)          | 0.00         | 0.24        | 0.24        | 0.24         |
+| Total TransformerWeights Memory (per model) | 1.12         | 93.11       | 221.53      | 511.57       |
+| Total RunState Memory (per user)            | 0.65         | 3.52        | 32.18       | 72.20        |
+| Overall Total Memory                        | 1.76         | 96.62       | 253.71      | 583.78       |
+
+### Canister Metrics
+
+| Canister Metrics               | 260K<br>(MB) | 15M<br>(MB) | 42M<br>(MB) | 110M<br>(MB) |
+| ------------------------------ | ------------ | ----------- | ----------- | ------------ |
+| Max number of concurrent users | 6347         | 1138        | 120         | 49           |

icpp_llama2/scripts/icpp_llama2_sizer.py → llama2_c/scripts/llama2_c_sizer.py

@@ -245,7 +245,7 @@ def main() -> int:
 
     output_path = ROOT_PATH / "README_icpp_llama2_resource_requirements.md"
     with open(output_path, "w", encoding="utf-8") as file:
-        file.write("# Canister resource requirements for icpp_llama2.")
+        file.write("# Canister resource requirements for llama2_c.")
         file.write("\n")
         file.write("\nDo not edit this file. It is created with the command: ")
         file.write("\n```bash")

requirements.txt

@@ -1,4 +1,4 @@
--r icpp_llama2/requirements.txt
+-r llama2_c/requirements.txt
 
 # to lint python scripts
 black

test_all_llms.ps1

@@ -14,8 +14,8 @@ wsl --% dfx identity use default
 
 Write-Host " "
 Write-Host "--------------------------------------------------"
-Write-Host "Testing icpp_llama2"
-Set-Location -Path .\icpp_llama2
+Write-Host "Testing llama2_c"
+Set-Location -Path .\llama2_c
 .\demo.ps1
 
 # Change directory back to the root