Merge pull request #250 from kreneskyp/vault

Adding vault container

.gitignore

@@ -32,3 +32,8 @@ langchain
 
 # local models
 llama
+
+# vault setup
+.vault.env
+.certs
+.vault_file

Makefile

@@ -97,7 +97,7 @@ endif
 
 # setup target for docker-compose, add deps here to apply to all compose sessions
 .PHONY: compose
-compose: image
+compose: image .vault.env
 
 # =========================================================
 # Build
@@ -323,4 +323,45 @@ prettier: nodejs
 
 .PHONY: clean
 clean:
-	rm -rf .sentinel
+	rm -rf .sentinel
+	rm -rf .certs
+
+.vault.env:
+	@python ./bin/get_uuid.py > .vault.env
+	@echo ".vault.env file has been generated with a UUID key."
+
+
+# =========================================================
+# Misc
+# =========================================================
+
+# commands to generate all the certs needed for local development
+
+.certs/sentinel:
+	@mkdir -p .certs
+
+	@if [ ! -f .certs/ca.crt ]; then \
+		echo "Generating CA and server certificates..."; \
+		openssl genpkey -algorithm RSA -out .certs/ca.key; \
+		openssl req -x509 -new -nodes -key .certs/ca.key -subj "/CN=Vault CA" -days 3650 -out .certs/ca.crt; \
+	fi
+	@if [ ! -f .certs/server.crt ]; then \
+		echo "Generating Vault server certificate..."; \
+		openssl genpkey -algorithm RSA -out .certs/server.key; \
+		openssl req -new -key .certs/server.key -subj "/CN=localhost" -out .certs/server.csr; \
+		openssl x509 -req -in .certs/server.csr -CA .certs/ca.crt -CAkey .certs/ca.key -CAcreateserial -out .certs/server.crt -days 3650; \
+	fi
+	@if [ ! -f .certs/client.crt ]; then \
+		echo "Generating Vault client certificate..."; \
+		openssl genpkey -algorithm RSA -out .certs/client.key; \
+		openssl req -new -key .certs/client.key -subj "/CN=Vault Client" -out .certs/client.csr; \
+		openssl x509 -req -in .certs/client.csr -CA .certs/ca.crt -CAkey .certs/ca.key -CAcreateserial -out .certs/client.crt -days 3650; \
+	fi
+
+	@rm -f .certs/*.csr
+	@rm -f .certs/ca.srl
+	@touch .certs/sentinel
+
+
+.PHONY: certs
+certs: .certs/sentinel

bin/get_uuid.py

@@ -0,0 +1,9 @@
+from uuid import uuid4
+
+
+def main():
+    print(f"VAULT_UUID_KEY={uuid4()}")
+
+
+if __name__ == "__main__":
+    main()

docker-compose.yml

@@ -21,11 +21,13 @@ services:
     links:
       - db
       - redis
+      - vault
     volumes:
       - .:/var/app
       - ./bin:/usr/bin/ix
       - .bash_profile:/root/.bash_profile
       - .ipython:/root/.ipython
+      - ./.certs/:/var/vault/certs:ro,Z
     env_file:
       - .env
     environment:
@@ -49,12 +51,14 @@ services:
     links:
       - db
       - redis
+      - vault
       - chroma
     volumes:
       - .:/var/app
       - .node_modules:/var/npm/node_modules
       - ./bin:/usr/bin/ix
       - .bash_profile:/root/.bash_profile
+      - ./.certs/:/var/vault/certs:ro,Z
     env_file:
       - .env
 
@@ -80,5 +84,19 @@ services:
     ports:
       - "8020:8000"
 
+  vault:
+    image: hashicorp/vault:1.14
+    volumes:
+    - ./vault/config.hcl:/vault/config/config.hcl:ro
+    - ./.certs/:/vault/certs:ro,Z
+    - .vault_file:/vault/file
+    env_file:
+      - .vault.env
+    command: "server -dev -dev-tls -config=/vault/config/config.hcl"
+    ports:
+      - "8200:8200"
+    cap_add:
+      - IPC_LOCK
+
 volumes:
   postgres_data:

ix/server/settings.py

@@ -215,3 +215,19 @@
     "b7d8f662-12f6-4525-b07b-c9ea7c10010a",  # @code
     "cc054ff5-67cd-4489-b0f1-b8b62af2d825",  # @readme
 ]
+
+
+# Dev environment can load from .vault.env
+# TODO: move this to dev_settings if possible.
+if os.path.exists("/var/app/.vault.env"):
+    with open("/var/app/.vault.env", "r") as f:
+        key_file = f.read()
+        key, value = key_file.strip().split("=")
+        os.environ["VAULT_ROOT_KEY"] = value
+
+VAULT_ROOT_KEY = "myroot"
+VAULT_SERVER = os.environ.get("VAULT_SERVER", "https://vault:8200")
+VAULT_TOKEN__USER_TOKENS = VAULT_ROOT_KEY
+VAULT_CLIENT_CRT = "/var/vault/certs/client.crt"
+VAULT_CLIENT_KEY = "/var/vault/certs/client.key"
+VAULT_TLS_VERIFY = False

vault/config.hcl

@@ -0,0 +1,10 @@
+disable_mlock = true
+
+# vault-config.hcl
+
+
+storage "file" {
+  path    = "/vault/file"
+}
+
+