initial contents, working with bluesky codes asof 2014-01-06

.gitignore

@@ -0,0 +1,5 @@
+**~
+repos/**
+data/**
+certs/**
+config/**secret**

Makefile

@@ -0,0 +1,73 @@
+
+# definitions, dirs for top level and repo dirs
+wDir :=${PWD}
+rDir :=${wDir}/repos
+
+# dirs of repo
+nrepo    :=atproto indigo social-app pds did-method-plc  
+repoDirs :=$(addprefix ${rDir}/, ${nrepo})
+# repoDirs: ${rDir}/atproto, ... etc.
+
+
+# variables for github (gh=https://github.com/ | [email protected]:)
+gh   =$(addsuffix /, https://github.com)
+gh   =$(addsuffix :, [email protected])
+
+LOG_LEVEL_DEFAULT  ?=debug
+
+#         domain name of self hosting(NEED to care TLD, ie: NG=>.local)
+DOMAIN  ?=mybluesky.local.com
+
+# EMAIL4CERTS:  email address to lets encript or "internal"( caddy builtin CA)
+EMAIL4CERTS   ?=internal
+
+# docker composer related
+f      ?=docker-compose-starter.yaml
+Sdep  ?=caddy test-caddy test-ws database redis opensearch
+Sbsky ?=plc pds bgs bsky bsky-daemon bsky-indexer bsky-ingester bsky-cdn social-app search mod mod-daemon
+
+# password for bluesky components
+passfile=config/secrets-passwords.env
+
+# get source from github
+cloneAll:   ${repoDirs}
+${rDir}/atproto:
+	git clone ${gh}bluesky-social/atproto.git $@
+${rDir}/indigo:
+	git clone ${gh}bluesky-social/indigo.git $@
+${rDir}/social-app:
+	git clone ${gh}bluesky-social/social-app.git $@
+${rDir}/pds:
+	git clone ${gh}bluesky-social/pds.git $@
+${rDir}/did-method-plc:
+	git clone ${gh}did-method-plc/did-method-plc $@
+delRepoDirAll:
+	rm -rf ${rDir}/*
+# make clone_one d=social-app
+clone_one:  ${rDir}/${d}
+
+
+# generation for test env
+${passfile}:
+	./config/pass-gen/gen.sh > $@
+genPass: ${passfile}
+certs/ca-certificates.crt:
+	cp -p /etc/ssl/certs/ca-certificates.crt $@
+
+include ops/git.mk
+include ops/docker.mk
+include ops/patch.mk
+
+echo:
+	@echo "nrepo:    ${nrepo}"
+	@echo "repoDirs: ${repoDirs}"
+	@echo "gh:       ${gh}"
+	@echo "f:        ${f}"
+
+# make exec under=./repos/* cmd='git status|cat`
+# make exec under=./repos/* cmd='git checkout main'
+exec: ${under}
+	for d in ${under}; do \
+		echo "### exec cmd @ $${d}" ;\
+		(cd $${d};   ${cmd} ); \
+	done;

README.md

@@ -0,0 +1,150 @@
+# bluesky selfhost environment
+
+NOTE: testing with code asof 2024-01-06 of bluesky-social codes.
+under investigating for recent codes.
+
+## references
+
+special thanks to below prior works on selfhosting.
+   - https://github.com/bluesky-social/atproto/discussions/2026
+   - https://syui.ai/blog/post/2024/01/08/bluesky/
+   - https://github.com/ikuradon/atproto-starter-kit/tree/main
+
+## source code to use
+
+| components     | url (origin)                                           |
+|----------------|--------------------------------------------------------|
+| did-method-plc | https://github.com/did-method-plc/did-method-plc.git   |
+| atproto        | https://github.com/bluesky-social/atproto.git          |
+| indigo         | https://github.com/bluesky-social/indigo.git           |
+| social-app     | https://github.com/bluesky-social/social-app.git       |
+| caddy(revProxy)| official docker image of cady:2                        |
+| bind9(DNS srv) | https://github.com/itaru2622/docker-bind9.git or others|
+
+
+below ops assumes your self hosting domain is: mybluesky.local.com
+
+## ops powered by Makefile
+
+1) get codes and checkout by DayTime(2024-01-06)
+
+```bash
+# clone codes from all repos
+make    cloneAll
+
+# checkout codes asof 2024-01-06 for all sources.
+make    mkBranch_asof asof=2024-01-06 branch=work
+```
+
+
+2) prepare for your network
+
+```
+2.1) make DNS A recods for your self hosting domain, at least:
+     -    mybluesky.local.com
+     -  *.mybluesky.local.com
+
+2.2) prepare CA certificate (if self-signed )
+     -  put it into ./certs/root.{crt,key}
+     -  you also needs to deploy certificates to your hostmachine and browser.
+```
+
+3) test your network if it is ready to selfhost bluesky.
+
+```bash
+# check DNS server responses for your selfhost domain
+dig  mybluesky.local.com
+dig  any.mybluesky.local.com
+
+# start containers for test
+export  DOMAIN=mybluesky.local.com
+make    docker-start f=docker-compose-debug-caddy.yaml Sdep=
+
+# check HTTPS and WSS with your docker environment
+curl https://test-caddy.mybluesky.local.com/
+open https://test-ws.mybluesky.local.com/ on browser.
+
+# stop test containers.
+make    docker-stop f=docker-compose-debug-caddy.yaml
+```
+=> if testOK then go ahead, otherwise check your environment.
+
+
+4) prepare selfhosting...
+
+```bash
+# 4.1) build docker images for bluesky (with original code)
+DOMAIN= docker-compose -f docker-compose-starter.yaml build
+
+# 4.2) apply patch (as described in https://syui.ai/blog/post/2024/01/08/bluesky/)
+make patch-selfhost
+
+# 4.3) build social-app for selfhosting...
+make build-social-app
+```
+
+5) run bluesky with selfhosting
+
+```bash
+export  DOMAIN=mybluesky.local.com
+
+# start required containers.
+make docker-start f=./docker-compose-starter.yaml
+
+# wait until log message becomes silent.
+
+# start main containers.
+make docker-start-bsky f=./docker-compose-starter.yaml
+```
+
+## play with https://social-app.mybluesky.local.com/  in your browser.
+
+```bash
+# stop all containers.
+make docker-stop f=./docker-compose-starter.yaml
+```
+
+## sample of bind9 DNS server configuration
+
+```
+::::::::::::::
+/etc/bind/named.conf
+::::::::::::::
+include "/etc/bind/rndc.key";
+controls {
+        inet 127.0.0.1 allow { 127.0.0.1; } keys { "rndc-key"; };
+};
+options {
+        directory         "/etc/bind";
+        // UDP 53, from any
+        listen-on         { any; };
+        // HTTP 80, from any
+        listen-on  port 80  tls none http default  { any; };
+        listen-on-v6      { none; };
+        forwarders        { 8.8.8.8 ; };  # { 8.8.8.8; };
+        allow-recursion   { any; };
+        allow-query       { any; };
+        allow-query-cache { any; };
+        allow-transfer    { any; };
+};
+zone "local.com" { type master; file "zone-local.com"; allow-query { 0.0.0.0/0; }; allow-update { 0.0.0.0/0; }; allow-transfer { 0.0.0.0/0; }; };
+::::::::::::::
+/etc/bind/zone-local.com
+::::::::::::::
+$ORIGIN .
+$TTL 259200	; 3 days
+local.com		IN SOA	local.com. root.local.com. (
+				2024022809 ; serial
+				3600       ; refresh (1 hour)
+				900        ; retry (15 minutes)
+				86400      ; expire (1 day)
+				3600       ; minimum (1 hour)
+				)
+			NS	local.com.
+			A	192.168.1.27
+$ORIGIN local.com.
+$TTL 3600	; 1 hour
+mybluesky		A	192.168.1.51
+$ORIGIN mybluesky.local.com.
+*			A	192.168.1.51
+```

config/caddy/Caddyfile

@@ -0,0 +1,116 @@
+#
+# cf. https://caddyserver.com/docs/caddyfile
+#     https://caddyserver.com/docs/caddyfile/directives/tls
+#
+# global options: starts >>>>
+{
+#	debug
+        on_demand_tls {
+          ask http://test-caddy:80
+        }
+}
+# global options: ends   <<<<
+
+# self hosting CA with ACME,  for self signed certicates >>>>
+#   accessible as https://ca.DOMAIN:9000/acme/local/directory
+#   cf. https://blog.kurokobo.com/archives/3669#Caddy_acme_server
+#
+ca.{$DOMAIN}:9000 {
+	tls internal
+	acme_server
+}
+# self hosting CA with ACME,  for self signed certicates <<<<
+
+# reverse proxying rules: start >>>>>
+#
+#  FQDN_FOR_OUTER {
+#     tls  internal(for self-signed) | [email protected](for public-signed)
+#     reverse_proxy http://container:port
+# }
+
+# for debug HTTPS=>HTTP proxying  >>>
+#     try curl https://test-caddy.DOMAIN/
+test-caddy.{$DOMAIN} {
+	tls {$EMAIL4CERTS} {
+          on_demand
+        }
+	reverse_proxy http://test-caddy:80
+}
+
+test-ws.{$DOMAIN} {
+	tls {$EMAIL4CERTS} {
+          on_demand
+        }
+# it seems no need below lines, even https://caddy.community/t/websockets-over-https/16871 says it needed... <<<<<
+#	@ws {
+#		header Connection *Upgrade*
+#		header Upgrade websocket
+#	}
+# <<<<<<
+	reverse_proxy http://test-ws:8080
+# it seems no need below line
+#	reverse_proxy @ws http://test-ws:8080
+}
+
+# for debug HTTPS=>HTTP proxying  <<<
+
+# added by debugging >>>>>>
+search.{$DOMAIN} {
+	tls {$EMAIL4CERTS} {
+          on_demand
+        }
+	reverse_proxy http://search:3999
+}
+# added by debugging <<<<<<<
+
+plc.{$DOMAIN} {
+	tls {$EMAIL4CERTS} {
+          on_demand
+        }
+	reverse_proxy http://plc:3000
+}
+
+bgs.{$DOMAIN} {
+	tls {$EMAIL4CERTS} {
+          on_demand
+        }
+	reverse_proxy http://bgs:2470
+}
+
+api.{$DOMAIN}, bsky.{$DOMAIN} {
+	tls {$EMAIL4CERTS} {
+          on_demand
+        }
+	reverse_proxy http://bsky:3000
+}
+
+mod.{$DOMAIN} {
+	tls {$EMAIL4CERTS} {
+          on_demand
+        }
+	reverse_proxy http://mod:3000
+}
+
+socialapp.{$DOMAIN} {
+	tls {$EMAIL4CERTS} {
+          on_demand
+        }
+	reverse_proxy http://social-app:8100
+}
+
+pds.{$DOMAIN}, {$DOMAIN}, *.{$DOMAIN} {
+	tls {$EMAIL4CERTS} {
+          on_demand
+        }
+	reverse_proxy http://pds:3000
+}
+
+# not sure, for safe, below >>>
+public.api.{$DOMAIN} {
+	tls {$EMAIL4CERTS} {
+          on_demand
+        }
+	reverse_proxy http://pds:3000
+}
+
+# reverse proxying rules: ends  <<<<

config/init-postgres/init.sql

@@ -0,0 +1,25 @@
+-- PLC
+CREATE DATABASE plc;
+GRANT ALL PRIVILEGES ON DATABASE plc TO postgres;
+
+-- BGS
+CREATE DATABASE bgs;
+GRANT ALL PRIVILEGES ON DATABASE bgs TO postgres;
+CREATE DATABASE carstore;
+GRANT ALL PRIVILEGES ON DATABASE carstore TO postgres;
+
+-- bsky(AppView)
+CREATE DATABASE bsky;
+GRANT ALL PRIVILEGES ON DATABASE bsky TO postgres;
+
+-- ozone(Moderation)
+CREATE DATABASE mod;
+GRANT ALL PRIVILEGES ON DATABASE mod TO postgres;
+
+-- search(palomar)
+CREATE DATABASE search;
+GRANT ALL PRIVILEGES ON DATABASE search TO postgres;
+
+-- PDS
+-- CREATE DATABASE pds;
+-- GRANT ALL PRIVILEGES ON DATABASE pds TO postgres;