move kssh config from kbfs to kv store (#97)

- replace kbfs kssh config code with kvstore code
- move shared kbfs.go file into keybaseca
- update tests: change keybase fs run_commands to instead use keybase kvstore. lots of linting.
- update documentation to not refer to "config files" in KBFS.
- update circleci: clean up go linting, and add python linting
- some renaming and refactoring

Author: mmou

.circleci/config.yml

@@ -1,31 +1,34 @@
 version: 2
+commands:
 jobs:
-  integration:
-    machine:
-      image: circleci/classic:latest
+  go-lint:
+    docker:
+      - image: circleci/golang:1.11
     steps:
       - checkout
-      - run: "./integrationTest.sh"
+      - run: "! go mod tidy -v 2>&1 | read"
+      - run: go get github.com/golangci/golangci-lint/cmd/[email protected]
+      - run: "golangci-lint run"
+  py-lint:
+    docker:
+      - image: circleci/python:3.6.8
+    steps:
+      - run: sudo pip3 install isort flake8 black
+      - run: "isort -rc . --check-only"
+      - run: "flake8 . --max-line-length=88"
+      - run: "black . --check"
   unit:
     docker:
       - image: circleci/golang:1.11
     steps:
       - checkout
       - run: go test ./...
-  lint:
-    docker:
-      - image: circleci/golang:1.11
+  integration:
+    machine:
+      image: circleci/classic:latest
     steps:
       - checkout
-      - run: "! go fmt ./... 2>&1 | read"
-      - run: "! go vet ./... 2>&1 | read"
-      - run: go get -u golang.org/x/lint/golint
-      - run: "! golint ./... 2>&1 | read"
-      - run: go get golang.org/x/tools/cmd/goimports
-      - run: "! goimports ./... 2>&1 | read"
-      - run: "! go mod tidy ./... 2>&1 | read"
-      - run: go get github.com/golangci/golangci-lint/cmd/[email protected]
-      - run: "! golangci-lint 2>&1 | read"
+      - run: "./integrationTest.sh"
   sphinx-build:
     docker:
       - image: circleci/python:2.7.16
@@ -48,15 +51,17 @@ workflows:
   version: 2
   build:
     jobs:
-      - integration
+      - go-lint
+      - py-lint
       - unit
-      - lint
+      - integration
       - sphinx-build
       - publish-github-release:
           requires:
-            - integration
+            - go-lint
+            - py-lint
             - unit
-            - lint
+            - integration
           filters:
             branches:
               only: master

.isort.cfg

@@ -0,0 +1,5 @@
+[settings]
+combine_as_imports = true
+include_trailing_comma = true
+line_length = 88
+multi_line_output = 3

docker/Makefile

@@ -7,7 +7,7 @@ endif
 
 SHELL := /bin/bash
 
-.PHONY: build generate serve clean reset-permissions confirm-clean env-file-exists ca-key-exists
+.PHONY: build lint go-lint py-lint generate serve clean reset-permissions confirm-clean env-file-exists ca-key-exists
 
 # Build a new docker image for the CA bot
 build: reset-permissions
@@ -18,6 +18,18 @@ else
 	docker build -t ca -f Dockerfile-ca ..
 endif
 
+lint: go-lint py-lint
+
+go-lint:
+	go fmt ../...
+	go vet ../...
+	go mod tidy
+	golangci-lint run ../...
+
+py-lint:
+	isort -rc ..
+	flake8 .. --max-line-length=88
+	black ..
 
 # Generate a new CA key
 generate: env-file-exists build
@@ -45,7 +57,7 @@ restart: stop serve
 
 # Delete all kssh config files
 clean-kssh: env-file-exists
-	docker run --init -e FORCE_WRITE=$(FORCE_WRITE) --env-file ./env.list -v $(CURDIR)/example-keybaseca-volume:/mnt:rw ca:latest ./entrypoint-cleanup.sh
+	docker run --init --env-file ./env.list -v $(CURDIR)/example-keybaseca-volume:/mnt:rw ca:latest ./entrypoint-cleanup.sh
 
 # Delete all CA data
 clean: confirm-clean reset-permissions

docker/entrypoint-cleanup.sh

@@ -7,12 +7,11 @@ chown -R keybase:keybase /mnt
 
 # Run everything else as the keybase user
 sudo -i -u keybase bash << EOF
-export "FORCE_WRITE=$FORCE_WRITE"
+export "TEAMS=$TEAMS"
 export "KEYBASE_USERNAME=$KEYBASE_USERNAME"
 export "KEYBASE_PAPERKEY=$KEYBASE_PAPERKEY"
 nohup bash -c "KEYBASE_RUN_MODE=prod kbfsfuse /keybase | grep -v 'ERROR Mounting the filesystem failed' &"
 sleep ${KEYBASE_TIMEOUT:-5}
 keybase oneshot
 bin/keybaseca --wipe-all-configs
-sleep ${KEYBASE_TIMEOUT:-5}
 EOF

docs/best_practices.md

@@ -2,30 +2,37 @@
 
 ## Teams and Channels
 
-The SSH CA bot user needs to have write access in all of the teams used for granting SSH access in order for it to
-be able to store config files associated with each team. Since access to a team grants SSH access to servers, it is
-recommended to minimize the number of users with admin or owner permissions in the teams. Individual users of kssh
-only need to be given the read permission since they do not need to be able to edit or create files associated with
-a team. 
-
-It is also recommended to mute all notifications in the configured teams in order to minimize the number of notifications
-you get. 
-
-If you are using other bots in the same teams as the SSH CA bot (or if you wish to have normal conversation in those 
-teams), you can use the `CHAT_CHANNEL` environment variable in order to configure a specific chat channel for all
-SSH CA messages. 
+The SSH CA bot user needs to have write access in all of the teams used for
+granting SSH access in order for it to be able to store kssh client configs
+associated with each team. Since access to a team grants SSH access to servers,
+it is recommended to minimize the number of users with admin or owner
+permissions in the teams. Individual users of kssh only need to be given the
+read permission since they do not need to be able to edit or create files
+associated with a team. 
+
+It is also recommended to mute all notifications in the configured teams in
+order to minimize the number of notifications you get. 
+
+If you are using other bots in the same teams as the SSH CA bot (or if you wish
+to have normal conversation in those teams), you can use the `CHAT_CHANNEL`
+environment variable in order to configure a specific chat channel for all SSH
+CA messages. 
 
 ## Network Isolation
 
-Due to the highly sensitive nature of the SSH CA bot, it is recommended to configure firewalls in order to block all
-access to the server running the CA bot. It is not recommended to use kssh to access the server of the CA bot itself
-in order to make it easier to respond to any outages. 
+Due to the highly sensitive nature of the SSH CA bot, it is recommended to
+configure firewalls in order to block all access to the server running the CA
+bot. It is not recommended to use kssh to access the server of the CA bot
+itself in order to make it easier to respond to any outages. 
 
 ## Realms
 
-There are two general approaches one can take when defining realms of servers. The first approach (described in the
-getting started directions) is to define realms for staging and production. This approach is useful for the common
-scenario where all developers should be given access to the staging environment but only certain people should be given
-access to production. The second approach is a more granular approach where you can define realms associated with teams.
-For example, one could have a realm of web servers, a realm of database servers, ... where a specific group of people 
-is responsible for each class of server. 
+There are two general approaches one can take when defining realms of servers.
+The first approach (described in the getting started directions) is to define
+realms for staging and production. This approach is useful for the common
+scenario where all developers should be given access to the staging environment
+but only certain people should be given access to production. The second
+approach is a more granular approach where you can define realms associated
+with teams.  For example, one could have a realm of web servers, a realm of
+database servers, ... where a specific group of people is responsible for each
+class of server. 

docs/conf.py

@@ -18,14 +18,14 @@
 
 # -- Project information -----------------------------------------------------
 
-project = u'Keybase SSH CA Bot'
-copyright = u'2019, keybase.io'
-author = u'keybase.io'
+project = u"Keybase SSH CA Bot"
+copyright = u"2019, keybase.io"
+author = u"keybase.io"
 
 # The short X.Y version
-version = u''
+version = u""
 # The full version, including alpha/beta/rc tags
-release = u''
+release = u""
 
 
 # -- General configuration ---------------------------------------------------
@@ -40,16 +40,16 @@
 extensions = ["recommonmark", "sphinx_rtd_theme"]
 
 # Add any paths that contain templates here, relative to this directory.
-templates_path = ['_templates']
+templates_path = ["_templates"]
 
 # The suffix(es) of source filenames.
 # You can specify multiple suffix as a list of string:
 #
 # source_suffix = ['.rst', '.md']
-source_suffix = '.rst'
+source_suffix = ".rst"
 
 # The master toctree document.
-master_doc = 'index'
+master_doc = "index"
 
 # The language for content autogenerated by Sphinx. Refer to documentation
 # for a list of supported languages.
@@ -61,7 +61,7 @@
 # List of patterns, relative to source directory, that match files and
 # directories to ignore when looking for source files.
 # This pattern also affects html_static_path and html_extra_path.
-exclude_patterns = [u'_build', 'Thumbs.db', '.DS_Store']
+exclude_patterns = [u"_build", "Thumbs.db", ".DS_Store"]
 
 # The name of the Pygments (syntax highlighting) style to use.
 pygments_style = None
@@ -84,7 +84,7 @@
 # Add any paths that contain custom static files (such as style sheets) here,
 # relative to this directory. They are copied after the builtin static files,
 # so a file named "default.css" will overwrite the builtin "default.css".
-html_static_path = ['_static']
+html_static_path = ["_static"]
 
 # Custom sidebar templates, must be a dictionary that maps document names
 # to template names.
@@ -100,7 +100,7 @@
 # -- Options for HTMLHelp output ---------------------------------------------
 
 # Output file base name for HTML help builder.
-htmlhelp_basename = 'KeybaseSSHCABotdoc'
+htmlhelp_basename = "KeybaseSSHCABotdoc"
 
 
 # -- Options for LaTeX output ------------------------------------------------
@@ -109,15 +109,12 @@
     # The paper size ('letterpaper' or 'a4paper').
     #
     # 'papersize': 'letterpaper',
-
     # The font size ('10pt', '11pt' or '12pt').
     #
     # 'pointsize': '10pt',
-
     # Additional stuff for the LaTeX preamble.
     #
     # 'preamble': '',
-
     # Latex figure (float) alignment
     #
     # 'figure_align': 'htbp',
@@ -127,8 +124,13 @@
 # (source start file, target name, title,
 #  author, documentclass [howto, manual, or own class]).
 latex_documents = [
-    (master_doc, 'KeybaseSSHCABot.tex', u'Keybase SSH CA Bot Documentation',
-     u'keybase.io', 'manual'),
+    (
+        master_doc,
+        "KeybaseSSHCABot.tex",
+        u"Keybase SSH CA Bot Documentation",
+        u"keybase.io",
+        "manual",
+    ),
 ]
 
 
@@ -137,8 +139,7 @@
 # One entry per manual page. List of tuples
 # (source start file, name, description, authors, manual section).
 man_pages = [
-    (master_doc, 'keybasesshcabot', u'Keybase SSH CA Bot Documentation',
-     [author], 1)
+    (master_doc, "keybasesshcabot", u"Keybase SSH CA Bot Documentation", [author], 1,)
 ]
 
 
@@ -148,9 +149,15 @@
 # (source start file, target name, title, author,
 #  dir menu entry, description, category)
 texinfo_documents = [
-    (master_doc, 'KeybaseSSHCABot', u'Keybase SSH CA Bot Documentation',
-     author, 'KeybaseSSHCABot', 'One line description of project.',
-     'Miscellaneous'),
+    (
+        master_doc,
+        "KeybaseSSHCABot",
+        u"Keybase SSH CA Bot Documentation",
+        author,
+        "KeybaseSSHCABot",
+        "One line description of project.",
+        "Miscellaneous",
+    ),
 ]
 
 
@@ -169,6 +176,6 @@
 # epub_uid = ''
 
 # A list of files that should not be packed into the epub file.
-epub_exclude_files = ['search.html']
+epub_exclude_files = ["search.html"]
 
 collapse_navigation = False