update/add docs (#9)

* update/add docs

* add more package infrastructure

CONTRIBUTING.rst

@@ -83,9 +83,9 @@ We may ask you to sign a `Contributor License Agreement (CLA)`_ for larger chang
 .. _recently closed: https://github.com/awslabs/pipeformer/issues?utf8=%E2%9C%93&q=is%3Aissue%20is%3Aclosed%20
 .. _help wanted: https://github.com/awslabs/pipeformer/labels/help%20wanted
 .. _LICENSE: https://github.com/awslabs/pipeformer/blob/master/LICENSE
-.. _forking a repository: https://help.github.com/articles/fork-a-repo/
-.. _creating a pull request: https://help.github.com/articles/creating-a-pull-request/
+.. _forking a repository: https://help.github.com/en/articles/fork-a-repo
+.. _creating a pull request: https://help.github.com/en/articles/creating-a-pull-request
 .. _Amazon Open Source Code of Conduct: https://aws.github.io/code-of-conduct
 .. _Code of Conduct FAQ: https://aws.github.io/code-of-conduct-faq
-.. _vulnerability reporting page: http://aws.amazon.com/security/vulnerability-reporting/
-.. _Contributor License Agreement (CLA): http://en.wikipedia.org/wiki/Contributor_License_Agreement
+.. _vulnerability reporting page: https://aws.amazon.com/security/vulnerability-reporting/
+.. _Contributor License Agreement (CLA): https://en.wikipedia.org/wiki/Contributor_License_Agreement

README.rst

@@ -64,7 +64,7 @@ How to Use
 **********
 
 1. Define your configuration file.
-1. Deploy with pipeformer.
+2. Deploy with pipeformer.
 
 User Experience
 ===============
@@ -171,7 +171,7 @@ Before submitting a pull request
 
 Before submitting a pull request, please run the ``lint`` tox environment.
 This will ensure that your submission meets our code formatting requirements
-and will pass our continous integration code formatting tests.
+and will pass our continuous integration code formatting tests.
 
 
 .. _tox: http://tox.readthedocs.io/

doc/code_of_conduct.rst

@@ -0,0 +1 @@
+.. include:: ../CODE_OF_CONDUCT.rst

doc/conf.py

@@ -44,6 +44,8 @@ def get_version():
     "sphinx.ext.autosummary",
     "sphinx.ext.napoleon",
     "sphinx.ext.viewcode",
+    "sphinx_autodoc_typehints",
+    "sphinxcontrib.spelling",
 ]
 napoleon_include_special_with_doc = False
 
@@ -74,3 +76,9 @@ def get_version():
 
 # autosummary
 autosummary_generate = True
+
+# Spellchecker
+spelling_word_list_filename = "spelling_wordlist.txt"
+spelling_lang = "en_US"
+spelling_ignore_python_builtins = True
+spelling_ignore_importable_modules = True

doc/config_format.rst

@@ -0,0 +1 @@
+.. include:: ../CONFIG_FORMAT.rst

doc/contributing.rst

@@ -0,0 +1 @@
+.. include:: ../CONTRIBUTING.rst

doc/index.rst

@@ -1,27 +1,29 @@
 .. include:: ../README.rst
 
 .. toctree::
-   :maxdepth: 2
-   :caption: Contributing
+    :maxdepth: 2
+    :caption: Project Operations
 
-    ../CODE_OF_CONDUCT.rst
-    ../CONTRIBUTING.rst
+    code_of_conduct
+    contributing
+    versioning
 
 .. toctree::
-   :maxdepth: 2
-   :caption: REPLACEME
+    :maxdepth: 2
+    :caption: Project Details
+
+    config_format
+    resources
 
-   lib/REPLACEME
-   lib/REPLACEME
-   lib/REPLACEME
+***
+API
+***
 
 .. toctree::
    :maxdepth: 2
-   :caption: REPLACEME
+   :caption: Public API
 
-   lib/REPLACEME
-   lib/REPLACEME
-   lib/REPLACEME
+   lib/api/index
 
 .. toctree::
    :maxdepth: 1

doc/lib/api/index.rst

@@ -0,0 +1,9 @@
+Public API
+==========
+
+.. autosummary::
+    :toctree: generated
+
+    pipeformer.deploy
+    pipeformer.identifiers
+    pipeformer.input_handling

doc/lib/internal.rst

@@ -10,4 +10,15 @@ Internal Resources
    :toctree: generated
 
 
-    pipeformer.internal
+    pipeformer.internal.arg_parsing
+    pipeformer.internal.logging_utils
+    pipeformer.internal.resolve
+    pipeformer.internal.structures
+    pipeformer.internal.template_builder
+    pipeformer.internal.util
+    pipeformer.internal.templates
+    pipeformer.internal.templates.codebuild
+    pipeformer.internal.templates.codepipeline
+    pipeformer.internal.templates.core
+    pipeformer.internal.templates.iam
+    pipeformer.internal.templates.inputs

doc/requirements.txt

@@ -1,2 +1,4 @@
 sphinx>=1.3.0
 sphinx_rtd_theme
+sphinx-autodoc-typehints
+sphinxcontrib-spelling

doc/resources.rst

@@ -0,0 +1 @@
+.. include:: ../RESOURCES.rst

doc/spelling_wordlist.txt

@@ -0,0 +1,33 @@
+# Python/dev-tools
+tox
+pyenv
+cpython
+CPython
+
+# Project-specific
+pipeformer
+
+# AWS
+arn
+aws
+buildspec
+cmk
+codebuild
+codepipeline
+iam
+
+# GitHub tags
+wontfix
+
+# General dev words
+arg
+args
+config
+changelog
+github
+plugin
+pre
+runtimes
+util
+utils
+versioning

doc/versioning.rst

@@ -0,0 +1 @@
+.. include:: ../VERSIONING.rst

src/pipeformer/__init__.py

@@ -0,0 +1,43 @@
+# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""pipeformer."""
+import uuid
+from typing import Iterator, Optional
+
+from .deploy import Deployer
+from .identifiers import __version__
+from .internal.arg_parsing import parse_args
+from .internal.logging_utils import setup_logger
+from .internal.structures import Config
+
+__all__ = ("__version__", "cli")
+
+
+def cli(raw_args: Optional[Iterator[str]] = None):
+    """CLI entry point.  Processes arguments, sets up the key provider, and processes requested action.
+
+    :returns: Execution return value intended for ``sys.exit()``
+    """
+    args = parse_args(raw_args)
+
+    setup_logger(args.verbosity, args.quiet)
+
+    # 1. parse config file
+    project = Config.from_file(args.config)
+
+    # TODO: Use a better prefix
+    prefix = "pipeformer-" + str(uuid.uuid4()).split("-")[-1]
+
+    project_deployer = Deployer(project=project, stack_prefix=prefix)
+
+    project_deployer.deploy_standalone()

src/pipeformer/identifiers.py

@@ -0,0 +1,16 @@
+# Copyright 2018 Amazon.com, Inc. or its affiliates. All Rights Reserved.
+#
+# Licensed under the Apache License, Version 2.0 (the "License"). You
+# may not use this file except in compliance with the License. A copy of
+# the License is located at
+#
+# http://aws.amazon.com/apache2.0/
+#
+# or in the "license" file accompanying this file. This file is
+# distributed on an "AS IS" BASIS, WITHOUT WARRANTIES OR CONDITIONS OF
+# ANY KIND, either express or implied. See the License for the specific
+# language governing permissions and limitations under the License.
+"""Unique identifiers used by pipeformer."""
+__all__ = ("__version__", "LOGGER_NAME")
+__version__ = "0.0.1"
+LOGGER_NAME = "pipeformer"

src/pylintrc

@@ -0,0 +1,37 @@
+[MESSAGE CONTROL]
+# Disabling messages that either we don't care about we intentionally break.
+disable =
+    bad-continuation,  # we let black handle this
+    ungrouped-imports,  # we let black handle this
+    no-member,  # breaks with attrs
+    too-few-public-methods,  # does not allow data classes
+    no-else-return,  # we omit this on purpose for brevity where it would add no value
+    attribute-defined-outside-init,  # breaks with attrs_post_init
+    redefined-outer-name,  # we do this on purpose in multiple places
+
+[BASIC]
+# Allow function names up to 50 characters
+function-rgx = [a-z_][a-z0-9_]{2,50}$
+# Allow method names up to 50 characters
+method-rgx = [a-z_][a-z0-9_]{2,50}$
+# Allow class attribute names up to 50 characters
+# Whitelist class attribute names:
+class-attribute-rgx = (([A-Za-z_][A-Za-z0-9_]{2,50}|(__.*__))$)
+# Whitelist attribute names:
+attr-rgx = ([a-z_][a-z0-9_]{2,30}$)
+# Whitelist argument names: iv
+argument-rgx = ([a-z_][a-z0-9_]{2,30}$)|(^iv$)
+# Whitelist variable names: iv
+variable-rgx = ([a-z_][a-z0-9_]{2,30}$)|(^iv$)
+
+[VARIABLES]
+additional-builtins = raw_input
+
+[DESIGN]
+max-args = 10
+
+[FORMAT]
+max-line-length = 120
+
+[REPORTS]
+msg-template = {path}:{line}: [{msg_id}({symbol}), {obj}] {msg}

test/pylintrc

@@ -0,0 +1,29 @@
+[MESSAGES CONTROL]
+# Disabling messages that we either don't care about for tests or are necessary to break for tests.
+disable =
+    bad-continuation,  # we let black handle this
+    ungrouped-imports,  # we let black handle this
+    no-member,  # breaks with attrs
+    invalid-name,  # naming in tests often need to violate many common rules
+    too-few-public-methods,  # common when setting up mock classes
+    redefined-outer-name,  # raises false positives with fixtures
+    missing-docstring,  # we don't write docstrings for tests
+    abstract-class-instantiated,  # we do this on purpose to test that they are enforced
+    duplicate-code,  # unit tests for similar things tend to be similar
+    protected-access,  # raised when calling _ methods
+    abstract-method,  # we do this on purpose to test that they are enforced
+    unused-argument,  # raised when patches are needed but not called
+    no-self-use,  # raised when pytest tests are grouped in classes
+    too-many-public-methods,  # raised when pytest tests are grouped in classes
+
+[VARIABLES]
+additional-builtins = raw_input
+
+[DESIGN]
+max-args = 10
+
+[FORMAT]
+max-line-length = 120
+
+[REPORTS]
+msg-template = {path}:{line}: [{msg_id}({symbol}), {obj}] {msg}

test/requirements.txt

@@ -0,0 +1,4 @@
+mock
+pytest>=3.3.1
+pytest-cov
+pytest-mock

test/source-build-check.sh

@@ -0,0 +1,26 @@
+#!/bin/bash
+# Verify that tests can be successfully run from the source build.
+#
+# NOTE: Humans should not run this file directly. If you want to run this check, run "tox -e sourcebuildcheck".
+
+WORKINGDIR=$1
+DISTDIR=$2
+
+echo "Locating the source build and copying it into the working directory."
+DISTFILE=`ls $DISTDIR/pipeformer-*.tar.gz | tail -1`
+cp $DISTFILE $WORKINGDIR
+DISTFILE=`ls $WORKINGDIR/pipeformer-*.tar.gz | tail -1`
+
+echo "Extracting the source build."
+cd $WORKINGDIR
+tar xzvf $DISTFILE
+rm $DISTFILE
+EXTRACTEDDIR=`ls | tail -1`
+cd $EXTRACTEDDIR
+
+echo "Installing requirements from extracted source build."
+pip install -r test/requirements.txt
+pip install -e .
+
+echo "Running tests from extracted source build."
+pytest --cov pipeformer -m local